Multi-thread remapping, fix parameter annotations for forge, update CleanLoomBinaries to include forge jars, make forge jars respect refresh dependencies.

2026-04-03 05:57:42 -05:00 · 2020-12-28 20:17:19 +08:00
parent e4682d562c
commit e507000686
6 changed files with 232 additions and 129 deletions
--- a/src/main/java/net/fabricmc/loom/providers/ForgeUniversalProvider.java
+++ b/src/main/java/net/fabricmc/loom/providers/ForgeUniversalProvider.java
@@ -27,7 +27,7 @@ package net.fabricmc.loom.providers;
 import java.io.File;
 import java.util.function.Consumer;

-import com.google.common.io.Files;
+import org.apache.commons.io.FileUtils;
 import org.gradle.api.Project;

 import net.fabricmc.loom.util.Constants;
@@ -49,7 +49,7 @@ public class ForgeUniversalProvider extends DependencyProvider {

 		if (!forge.exists() || isRefreshDeps()) {
 			File dep = dependency.resolveFile().orElseThrow(() -> new RuntimeException("Could not resolve Forge"));
-			Files.copy(dep, forge);
+			FileUtils.copyFile(dep, forge);
 		}

 		if (!forgeManifest.exists() || isRefreshDeps()) {
--- a/src/main/java/net/fabricmc/loom/providers/MinecraftMappedProvider.java
+++ b/src/main/java/net/fabricmc/loom/providers/MinecraftMappedProvider.java
@@ -43,9 +43,10 @@ import java.nio.file.FileSystem;
 import java.nio.file.FileSystems;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Map;
+import java.util.*;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 import java.util.function.Consumer;
 import java.util.jar.Attributes;
 import java.util.jar.Manifest;
@@ -116,7 +117,7 @@ public class MinecraftMappedProvider extends DependencyProvider {
 		addDependencies(dependency, postPopulationScheduler);
 	}

-	private void mapMinecraftJar() throws IOException {
+	private void mapMinecraftJar() throws Exception {
 		String fromM = "official";

 		MappingsProvider mappingsProvider = getExtension().getMappingsProvider();
@@ -126,63 +127,76 @@ public class MinecraftMappedProvider extends DependencyProvider {
 		Path outputIntermediary = minecraftIntermediaryJar.toPath();
 		Path outputSrg = minecraftSrgJar == null ? null : minecraftSrgJar.toPath();

+		ExecutorService executor = Executors.newFixedThreadPool(Math.min(10, Math.max(Runtime.getRuntime().availableProcessors() / 2, 1)));
+		List<Future<?>> futures = new LinkedList<>();
+
 		for (String toM : (getExtension().isForge() ? Arrays.asList("named", "intermediary", "srg") : Arrays.asList("named", "intermediary"))) {
-			Path output = "named".equals(toM) ? outputMapped : "srg".equals(toM) ? outputSrg : outputIntermediary;
+			futures.add(executor.submit(() -> {
+				try {
+					Path output = "named".equals(toM) ? outputMapped : "srg".equals(toM) ? outputSrg : outputIntermediary;

-			getProject().getLogger().lifecycle(":remapping minecraft (TinyRemapper, " + fromM + " -> " + toM + ")");
+					getProject().getLogger().lifecycle(":remapping minecraft (TinyRemapper, " + fromM + " -> " + toM + ")");

-			TinyRemapper remapper = getTinyRemapper(fromM, toM);
+					TinyRemapper remapper = getTinyRemapper(fromM, toM);

-			try (OutputConsumerPath outputConsumer = new OutputConsumerPath.Builder(output).build()) {
-				if (getExtension().isForge()) {
-					outputConsumer.addNonClassFiles(input, NonClassCopyMode.FIX_META_INF, remapper);
-				} else {
-					outputConsumer.addNonClassFiles(input);
-				}
-
-				remapper.readClassPath(getRemapClasspath());
-				remapper.readInputs(input);
-				remapper.apply(outputConsumer);
-			} catch (Exception e) {
-				throw new RuntimeException("Failed to remap JAR " + input + " with mappings from " + mappingsProvider.tinyMappings, e);
-			} finally {
-				remapper.finish();
-			}
-
-			if (getExtension().isForge() && !"srg".equals(toM)) {
-				getProject().getLogger().lifecycle(":running forge finalising tasks");
-
-				// TODO: Relocate this to its own class
-				try (FileSystem fs = FileSystems.newFileSystem(URI.create("jar:" + output.toUri()), ImmutableMap.of("create", false))) {
-					Path manifestPath = fs.getPath("META-INF", "MANIFEST.MF");
-					Manifest minecraftManifest;
-					Manifest forgeManifest;
-
-					try (InputStream in = Files.newInputStream(manifestPath)) {
-						minecraftManifest = new Manifest(in);
-					}
-
-					try (InputStream in = new FileInputStream(getExtension().getForgeUniversalProvider().getForgeManifest())) {
-						forgeManifest = new Manifest(in);
-					}
-
-					for (Map.Entry<String, Attributes> forgeEntry : forgeManifest.getEntries().entrySet()) {
-						if (forgeEntry.getKey().endsWith("/")) {
-							minecraftManifest.getEntries().put(forgeEntry.getKey(), forgeEntry.getValue());
+					try (OutputConsumerPath outputConsumer = new OutputConsumerPath.Builder(output).build()) {
+						if (getExtension().isForge()) {
+							outputConsumer.addNonClassFiles(input, NonClassCopyMode.FIX_META_INF, remapper);
+						} else {
+							outputConsumer.addNonClassFiles(input);
 						}
+
+						remapper.readClassPath(getRemapClasspath());
+						remapper.readInputs(input);
+						remapper.apply(outputConsumer);
+					} catch (Exception e) {
+						throw new RuntimeException("Failed to remap JAR " + input + " with mappings from " + mappingsProvider.tinyMappings, e);
+					} finally {
+						remapper.finish();
 					}

-					Files.delete(manifestPath);
+					if (getExtension().isForge() && !"srg".equals(toM)) {
+						getProject().getLogger().info(":running forge finalising tasks");

-					try (OutputStream out = Files.newOutputStream(manifestPath)) {
-						minecraftManifest.write(out);
+						// TODO: Relocate this to its own class
+						try (FileSystem fs = FileSystems.newFileSystem(URI.create("jar:" + output.toUri()), ImmutableMap.of("create", false))) {
+							Path manifestPath = fs.getPath("META-INF", "MANIFEST.MF");
+							Manifest minecraftManifest;
+							Manifest forgeManifest;
+
+							try (InputStream in = Files.newInputStream(manifestPath)) {
+								minecraftManifest = new Manifest(in);
+							}
+
+							try (InputStream in = new FileInputStream(getExtension().getForgeUniversalProvider().getForgeManifest())) {
+								forgeManifest = new Manifest(in);
+							}
+
+							for (Map.Entry<String, Attributes> forgeEntry : forgeManifest.getEntries().entrySet()) {
+								if (forgeEntry.getKey().endsWith("/")) {
+									minecraftManifest.getEntries().put(forgeEntry.getKey(), forgeEntry.getValue());
+								}
+							}
+
+							Files.delete(manifestPath);
+
+							try (OutputStream out = Files.newOutputStream(manifestPath)) {
+								minecraftManifest.write(out);
+							}
+						}
+
+						TinyTree yarnWithSrg = getExtension().getMappingsProvider().getMappingsWithSrg();
+						AtRemapper.remap(getProject().getLogger(), output, yarnWithSrg);
+						CoreModClassRemapper.remapJar(output, yarnWithSrg, getProject().getLogger());
 					}
+				} catch (IOException e) {
+					throw new RuntimeException(e);
 				}
+			}));
+		}

-				TinyTree yarnWithSrg = getExtension().getMappingsProvider().getMappingsWithSrg();
-				AtRemapper.remap(getProject().getLogger(), output, yarnWithSrg);
-				CoreModClassRemapper.remapJar(output, yarnWithSrg, getProject().getLogger());
-			}
+		for (Future<?> future : futures) {
+			future.get();
 		}
 	}

--- a/src/main/java/net/fabricmc/loom/providers/MinecraftPatchedProvider.java
+++ b/src/main/java/net/fabricmc/loom/providers/MinecraftPatchedProvider.java
@@ -24,11 +24,30 @@

 package net.fabricmc.loom.providers;

-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.UncheckedIOException;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.io.Files;
+import de.oceanlabs.mcp.mcinjector.adaptors.ParameterAnnotationFixer;
+import net.fabricmc.loom.util.*;
+import net.fabricmc.loom.util.function.FsPathConsumer;
+import net.fabricmc.tinyremapper.OutputConsumerPath;
+import net.fabricmc.tinyremapper.TinyRemapper;
+import net.minecraftforge.accesstransformer.TransformerProcessor;
+import net.minecraftforge.binarypatcher.ConsoleTool;
+import net.minecraftforge.gradle.mcp.util.MCPRuntime;
+import net.minecraftforge.gradle.mcp.util.MCPWrapper;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.output.NullOutputStream;
+import org.gradle.api.Project;
+import org.gradle.api.logging.Logger;
+import org.gradle.api.plugins.JavaPluginConvention;
+import org.gradle.api.tasks.SourceSet;
+import org.jetbrains.annotations.Nullable;
+import org.objectweb.asm.ClassReader;
+import org.objectweb.asm.ClassVisitor;
+import org.objectweb.asm.ClassWriter;
+import org.objectweb.asm.tree.ClassNode;
+
+import java.io.*;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.nio.file.FileSystem;
@@ -37,31 +56,15 @@ import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.Predicate;
-
-import com.google.common.collect.ImmutableMap;
-import com.google.common.io.Files;
-import net.minecraftforge.accesstransformer.TransformerProcessor;
-import net.minecraftforge.binarypatcher.ConsoleTool;
-import net.minecraftforge.gradle.mcp.util.MCPRuntime;
-import net.minecraftforge.gradle.mcp.util.MCPWrapper;
-import org.apache.commons.io.FileUtils;
-import org.gradle.api.Project;
-import org.gradle.api.logging.Logger;
-import org.gradle.api.plugins.JavaPluginConvention;
-import org.gradle.api.tasks.SourceSet;
-import org.jetbrains.annotations.Nullable;
-
-import net.fabricmc.loom.util.Checksum;
-import net.fabricmc.loom.util.Constants;
-import net.fabricmc.loom.util.DependencyProvider;
-import net.fabricmc.loom.util.JarUtil;
-import net.fabricmc.loom.util.TinyRemapperMappingsHelper;
-import net.fabricmc.loom.util.function.FsPathConsumer;
-import net.fabricmc.tinyremapper.OutputConsumerPath;
-import net.fabricmc.tinyremapper.TinyRemapper;
+import java.util.stream.Stream;

 public class MinecraftPatchedProvider extends DependencyProvider {
 	private File minecraftClientSrgJar;
@@ -80,29 +83,7 @@ public class MinecraftPatchedProvider extends DependencyProvider {
 		super(project);
 	}

-	@Override
-	public void provide(DependencyInfo dependency, Consumer<Runnable> postPopulationScheduler) throws Exception {
-		initFiles();
-
-		if (atDirty || !minecraftClientPatchedJar.exists() || !minecraftServerPatchedJar.exists()) {
-			if (!minecraftClientSrgJar.exists() || !minecraftServerSrgJar.exists()) {
-				createSrgJars(getProject().getLogger());
-			}
-
-			if (atDirty || !minecraftClientPatchedSrgJar.exists() || !minecraftServerPatchedSrgJar.exists()) {
-				patchJars(getProject().getLogger());
-				injectForgeClasses(getProject().getLogger());
-			}
-
-			remapPatchedJars(getProject().getLogger());
-		}
-
-		if (atDirty || !minecraftMergedPatchedJar.exists()) {
-			mergeJars(getProject().getLogger());
-		}
-	}
-
-	private void initFiles() throws IOException {
+	public void initFiles() throws IOException {
 		projectAtHash = new File(getExtension().getProjectPersistentCache(), "at.sha256");

 		SourceSet main = getProject().getConvention().findPlugin(JavaPluginConvention.class).getSourceSets().getByName("main");
@@ -116,7 +97,7 @@ public class MinecraftPatchedProvider extends DependencyProvider {
 			}
 		}

-		if (!projectAtHash.exists()) {
+		if (isRefreshDeps() || !projectAtHash.exists()) {
 			writeAtHash();
 			atDirty = projectAt != null;
 		} else {
@@ -146,6 +127,52 @@ public class MinecraftPatchedProvider extends DependencyProvider {
 		minecraftClientPatchedSrgJar = new File(cache, "minecraft-" + minecraftVersion + "-client-srg" + jarSuffix + ".jar");
 		minecraftServerPatchedSrgJar = new File(cache, "minecraft-" + minecraftVersion + "-server-srg" + jarSuffix + ".jar");
 		minecraftMergedPatchedJar = new File(cache, "minecraft-" + minecraftVersion + "-merged" + jarSuffix + ".jar");
+		
+		if (isRefreshDeps()) {
+			cleanCache();
+		}
+
+		if (!minecraftClientSrgJar.exists() || !minecraftServerSrgJar.exists()) {
+			minecraftClientPatchedJar.delete();
+			minecraftServerPatchedJar.delete();
+		}
+	}
+	
+	public void cleanCache() {
+		for (File file : Arrays.asList(
+				projectAtHash,
+				minecraftClientSrgJar,
+				minecraftServerSrgJar,
+				minecraftClientPatchedSrgJar,
+				minecraftServerPatchedSrgJar,
+				minecraftClientPatchedJar,
+				minecraftServerPatchedJar,
+				minecraftMergedPatchedJar
+		)) {
+			file.delete();
+		}
+	}
+
+	@Override
+	public void provide(DependencyInfo dependency, Consumer<Runnable> postPopulationScheduler) throws Exception {
+		initFiles();
+
+		if (atDirty || !minecraftClientPatchedJar.exists() || !minecraftServerPatchedJar.exists()) {
+			if (!minecraftClientSrgJar.exists() || !minecraftServerSrgJar.exists()) {
+				createSrgJars(getProject().getLogger());
+			}
+
+			if (atDirty || !minecraftClientPatchedSrgJar.exists() || !minecraftServerPatchedSrgJar.exists()) {
+				patchJars(getProject().getLogger());
+				injectForgeClasses(getProject().getLogger());
+			}
+
+			remapPatchedJars(getProject().getLogger());
+		}
+
+		if (atDirty || !minecraftMergedPatchedJar.exists()) {
+			mergeJars(getProject().getLogger());
+		}
 	}

 	private void writeAtHash() throws IOException {
@@ -171,14 +198,38 @@ public class MinecraftPatchedProvider extends DependencyProvider {
 		{
 			MCPRuntime runtime = wrapper.getRuntime(getProject(), "client");
 			File output = runtime.execute(logger, "rename");
-			Files.copy(output, minecraftClientSrgJar);
+			FileUtils.copyFile(output, minecraftClientSrgJar);
 		}

 		// Server
 		{
 			MCPRuntime runtime = wrapper.getRuntime(getProject(), "server");
 			File output = runtime.execute(logger, "rename");
-			Files.copy(output, minecraftServerSrgJar);
+			FileUtils.copyFile(output, minecraftServerSrgJar);
+		}
+	}
+	
+	private void fixParameterAnnotation(File jarFile) throws Exception {
+		getProject().getLogger().info(":fixing parameter annotations for " + jarFile.toString());
+		try (FileSystem fs = FileSystems.newFileSystem(new URI("jar:" + jarFile.toURI()), ImmutableMap.of("create", false))) {
+			for (Path rootDir : fs.getRootDirectories()) {
+				for (Path file : (Iterable<? extends Path>) java.nio.file.Files.walk(rootDir)::iterator) {
+					if (!file.toString().endsWith(".class")) continue;
+					byte[] bytes = java.nio.file.Files.readAllBytes(file);
+					ClassReader reader = new ClassReader(bytes);
+					ClassNode cn = new ClassNode();
+					ClassVisitor visitor = new ParameterAnnotationFixer(cn, null);
+					reader.accept(visitor, 0);
+
+					ClassWriter writer = new ClassWriter(ClassWriter.COMPUTE_MAXS);
+					cn.accept(writer);
+					byte[] out = writer.toByteArray();
+					if (!Arrays.equals(bytes, out)) {
+						java.nio.file.Files.delete(file);
+						java.nio.file.Files.write(file, out);
+					}
+				}
+			}
 		}
 	}

@@ -228,7 +279,7 @@ public class MinecraftPatchedProvider extends DependencyProvider {
 		}
 	}

-	private void remapPatchedJars(Logger logger) throws IOException {
+	private void remapPatchedJars(Logger logger) throws Exception {
 		boolean[] bools = { true, false };
 		Path[] libraries = getExtension()
 				.getMinecraftProvider()
@@ -238,31 +289,44 @@ public class MinecraftPatchedProvider extends DependencyProvider {
 				.map(File::toPath)
 				.toArray(Path[]::new);

+		ExecutorService service = Executors.newFixedThreadPool(2);
+		List<Future<?>> futures = new LinkedList<>();
+		
 		for (boolean isClient : bools) {
-			logger.lifecycle(":remapping minecraft (TinyRemapper, " + (isClient ? "client" : "server") + ", srg -> official)");
+			futures.add(service.submit(() -> {
+				try {
+					logger.lifecycle(":remapping minecraft (TinyRemapper, " + (isClient ? "client" : "server") + ", srg -> official)");

-			TinyRemapper remapper = TinyRemapper.newRemapper()
-					.withMappings(TinyRemapperMappingsHelper.create(getExtension().getMappingsProvider().getMappingsWithSrg(), "srg", "official", true))
-					.renameInvalidLocals(true)
-					.rebuildSourceFilenames(true)
-					.build();
+					TinyRemapper remapper = TinyRemapper.newRemapper()
+							.withMappings(TinyRemapperMappingsHelper.create(getExtension().getMappingsProvider().getMappingsWithSrg(), "srg", "official", true))
+							.renameInvalidLocals(true)
+							.rebuildSourceFilenames(true)
+							.build();

-			Path input = (isClient ? minecraftClientPatchedSrgJar : minecraftServerPatchedSrgJar).toPath();
-			Path output = (isClient ? minecraftClientPatchedJar : minecraftServerPatchedJar).toPath();
+					Path input = (isClient ? minecraftClientPatchedSrgJar : minecraftServerPatchedSrgJar).toPath();
+					Path output = (isClient ? minecraftClientPatchedJar : minecraftServerPatchedJar).toPath();

-			try (OutputConsumerPath outputConsumer = new OutputConsumerPath.Builder(output).build()) {
-				outputConsumer.addNonClassFiles(input);
+					try (OutputConsumerPath outputConsumer = new OutputConsumerPath.Builder(output).build()) {
+						outputConsumer.addNonClassFiles(input);

-				remapper.readClassPath(libraries);
-				remapper.readInputs(input);
-				remapper.apply(outputConsumer);
-			} finally {
-				remapper.finish();
-			}
+						remapper.readClassPath(libraries);
+						remapper.readInputs(input);
+						remapper.apply(outputConsumer);
+					} finally {
+						remapper.finish();
+					}
+				} catch (IOException e) {
+					throw new RuntimeException(e);
+				}
+			}));
+		}
+
+		for (Future<?> future : futures) {
+			future.get();
 		}
 	}

-	private void patchJars(Logger logger) throws IOException {
+	private void patchJars(Logger logger) throws Exception {
 		logger.lifecycle(":patching jars");

 		PatchProvider patchProvider = getExtension().getPatchProvider();
@@ -272,19 +336,31 @@ public class MinecraftPatchedProvider extends DependencyProvider {
 		logger.lifecycle(":copying missing classes into patched jars");
 		copyMissingClasses(minecraftClientSrgJar, minecraftClientPatchedSrgJar);
 		copyMissingClasses(minecraftServerSrgJar, minecraftServerPatchedSrgJar);
+		
+		fixParameterAnnotation(minecraftClientPatchedSrgJar);
+		fixParameterAnnotation(minecraftServerPatchedSrgJar);
 	}

 	private void patchJars(File clean, File output, Path patches) throws IOException {
+		PrintStream previous = System.out;
+		try {
+			System.setOut(new PrintStream(new NullOutputStream()));
+		} catch (SecurityException ignored) {
+		}
 		ConsoleTool.main(new String[]{
 				"--clean", clean.getAbsolutePath(),
 				"--output", output.getAbsolutePath(),
 				"--apply", patches.toAbsolutePath().toString()
 		});
+		try {
+			System.setOut(previous);
+		} catch (SecurityException ignored) {
+		}
 	}

 	private void mergeJars(Logger logger) throws IOException {
 		// FIXME: Hack here: There are no server-only classes so we can just copy the client JAR.
-		Files.copy(minecraftClientPatchedJar, minecraftMergedPatchedJar);
+		FileUtils.copyFile(minecraftClientPatchedJar, minecraftMergedPatchedJar);

 		logger.lifecycle(":copying resources");

--- a/src/main/java/net/fabricmc/loom/task/CleanLoomBinaries.java
+++ b/src/main/java/net/fabricmc/loom/task/CleanLoomBinaries.java
@@ -24,12 +24,14 @@

 package net.fabricmc.loom.task;

-import java.io.IOException;
-
+import net.fabricmc.loom.LoomGradleExtension;
+import net.fabricmc.loom.providers.MinecraftPatchedProvider;
 import org.apache.commons.io.FileUtils;
 import org.gradle.api.tasks.TaskAction;

-import net.fabricmc.loom.LoomGradleExtension;
+import java.io.File;
+import java.io.IOException;
+import java.util.Optional;

 public class CleanLoomBinaries extends AbstractLoomTask {
 	@TaskAction
@@ -38,6 +40,16 @@ public class CleanLoomBinaries extends AbstractLoomTask {
 		extension.getMinecraftProvider().getMergedJar().delete();
 		extension.getMinecraftMappedProvider().getIntermediaryJar().delete();
 		extension.getMinecraftMappedProvider().getMappedJar().delete();
+		Optional.ofNullable(extension.getMinecraftMappedProvider().getSrgJar()).ifPresent(File::delete);
+		if (extension.isForge()) {
+			try {
+				MinecraftPatchedProvider provider = new MinecraftPatchedProvider(getProject());
+				provider.initFiles();
+				provider.cleanCache();
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
+		}

 		try {
 			FileUtils.deleteDirectory(extension.getNativesDirectory());
--- a/src/main/java/net/fabricmc/loom/util/srg/CoreModClassRemapper.java
+++ b/src/main/java/net/fabricmc/loom/util/srg/CoreModClassRemapper.java
@@ -58,7 +58,7 @@ public final class CoreModClassRemapper {
 			Path coremodsJsonPath = fs.getPath("META-INF", "coremods.json");

 			if (Files.notExists(coremodsJsonPath)) {
-				logger.lifecycle(":no coremods in " + jar.getFileName());
+				logger.info(":no coremods in " + jar.getFileName());
 				return;
 			}

@@ -73,7 +73,7 @@ public final class CoreModClassRemapper {
 				Path js = fs.getPath(file);

 				if (Files.exists(js)) {
-					logger.lifecycle(":remapping coremod '" + file + "'");
+					logger.info(":remapping coremod '" + file + "'");
 					remap(js, mappings);
 				} else {
 					logger.warn("Coremod '" + file + "' listed in coremods.json but not found");