From 001f4b58f6b56b6e89be9c8ebfa3bd73868280e4 Mon Sep 17 00:00:00 2001 From: shedaniel Date: Thu, 22 Apr 2021 23:41:35 +0800 Subject: [PATCH 1/6] Performance Optimisations on Project Setup "Sorry I kinda put them into a single commit" - shedaniel - Fix AtRemapper into not using ZipUtil#transformEntries, which is slow as it transverses the whole zip tree - Optimises InnerClassRemapper into not using ZipUtil#iterate, which provides the InputStream of the ZipEntry, data that we don't utilize. Switch to a simple FileSystem to traverse through the list of files - Make MappingsProvider respect mergedv2 yarn files, skipping merging and reordering namespaces in the process. Users that wish to benefit from this should switch to using mergedv2 yarn artifacts - Make MinecraftMappedProvider only read inputs once - Replace (TinyRemapper, official -> named) to (AsmRemapper, intermediary -> named), and chain it to the (TinyRemapper, official -> intermediary) to avoid reading the Minecraft jar again - Multi-thread MinecraftPatchedProvider#fixParameterAnnotation properly On a typical project, these changes can save up to 50% of the import time. Tested on architectury example mod, which brought import times from 1m 51s to 48s. --- .../providers/MinecraftProvider.java | 5 ++ .../forge/MinecraftPatchedProvider.java | 19 ++++-- .../providers/mappings/MappingsProvider.java | 18 ++++- .../minecraft/MinecraftMappedProvider.java | 66 +++++++++++++------ .../tr/CompiledMappedClassRemapper.java | 52 +++++++++++++++ .../minecraft/tr/MappingsCompiled.java | 65 ++++++++++++++++++ .../minecraft/tr/OutputRemappingHandler.java | 51 ++++++++++++++ .../fabricmc/loom/util/ThreadingUtils.java | 30 +++++++++ .../fabricmc/loom/util/srg/AtRemapper.java | 26 +++++--- .../loom/util/srg/InnerClassRemapper.java | 22 ++++--- 10 files changed, 311 insertions(+), 43 deletions(-) create mode 100644 src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/CompiledMappedClassRemapper.java create mode 100644 src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/MappingsCompiled.java create mode 100644 src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/OutputRemappingHandler.java diff --git a/src/main/java/net/fabricmc/loom/configuration/providers/MinecraftProvider.java b/src/main/java/net/fabricmc/loom/configuration/providers/MinecraftProvider.java index 226f3f17..67fa250b 100644 --- a/src/main/java/net/fabricmc/loom/configuration/providers/MinecraftProvider.java +++ b/src/main/java/net/fabricmc/loom/configuration/providers/MinecraftProvider.java @@ -33,6 +33,7 @@ import java.util.Optional; import java.util.function.Consumer; import java.util.zip.ZipError; +import com.google.common.base.Stopwatch; import com.google.common.io.Files; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -112,6 +113,7 @@ public class MinecraftProvider extends DependencyProvider { } catch (ZipError e) { HashedDownloadUtil.delete(minecraftClientJar); HashedDownloadUtil.delete(minecraftServerJar); + HashedDownloadUtil.delete(minecraftMergedJar); getProject().getLogger().error("Could not merge JARs! Deleting source JARs - please re-run the command and move on.", e); throw new RuntimeException(); @@ -232,11 +234,14 @@ public class MinecraftProvider extends DependencyProvider { private void mergeJars(Logger logger) throws IOException { logger.info(":merging jars"); + Stopwatch stopwatch = Stopwatch.createStarted(); try (JarMerger jarMerger = new JarMerger(minecraftClientJar, minecraftServerJar, minecraftMergedJar)) { jarMerger.enableSyntheticParamsOffset(); jarMerger.merge(); } + + logger.info(":merged jars in " + stopwatch); } public File getMergedJar() { diff --git a/src/main/java/net/fabricmc/loom/configuration/providers/forge/MinecraftPatchedProvider.java b/src/main/java/net/fabricmc/loom/configuration/providers/forge/MinecraftPatchedProvider.java index cccfc7f8..b53154d1 100644 --- a/src/main/java/net/fabricmc/loom/configuration/providers/forge/MinecraftPatchedProvider.java +++ b/src/main/java/net/fabricmc/loom/configuration/providers/forge/MinecraftPatchedProvider.java @@ -51,6 +51,7 @@ import java.util.function.Predicate; import java.util.stream.Stream; import com.google.common.base.Predicates; +import com.google.common.base.Stopwatch; import com.google.common.collect.ImmutableMap; import com.google.gson.JsonParser; import de.oceanlabs.mcp.mcinjector.adaptors.ParameterAnnotationFixer; @@ -291,12 +292,16 @@ public class MinecraftPatchedProvider extends DependencyProvider { } private void fixParameterAnnotation(File jarFile) throws Exception { - getProject().getLogger().info(":fixing parameter annotations for " + jarFile.toString()); + getProject().getLogger().info(":fixing parameter annotations for " + jarFile.getAbsolutePath()); + Stopwatch stopwatch = Stopwatch.createStarted(); try (FileSystem fs = FileSystems.newFileSystem(new URI("jar:" + jarFile.toURI()), ImmutableMap.of("create", false))) { - for (Path rootDir : fs.getRootDirectories()) { - for (Path file : (Iterable) Files.walk(rootDir)::iterator) { - if (!file.toString().endsWith(".class")) continue; + ThreadingUtils.TaskCompleter completer = ThreadingUtils.taskCompleter(); + + for (Path file : (Iterable) Files.walk(fs.getPath("/"))::iterator) { + if (!file.toString().endsWith(".class")) continue; + + completer.add(() -> { byte[] bytes = Files.readAllBytes(file); ClassReader reader = new ClassReader(bytes); ClassNode node = new ClassNode(); @@ -311,9 +316,13 @@ public class MinecraftPatchedProvider extends DependencyProvider { Files.delete(file); Files.write(file, out); } - } + }); } + + completer.complete(); } + + getProject().getLogger().info(":fixing parameter annotations for " + jarFile.getAbsolutePath() + " in " + stopwatch); } private void injectForgeClasses(Logger logger) throws IOException { diff --git a/src/main/java/net/fabricmc/loom/configuration/providers/mappings/MappingsProvider.java b/src/main/java/net/fabricmc/loom/configuration/providers/mappings/MappingsProvider.java index 40ae05d0..c7160bea 100644 --- a/src/main/java/net/fabricmc/loom/configuration/providers/mappings/MappingsProvider.java +++ b/src/main/java/net/fabricmc/loom/configuration/providers/mappings/MappingsProvider.java @@ -37,6 +37,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.function.Consumer; @@ -67,6 +68,7 @@ import net.fabricmc.loom.util.DownloadUtil; import net.fabricmc.loom.util.srg.MCPReader; import net.fabricmc.loom.util.srg.SrgMerger; import net.fabricmc.loom.util.srg.SrgNamedWriter; +import net.fabricmc.mapping.reader.v2.TinyMetadata; import net.fabricmc.mapping.reader.v2.TinyV2Factory; import net.fabricmc.mapping.tree.TinyTree; import net.fabricmc.stitch.Command; @@ -274,7 +276,12 @@ public class MappingsProvider extends DependencyProvider { extractUnpickDefinitions(fileSystem, unpickDefinitionsFile.toPath()); } - if (baseMappingsAreV2()) { + if (baseMappingsAreMergedV2()) { + // Architectury Loom Patch + // If a merged tiny v2 mappings file is provided + // Skip merging, should save a lot of time + Files.copy(baseTinyMappings, tinyMappings.toPath(), StandardCopyOption.REPLACE_EXISTING); + } else if (baseMappingsAreV2()) { // These are unmerged v2 mappings mergeAndSaveMappings(project, yarnJar); } else { @@ -337,6 +344,15 @@ public class MappingsProvider extends DependencyProvider { } } + private boolean baseMappingsAreMergedV2() throws IOException { + try (BufferedReader reader = Files.newBufferedReader(baseTinyMappings)) { + TinyMetadata metadata = TinyV2Factory.readMetadata(reader); + return metadata.getNamespaces().containsAll(Arrays.asList("named", "intermediary", "official")); + } catch (IllegalArgumentException e) { + return false; + } + } + public static void extractMappings(FileSystem jar, Path extractTo) throws IOException { Files.copy(jar.getPath("mappings/mappings.tiny"), extractTo, StandardCopyOption.REPLACE_EXISTING); } diff --git a/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/MinecraftMappedProvider.java b/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/MinecraftMappedProvider.java index 88fd4338..994f9577 100644 --- a/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/MinecraftMappedProvider.java +++ b/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/MinecraftMappedProvider.java @@ -29,36 +29,48 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.UncheckedIOException; import java.net.URI; import java.nio.file.FileSystem; import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.function.Consumer; import java.util.jar.Attributes; import java.util.jar.Manifest; +import com.google.common.base.Stopwatch; import com.google.common.collect.ImmutableMap; import org.gradle.api.Project; import org.jetbrains.annotations.Nullable; +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassWriter; +import org.objectweb.asm.commons.ClassRemapper; import net.fabricmc.loom.configuration.DependencyProvider; import net.fabricmc.loom.configuration.providers.MinecraftProvider; import net.fabricmc.loom.configuration.providers.mappings.MappingsProvider; +import net.fabricmc.loom.configuration.providers.minecraft.tr.CompiledMappedClassRemapper; +import net.fabricmc.loom.configuration.providers.minecraft.tr.MappingsCompiled; +import net.fabricmc.loom.configuration.providers.minecraft.tr.OutputRemappingHandler; import net.fabricmc.loom.util.Constants; import net.fabricmc.loom.util.DownloadUtil; +import net.fabricmc.loom.util.FileSystemUtil; +import net.fabricmc.loom.util.ThreadingUtils; import net.fabricmc.loom.util.TinyRemapperMappingsHelper; import net.fabricmc.loom.util.srg.AtRemapper; import net.fabricmc.loom.util.srg.CoreModClassRemapper; import net.fabricmc.loom.util.srg.InnerClassRemapper; import net.fabricmc.mapping.tree.TinyTree; import net.fabricmc.tinyremapper.IMappingProvider; -import net.fabricmc.tinyremapper.NonClassCopyMode; -import net.fabricmc.tinyremapper.OutputConsumerPath; import net.fabricmc.tinyremapper.TinyRemapper; public class MinecraftMappedProvider extends DependencyProvider { @@ -144,30 +156,41 @@ public class MinecraftMappedProvider extends DependencyProvider { TinyRemapper remapper = getTinyRemapper(); remapper.readClassPath(libraries); remapper.prepareClasses(); + remapper.readInputs(input); - for (String toM : getExtension().isForge() ? Arrays.asList("named", "intermediary", "srg") : Arrays.asList("named", "intermediary")) { - Path output = "named".equals(toM) ? outputMapped : "srg".equals(toM) ? outputSrg : outputIntermediary; + Files.copy(input, outputMapped, StandardCopyOption.REPLACE_EXISTING); + FileSystemUtil.FileSystemDelegate systemMapped = FileSystemUtil.getJarFileSystem(outputMapped, true); + ThreadingUtils.TaskCompleter taskCompleter = ThreadingUtils.taskCompleter(); + MappingsCompiled compiledMapped = new MappingsCompiled(getMappings(input, "intermediary", "named")); + for (String toM : getExtension().isForge() ? Arrays.asList("intermediary", "srg") : Collections.singletonList("intermediary")) { + Path output = "srg".equals(toM) ? outputSrg : outputIntermediary; + Stopwatch stopwatch = Stopwatch.createStarted(); getProject().getLogger().lifecycle(":remapping minecraft (TinyRemapper, " + fromM + " -> " + toM + ")"); - Files.deleteIfExists(output); + remapper.replaceMappings(getMappings(input, fromM, toM)); + OutputRemappingHandler.remap(remapper, input, output, toM.equals("intermediary") ? (path, bytes) -> { + try { + Path fsPath = systemMapped.get().getPath(compiledMapped.mapClass(path) + ".class"); - try (OutputConsumerPath outputConsumer = new OutputConsumerPath.Builder(output).build()) { - if (getExtension().isForge()) { - outputConsumer.addNonClassFiles(input, NonClassCopyMode.FIX_META_INF, remapper); - } else { - outputConsumer.addNonClassFiles(input); + if (fsPath.getParent() != null) { + Files.createDirectories(fsPath.getParent()); + } + + taskCompleter.add(() -> { + ClassWriter writer = new ClassWriter(ClassWriter.COMPUTE_MAXS); + ClassRemapper classRemapper = new CompiledMappedClassRemapper(writer, compiledMapped); + new ClassReader(bytes).accept(classRemapper, ClassReader.EXPAND_FRAMES); + + Files.write(fsPath, writer.toByteArray(), StandardOpenOption.CREATE); + }); + } catch (IOException e) { + throw new UncheckedIOException(e); } + } : (path, bytes) -> { + }); - remapper.replaceMappings(getMappings(input, fromM, toM)); - remapper.readInputs(input); - remapper.apply(outputConsumer); - } catch (Exception e) { - Files.deleteIfExists(output); - throw new RuntimeException("Failed to remap JAR " + input + " with mappings from " + mappingsProvider.tinyMappings, e); - } finally { - remapper.removeInput(); - } + getProject().getLogger().info(":remapped minecraft (TinyRemapper, " + fromM + " -> " + toM + ") in " + stopwatch); if (getExtension().isForge() && !"srg".equals(toM)) { getProject().getLogger().info(":running forge finalising tasks"); @@ -206,6 +229,11 @@ public class MinecraftMappedProvider extends DependencyProvider { } remapper.finish(); + + getProject().getLogger().lifecycle(":remapping minecraft (AsmRemapper, intermediary -> named)"); + taskCompleter.complete(); + + systemMapped.close(); } public TinyRemapper getTinyRemapper() throws IOException { diff --git a/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/CompiledMappedClassRemapper.java b/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/CompiledMappedClassRemapper.java new file mode 100644 index 00000000..eff9af23 --- /dev/null +++ b/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/CompiledMappedClassRemapper.java @@ -0,0 +1,52 @@ +package net.fabricmc.loom.configuration.providers.minecraft.tr; + +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.Label; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; +import org.objectweb.asm.commons.ClassRemapper; +import org.objectweb.asm.commons.MethodRemapper; +import org.objectweb.asm.commons.Remapper; + +public class CompiledMappedClassRemapper extends ClassRemapper { + private final MappingsCompiled compiled; + private String lastMethodName; + + public CompiledMappedClassRemapper(ClassVisitor classVisitor, MappingsCompiled compiled) { + super(Opcodes.ASM9, classVisitor, new Remapper() { + @Override + public String map(String internalName) { + return compiled.mapClass(internalName); + } + + @Override + public String mapFieldName(String owner, String name, String descriptor) { + return compiled.mapField(name); + } + + @Override + public String mapMethodName(String owner, String name, String descriptor) { + return compiled.mapMethod(name); + } + }); + + this.compiled = compiled; + } + + @Override + public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) { + lastMethodName = name; + return super.visitMethod(access, name, descriptor, signature, exceptions); + } + + @Override + protected MethodVisitor createMethodRemapper(MethodVisitor methodVisitor) { + return new MethodRemapper(api, methodVisitor, remapper) { + @Override + public void visitLocalVariable(String name, String descriptor, String signature, Label start, Label end, int index) { + super.visitLocalVariable(compiled.mapMethodArg(lastMethodName, index, name), + descriptor, signature, start, end, index); + } + }; + } +} diff --git a/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/MappingsCompiled.java b/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/MappingsCompiled.java new file mode 100644 index 00000000..9f5f96d6 --- /dev/null +++ b/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/MappingsCompiled.java @@ -0,0 +1,65 @@ +package net.fabricmc.loom.configuration.providers.minecraft.tr; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import net.fabricmc.tinyremapper.IMappingProvider; + +public class MappingsCompiled { + private final Map classes; + private final Map fields; + private final Map methods; + private final Map methodArgs; + + public MappingsCompiled(Set mappings) { + this.classes = new HashMap<>(); + this.fields = new HashMap<>(); + this.methods = new HashMap<>(); + this.methodArgs = new HashMap<>(); + + for (IMappingProvider mapping : mappings) { + mapping.load(new IMappingProvider.MappingAcceptor() { + @Override + public void acceptClass(String srcName, String dstName) { + classes.put(srcName, dstName); + } + + @Override + public void acceptMethod(IMappingProvider.Member method, String dstName) { + methods.put(method.name, dstName); + } + + @Override + public void acceptMethodArg(IMappingProvider.Member method, int lvIndex, String dstName) { + methodArgs.put(method.name + "|" + lvIndex, dstName); + } + + @Override + public void acceptMethodVar(IMappingProvider.Member method, int lvIndex, int startOpIdx, int asmIndex, String dstName) { + } + + @Override + public void acceptField(IMappingProvider.Member field, String dstName) { + fields.put(field.name, dstName); + } + }); + } + } + + public String mapClass(String name) { + return classes.getOrDefault(name, name); + } + + public String mapField(String name) { + return fields.getOrDefault(name, name); + } + + public String mapMethod(String name) { + return methods.getOrDefault(name, name); + } + + public String mapMethodArg(String methodName, int lvIndex, String def) { + return methodArgs.getOrDefault(methodName + "|" + lvIndex, def); + } +} diff --git a/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/OutputRemappingHandler.java b/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/OutputRemappingHandler.java new file mode 100644 index 00000000..de0a71e3 --- /dev/null +++ b/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/OutputRemappingHandler.java @@ -0,0 +1,51 @@ +package net.fabricmc.loom.configuration.providers.minecraft.tr; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.function.BiConsumer; + +import net.fabricmc.loom.util.FileSystemUtil; +import net.fabricmc.loom.util.FileSystemUtil.FileSystemDelegate; +import net.fabricmc.loom.util.ThreadingUtils; +import net.fabricmc.tinyremapper.TinyRemapper; + +public class OutputRemappingHandler { + public static void remap(TinyRemapper remapper, Path input, Path output) throws IOException { + remap(remapper, input, output, (path, bytes) -> { + }); + } + + public static void remap(TinyRemapper remapper, Path input, Path output, BiConsumer then) throws IOException { + Files.copy(input, output, StandardCopyOption.REPLACE_EXISTING); + + try (FileSystemDelegate system = FileSystemUtil.getJarFileSystem(output, true)) { + ThreadingUtils.TaskCompleter taskCompleter = ThreadingUtils.taskCompleter(); + + remapper.apply((path, bytes) -> { + if (path.startsWith("/")) path = path.substring(1); + + try { + Path fsPath = system.get().getPath(path + ".class"); + + if (fsPath.getParent() != null) { + Files.createDirectories(fsPath.getParent()); + } + + taskCompleter.add(() -> { + Files.write(fsPath, bytes, StandardOpenOption.CREATE); + }); + + then.accept(path, bytes); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + + taskCompleter.complete(); + } + } +} diff --git a/src/main/java/net/fabricmc/loom/util/ThreadingUtils.java b/src/main/java/net/fabricmc/loom/util/ThreadingUtils.java index 0e58cfa5..79b67929 100644 --- a/src/main/java/net/fabricmc/loom/util/ThreadingUtils.java +++ b/src/main/java/net/fabricmc/loom/util/ThreadingUtils.java @@ -29,6 +29,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.LinkedList; import java.util.List; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -121,4 +122,33 @@ public class ThreadingUtils { public interface UnsafeConsumer { void accept(T value) throws Throwable; } + + public static TaskCompleter taskCompleter() { + return new TaskCompleter(); + } + + public static class TaskCompleter { + List> tasks = new ArrayList<>(); + ExecutorService service = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); + + public TaskCompleter add(UnsafeRunnable job) { + tasks.add(CompletableFuture.runAsync(() -> { + try { + job.run(); + } catch (Throwable throwable) { + throw new RuntimeException(throwable); + } + }, service)); + return this; + } + + public void complete() { + try { + CompletableFuture.allOf(tasks.toArray(new CompletableFuture[0])).get(); + service.shutdownNow(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + } + } } diff --git a/src/main/java/net/fabricmc/loom/util/srg/AtRemapper.java b/src/main/java/net/fabricmc/loom/util/srg/AtRemapper.java index 467c15a4..76b7e87c 100644 --- a/src/main/java/net/fabricmc/loom/util/srg/AtRemapper.java +++ b/src/main/java/net/fabricmc/loom/util/srg/AtRemapper.java @@ -26,17 +26,20 @@ package net.fabricmc.loom.util.srg; import java.io.IOException; import java.io.StringReader; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.List; import java.util.function.UnaryOperator; -import java.util.zip.ZipEntry; +import com.google.common.collect.ImmutableMap; import org.apache.logging.log4j.util.Strings; import org.gradle.api.logging.Logger; -import org.zeroturnaround.zip.ZipUtil; -import org.zeroturnaround.zip.transform.StringZipEntryTransformer; -import org.zeroturnaround.zip.transform.ZipEntryTransformerEntry; import net.fabricmc.loom.util.function.CollectionUtil; import net.fabricmc.mapping.tree.TinyTree; @@ -48,10 +51,13 @@ import net.fabricmc.mapping.tree.TinyTree; */ public final class AtRemapper { public static void remap(Logger logger, Path jar, TinyTree mappings) throws IOException { - ZipUtil.transformEntries(jar.toFile(), new ZipEntryTransformerEntry[] {(new ZipEntryTransformerEntry("META-INF/accesstransformer.cfg", new StringZipEntryTransformer() { - @Override - protected String transform(ZipEntry zipEntry, String input) { - String[] lines = input.split("\n"); + try (FileSystem fs = FileSystems.newFileSystem(URI.create("jar:" + jar.toUri()), ImmutableMap.of("create", false))) { + Path atPath = fs.getPath("META-INF/accesstransformer.cfg"); + + if (Files.exists(atPath)) { + String atContent = new String(Files.readAllBytes(atPath), StandardCharsets.UTF_8); + + String[] lines = atContent.split("\n"); List output = new ArrayList<>(lines.length); for (int i = 0; i < lines.length; i++) { @@ -90,9 +96,9 @@ public final class AtRemapper { output.add(i, String.join(" ", parts)); } - return String.join("\n", output); + Files.write(atPath, String.join("\n", output).getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE); } - }))}); + } } private static String remapDescriptor(String original, UnaryOperator classMappings) { diff --git a/src/main/java/net/fabricmc/loom/util/srg/InnerClassRemapper.java b/src/main/java/net/fabricmc/loom/util/srg/InnerClassRemapper.java index 893dfe1b..d0bc7864 100644 --- a/src/main/java/net/fabricmc/loom/util/srg/InnerClassRemapper.java +++ b/src/main/java/net/fabricmc/loom/util/srg/InnerClassRemapper.java @@ -25,15 +25,15 @@ package net.fabricmc.loom.util.srg; import java.io.IOException; -import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Iterator; import java.util.Map; import java.util.function.BiConsumer; import java.util.stream.Collectors; -import org.zeroturnaround.zip.ZipUtil; - +import net.fabricmc.loom.util.FileSystemUtil; +import net.fabricmc.loom.util.FileSystemUtil.FileSystemDelegate; import net.fabricmc.mapping.tree.ClassDef; import net.fabricmc.mapping.tree.TinyTree; import net.fabricmc.tinyremapper.IMappingProvider; @@ -46,7 +46,7 @@ public class InnerClassRemapper { } private static void remapInnerClass(Path fromJar, TinyTree mappingsWithSrg, String from, String to, BiConsumer action) { - try (InputStream inputStream = Files.newInputStream(fromJar)) { + try (FileSystemDelegate system = FileSystemUtil.getJarFileSystem(fromJar, false)) { Map availableClasses = mappingsWithSrg.getClasses().stream() .collect(Collectors.groupingBy(classDef -> classDef.getName(from), Collectors.reducing( @@ -55,9 +55,15 @@ public class InnerClassRemapper { (first, last) -> last )) ); - ZipUtil.iterate(inputStream, (in, zipEntry) -> { - if (!zipEntry.isDirectory() && zipEntry.getName().contains("$") && zipEntry.getName().endsWith(".class")) { - String className = zipEntry.getName().substring(0, zipEntry.getName().length() - 6); + Iterator iterator = Files.walk(system.get().getPath("/")).iterator(); + + while (iterator.hasNext()) { + Path path = iterator.next(); + String name = path.toString(); + if (name.startsWith("/")) name = name.substring(1); + + if (!Files.isDirectory(path) && name.contains("$") && name.endsWith(".class")) { + String className = name.substring(0, name.length() - 6); if (!availableClasses.containsKey(className)) { String parentName = className.substring(0, className.indexOf('$')); @@ -70,7 +76,7 @@ public class InnerClassRemapper { } } } - }); + } } catch (IOException e) { throw new RuntimeException(e); } From 77e71acd99b735e3f4fea3b15149c2f351c1cdb8 Mon Sep 17 00:00:00 2001 From: shedaniel Date: Sat, 1 May 2021 19:40:27 +0800 Subject: [PATCH 2/6] a --- build.gradle | 27 +++------ .../net/fabricmc/loom/build/JarRemapper.java | 8 +-- .../AccessWidenerJarProcessor.java | 2 +- .../loom/configuration/mods/ModProcessor.java | 9 ++- .../forge/MinecraftPatchedProvider.java | 4 +- .../minecraft/MinecraftMappedProvider.java | 50 ++++++++++++---- .../tr/CompiledMappedClassRemapper.java | 47 ++++++++------- .../minecraft/tr/MappingsCompiled.java | 58 ++++++++++++++++--- .../providers/minecraft/tr/MercuryUtils.java | 18 ++++++ .../minecraft/tr/OutputRemappingHandler.java | 22 ++++--- .../fabricmc/loom/task/AbstractLoomTask.java | 3 +- .../fabricmc/loom/task/AbstractRunTask.java | 3 +- .../loom/task/GenerateSourcesTask.java | 1 - .../net/fabricmc/loom/task/LoomTasks.java | 4 +- .../net/fabricmc/loom/task/RemapJarTask.java | 23 ++++---- .../net/fabricmc/loom/util/Constants.java | 1 + .../fabricmc/loom/util/SourceRemapper.java | 27 ++++----- .../fabricmc/loom/util/ThreadingUtils.java | 20 +++++++ .../loom/util/TinyRemapperMappingsHelper.java | 3 +- .../loom/util/srg/InnerClassRemapper.java | 3 +- .../net/fabricmc/loom/util/srg/MCPReader.java | 12 +++- 21 files changed, 229 insertions(+), 116 deletions(-) create mode 100644 src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/MercuryUtils.java diff --git a/build.gradle b/build.gradle index 39c4769a..532b54a8 100644 --- a/build.gradle +++ b/build.gradle @@ -35,7 +35,6 @@ if (!isSnapshot) { logger.lifecycle(":building plugin v${version}") configurations { - shadowArchitectury forgeInjectShadow forgeInjectCompileClasspath.extendsFrom(forgeInjectShadow) forgeInjectRuntimeClasspath.extendsFrom(forgeInjectShadow) @@ -48,8 +47,9 @@ sourceSets { repositories { maven { url "https://maven.fabricmc.net/" } maven { url "https://maven.minecraftforge.net/" } - maven { url "https://maven.shedaniel.me/" } + maven { url "https://maven.architectury.dev/" } mavenCentral() + mavenLocal() } dependencies { @@ -77,10 +77,7 @@ dependencies { } // tinyfile management - compileOnly ('net.fabricmc:tiny-remapper:0.3.2-architectury.7') - shadowArchitectury ('net.fabricmc:tiny-remapper:0.3.2-architectury.7') { - transitive = false - } + implementation ('dev.architectury:tiny-remapper:1.0.0') implementation ('net.fabricmc:tiny-mappings-parser:0.3.0+build.17') implementation 'net.fabricmc:access-widener:1.0.0' @@ -89,10 +86,7 @@ dependencies { transitive = false } implementation ('org.cadixdev:lorenz-io-proguard:0.5.6') - compileOnly "me.shedaniel.architectury:refmap-remapper:1.0.4" - shadowArchitectury ("me.shedaniel.architectury:refmap-remapper:1.0.4") { - transitive = false - } + implementation "dev.architectury:refmap-remapper:1.0.5" // decompilers implementation ('net.fabricmc:fabric-fernflower:1.3.0') @@ -110,7 +104,7 @@ dependencies { implementation ('org.cadixdev:lorenz-asm:0.5.3') implementation ('net.minecraftforge:accesstransformers:2.2.0') implementation ('de.oceanlabs.mcp:mcinjector:3.8.0') - implementation ('net.md-5:SpecialSource:1.8.3') + implementation ('com.opencsv:opencsv:5.4') // Forge injection forgeInjectShadow ('net.fabricmc:tiny-mappings-parser:0.2.2.14') @@ -141,17 +135,10 @@ jar { classifier 'jar' } -shadowJar { - relocate "net.fabricmc.tinyremapper", "me.shedaniel.architectury.loom.shadowed.impl.net.fabricmc.tinyremapper" - relocate "me.shedaniel.architectury.refmapremapper", "me.shedaniel.architectury.loom.shadowed.impl.me.shedaniel.architectury.refmapremapper" - configurations = [project.configurations.shadowArchitectury] - classifier "shadow" -} - -task mainJar(type: Jar, dependsOn: shadowJar) { +task mainJar(type: Jar, dependsOn: jar) { dependsOn forgeInjectJar - from zipTree(shadowJar.archivePath) + from zipTree(jar.archivePath) from(forgeInjectJar.outputs) { into "inject" rename { "injection.jar" } diff --git a/src/main/java/net/fabricmc/loom/build/JarRemapper.java b/src/main/java/net/fabricmc/loom/build/JarRemapper.java index 6c7f0179..116fb135 100644 --- a/src/main/java/net/fabricmc/loom/build/JarRemapper.java +++ b/src/main/java/net/fabricmc/loom/build/JarRemapper.java @@ -35,16 +35,16 @@ import java.util.Set; import java.util.function.BiConsumer; import java.util.function.BiFunction; +import dev.architectury.tinyremapper.IMappingProvider; +import dev.architectury.tinyremapper.InputTag; +import dev.architectury.tinyremapper.OutputConsumerPath; +import dev.architectury.tinyremapper.TinyRemapper; import org.gradle.api.Action; import org.gradle.api.Project; import org.objectweb.asm.commons.Remapper; import net.fabricmc.loom.util.LoggerFilter; import net.fabricmc.stitch.util.Pair; -import net.fabricmc.tinyremapper.IMappingProvider; -import net.fabricmc.tinyremapper.InputTag; -import net.fabricmc.tinyremapper.OutputConsumerPath; -import net.fabricmc.tinyremapper.TinyRemapper; public class JarRemapper { private final List mappingProviders = new ArrayList<>(); diff --git a/src/main/java/net/fabricmc/loom/configuration/accesswidener/AccessWidenerJarProcessor.java b/src/main/java/net/fabricmc/loom/configuration/accesswidener/AccessWidenerJarProcessor.java index 42f0e7bf..79695d45 100644 --- a/src/main/java/net/fabricmc/loom/configuration/accesswidener/AccessWidenerJarProcessor.java +++ b/src/main/java/net/fabricmc/loom/configuration/accesswidener/AccessWidenerJarProcessor.java @@ -38,6 +38,7 @@ import java.util.zip.ZipEntry; import com.google.gson.Gson; import com.google.gson.JsonObject; +import dev.architectury.tinyremapper.TinyRemapper; import org.gradle.api.Project; import org.objectweb.asm.ClassReader; import org.objectweb.asm.ClassVisitor; @@ -58,7 +59,6 @@ import net.fabricmc.loom.configuration.processors.JarProcessor; import net.fabricmc.loom.configuration.providers.minecraft.MinecraftMappedProvider; import net.fabricmc.loom.util.Checksum; import net.fabricmc.loom.util.Constants; -import net.fabricmc.tinyremapper.TinyRemapper; public class AccessWidenerJarProcessor implements JarProcessor { private AccessWidener accessWidener = new AccessWidener(); diff --git a/src/main/java/net/fabricmc/loom/configuration/mods/ModProcessor.java b/src/main/java/net/fabricmc/loom/configuration/mods/ModProcessor.java index 990c042c..90c71683 100644 --- a/src/main/java/net/fabricmc/loom/configuration/mods/ModProcessor.java +++ b/src/main/java/net/fabricmc/loom/configuration/mods/ModProcessor.java @@ -45,7 +45,11 @@ import java.util.jar.Manifest; import java.util.stream.Collectors; import java.util.zip.ZipEntry; +import com.google.common.base.Stopwatch; import com.google.gson.JsonObject; +import dev.architectury.tinyremapper.InputTag; +import dev.architectury.tinyremapper.OutputConsumerPath; +import dev.architectury.tinyremapper.TinyRemapper; import org.apache.commons.io.IOUtils; import org.gradle.api.Project; import org.objectweb.asm.commons.Remapper; @@ -69,9 +73,6 @@ import net.fabricmc.loom.util.TinyRemapperMappingsHelper; import net.fabricmc.loom.util.srg.AtRemapper; import net.fabricmc.loom.util.srg.CoreModClassRemapper; import net.fabricmc.mapping.tree.TinyTree; -import net.fabricmc.tinyremapper.InputTag; -import net.fabricmc.tinyremapper.OutputConsumerPath; -import net.fabricmc.tinyremapper.TinyRemapper; public class ModProcessor { public static void processMods(Project project, List processList) throws IOException { @@ -150,6 +151,7 @@ public class ModProcessor { List remapList = processList.stream().filter(ModDependencyInfo::requiresRemapping).collect(Collectors.toList()); + Stopwatch stopwatch = Stopwatch.createStarted(); project.getLogger().lifecycle(":remapping " + remapList.size() + " mods (TinyRemapper, " + fromM + " -> " + toM + ")"); TinyTree mappings = extension.isForge() ? mappingsProvider.getMappingsWithSrg() : mappingsProvider.getMappings(); @@ -201,6 +203,7 @@ public class ModProcessor { } remapper.finish(); + project.getLogger().lifecycle(":remapped " + remapList.size() + " mods (TinyRemapper, " + fromM + " -> " + toM + ") in " + stopwatch.stop()); for (ModDependencyInfo info : remapList) { outputConsumerMap.get(info).close(); diff --git a/src/main/java/net/fabricmc/loom/configuration/providers/forge/MinecraftPatchedProvider.java b/src/main/java/net/fabricmc/loom/configuration/providers/forge/MinecraftPatchedProvider.java index b53154d1..170f2b65 100644 --- a/src/main/java/net/fabricmc/loom/configuration/providers/forge/MinecraftPatchedProvider.java +++ b/src/main/java/net/fabricmc/loom/configuration/providers/forge/MinecraftPatchedProvider.java @@ -55,6 +55,8 @@ import com.google.common.base.Stopwatch; import com.google.common.collect.ImmutableMap; import com.google.gson.JsonParser; import de.oceanlabs.mcp.mcinjector.adaptors.ParameterAnnotationFixer; +import dev.architectury.tinyremapper.OutputConsumerPath; +import dev.architectury.tinyremapper.TinyRemapper; import net.minecraftforge.accesstransformer.AccessTransformerEngine; import net.minecraftforge.accesstransformer.TransformerProcessor; import net.minecraftforge.accesstransformer.parser.AccessTransformerList; @@ -88,8 +90,6 @@ import net.fabricmc.loom.util.function.FsPathConsumer; import net.fabricmc.loom.util.srg.InnerClassRemapper; import net.fabricmc.loom.util.srg.SpecialSourceExecutor; import net.fabricmc.mapping.tree.TinyTree; -import net.fabricmc.tinyremapper.OutputConsumerPath; -import net.fabricmc.tinyremapper.TinyRemapper; public class MinecraftPatchedProvider extends DependencyProvider { private final MappingsProvider mappingsProvider; diff --git a/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/MinecraftMappedProvider.java b/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/MinecraftMappedProvider.java index 994f9577..3be5e9f8 100644 --- a/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/MinecraftMappedProvider.java +++ b/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/MinecraftMappedProvider.java @@ -42,13 +42,14 @@ import java.util.Collections; import java.util.HashSet; import java.util.Map; import java.util.Set; -import java.util.concurrent.CompletableFuture; import java.util.function.Consumer; import java.util.jar.Attributes; import java.util.jar.Manifest; import com.google.common.base.Stopwatch; import com.google.common.collect.ImmutableMap; +import dev.architectury.tinyremapper.IMappingProvider; +import dev.architectury.tinyremapper.TinyRemapper; import org.gradle.api.Project; import org.jetbrains.annotations.Nullable; import org.objectweb.asm.ClassReader; @@ -70,8 +71,6 @@ import net.fabricmc.loom.util.srg.AtRemapper; import net.fabricmc.loom.util.srg.CoreModClassRemapper; import net.fabricmc.loom.util.srg.InnerClassRemapper; import net.fabricmc.mapping.tree.TinyTree; -import net.fabricmc.tinyremapper.IMappingProvider; -import net.fabricmc.tinyremapper.TinyRemapper; public class MinecraftMappedProvider extends DependencyProvider { private static final Map JSR_TO_JETBRAINS = new ImmutableMap.Builder() @@ -158,9 +157,37 @@ public class MinecraftMappedProvider extends DependencyProvider { remapper.prepareClasses(); remapper.readInputs(input); - Files.copy(input, outputMapped, StandardCopyOption.REPLACE_EXISTING); + Path tmpAssets = Files.createTempFile("tmpAssets", null); + Files.deleteIfExists(tmpAssets); + tmpAssets.toFile().deleteOnExit(); + + try (FileSystemUtil.FileSystemDelegate inputFs = FileSystemUtil.getJarFileSystem(input, false)) { + ThreadingUtils.TaskCompleter taskCompleter = ThreadingUtils.taskCompleter(); + + try (FileSystemUtil.FileSystemDelegate assetsFs = FileSystemUtil.getJarFileSystem(tmpAssets, true)) { + for (Path path : (Iterable) Files.walk(inputFs.get().getPath("/"))::iterator) { + if (Files.isRegularFile(path) && !path.getFileName().toString().endsWith(".class")) { + Path p = assetsFs.get().getPath(path.toString()); + + if (p.getParent() != null) { + Files.createDirectories(p.getParent()); + } + + taskCompleter.add(() -> { + Files.copy(path, p); + }); + } + } + + taskCompleter.complete(); + } + } + + Files.copy(tmpAssets, outputMapped, StandardCopyOption.REPLACE_EXISTING); FileSystemUtil.FileSystemDelegate systemMapped = FileSystemUtil.getJarFileSystem(outputMapped, true); - ThreadingUtils.TaskCompleter taskCompleter = ThreadingUtils.taskCompleter(); + ThreadingUtils.TaskCompleter mappedRemapper = ThreadingUtils.taskCompleter().onComplete(stopwatch -> { + getProject().getLogger().lifecycle(":remapped minecraft (AsmRemapper, intermediary -> named) in " + stopwatch); + }); MappingsCompiled compiledMapped = new MappingsCompiled(getMappings(input, "intermediary", "named")); for (String toM : getExtension().isForge() ? Arrays.asList("intermediary", "srg") : Collections.singletonList("intermediary")) { @@ -169,15 +196,15 @@ public class MinecraftMappedProvider extends DependencyProvider { getProject().getLogger().lifecycle(":remapping minecraft (TinyRemapper, " + fromM + " -> " + toM + ")"); remapper.replaceMappings(getMappings(input, fromM, toM)); - OutputRemappingHandler.remap(remapper, input, output, toM.equals("intermediary") ? (path, bytes) -> { + OutputRemappingHandler.remap(remapper, tmpAssets, output, toM.equals("intermediary") ? (path, bytes) -> { try { - Path fsPath = systemMapped.get().getPath(compiledMapped.mapClass(path) + ".class"); + Path fsPath = systemMapped.get().getPath(compiledMapped.map(path) + ".class"); if (fsPath.getParent() != null) { Files.createDirectories(fsPath.getParent()); } - taskCompleter.add(() -> { + mappedRemapper.add(() -> { ClassWriter writer = new ClassWriter(ClassWriter.COMPUTE_MAXS); ClassRemapper classRemapper = new CompiledMappedClassRemapper(writer, compiledMapped); new ClassReader(bytes).accept(classRemapper, ClassReader.EXPAND_FRAMES); @@ -187,10 +214,9 @@ public class MinecraftMappedProvider extends DependencyProvider { } catch (IOException e) { throw new UncheckedIOException(e); } - } : (path, bytes) -> { - }); + } : null); - getProject().getLogger().info(":remapped minecraft (TinyRemapper, " + fromM + " -> " + toM + ") in " + stopwatch); + getProject().getLogger().lifecycle(":remapped minecraft (TinyRemapper, " + fromM + " -> " + toM + ") in " + stopwatch); if (getExtension().isForge() && !"srg".equals(toM)) { getProject().getLogger().info(":running forge finalising tasks"); @@ -231,7 +257,7 @@ public class MinecraftMappedProvider extends DependencyProvider { remapper.finish(); getProject().getLogger().lifecycle(":remapping minecraft (AsmRemapper, intermediary -> named)"); - taskCompleter.complete(); + mappedRemapper.complete(); systemMapped.close(); } diff --git a/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/CompiledMappedClassRemapper.java b/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/CompiledMappedClassRemapper.java index eff9af23..e9b484f4 100644 --- a/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/CompiledMappedClassRemapper.java +++ b/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/CompiledMappedClassRemapper.java @@ -1,51 +1,50 @@ package net.fabricmc.loom.configuration.providers.minecraft.tr; +import java.util.HashMap; + +import dev.architectury.tinyremapper.AsmClassRemapper; import org.objectweb.asm.ClassVisitor; -import org.objectweb.asm.Label; import org.objectweb.asm.MethodVisitor; import org.objectweb.asm.Opcodes; import org.objectweb.asm.commons.ClassRemapper; import org.objectweb.asm.commons.MethodRemapper; -import org.objectweb.asm.commons.Remapper; +import org.objectweb.asm.tree.MethodNode; public class CompiledMappedClassRemapper extends ClassRemapper { private final MappingsCompiled compiled; - private String lastMethodName; + private String lastName; + private MethodNode lastMethod; public CompiledMappedClassRemapper(ClassVisitor classVisitor, MappingsCompiled compiled) { - super(Opcodes.ASM9, classVisitor, new Remapper() { - @Override - public String map(String internalName) { - return compiled.mapClass(internalName); - } - - @Override - public String mapFieldName(String owner, String name, String descriptor) { - return compiled.mapField(name); - } - - @Override - public String mapMethodName(String owner, String name, String descriptor) { - return compiled.mapMethod(name); - } - }); + super(Opcodes.ASM9, classVisitor, compiled); this.compiled = compiled; } + @Override + public void visit(int version, int access, String name, String signature, String superName, String[] interfaces) { + lastName = name; + this.compiled.lastSuperClass = superName; + this.compiled.lastInterfaces = interfaces; + super.visit(version, access, name, signature, superName, interfaces); + } + @Override public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) { - lastMethodName = name; + lastMethod = new MethodNode(api, access, name, descriptor, signature, exceptions); return super.visitMethod(access, name, descriptor, signature, exceptions); } @Override protected MethodVisitor createMethodRemapper(MethodVisitor methodVisitor) { - return new MethodRemapper(api, methodVisitor, remapper) { + return new MethodRemapper(api, lastMethod, remapper) { @Override - public void visitLocalVariable(String name, String descriptor, String signature, Label start, Label end, int index) { - super.visitLocalVariable(compiled.mapMethodArg(lastMethodName, index, name), - descriptor, signature, start, end, index); + public void visitEnd() { + lastMethod.localVariables = null; + lastMethod.parameters = null; + AsmClassRemapper.AsmMethodRemapper.processLocals(compiled, lastName, lastMethod, false, true, new HashMap<>()); + lastMethod.visitEnd(); + lastMethod.accept(methodVisitor); } }; } diff --git a/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/MappingsCompiled.java b/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/MappingsCompiled.java index 9f5f96d6..c4ebb67e 100644 --- a/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/MappingsCompiled.java +++ b/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/MappingsCompiled.java @@ -4,19 +4,25 @@ import java.util.HashMap; import java.util.Map; import java.util.Set; -import net.fabricmc.tinyremapper.IMappingProvider; +import com.google.common.collect.HashBasedTable; +import com.google.common.collect.Table; +import dev.architectury.tinyremapper.IMappingProvider; +import dev.architectury.tinyremapper.MethodRemapperProvider; +import org.objectweb.asm.commons.Remapper; -public class MappingsCompiled { +public class MappingsCompiled extends Remapper implements MethodRemapperProvider { private final Map classes; private final Map fields; private final Map methods; - private final Map methodArgs; + private final Table methodArgs; + String lastSuperClass; + String[] lastInterfaces; public MappingsCompiled(Set mappings) { this.classes = new HashMap<>(); this.fields = new HashMap<>(); this.methods = new HashMap<>(); - this.methodArgs = new HashMap<>(); + this.methodArgs = HashBasedTable.create(); for (IMappingProvider mapping : mappings) { mapping.load(new IMappingProvider.MappingAcceptor() { @@ -32,7 +38,7 @@ public class MappingsCompiled { @Override public void acceptMethodArg(IMappingProvider.Member method, int lvIndex, String dstName) { - methodArgs.put(method.name + "|" + lvIndex, dstName); + methodArgs.put(method.owner + "|" + method.name, lvIndex, dstName); } @Override @@ -47,19 +53,55 @@ public class MappingsCompiled { } } - public String mapClass(String name) { + @Override + public String map(String name) { return classes.getOrDefault(name, name); } + @Override + public String mapFieldName(String owner, String name, String descriptor) { + return mapField(name); + } + public String mapField(String name) { return fields.getOrDefault(name, name); } + @Override + public String mapMethodName(String owner, String name, String descriptor) { + return mapMethod(name); + } + public String mapMethod(String name) { return methods.getOrDefault(name, name); } - public String mapMethodArg(String methodName, int lvIndex, String def) { - return methodArgs.getOrDefault(methodName + "|" + lvIndex, def); + public String mapMethodArg(String methodOwner, String methodName, int lvIndex, String def) { + String arg = methodArgs.get(methodOwner + "|" + methodName, lvIndex); + if (arg != null) return arg; + + if (lastSuperClass != null) { + arg = methodArgs.get(lastSuperClass + "|" + methodName, lvIndex); + if (arg != null) return arg; + } + + if (lastInterfaces != null) { + for (String lastInterface : lastInterfaces) { + arg = methodArgs.get(lastInterface + "|" + methodName, lvIndex); + if (arg != null) return arg; + } + } + + return def; + } + + @Override + public String mapMethodVar(String methodOwner, String methodName, String methodDesc, int lvIndex, int startOpIdx, int asmIndex, String name) { + return name; + } + + @Override + public String mapMethodArg(String methodOwner, String methodName, String methodDesc, int lvIndex, String name) { + return mapMethodArg(methodOwner, methodName, lvIndex, name); } } diff --git a/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/MercuryUtils.java b/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/MercuryUtils.java new file mode 100644 index 00000000..44af6b2c --- /dev/null +++ b/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/MercuryUtils.java @@ -0,0 +1,18 @@ +package net.fabricmc.loom.configuration.providers.minecraft.tr; + +import org.cadixdev.mercury.Mercury; + +public class MercuryUtils { + public static Mercury copyMercury(Mercury mercury) { + Mercury copy = new Mercury(); + copy.getClassPath().addAll(mercury.getClassPath()); + copy.getContext().putAll(mercury.getContext()); + copy.getProcessors().addAll(mercury.getProcessors()); + copy.setEncoding(mercury.getEncoding()); + copy.setFlexibleAnonymousClassMemberLookups(mercury.isFlexibleAnonymousClassMemberLookups()); + copy.setGracefulClasspathChecks(mercury.isGracefulClasspathChecks()); + copy.setGracefulJavadocClasspathChecks(mercury.isGracefulJavadocClasspathChecks()); + copy.setSourceCompatibility(mercury.getSourceCompatibility()); + return copy; + } +} diff --git a/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/OutputRemappingHandler.java b/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/OutputRemappingHandler.java index de0a71e3..d3a6c69d 100644 --- a/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/OutputRemappingHandler.java +++ b/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/OutputRemappingHandler.java @@ -8,19 +8,25 @@ import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.util.function.BiConsumer; +import dev.architectury.tinyremapper.InputTag; +import dev.architectury.tinyremapper.TinyRemapper; + import net.fabricmc.loom.util.FileSystemUtil; import net.fabricmc.loom.util.FileSystemUtil.FileSystemDelegate; import net.fabricmc.loom.util.ThreadingUtils; -import net.fabricmc.tinyremapper.TinyRemapper; public class OutputRemappingHandler { - public static void remap(TinyRemapper remapper, Path input, Path output) throws IOException { - remap(remapper, input, output, (path, bytes) -> { + public static void remap(TinyRemapper remapper, Path assets, Path output) throws IOException { + remap(remapper, assets, output, (path, bytes) -> { }); } - public static void remap(TinyRemapper remapper, Path input, Path output, BiConsumer then) throws IOException { - Files.copy(input, output, StandardCopyOption.REPLACE_EXISTING); + public static void remap(TinyRemapper remapper, Path assets, Path output, BiConsumer then) throws IOException { + remap(remapper, assets, output, then, (InputTag[]) null); + } + + public static void remap(TinyRemapper remapper, Path assets, Path output, BiConsumer then, InputTag... inputTags) throws IOException { + Files.copy(assets, output, StandardCopyOption.REPLACE_EXISTING); try (FileSystemDelegate system = FileSystemUtil.getJarFileSystem(output, true)) { ThreadingUtils.TaskCompleter taskCompleter = ThreadingUtils.taskCompleter(); @@ -39,11 +45,13 @@ public class OutputRemappingHandler { Files.write(fsPath, bytes, StandardOpenOption.CREATE); }); - then.accept(path, bytes); + if (then != null) { + then.accept(path, bytes); + } } catch (IOException e) { throw new UncheckedIOException(e); } - }); + }, inputTags); taskCompleter.complete(); } diff --git a/src/main/java/net/fabricmc/loom/task/AbstractLoomTask.java b/src/main/java/net/fabricmc/loom/task/AbstractLoomTask.java index 9dada304..e1f4db16 100644 --- a/src/main/java/net/fabricmc/loom/task/AbstractLoomTask.java +++ b/src/main/java/net/fabricmc/loom/task/AbstractLoomTask.java @@ -28,10 +28,11 @@ import org.gradle.api.DefaultTask; import org.gradle.api.tasks.Internal; import net.fabricmc.loom.LoomGradleExtension; +import net.fabricmc.loom.util.Constants; public abstract class AbstractLoomTask extends DefaultTask { public AbstractLoomTask() { - setGroup("fabric"); + setGroup(Constants.TASK_CATEGORY); } @Internal diff --git a/src/main/java/net/fabricmc/loom/task/AbstractRunTask.java b/src/main/java/net/fabricmc/loom/task/AbstractRunTask.java index 34f2dac8..111d074a 100644 --- a/src/main/java/net/fabricmc/loom/task/AbstractRunTask.java +++ b/src/main/java/net/fabricmc/loom/task/AbstractRunTask.java @@ -35,13 +35,14 @@ import org.gradle.api.Project; import org.gradle.api.tasks.JavaExec; import net.fabricmc.loom.configuration.ide.RunConfig; +import net.fabricmc.loom.util.Constants; public abstract class AbstractRunTask extends JavaExec { private final RunConfig config; public AbstractRunTask(Function configProvider) { super(); - setGroup("fabric"); + setGroup(Constants.TASK_CATEGORY); this.config = configProvider.apply(getProject()); setClasspath(config.sourceSet.getRuntimeClasspath()); diff --git a/src/main/java/net/fabricmc/loom/task/GenerateSourcesTask.java b/src/main/java/net/fabricmc/loom/task/GenerateSourcesTask.java index 25d6fe47..eb028d40 100644 --- a/src/main/java/net/fabricmc/loom/task/GenerateSourcesTask.java +++ b/src/main/java/net/fabricmc/loom/task/GenerateSourcesTask.java @@ -56,7 +56,6 @@ public class GenerateSourcesTask extends AbstractLoomTask { public GenerateSourcesTask(LoomDecompiler decompiler) { this.decompiler = decompiler; - setGroup("fabric"); getOutputs().upToDateWhen((o) -> false); } diff --git a/src/main/java/net/fabricmc/loom/task/LoomTasks.java b/src/main/java/net/fabricmc/loom/task/LoomTasks.java index 1b1de6a7..f61ec8f3 100644 --- a/src/main/java/net/fabricmc/loom/task/LoomTasks.java +++ b/src/main/java/net/fabricmc/loom/task/LoomTasks.java @@ -35,6 +35,7 @@ import net.fabricmc.loom.api.decompilers.LoomDecompiler; import net.fabricmc.loom.configuration.ide.RunConfigSettings; import net.fabricmc.loom.configuration.providers.mappings.MappingsProvider; import net.fabricmc.loom.decompilers.fernflower.FabricFernFlowerDecompiler; +import net.fabricmc.loom.util.Constants; public final class LoomTasks { private LoomTasks() { @@ -50,7 +51,7 @@ public final class LoomTasks { tasks.register("remapJar", RemapJarTask.class, t -> { t.setDescription("Remaps the built project jar to intermediary mappings."); - t.setGroup("fabric"); + t.setGroup(Constants.TASK_CATEGORY); }); tasks.register("downloadAssets", DownloadAssetsTask.class, t -> t.setDescription("Downloads required assets for Fabric.")); @@ -97,7 +98,6 @@ public final class LoomTasks { tasks.register(taskName, RunGameTask.class, config).configure(t -> { t.setDescription("Starts the '" + config.getConfigName() + "' run configuration"); - t.setGroup("fabric"); if (config.getEnvironment().equals("client")) { t.dependsOn("downloadAssets"); diff --git a/src/main/java/net/fabricmc/loom/task/RemapJarTask.java b/src/main/java/net/fabricmc/loom/task/RemapJarTask.java index cf03e57c..723c6ec4 100644 --- a/src/main/java/net/fabricmc/loom/task/RemapJarTask.java +++ b/src/main/java/net/fabricmc/loom/task/RemapJarTask.java @@ -39,21 +39,24 @@ import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; -import java.util.Set; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonObject; -import me.shedaniel.architectury.refmapremapper.RefmapRemapper; -import me.shedaniel.architectury.refmapremapper.remapper.MappingsRemapper; -import me.shedaniel.architectury.refmapremapper.remapper.ReferenceRemapper; -import me.shedaniel.architectury.refmapremapper.remapper.Remapper; -import me.shedaniel.architectury.refmapremapper.remapper.SimpleReferenceRemapper; +import dev.architectury.refmapremapper.RefmapRemapper; +import dev.architectury.refmapremapper.remapper.MappingsRemapper; +import dev.architectury.refmapremapper.remapper.ReferenceRemapper; +import dev.architectury.refmapremapper.remapper.Remapper; +import dev.architectury.refmapremapper.remapper.SimpleReferenceRemapper; +import dev.architectury.tinyremapper.IMappingProvider; +import dev.architectury.tinyremapper.TinyRemapper; +import dev.architectury.tinyremapper.TinyUtils; import org.gradle.api.Action; import org.gradle.api.Project; import org.gradle.api.artifacts.Configuration; @@ -72,27 +75,23 @@ import net.fabricmc.loom.LoomGradleExtension; import net.fabricmc.loom.build.JarRemapper; import net.fabricmc.loom.build.MixinRefmapHelper; import net.fabricmc.loom.build.nesting.EmptyNestedJarProvider; -import net.fabricmc.loom.build.nesting.NestedJarPathProvider; import net.fabricmc.loom.build.nesting.JarNester; import net.fabricmc.loom.build.nesting.MergedNestedJarProvider; import net.fabricmc.loom.build.nesting.NestedDependencyProvider; +import net.fabricmc.loom.build.nesting.NestedJarPathProvider; import net.fabricmc.loom.build.nesting.NestedJarProvider; import net.fabricmc.loom.configuration.accesswidener.AccessWidenerJarProcessor; import net.fabricmc.loom.configuration.providers.mappings.MappingsProvider; import net.fabricmc.loom.util.Constants; import net.fabricmc.loom.util.SourceRemapper; import net.fabricmc.loom.util.TinyRemapperMappingsHelper; -import net.fabricmc.loom.util.gradle.GradleSupport; import net.fabricmc.loom.util.ZipReprocessorUtil; +import net.fabricmc.loom.util.gradle.GradleSupport; import net.fabricmc.mapping.tree.ClassDef; import net.fabricmc.mapping.tree.FieldDef; import net.fabricmc.mapping.tree.MethodDef; import net.fabricmc.mapping.tree.TinyTree; import net.fabricmc.stitch.util.Pair; -import net.fabricmc.tinyremapper.IMappingProvider; -import net.fabricmc.tinyremapper.OutputConsumerPath; -import net.fabricmc.tinyremapper.TinyRemapper; -import net.fabricmc.tinyremapper.TinyUtils; public class RemapJarTask extends Jar { private final RegularFileProperty input; diff --git a/src/main/java/net/fabricmc/loom/util/Constants.java b/src/main/java/net/fabricmc/loom/util/Constants.java index cc33b6fe..046a2759 100644 --- a/src/main/java/net/fabricmc/loom/util/Constants.java +++ b/src/main/java/net/fabricmc/loom/util/Constants.java @@ -40,6 +40,7 @@ public class Constants { public static final String VERSION_MANIFESTS = "https://launchermeta.mojang.com/mc/game/version_manifest_v2.json"; public static final String SYSTEM_ARCH = System.getProperty("os.arch").equals("64") ? "64" : "32"; + public static final String TASK_CATEGORY = "loom"; public static final int ASM_VERSION = Opcodes.ASM9; diff --git a/src/main/java/net/fabricmc/loom/util/SourceRemapper.java b/src/main/java/net/fabricmc/loom/util/SourceRemapper.java index f9573bf5..22ba7f70 100644 --- a/src/main/java/net/fabricmc/loom/util/SourceRemapper.java +++ b/src/main/java/net/fabricmc/loom/util/SourceRemapper.java @@ -33,6 +33,7 @@ import java.util.List; import java.util.Set; import java.util.function.Consumer; +import com.google.common.base.Stopwatch; import org.cadixdev.lorenz.MappingSet; import org.cadixdev.mercury.Mercury; import org.cadixdev.mercury.remapper.MercuryRemapper; @@ -45,7 +46,7 @@ import net.fabricmc.loom.LoomGradleExtension; import net.fabricmc.loom.configuration.RemappedConfigurationEntry; import net.fabricmc.loom.configuration.providers.LaunchProvider; import net.fabricmc.loom.configuration.providers.mappings.MappingsProvider; -import net.fabricmc.loom.util.gradle.ProgressLogger; +import net.fabricmc.loom.configuration.providers.minecraft.tr.MercuryUtils; import net.fabricmc.lorenztiny.TinyMappingsReader; import net.fabricmc.mapping.tree.TinyTree; import net.fabricmc.stitch.util.StitchUtil; @@ -54,7 +55,7 @@ public class SourceRemapper { private final Project project; private String from; private String to; - private final List> remapTasks = new ArrayList<>(); + private final List> remapTasks = new ArrayList<>(); private Mercury mercury; @@ -80,10 +81,9 @@ public class SourceRemapper { } public void scheduleRemapSources(File source, File destination, boolean reproducibleFileOrder, boolean preserveFileTimestamps) { - remapTasks.add((logger) -> { + remapTasks.add((mercury) -> { try { - logger.progress("remapping sources - " + source.getName()); - remapSourcesInner(source, destination); + remapSourcesInner(mercury, source, destination); ZipReprocessorUtil.reprocessZip(destination, reproducibleFileOrder, preserveFileTimestamps); // Set the remapped sources creation date to match the sources if we're likely succeeded in making it @@ -101,22 +101,21 @@ public class SourceRemapper { return; } - project.getLogger().lifecycle(":remapping sources"); + Stopwatch stopwatch = Stopwatch.createStarted(); + project.getLogger().lifecycle(":remapping " + remapTasks.size() + " sources"); - ProgressLogger progressLogger = ProgressLogger.getProgressFactory(project, SourceRemapper.class.getName()); - progressLogger.start("Remapping dependency sources", "sources"); + Mercury mercury = getMercuryInstance(); + ThreadingUtils.run(remapTasks, consumer -> consumer.accept(MercuryUtils.copyMercury(mercury))); - remapTasks.forEach(consumer -> consumer.accept(progressLogger)); - - progressLogger.completed(); + project.getLogger().lifecycle(":remapped " + remapTasks.size() + " sources in " + stopwatch.stop()); // TODO: FIXME - WORKAROUND https://github.com/FabricMC/fabric-loom/issues/45 System.gc(); } - private void remapSourcesInner(File source, File destination) throws Exception { + private void remapSourcesInner(Mercury mercury, File source, File destination) throws Exception { + Stopwatch stopwatch = Stopwatch.createStarted(); project.getLogger().info(":remapping source jar " + source.getName() + " from " + from + " to " + to); - Mercury mercury = getMercuryInstance(); if (source.equals(destination)) { if (source.isDirectory()) { @@ -166,6 +165,8 @@ public class SourceRemapper { if (isSrcTmp) { Files.walkFileTree(srcPath, new DeletingFileVisitor()); } + + project.getLogger().info(":remapped source jar " + source.getName() + " from " + from + " to " + to + " in " + stopwatch.stop()); } private Mercury getMercuryInstance() { diff --git a/src/main/java/net/fabricmc/loom/util/ThreadingUtils.java b/src/main/java/net/fabricmc/loom/util/ThreadingUtils.java index 79b67929..f0c9ff9c 100644 --- a/src/main/java/net/fabricmc/loom/util/ThreadingUtils.java +++ b/src/main/java/net/fabricmc/loom/util/ThreadingUtils.java @@ -34,9 +34,12 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Collectors; +import com.google.common.base.Stopwatch; + public class ThreadingUtils { public static void run(Collection values, UnsafeConsumer action) { run(values.stream() @@ -128,10 +131,16 @@ public class ThreadingUtils { } public static class TaskCompleter { + Stopwatch stopwatch = Stopwatch.createUnstarted(); List> tasks = new ArrayList<>(); ExecutorService service = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); + List> completionListener = new ArrayList<>(); public TaskCompleter add(UnsafeRunnable job) { + if (!stopwatch.isRunning()) { + stopwatch.start(); + } + tasks.add(CompletableFuture.runAsync(() -> { try { job.run(); @@ -139,6 +148,12 @@ public class ThreadingUtils { throw new RuntimeException(throwable); } }, service)); + + return this; + } + + public TaskCompleter onComplete(Consumer consumer) { + completionListener.add(consumer); return this; } @@ -146,6 +161,11 @@ public class ThreadingUtils { try { CompletableFuture.allOf(tasks.toArray(new CompletableFuture[0])).get(); service.shutdownNow(); + stopwatch.stop(); + + for (Consumer consumer : completionListener) { + consumer.accept(stopwatch); + } } catch (InterruptedException | ExecutionException e) { throw new RuntimeException(e); } diff --git a/src/main/java/net/fabricmc/loom/util/TinyRemapperMappingsHelper.java b/src/main/java/net/fabricmc/loom/util/TinyRemapperMappingsHelper.java index b9964020..92e5bf94 100644 --- a/src/main/java/net/fabricmc/loom/util/TinyRemapperMappingsHelper.java +++ b/src/main/java/net/fabricmc/loom/util/TinyRemapperMappingsHelper.java @@ -24,13 +24,14 @@ package net.fabricmc.loom.util; +import dev.architectury.tinyremapper.IMappingProvider; + import net.fabricmc.mapping.tree.ClassDef; import net.fabricmc.mapping.tree.FieldDef; import net.fabricmc.mapping.tree.LocalVariableDef; import net.fabricmc.mapping.tree.MethodDef; import net.fabricmc.mapping.tree.ParameterDef; import net.fabricmc.mapping.tree.TinyTree; -import net.fabricmc.tinyremapper.IMappingProvider; public class TinyRemapperMappingsHelper { private TinyRemapperMappingsHelper() { } diff --git a/src/main/java/net/fabricmc/loom/util/srg/InnerClassRemapper.java b/src/main/java/net/fabricmc/loom/util/srg/InnerClassRemapper.java index d0bc7864..021410b6 100644 --- a/src/main/java/net/fabricmc/loom/util/srg/InnerClassRemapper.java +++ b/src/main/java/net/fabricmc/loom/util/srg/InnerClassRemapper.java @@ -32,11 +32,12 @@ import java.util.Map; import java.util.function.BiConsumer; import java.util.stream.Collectors; +import dev.architectury.tinyremapper.IMappingProvider; + import net.fabricmc.loom.util.FileSystemUtil; import net.fabricmc.loom.util.FileSystemUtil.FileSystemDelegate; import net.fabricmc.mapping.tree.ClassDef; import net.fabricmc.mapping.tree.TinyTree; -import net.fabricmc.tinyremapper.IMappingProvider; public class InnerClassRemapper { public static IMappingProvider of(Path fromJar, TinyTree mappingsWithSrg, String from, String to) throws IOException { diff --git a/src/main/java/net/fabricmc/loom/util/srg/MCPReader.java b/src/main/java/net/fabricmc/loom/util/srg/MCPReader.java index 7639eb00..777a9007 100644 --- a/src/main/java/net/fabricmc/loom/util/srg/MCPReader.java +++ b/src/main/java/net/fabricmc/loom/util/srg/MCPReader.java @@ -39,7 +39,8 @@ import java.util.Objects; import java.util.regex.Matcher; import java.util.regex.Pattern; -import au.com.bytecode.opencsv.CSVReader; +import com.opencsv.CSVReader; +import com.opencsv.exceptions.CsvValidationException; import org.cadixdev.lorenz.MappingSet; import org.cadixdev.lorenz.io.srg.tsrg.TSrgReader; import org.cadixdev.lorenz.model.ClassMapping; @@ -71,7 +72,12 @@ public class MCPReader { Map intermediaryToMCPMap = createIntermediaryToMCPMap(intermediaryTiny, srgTokens); Map intermediaryToDocsMap = new HashMap<>(); Map> intermediaryToParamsMap = new HashMap<>(); - injectMcp(mcpJar, intermediaryToMCPMap, intermediaryToDocsMap, intermediaryToParamsMap); + + try { + injectMcp(mcpJar, intermediaryToMCPMap, intermediaryToDocsMap, intermediaryToParamsMap); + } catch (CsvValidationException e) { + throw new RuntimeException(e); + } mergeTokensIntoIntermediary(intermediaryTiny, intermediaryToMCPMap, intermediaryToDocsMap, intermediaryToParamsMap); return intermediaryTiny; @@ -186,7 +192,7 @@ public class MCPReader { } private void injectMcp(Path mcpJar, Map intermediaryToSrgMap, Map intermediaryToDocsMap, Map> intermediaryToParamsMap) - throws IOException { + throws IOException, CsvValidationException { Map> srgToIntermediary = inverseMap(intermediaryToSrgMap); Map> simpleSrgToIntermediary = new HashMap<>(); Pattern methodPattern = Pattern.compile("(func_\\d*)_.*"); From 6f2ec219e70b9c00d18090b2355f18eb81329af1 Mon Sep 17 00:00:00 2001 From: shedaniel Date: Sat, 1 May 2021 21:16:42 +0800 Subject: [PATCH 3/6] Use TinyRemapper directly --- .../minecraft/MinecraftMappedProvider.java | 78 +++++-------- .../tr/CompiledMappedClassRemapper.java | 51 --------- .../minecraft/tr/MappingsCompiled.java | 107 ------------------ .../minecraft/tr/OutputRemappingHandler.java | 3 +- .../fabricmc/loom/util/ThreadingUtils.java | 12 +- 5 files changed, 38 insertions(+), 213 deletions(-) delete mode 100644 src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/CompiledMappedClassRemapper.java delete mode 100644 src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/MappingsCompiled.java diff --git a/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/MinecraftMappedProvider.java b/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/MinecraftMappedProvider.java index 3be5e9f8..90af6c80 100644 --- a/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/MinecraftMappedProvider.java +++ b/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/MinecraftMappedProvider.java @@ -29,17 +29,15 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.io.UncheckedIOException; import java.net.URI; import java.nio.file.FileSystem; import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.StandardCopyOption; -import java.nio.file.StandardOpenOption; +import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.Consumer; @@ -52,15 +50,10 @@ import dev.architectury.tinyremapper.IMappingProvider; import dev.architectury.tinyremapper.TinyRemapper; import org.gradle.api.Project; import org.jetbrains.annotations.Nullable; -import org.objectweb.asm.ClassReader; -import org.objectweb.asm.ClassWriter; -import org.objectweb.asm.commons.ClassRemapper; import net.fabricmc.loom.configuration.DependencyProvider; import net.fabricmc.loom.configuration.providers.MinecraftProvider; import net.fabricmc.loom.configuration.providers.mappings.MappingsProvider; -import net.fabricmc.loom.configuration.providers.minecraft.tr.CompiledMappedClassRemapper; -import net.fabricmc.loom.configuration.providers.minecraft.tr.MappingsCompiled; import net.fabricmc.loom.configuration.providers.minecraft.tr.OutputRemappingHandler; import net.fabricmc.loom.util.Constants; import net.fabricmc.loom.util.DownloadUtil; @@ -155,27 +148,38 @@ public class MinecraftMappedProvider extends DependencyProvider { TinyRemapper remapper = getTinyRemapper(); remapper.readClassPath(libraries); remapper.prepareClasses(); - remapper.readInputs(input); Path tmpAssets = Files.createTempFile("tmpAssets", null); Files.deleteIfExists(tmpAssets); tmpAssets.toFile().deleteOnExit(); + List inputByteList = new ArrayList<>(); + try (FileSystemUtil.FileSystemDelegate inputFs = FileSystemUtil.getJarFileSystem(input, false)) { ThreadingUtils.TaskCompleter taskCompleter = ThreadingUtils.taskCompleter(); try (FileSystemUtil.FileSystemDelegate assetsFs = FileSystemUtil.getJarFileSystem(tmpAssets, true)) { for (Path path : (Iterable) Files.walk(inputFs.get().getPath("/"))::iterator) { - if (Files.isRegularFile(path) && !path.getFileName().toString().endsWith(".class")) { - Path p = assetsFs.get().getPath(path.toString()); + if (Files.isRegularFile(path)) { + if (path.getFileName().toString().endsWith(".class")) { + taskCompleter.add(() -> { + byte[] bytes = Files.readAllBytes(path); - if (p.getParent() != null) { - Files.createDirectories(p.getParent()); + synchronized (inputByteList) { + inputByteList.add(bytes); + } + }); + } else { + Path p = assetsFs.get().getPath(path.toString()); + + if (p.getParent() != null) { + Files.createDirectories(p.getParent()); + } + + taskCompleter.add(() -> { + Files.copy(path, p); + }); } - - taskCompleter.add(() -> { - Files.copy(path, p); - }); } } @@ -183,40 +187,19 @@ public class MinecraftMappedProvider extends DependencyProvider { } } - Files.copy(tmpAssets, outputMapped, StandardCopyOption.REPLACE_EXISTING); - FileSystemUtil.FileSystemDelegate systemMapped = FileSystemUtil.getJarFileSystem(outputMapped, true); - ThreadingUtils.TaskCompleter mappedRemapper = ThreadingUtils.taskCompleter().onComplete(stopwatch -> { - getProject().getLogger().lifecycle(":remapped minecraft (AsmRemapper, intermediary -> named) in " + stopwatch); - }); - MappingsCompiled compiledMapped = new MappingsCompiled(getMappings(input, "intermediary", "named")); + byte[][] inputBytes = inputByteList.toArray(new byte[0][0]); - for (String toM : getExtension().isForge() ? Arrays.asList("intermediary", "srg") : Collections.singletonList("intermediary")) { - Path output = "srg".equals(toM) ? outputSrg : outputIntermediary; + for (String toM : getExtension().isForge() ? Arrays.asList("intermediary", "srg", "named") : Arrays.asList("intermediary", "named")) { + Path output = "named".equals(toM) ? outputMapped : "srg".equals(toM) ? outputSrg : outputIntermediary; Stopwatch stopwatch = Stopwatch.createStarted(); getProject().getLogger().lifecycle(":remapping minecraft (TinyRemapper, " + fromM + " -> " + toM + ")"); + remapper.readInputs(inputBytes); remapper.replaceMappings(getMappings(input, fromM, toM)); - OutputRemappingHandler.remap(remapper, tmpAssets, output, toM.equals("intermediary") ? (path, bytes) -> { - try { - Path fsPath = systemMapped.get().getPath(compiledMapped.map(path) + ".class"); - - if (fsPath.getParent() != null) { - Files.createDirectories(fsPath.getParent()); - } - - mappedRemapper.add(() -> { - ClassWriter writer = new ClassWriter(ClassWriter.COMPUTE_MAXS); - ClassRemapper classRemapper = new CompiledMappedClassRemapper(writer, compiledMapped); - new ClassReader(bytes).accept(classRemapper, ClassReader.EXPAND_FRAMES); - - Files.write(fsPath, writer.toByteArray(), StandardOpenOption.CREATE); - }); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } : null); + OutputRemappingHandler.remap(remapper, tmpAssets, output); getProject().getLogger().lifecycle(":remapped minecraft (TinyRemapper, " + fromM + " -> " + toM + ") in " + stopwatch); + remapper.removeInput(); if (getExtension().isForge() && !"srg".equals(toM)) { getProject().getLogger().info(":running forge finalising tasks"); @@ -255,11 +238,6 @@ public class MinecraftMappedProvider extends DependencyProvider { } remapper.finish(); - - getProject().getLogger().lifecycle(":remapping minecraft (AsmRemapper, intermediary -> named)"); - mappedRemapper.complete(); - - systemMapped.close(); } public TinyRemapper getTinyRemapper() throws IOException { diff --git a/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/CompiledMappedClassRemapper.java b/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/CompiledMappedClassRemapper.java deleted file mode 100644 index e9b484f4..00000000 --- a/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/CompiledMappedClassRemapper.java +++ /dev/null @@ -1,51 +0,0 @@ -package net.fabricmc.loom.configuration.providers.minecraft.tr; - -import java.util.HashMap; - -import dev.architectury.tinyremapper.AsmClassRemapper; -import org.objectweb.asm.ClassVisitor; -import org.objectweb.asm.MethodVisitor; -import org.objectweb.asm.Opcodes; -import org.objectweb.asm.commons.ClassRemapper; -import org.objectweb.asm.commons.MethodRemapper; -import org.objectweb.asm.tree.MethodNode; - -public class CompiledMappedClassRemapper extends ClassRemapper { - private final MappingsCompiled compiled; - private String lastName; - private MethodNode lastMethod; - - public CompiledMappedClassRemapper(ClassVisitor classVisitor, MappingsCompiled compiled) { - super(Opcodes.ASM9, classVisitor, compiled); - - this.compiled = compiled; - } - - @Override - public void visit(int version, int access, String name, String signature, String superName, String[] interfaces) { - lastName = name; - this.compiled.lastSuperClass = superName; - this.compiled.lastInterfaces = interfaces; - super.visit(version, access, name, signature, superName, interfaces); - } - - @Override - public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) { - lastMethod = new MethodNode(api, access, name, descriptor, signature, exceptions); - return super.visitMethod(access, name, descriptor, signature, exceptions); - } - - @Override - protected MethodVisitor createMethodRemapper(MethodVisitor methodVisitor) { - return new MethodRemapper(api, lastMethod, remapper) { - @Override - public void visitEnd() { - lastMethod.localVariables = null; - lastMethod.parameters = null; - AsmClassRemapper.AsmMethodRemapper.processLocals(compiled, lastName, lastMethod, false, true, new HashMap<>()); - lastMethod.visitEnd(); - lastMethod.accept(methodVisitor); - } - }; - } -} diff --git a/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/MappingsCompiled.java b/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/MappingsCompiled.java deleted file mode 100644 index c4ebb67e..00000000 --- a/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/MappingsCompiled.java +++ /dev/null @@ -1,107 +0,0 @@ -package net.fabricmc.loom.configuration.providers.minecraft.tr; - -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - -import com.google.common.collect.HashBasedTable; -import com.google.common.collect.Table; -import dev.architectury.tinyremapper.IMappingProvider; -import dev.architectury.tinyremapper.MethodRemapperProvider; -import org.objectweb.asm.commons.Remapper; - -public class MappingsCompiled extends Remapper implements MethodRemapperProvider { - private final Map classes; - private final Map fields; - private final Map methods; - private final Table methodArgs; - String lastSuperClass; - String[] lastInterfaces; - - public MappingsCompiled(Set mappings) { - this.classes = new HashMap<>(); - this.fields = new HashMap<>(); - this.methods = new HashMap<>(); - this.methodArgs = HashBasedTable.create(); - - for (IMappingProvider mapping : mappings) { - mapping.load(new IMappingProvider.MappingAcceptor() { - @Override - public void acceptClass(String srcName, String dstName) { - classes.put(srcName, dstName); - } - - @Override - public void acceptMethod(IMappingProvider.Member method, String dstName) { - methods.put(method.name, dstName); - } - - @Override - public void acceptMethodArg(IMappingProvider.Member method, int lvIndex, String dstName) { - methodArgs.put(method.owner + "|" + method.name, lvIndex, dstName); - } - - @Override - public void acceptMethodVar(IMappingProvider.Member method, int lvIndex, int startOpIdx, int asmIndex, String dstName) { - } - - @Override - public void acceptField(IMappingProvider.Member field, String dstName) { - fields.put(field.name, dstName); - } - }); - } - } - - @Override - public String map(String name) { - return classes.getOrDefault(name, name); - } - - @Override - public String mapFieldName(String owner, String name, String descriptor) { - return mapField(name); - } - - public String mapField(String name) { - return fields.getOrDefault(name, name); - } - - @Override - public String mapMethodName(String owner, String name, String descriptor) { - return mapMethod(name); - } - - public String mapMethod(String name) { - return methods.getOrDefault(name, name); - } - - public String mapMethodArg(String methodOwner, String methodName, int lvIndex, String def) { - String arg = methodArgs.get(methodOwner + "|" + methodName, lvIndex); - if (arg != null) return arg; - - if (lastSuperClass != null) { - arg = methodArgs.get(lastSuperClass + "|" + methodName, lvIndex); - if (arg != null) return arg; - } - - if (lastInterfaces != null) { - for (String lastInterface : lastInterfaces) { - arg = methodArgs.get(lastInterface + "|" + methodName, lvIndex); - if (arg != null) return arg; - } - } - - return def; - } - - @Override - public String mapMethodVar(String methodOwner, String methodName, String methodDesc, int lvIndex, int startOpIdx, int asmIndex, String name) { - return name; - } - - @Override - public String mapMethodArg(String methodOwner, String methodName, String methodDesc, int lvIndex, String name) { - return mapMethodArg(methodOwner, methodName, lvIndex, name); - } -} diff --git a/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/OutputRemappingHandler.java b/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/OutputRemappingHandler.java index d3a6c69d..f69beb0c 100644 --- a/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/OutputRemappingHandler.java +++ b/src/main/java/net/fabricmc/loom/configuration/providers/minecraft/tr/OutputRemappingHandler.java @@ -17,8 +17,7 @@ import net.fabricmc.loom.util.ThreadingUtils; public class OutputRemappingHandler { public static void remap(TinyRemapper remapper, Path assets, Path output) throws IOException { - remap(remapper, assets, output, (path, bytes) -> { - }); + remap(remapper, assets, output, null); } public static void remap(TinyRemapper remapper, Path assets, Path output, BiConsumer then) throws IOException { diff --git a/src/main/java/net/fabricmc/loom/util/ThreadingUtils.java b/src/main/java/net/fabricmc/loom/util/ThreadingUtils.java index f0c9ff9c..53c2d221 100644 --- a/src/main/java/net/fabricmc/loom/util/ThreadingUtils.java +++ b/src/main/java/net/fabricmc/loom/util/ThreadingUtils.java @@ -130,7 +130,7 @@ public class ThreadingUtils { return new TaskCompleter(); } - public static class TaskCompleter { + public static class TaskCompleter implements Function { Stopwatch stopwatch = Stopwatch.createUnstarted(); List> tasks = new ArrayList<>(); ExecutorService service = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); @@ -147,7 +147,7 @@ public class ThreadingUtils { } catch (Throwable throwable) { throw new RuntimeException(throwable); } - }, service)); + }, service).exceptionally(this)); return this; } @@ -159,7 +159,7 @@ public class ThreadingUtils { public void complete() { try { - CompletableFuture.allOf(tasks.toArray(new CompletableFuture[0])).get(); + CompletableFuture.allOf(tasks.toArray(new CompletableFuture[0])).exceptionally(this).get(); service.shutdownNow(); stopwatch.stop(); @@ -170,5 +170,11 @@ public class ThreadingUtils { throw new RuntimeException(e); } } + + @Override + public Void apply(Throwable throwable) { + throwable.printStackTrace(); + return null; + } } } From 4c71989d2db01f926c5b1993d57b6b87f3d7b3f0 Mon Sep 17 00:00:00 2001 From: shedaniel Date: Sun, 2 May 2021 00:37:26 +0800 Subject: [PATCH 4/6] Remove mavenLocal() --- build.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/build.gradle b/build.gradle index 532b54a8..df9bcbe0 100644 --- a/build.gradle +++ b/build.gradle @@ -49,7 +49,6 @@ repositories { maven { url "https://maven.minecraftforge.net/" } maven { url "https://maven.architectury.dev/" } mavenCentral() - mavenLocal() } dependencies { From 24d40437645f410b684bc62717e395883e0d17e9 Mon Sep 17 00:00:00 2001 From: shedaniel Date: Mon, 3 May 2021 18:36:55 +0800 Subject: [PATCH 5/6] Bump to 0.7.1 & Update mercury --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index df9bcbe0..dc65da0f 100644 --- a/build.gradle +++ b/build.gradle @@ -19,7 +19,7 @@ targetCompatibility = 1.8 group = 'me.shedaniel' archivesBaseName = project.name -def baseVersion = '0.7.0' +def baseVersion = '0.7.1' def runNumber = System.getenv("GITHUB_RUN_NUMBER") ?: "9999" def isSnapshot = System.getenv("PR_NUM") != null @@ -92,7 +92,7 @@ dependencies { implementation ('org.benf:cfr:0.150') // source code remapping - implementation ('org.cadixdev:mercury:0.2.8') + implementation ('org.cadixdev:mercury:0.2.9-architectury') // Kapt integration compileOnly('org.jetbrains.kotlin:kotlin-gradle-plugin:1.4.21') From da286b6c864135522b8fcd3a1862ba3db6041d12 Mon Sep 17 00:00:00 2001 From: shedaniel Date: Mon, 3 May 2021 18:40:02 +0800 Subject: [PATCH 6/6] Change plugin ID to "dev.architectury.loom" --- build.gradle | 12 ++++++------ settings.gradle | 2 +- src/main/java/net/fabricmc/loom/util/Constants.java | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/build.gradle b/build.gradle index dc65da0f..074df6a5 100644 --- a/build.gradle +++ b/build.gradle @@ -17,7 +17,7 @@ plugins { sourceCompatibility = 1.8 targetCompatibility = 1.8 -group = 'me.shedaniel' +group = "dev.architectury" archivesBaseName = project.name def baseVersion = '0.7.1' def runNumber = System.getenv("GITHUB_RUN_NUMBER") ?: "9999" @@ -181,7 +181,7 @@ codenarc { gradlePlugin { plugins { fabricLoom { - id = 'forgified-fabric-loom' + id = 'dev.architectury.loom' implementationClass = 'net.fabricmc.loom.LoomGradlePlugin' } } @@ -214,8 +214,8 @@ import org.w3c.dom.Node publishing { publications { plugin(MavenPublication) { - groupId 'forgified-fabric-loom' - artifactId 'forgified-fabric-loom.gradle.plugin' + groupId 'dev.architectury.loom' + artifactId 'dev.architectury.loom.gradle.plugin' from components.java artifact mainJar @@ -246,8 +246,8 @@ publishing { } pluginSnapshot(MavenPublication) { - groupId 'forgified-fabric-loom' - artifactId 'forgified-fabric-loom.gradle.plugin' + groupId 'dev.architectury.loom' + artifactId 'dev.architectury.loom.gradle.plugin' version baseVersion + '-SNAPSHOT' pom.withXml { diff --git a/settings.gradle b/settings.gradle index 78daf2cb..960cc9c4 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1 +1 @@ -rootProject.name = "forgified-fabric-loom" +rootProject.name = "architectury-loom" diff --git a/src/main/java/net/fabricmc/loom/util/Constants.java b/src/main/java/net/fabricmc/loom/util/Constants.java index 046a2759..a9404c44 100644 --- a/src/main/java/net/fabricmc/loom/util/Constants.java +++ b/src/main/java/net/fabricmc/loom/util/Constants.java @@ -34,7 +34,7 @@ import net.fabricmc.loom.configuration.RemappedConfigurationEntry; import net.fabricmc.loom.util.gradle.GradleSupport; public class Constants { - public static final String PLUGIN_ID = "forgified-fabric-loom"; + public static final String PLUGIN_ID = "dev.architectury.loom"; public static final String LIBRARIES_BASE = "https://libraries.minecraft.net/"; public static final String RESOURCES_BASE = "http://resources.download.minecraft.net/"; public static final String VERSION_MANIFESTS = "https://launchermeta.mojang.com/mc/game/version_manifest_v2.json";