Performance Optimisations on Project Setup

"Sorry I kinda put them into a single commit" - shedaniel

- Fix AtRemapper into not using ZipUtil#transformEntries, which is slow as it transverses the whole zip tree
- Optimises InnerClassRemapper into not using ZipUtil#iterate, which provides the InputStream of the ZipEntry, data that we don't utilize. Switch to a simple FileSystem to traverse through the list of files
- Make MappingsProvider respect mergedv2 yarn files, skipping merging and reordering namespaces in the process. Users that wish to benefit from this should switch to using mergedv2 yarn artifacts
- Make MinecraftMappedProvider only read inputs once
- Replace (TinyRemapper, official -> named) to (AsmRemapper, intermediary -> named), and chain it to the (TinyRemapper, official -> intermediary) to avoid reading the Minecraft jar again
- Multi-thread MinecraftPatchedProvider#fixParameterAnnotation properly

On a typical project, these changes can save up to 50% of the import time. Tested on architectury example mod, which brought import times from 1m 51s to 48s.
This commit is contained in:
shedaniel
2021-04-22 23:41:35 +08:00
parent b200c4ff62
commit 001f4b58f6
10 changed files with 311 additions and 43 deletions

View File

@@ -33,6 +33,7 @@ import java.util.Optional;
import java.util.function.Consumer;
import java.util.zip.ZipError;
import com.google.common.base.Stopwatch;
import com.google.common.io.Files;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
@@ -112,6 +113,7 @@ public class MinecraftProvider extends DependencyProvider {
} catch (ZipError e) {
HashedDownloadUtil.delete(minecraftClientJar);
HashedDownloadUtil.delete(minecraftServerJar);
HashedDownloadUtil.delete(minecraftMergedJar);
getProject().getLogger().error("Could not merge JARs! Deleting source JARs - please re-run the command and move on.", e);
throw new RuntimeException();
@@ -232,11 +234,14 @@ public class MinecraftProvider extends DependencyProvider {
private void mergeJars(Logger logger) throws IOException {
logger.info(":merging jars");
Stopwatch stopwatch = Stopwatch.createStarted();
try (JarMerger jarMerger = new JarMerger(minecraftClientJar, minecraftServerJar, minecraftMergedJar)) {
jarMerger.enableSyntheticParamsOffset();
jarMerger.merge();
}
logger.info(":merged jars in " + stopwatch);
}
public File getMergedJar() {

View File

@@ -51,6 +51,7 @@ import java.util.function.Predicate;
import java.util.stream.Stream;
import com.google.common.base.Predicates;
import com.google.common.base.Stopwatch;
import com.google.common.collect.ImmutableMap;
import com.google.gson.JsonParser;
import de.oceanlabs.mcp.mcinjector.adaptors.ParameterAnnotationFixer;
@@ -291,12 +292,16 @@ public class MinecraftPatchedProvider extends DependencyProvider {
}
private void fixParameterAnnotation(File jarFile) throws Exception {
getProject().getLogger().info(":fixing parameter annotations for " + jarFile.toString());
getProject().getLogger().info(":fixing parameter annotations for " + jarFile.getAbsolutePath());
Stopwatch stopwatch = Stopwatch.createStarted();
try (FileSystem fs = FileSystems.newFileSystem(new URI("jar:" + jarFile.toURI()), ImmutableMap.of("create", false))) {
for (Path rootDir : fs.getRootDirectories()) {
for (Path file : (Iterable<? extends Path>) Files.walk(rootDir)::iterator) {
if (!file.toString().endsWith(".class")) continue;
ThreadingUtils.TaskCompleter completer = ThreadingUtils.taskCompleter();
for (Path file : (Iterable<? extends Path>) Files.walk(fs.getPath("/"))::iterator) {
if (!file.toString().endsWith(".class")) continue;
completer.add(() -> {
byte[] bytes = Files.readAllBytes(file);
ClassReader reader = new ClassReader(bytes);
ClassNode node = new ClassNode();
@@ -311,9 +316,13 @@ public class MinecraftPatchedProvider extends DependencyProvider {
Files.delete(file);
Files.write(file, out);
}
}
});
}
completer.complete();
}
getProject().getLogger().info(":fixing parameter annotations for " + jarFile.getAbsolutePath() + " in " + stopwatch);
}
private void injectForgeClasses(Logger logger) throws IOException {

View File

@@ -37,6 +37,7 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.function.Consumer;
@@ -67,6 +68,7 @@ import net.fabricmc.loom.util.DownloadUtil;
import net.fabricmc.loom.util.srg.MCPReader;
import net.fabricmc.loom.util.srg.SrgMerger;
import net.fabricmc.loom.util.srg.SrgNamedWriter;
import net.fabricmc.mapping.reader.v2.TinyMetadata;
import net.fabricmc.mapping.reader.v2.TinyV2Factory;
import net.fabricmc.mapping.tree.TinyTree;
import net.fabricmc.stitch.Command;
@@ -274,7 +276,12 @@ public class MappingsProvider extends DependencyProvider {
extractUnpickDefinitions(fileSystem, unpickDefinitionsFile.toPath());
}
if (baseMappingsAreV2()) {
if (baseMappingsAreMergedV2()) {
// Architectury Loom Patch
// If a merged tiny v2 mappings file is provided
// Skip merging, should save a lot of time
Files.copy(baseTinyMappings, tinyMappings.toPath(), StandardCopyOption.REPLACE_EXISTING);
} else if (baseMappingsAreV2()) {
// These are unmerged v2 mappings
mergeAndSaveMappings(project, yarnJar);
} else {
@@ -337,6 +344,15 @@ public class MappingsProvider extends DependencyProvider {
}
}
private boolean baseMappingsAreMergedV2() throws IOException {
try (BufferedReader reader = Files.newBufferedReader(baseTinyMappings)) {
TinyMetadata metadata = TinyV2Factory.readMetadata(reader);
return metadata.getNamespaces().containsAll(Arrays.asList("named", "intermediary", "official"));
} catch (IllegalArgumentException e) {
return false;
}
}
public static void extractMappings(FileSystem jar, Path extractTo) throws IOException {
Files.copy(jar.getPath("mappings/mappings.tiny"), extractTo, StandardCopyOption.REPLACE_EXISTING);
}

View File

@@ -29,36 +29,48 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.UncheckedIOException;
import java.net.URI;
import java.nio.file.FileSystem;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.nio.file.StandardOpenOption;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.function.Consumer;
import java.util.jar.Attributes;
import java.util.jar.Manifest;
import com.google.common.base.Stopwatch;
import com.google.common.collect.ImmutableMap;
import org.gradle.api.Project;
import org.jetbrains.annotations.Nullable;
import org.objectweb.asm.ClassReader;
import org.objectweb.asm.ClassWriter;
import org.objectweb.asm.commons.ClassRemapper;
import net.fabricmc.loom.configuration.DependencyProvider;
import net.fabricmc.loom.configuration.providers.MinecraftProvider;
import net.fabricmc.loom.configuration.providers.mappings.MappingsProvider;
import net.fabricmc.loom.configuration.providers.minecraft.tr.CompiledMappedClassRemapper;
import net.fabricmc.loom.configuration.providers.minecraft.tr.MappingsCompiled;
import net.fabricmc.loom.configuration.providers.minecraft.tr.OutputRemappingHandler;
import net.fabricmc.loom.util.Constants;
import net.fabricmc.loom.util.DownloadUtil;
import net.fabricmc.loom.util.FileSystemUtil;
import net.fabricmc.loom.util.ThreadingUtils;
import net.fabricmc.loom.util.TinyRemapperMappingsHelper;
import net.fabricmc.loom.util.srg.AtRemapper;
import net.fabricmc.loom.util.srg.CoreModClassRemapper;
import net.fabricmc.loom.util.srg.InnerClassRemapper;
import net.fabricmc.mapping.tree.TinyTree;
import net.fabricmc.tinyremapper.IMappingProvider;
import net.fabricmc.tinyremapper.NonClassCopyMode;
import net.fabricmc.tinyremapper.OutputConsumerPath;
import net.fabricmc.tinyremapper.TinyRemapper;
public class MinecraftMappedProvider extends DependencyProvider {
@@ -144,30 +156,41 @@ public class MinecraftMappedProvider extends DependencyProvider {
TinyRemapper remapper = getTinyRemapper();
remapper.readClassPath(libraries);
remapper.prepareClasses();
remapper.readInputs(input);
for (String toM : getExtension().isForge() ? Arrays.asList("named", "intermediary", "srg") : Arrays.asList("named", "intermediary")) {
Path output = "named".equals(toM) ? outputMapped : "srg".equals(toM) ? outputSrg : outputIntermediary;
Files.copy(input, outputMapped, StandardCopyOption.REPLACE_EXISTING);
FileSystemUtil.FileSystemDelegate systemMapped = FileSystemUtil.getJarFileSystem(outputMapped, true);
ThreadingUtils.TaskCompleter taskCompleter = ThreadingUtils.taskCompleter();
MappingsCompiled compiledMapped = new MappingsCompiled(getMappings(input, "intermediary", "named"));
for (String toM : getExtension().isForge() ? Arrays.asList("intermediary", "srg") : Collections.singletonList("intermediary")) {
Path output = "srg".equals(toM) ? outputSrg : outputIntermediary;
Stopwatch stopwatch = Stopwatch.createStarted();
getProject().getLogger().lifecycle(":remapping minecraft (TinyRemapper, " + fromM + " -> " + toM + ")");
Files.deleteIfExists(output);
remapper.replaceMappings(getMappings(input, fromM, toM));
OutputRemappingHandler.remap(remapper, input, output, toM.equals("intermediary") ? (path, bytes) -> {
try {
Path fsPath = systemMapped.get().getPath(compiledMapped.mapClass(path) + ".class");
try (OutputConsumerPath outputConsumer = new OutputConsumerPath.Builder(output).build()) {
if (getExtension().isForge()) {
outputConsumer.addNonClassFiles(input, NonClassCopyMode.FIX_META_INF, remapper);
} else {
outputConsumer.addNonClassFiles(input);
if (fsPath.getParent() != null) {
Files.createDirectories(fsPath.getParent());
}
taskCompleter.add(() -> {
ClassWriter writer = new ClassWriter(ClassWriter.COMPUTE_MAXS);
ClassRemapper classRemapper = new CompiledMappedClassRemapper(writer, compiledMapped);
new ClassReader(bytes).accept(classRemapper, ClassReader.EXPAND_FRAMES);
Files.write(fsPath, writer.toByteArray(), StandardOpenOption.CREATE);
});
} catch (IOException e) {
throw new UncheckedIOException(e);
}
} : (path, bytes) -> {
});
remapper.replaceMappings(getMappings(input, fromM, toM));
remapper.readInputs(input);
remapper.apply(outputConsumer);
} catch (Exception e) {
Files.deleteIfExists(output);
throw new RuntimeException("Failed to remap JAR " + input + " with mappings from " + mappingsProvider.tinyMappings, e);
} finally {
remapper.removeInput();
}
getProject().getLogger().info(":remapped minecraft (TinyRemapper, " + fromM + " -> " + toM + ") in " + stopwatch);
if (getExtension().isForge() && !"srg".equals(toM)) {
getProject().getLogger().info(":running forge finalising tasks");
@@ -206,6 +229,11 @@ public class MinecraftMappedProvider extends DependencyProvider {
}
remapper.finish();
getProject().getLogger().lifecycle(":remapping minecraft (AsmRemapper, intermediary -> named)");
taskCompleter.complete();
systemMapped.close();
}
public TinyRemapper getTinyRemapper() throws IOException {

View File

@@ -0,0 +1,52 @@
package net.fabricmc.loom.configuration.providers.minecraft.tr;
import org.objectweb.asm.ClassVisitor;
import org.objectweb.asm.Label;
import org.objectweb.asm.MethodVisitor;
import org.objectweb.asm.Opcodes;
import org.objectweb.asm.commons.ClassRemapper;
import org.objectweb.asm.commons.MethodRemapper;
import org.objectweb.asm.commons.Remapper;
public class CompiledMappedClassRemapper extends ClassRemapper {
private final MappingsCompiled compiled;
private String lastMethodName;
public CompiledMappedClassRemapper(ClassVisitor classVisitor, MappingsCompiled compiled) {
super(Opcodes.ASM9, classVisitor, new Remapper() {
@Override
public String map(String internalName) {
return compiled.mapClass(internalName);
}
@Override
public String mapFieldName(String owner, String name, String descriptor) {
return compiled.mapField(name);
}
@Override
public String mapMethodName(String owner, String name, String descriptor) {
return compiled.mapMethod(name);
}
});
this.compiled = compiled;
}
@Override
public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) {
lastMethodName = name;
return super.visitMethod(access, name, descriptor, signature, exceptions);
}
@Override
protected MethodVisitor createMethodRemapper(MethodVisitor methodVisitor) {
return new MethodRemapper(api, methodVisitor, remapper) {
@Override
public void visitLocalVariable(String name, String descriptor, String signature, Label start, Label end, int index) {
super.visitLocalVariable(compiled.mapMethodArg(lastMethodName, index, name),
descriptor, signature, start, end, index);
}
};
}
}

View File

@@ -0,0 +1,65 @@
package net.fabricmc.loom.configuration.providers.minecraft.tr;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import net.fabricmc.tinyremapper.IMappingProvider;
public class MappingsCompiled {
private final Map<String, String> classes;
private final Map<String, String> fields;
private final Map<String, String> methods;
private final Map<String, String> methodArgs;
public MappingsCompiled(Set<IMappingProvider> mappings) {
this.classes = new HashMap<>();
this.fields = new HashMap<>();
this.methods = new HashMap<>();
this.methodArgs = new HashMap<>();
for (IMappingProvider mapping : mappings) {
mapping.load(new IMappingProvider.MappingAcceptor() {
@Override
public void acceptClass(String srcName, String dstName) {
classes.put(srcName, dstName);
}
@Override
public void acceptMethod(IMappingProvider.Member method, String dstName) {
methods.put(method.name, dstName);
}
@Override
public void acceptMethodArg(IMappingProvider.Member method, int lvIndex, String dstName) {
methodArgs.put(method.name + "|" + lvIndex, dstName);
}
@Override
public void acceptMethodVar(IMappingProvider.Member method, int lvIndex, int startOpIdx, int asmIndex, String dstName) {
}
@Override
public void acceptField(IMappingProvider.Member field, String dstName) {
fields.put(field.name, dstName);
}
});
}
}
public String mapClass(String name) {
return classes.getOrDefault(name, name);
}
public String mapField(String name) {
return fields.getOrDefault(name, name);
}
public String mapMethod(String name) {
return methods.getOrDefault(name, name);
}
public String mapMethodArg(String methodName, int lvIndex, String def) {
return methodArgs.getOrDefault(methodName + "|" + lvIndex, def);
}
}

View File

@@ -0,0 +1,51 @@
package net.fabricmc.loom.configuration.providers.minecraft.tr;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.nio.file.StandardOpenOption;
import java.util.function.BiConsumer;
import net.fabricmc.loom.util.FileSystemUtil;
import net.fabricmc.loom.util.FileSystemUtil.FileSystemDelegate;
import net.fabricmc.loom.util.ThreadingUtils;
import net.fabricmc.tinyremapper.TinyRemapper;
public class OutputRemappingHandler {
public static void remap(TinyRemapper remapper, Path input, Path output) throws IOException {
remap(remapper, input, output, (path, bytes) -> {
});
}
public static void remap(TinyRemapper remapper, Path input, Path output, BiConsumer<String, byte[]> then) throws IOException {
Files.copy(input, output, StandardCopyOption.REPLACE_EXISTING);
try (FileSystemDelegate system = FileSystemUtil.getJarFileSystem(output, true)) {
ThreadingUtils.TaskCompleter taskCompleter = ThreadingUtils.taskCompleter();
remapper.apply((path, bytes) -> {
if (path.startsWith("/")) path = path.substring(1);
try {
Path fsPath = system.get().getPath(path + ".class");
if (fsPath.getParent() != null) {
Files.createDirectories(fsPath.getParent());
}
taskCompleter.add(() -> {
Files.write(fsPath, bytes, StandardOpenOption.CREATE);
});
then.accept(path, bytes);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
});
taskCompleter.complete();
}
}
}

View File

@@ -29,6 +29,7 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -121,4 +122,33 @@ public class ThreadingUtils {
public interface UnsafeConsumer<T> {
void accept(T value) throws Throwable;
}
public static TaskCompleter taskCompleter() {
return new TaskCompleter();
}
public static class TaskCompleter {
List<CompletableFuture<?>> tasks = new ArrayList<>();
ExecutorService service = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
public TaskCompleter add(UnsafeRunnable job) {
tasks.add(CompletableFuture.runAsync(() -> {
try {
job.run();
} catch (Throwable throwable) {
throw new RuntimeException(throwable);
}
}, service));
return this;
}
public void complete() {
try {
CompletableFuture.allOf(tasks.toArray(new CompletableFuture[0])).get();
service.shutdownNow();
} catch (InterruptedException | ExecutionException e) {
throw new RuntimeException(e);
}
}
}
}

View File

@@ -26,17 +26,20 @@ package net.fabricmc.loom.util.srg;
import java.io.IOException;
import java.io.StringReader;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileSystem;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.List;
import java.util.function.UnaryOperator;
import java.util.zip.ZipEntry;
import com.google.common.collect.ImmutableMap;
import org.apache.logging.log4j.util.Strings;
import org.gradle.api.logging.Logger;
import org.zeroturnaround.zip.ZipUtil;
import org.zeroturnaround.zip.transform.StringZipEntryTransformer;
import org.zeroturnaround.zip.transform.ZipEntryTransformerEntry;
import net.fabricmc.loom.util.function.CollectionUtil;
import net.fabricmc.mapping.tree.TinyTree;
@@ -48,10 +51,13 @@ import net.fabricmc.mapping.tree.TinyTree;
*/
public final class AtRemapper {
public static void remap(Logger logger, Path jar, TinyTree mappings) throws IOException {
ZipUtil.transformEntries(jar.toFile(), new ZipEntryTransformerEntry[] {(new ZipEntryTransformerEntry("META-INF/accesstransformer.cfg", new StringZipEntryTransformer() {
@Override
protected String transform(ZipEntry zipEntry, String input) {
String[] lines = input.split("\n");
try (FileSystem fs = FileSystems.newFileSystem(URI.create("jar:" + jar.toUri()), ImmutableMap.of("create", false))) {
Path atPath = fs.getPath("META-INF/accesstransformer.cfg");
if (Files.exists(atPath)) {
String atContent = new String(Files.readAllBytes(atPath), StandardCharsets.UTF_8);
String[] lines = atContent.split("\n");
List<String> output = new ArrayList<>(lines.length);
for (int i = 0; i < lines.length; i++) {
@@ -90,9 +96,9 @@ public final class AtRemapper {
output.add(i, String.join(" ", parts));
}
return String.join("\n", output);
Files.write(atPath, String.join("\n", output).getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE);
}
}))});
}
}
private static String remapDescriptor(String original, UnaryOperator<String> classMappings) {

View File

@@ -25,15 +25,15 @@
package net.fabricmc.loom.util.srg;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Iterator;
import java.util.Map;
import java.util.function.BiConsumer;
import java.util.stream.Collectors;
import org.zeroturnaround.zip.ZipUtil;
import net.fabricmc.loom.util.FileSystemUtil;
import net.fabricmc.loom.util.FileSystemUtil.FileSystemDelegate;
import net.fabricmc.mapping.tree.ClassDef;
import net.fabricmc.mapping.tree.TinyTree;
import net.fabricmc.tinyremapper.IMappingProvider;
@@ -46,7 +46,7 @@ public class InnerClassRemapper {
}
private static void remapInnerClass(Path fromJar, TinyTree mappingsWithSrg, String from, String to, BiConsumer<String, String> action) {
try (InputStream inputStream = Files.newInputStream(fromJar)) {
try (FileSystemDelegate system = FileSystemUtil.getJarFileSystem(fromJar, false)) {
Map<String, String> availableClasses = mappingsWithSrg.getClasses().stream()
.collect(Collectors.groupingBy(classDef -> classDef.getName(from),
Collectors.<ClassDef, String>reducing(
@@ -55,9 +55,15 @@ public class InnerClassRemapper {
(first, last) -> last
))
);
ZipUtil.iterate(inputStream, (in, zipEntry) -> {
if (!zipEntry.isDirectory() && zipEntry.getName().contains("$") && zipEntry.getName().endsWith(".class")) {
String className = zipEntry.getName().substring(0, zipEntry.getName().length() - 6);
Iterator<Path> iterator = Files.walk(system.get().getPath("/")).iterator();
while (iterator.hasNext()) {
Path path = iterator.next();
String name = path.toString();
if (name.startsWith("/")) name = name.substring(1);
if (!Files.isDirectory(path) && name.contains("$") && name.endsWith(".class")) {
String className = name.substring(0, name.length() - 6);
if (!availableClasses.containsKey(className)) {
String parentName = className.substring(0, className.indexOf('$'));
@@ -70,7 +76,7 @@ public class InnerClassRemapper {
}
}
}
});
}
} catch (IOException e) {
throw new RuntimeException(e);
}