WIP AT -> JarProcessor

2026-04-02 21:47:42 -05:00 · 2021-04-04 20:19:18 +08:00
parent 6a25a40ff9
commit 4fe9b81e32
20 changed files with 706 additions and 228 deletions
--- a/src/main/java/net/fabricmc/loom/task/AbstractLoomTask.java
+++ b/src/main/java/net/fabricmc/loom/task/AbstractLoomTask.java
@@ -28,10 +28,11 @@ import org.gradle.api.DefaultTask;
 import org.gradle.api.tasks.Internal;

 import net.fabricmc.loom.LoomGradleExtension;
+import net.fabricmc.loom.util.Constants;

 public abstract class AbstractLoomTask extends DefaultTask {
 	public AbstractLoomTask() {
-		setGroup("fabric");
+		setGroup(Constants.TASK_CATEGORY);
 	}

 	@Internal
--- a/src/main/java/net/fabricmc/loom/task/AbstractRunTask.java
+++ b/src/main/java/net/fabricmc/loom/task/AbstractRunTask.java
@@ -35,13 +35,14 @@ import org.gradle.api.Project;
 import org.gradle.api.tasks.JavaExec;

 import net.fabricmc.loom.configuration.ide.RunConfig;
+import net.fabricmc.loom.util.Constants;

 public abstract class AbstractRunTask extends JavaExec {
 	private final RunConfig config;

 	public AbstractRunTask(Function<Project, RunConfig> configProvider) {
 		super();
-		setGroup("fabric");
+		setGroup(Constants.TASK_CATEGORY);
 		this.config = configProvider.apply(getProject());

 		setClasspath(config.sourceSet.getRuntimeClasspath());
--- a/src/main/java/net/fabricmc/loom/task/CleanSourcesTask.java
+++ b/src/main/java/net/fabricmc/loom/task/CleanSourcesTask.java
@@ -0,0 +1,15 @@
+package net.fabricmc.loom.task;
+
+import static net.fabricmc.loom.task.GenerateSourcesTask.getMappedJarFileWithSuffix;
+
+import java.nio.file.Files;
+
+import org.gradle.api.tasks.TaskAction;
+
+public class CleanSourcesTask extends AbstractLoomTask {
+	@TaskAction
+	public void doTask() throws Throwable {
+		Files.deleteIfExists(getMappedJarFileWithSuffix(getProject(), "-sources.jar", false).toPath());
+		Files.deleteIfExists(getMappedJarFileWithSuffix(getProject(), "-sources.jar", true).toPath());
+	}
+}
--- a/src/main/java/net/fabricmc/loom/task/GenerateIncrementalSourcesTask.java
+++ b/src/main/java/net/fabricmc/loom/task/GenerateIncrementalSourcesTask.java
@@ -0,0 +1,128 @@
+package net.fabricmc.loom.task;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.URI;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+
+import javax.inject.Inject;
+
+import org.gradle.api.tasks.TaskAction;
+
+import net.fabricmc.loom.api.decompilers.LoomDecompiler;
+import net.fabricmc.loom.configuration.processors.JarProcessorManager;
+import net.fabricmc.loom.configuration.processors.MinecraftProcessedProvider;
+import net.fabricmc.stitch.util.StitchUtil;
+
+public class GenerateIncrementalSourcesTask extends AbstractLoomTask {
+	public final LoomDecompiler decompiler;
+
+	@Inject
+	public GenerateIncrementalSourcesTask(LoomDecompiler decompiler) {
+		this.decompiler = decompiler;
+
+		getOutputs().upToDateWhen((o) -> false);
+	}
+
+	@TaskAction
+	public void doTask() throws Throwable {
+		GenerateSourcesTask.generateSources(getProject(), getClass(), decompiler, null, false, true);
+
+		if (getExtension().getMappingsProvider().mappedProvider instanceof MinecraftProcessedProvider) {
+			JarProcessorManager processorManager = getExtension().getJarProcessorManager();
+			Path unprocessedCompiledJar = GenerateSourcesTask.getMappedJarFileWithSuffix(getProject(), null, false).toPath();
+			Path compiledJar = GenerateSourcesTask.getMappedJarFileWithSuffix(getProject(), null, true).toPath();
+			Path unprocessedSourcesDestination = GenerateSourcesTask.getMappedJarFileWithSuffix(getProject(), "-sources.jar", false).toPath();
+			Map<String, byte[]> unprocessedSources = new HashMap<>();
+			Set<String> affectedSources = new HashSet<>();
+
+			try (FileSystem system = FileSystems.newFileSystem(URI.create("jar:" + unprocessedSourcesDestination.toUri()), new HashMap<>())) {
+				Files.walkFileTree(system.getPath("/"), new SimpleFileVisitor<Path>() {
+					@Override
+					public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+						if (file.toString().endsWith(".java")) {
+							String sourcePath = file.toString();
+
+							if (sourcePath.length() >= 1 && sourcePath.charAt(0) == '/') {
+								sourcePath = sourcePath.substring(1);
+							}
+
+							sourcePath = sourcePath.substring(0, sourcePath.length() - 5);
+							unprocessedSources.put(sourcePath, Files.readAllBytes(file));
+
+							if (processorManager.doesProcessClass(sourcePath)) {
+								affectedSources.add(sourcePath);
+							}
+						}
+
+						return FileVisitResult.CONTINUE;
+					}
+				});
+			} catch (IOException exception) {
+				exception.printStackTrace();
+				unprocessedSources.clear();
+			}
+
+			Consumer<Path> linemapConsumer = linemap -> {
+				try {
+					try (StitchUtil.FileSystemDelegate unprocessedFs = StitchUtil.getJarFileSystem(unprocessedCompiledJar.toFile(), true);
+							StitchUtil.FileSystemDelegate processedFs = StitchUtil.getJarFileSystem(compiledJar.toFile(), true)) {
+						int i = 0;
+
+						for (Path path : (Iterable<? extends Path>) Files.walk(processedFs.get().getPath("/"))::iterator) {
+							if (path.toString().endsWith(".class")) {
+								String sourcePath = path.toString();
+
+								if (sourcePath.length() >= 1 && sourcePath.charAt(0) == '/') {
+									sourcePath = sourcePath.substring(1);
+								}
+
+								sourcePath = sourcePath.substring(0, sourcePath.length() - 6);
+
+								Path unprocessedPath = unprocessedFs.get().getPath(path.toString());
+
+								if (Files.exists(unprocessedPath) && !processorManager.doesProcessClass(sourcePath)) {
+									i++;
+									Files.copy(unprocessedPath, path, StandardCopyOption.REPLACE_EXISTING);
+								}
+							}
+						}
+
+						getLogger().lifecycle(":copied {} linemap remapped classes from unprocessed jar", i);
+					}
+
+					GenerateSourcesTask.remapLinemap(getProject(), getClass(), compiledJar, linemap, true);
+				} catch (IOException exception) {
+					throw new UncheckedIOException(exception);
+				}
+			};
+
+			getLogger().lifecycle(":skipping {} unprocessed classes (excluding {} processed classes)", unprocessedSources.size() - affectedSources.size(), affectedSources.size());
+			GenerateSourcesTask.generateSources(getProject(), getClass(), decompiler, className -> {
+				int i = className.indexOf('$');
+				if (i >= 0) className = className.substring(0, i);
+				if (affectedSources.contains(className)) return GenerateSourcesTask.SkipState.GENERATE;
+				if (unprocessedSources.containsKey(className)) return GenerateSourcesTask.SkipState.SKIP;
+				return null;
+			}, true, true, linemapConsumer);
+		}
+	}
+
+	public GenerateIncrementalSourcesTask setInputJar(File inputJar) {
+		this.inputJar = inputJar;
+		return this;
+	}
+}
--- a/src/main/java/net/fabricmc/loom/task/GenerateSourcesTask.java
+++ b/src/main/java/net/fabricmc/loom/task/GenerateSourcesTask.java
@@ -26,19 +26,34 @@ package net.fabricmc.loom.task;

 import java.io.File;
 import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.URI;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.FileVisitResult;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
 import java.nio.file.StandardCopyOption;
+import java.nio.file.StandardOpenOption;
+import java.nio.file.attribute.BasicFileAttributes;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.Locale;
+import java.util.Map;
+import java.util.function.Consumer;
+import java.util.function.Function;
 import java.util.stream.Collectors;

 import javax.inject.Inject;

+import com.google.common.base.MoreObjects;
+import org.gradle.api.Project;
 import org.gradle.api.tasks.InputFile;
 import org.gradle.api.tasks.TaskAction;

 import net.fabricmc.loom.LoomGradleExtension;
+import net.fabricmc.loom.LoomGradlePlugin;
 import net.fabricmc.loom.api.decompilers.DecompilationMetadata;
 import net.fabricmc.loom.api.decompilers.LoomDecompiler;
 import net.fabricmc.loom.configuration.providers.mappings.MappingsProvider;
@@ -56,25 +71,100 @@ public class GenerateSourcesTask extends AbstractLoomTask {
 	public GenerateSourcesTask(LoomDecompiler decompiler) {
 		this.decompiler = decompiler;

-		setGroup("fabric");
 		getOutputs().upToDateWhen((o) -> false);
 	}

 	@TaskAction
 	public void doTask() throws Throwable {
-		int threads = Runtime.getRuntime().availableProcessors();
-		Path javaDocs = getExtension().getMappingsProvider().tinyMappings.toPath();
-		Collection<Path> libraries = getProject().getConfigurations().getByName(Constants.Configurations.MINECRAFT_DEPENDENCIES).getFiles()
-						.stream().map(File::toPath).collect(Collectors.toSet());
+		generateSources(getProject(), getClass(), decompiler, null, true, false);
+	}

-		DecompilationMetadata metadata = new DecompilationMetadata(threads, javaDocs, libraries);
-		Path runtimeJar = getExtension().getMappingsProvider().mappedProvider.getMappedJar().toPath();
-		Path sourcesDestination = getMappedJarFileWithSuffix("-sources.jar").toPath();
-		Path linemap = getMappedJarFileWithSuffix("-sources.lmap").toPath();
+	public static void generateSources(Project project, Class<?> taskClass, LoomDecompiler decompiler, Function<String, SkipState> additionalClassFilter, boolean processed, boolean incremental)
+			throws IOException {
+		generateSources(project, taskClass, decompiler, additionalClassFilter, processed, incremental, linemap -> {
+			try {
+				Path compiledJar = getMappedJarFileWithSuffix(project, null, processed).toPath();
+				remapLinemap(project, taskClass, compiledJar, linemap, processed);
+			} catch (IOException exception) {
+				throw new UncheckedIOException(exception);
+			}
+		});
+	}
+
+	public static void generateSources(Project project, Class<?> taskClass, LoomDecompiler decompiler, Function<String, SkipState> additionalClassFilter, boolean processed, boolean incremental, Consumer<Path> linemapConsumer)
+			throws IOException {
+		LoomGradleExtension extension = project.getExtensions().getByType(LoomGradleExtension.class);
+		int threads = Runtime.getRuntime().availableProcessors();
+		Path javaDocs = extension.getMappingsProvider().tinyMappings.toPath();
+		Collection<Path> libraries = project.getConfigurations().getByName(Constants.Configurations.MINECRAFT_DEPENDENCIES).getFiles()
+				.stream().map(File::toPath).collect(Collectors.toSet());
+
+		Path runtimeJar = getMappedJarFileWithSuffix(project, null, processed).toPath();
+		Path sourcesDestination = getMappedJarFileWithSuffix(project, "-sources.jar", processed).toPath();
+		Path linemap = getMappedJarFileWithSuffix(project, "-sources.lmap", processed).toPath();
+		Map<String, byte[]> remappedClasses = new HashMap<>();
+
+		if (!LoomGradlePlugin.refreshDeps && incremental && Files.exists(sourcesDestination)) {
+			try (FileSystem system = FileSystems.newFileSystem(URI.create("jar:" + sourcesDestination.toUri()), new HashMap<>())) {
+				Files.walkFileTree(system.getPath("/"), new SimpleFileVisitor<Path>() {
+					@Override
+					public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+						if (file.toString().endsWith(".java")) {
+							String sourcePath = file.toString();
+
+							if (sourcePath.length() >= 1 && sourcePath.charAt(0) == '/') {
+								sourcePath = sourcePath.substring(1);
+							}
+
+							remappedClasses.put(sourcePath.substring(0, sourcePath.length() - 5), Files.readAllBytes(file));
+						}
+
+						return FileVisitResult.CONTINUE;
+					}
+				});
+			} catch (IOException exception) {
+				exception.printStackTrace();
+				remappedClasses.clear();
+			}
+		}
+
+		Files.deleteIfExists(sourcesDestination);
+
+		if (incremental) {
+			project.getLogger().lifecycle(":incremental source generation: skipping {} classes", remappedClasses.size());
+		}
+
+		Function<String, SkipState> function = MoreObjects.firstNonNull(additionalClassFilter, s -> null);
+		Function<String, SkipState> classFilter = remappedClasses.isEmpty() ? function : s -> {
+			SkipState apply = function.apply(s);
+			if (apply != null) return apply;
+			int i = s.indexOf('$');
+			if (i >= 0) s = s.substring(0, i);
+			return remappedClasses.containsKey(s) ? SkipState.SKIP : SkipState.GENERATE;
+		};
+		DecompilationMetadata metadata = new DecompilationMetadata(threads, javaDocs, libraries, classFilter);
 		decompiler.decompile(inputJar.toPath(), sourcesDestination, linemap, metadata);

+		if (incremental && !remappedClasses.isEmpty()) {
+			try (FileSystem system = FileSystems.newFileSystem(URI.create("jar:" + sourcesDestination.toUri()), new HashMap<String, String>() {
+				{
+					put("create", "true");
+				}
+			})) {
+				for (Map.Entry<String, byte[]> entry : remappedClasses.entrySet()) {
+					if (additionalClassFilter != null && SkipState.SKIP != additionalClassFilter.apply(entry.getKey())) continue;
+					Path path = system.getPath(entry.getKey() + ".java");
+					Path parent = path.getParent();
+					if (parent != null) Files.createDirectories(parent);
+					Files.write(path, entry.getValue(), StandardOpenOption.CREATE);
+				}
+			}
+		}
+	}
+
+	public static void remapLinemap(Project project, Class<?> taskClass, Path compiledJar, Path linemap, boolean processed) throws IOException {
 		if (Files.exists(linemap)) {
-			Path linemappedJarDestination = getMappedJarFileWithSuffix("-linemapped.jar").toPath();
+			Path linemappedJarDestination = getMappedJarFileWithSuffix(project, "-linemapped.jar", processed).toPath();

 			// Line map the actually jar used to run the game, not the one used to decompile
 			remapLineNumbers(runtimeJar, linemap, linemappedJarDestination);
@@ -84,12 +174,13 @@ public class GenerateSourcesTask extends AbstractLoomTask {
 		}
 	}

-	private void remapLineNumbers(Path oldCompiledJar, Path linemap, Path linemappedJarDestination) throws IOException {
-		getProject().getLogger().info(":adjusting line numbers");
+	private static void remapLineNumbers(Project project, Class<?> taskClass, Path oldCompiledJar, Path linemap, Path linemappedJarDestination)
+			throws IOException {
+		project.getLogger().info(":adjusting line numbers");
 		LineNumberRemapper remapper = new LineNumberRemapper();
 		remapper.readMappings(linemap.toFile());

-		ProgressLogger progressLogger = ProgressLogger.getProgressFactory(getProject(), getClass().getName());
+		ProgressLogger progressLogger = ProgressLogger.getProgressFactory(project, taskClass.getName());
 		progressLogger.start("Adjusting line numbers", "linemap");

 		try (StitchUtil.FileSystemDelegate inFs = StitchUtil.getJarFileSystem(oldCompiledJar.toFile(), true);
@@ -100,19 +191,27 @@ public class GenerateSourcesTask extends AbstractLoomTask {
 		progressLogger.completed();
 	}

-	private File getMappedJarFileWithSuffix(String suffix) {
-		LoomGradleExtension extension = getProject().getExtensions().getByType(LoomGradleExtension.class);
+	public static File getMappedJarFileWithSuffix(Project project, String suffix, boolean processed) {
+		LoomGradleExtension extension = project.getExtensions().getByType(LoomGradleExtension.class);
 		MappingsProvider mappingsProvider = extension.getMappingsProvider();
-		File mappedJar = mappingsProvider.mappedProvider.getMappedJar();
+		File mappedJar = processed ? mappingsProvider.mappedProvider.getMappedJar() : mappingsProvider.mappedProvider.getUnprocessedMappedJar();
 		String path = mappedJar.getAbsolutePath();

 		if (!path.toLowerCase(Locale.ROOT).endsWith(".jar")) {
 			throw new RuntimeException("Invalid mapped JAR path: " + path);
 		}

+		if (suffix == null) {
+			return new File(path);
+		}
+
 		return new File(path.substring(0, path.length() - 4) + suffix);
 	}

+	public enum SkipState {
+		SKIP, GENERATE;
+	}
+
 	@InputFile
 	public File getInputJar() {
 		return inputJar;
--- a/src/main/java/net/fabricmc/loom/task/LoomTasks.java
+++ b/src/main/java/net/fabricmc/loom/task/LoomTasks.java
@@ -35,6 +35,7 @@ import net.fabricmc.loom.api.decompilers.LoomDecompiler;
 import net.fabricmc.loom.configuration.ide.RunConfigSettings;
 import net.fabricmc.loom.configuration.providers.mappings.MappingsProvider;
 import net.fabricmc.loom.decompilers.fernflower.FabricFernFlowerDecompiler;
+import net.fabricmc.loom.util.Constants;

 public final class LoomTasks {
 	private LoomTasks() {
@@ -50,7 +51,7 @@ public final class LoomTasks {

 		tasks.register("remapJar", RemapJarTask.class, t -> {
 			t.setDescription("Remaps the built project jar to intermediary mappings.");
-			t.setGroup("fabric");
+			t.setGroup(Constants.TASK_CATEGORY);
 		});

 		tasks.register("downloadAssets", DownloadAssetsTask.class, t -> t.setDescription("Downloads required assets for Fabric."));
@@ -59,6 +60,8 @@ public final class LoomTasks {
 		registerIDETasks(tasks);
 		registerRunTasks(tasks, project);
 		registerDecompileTasks(tasks, project);
+
+		tasks.register("cleanSources", CleanSourcesTask.class);
 	}

 	private static void registerIDETasks(TaskContainer tasks) {
@@ -97,7 +100,6 @@ public final class LoomTasks {

 			tasks.register(taskName, RunGameTask.class, config).configure(t -> {
 				t.setDescription("Starts the '" + config.getConfigName() + "' run configuration");
-				t.setGroup("fabric");

 				if (config.getEnvironment().equals("client")) {
 					t.dependsOn("downloadAssets");
@@ -130,6 +132,7 @@ public final class LoomTasks {

 			for (LoomDecompiler decompiler : extension.getDecompilers()) {
 				String taskName = decompiler instanceof FabricFernFlowerDecompiler ? "genSources" : "genSourcesWith" + decompiler.name();
+				String incrementalTaskName = decompiler instanceof FabricFernFlowerDecompiler ? "genIncrementalSources" : "genIncrementalSourcesWith" + decompiler.name();
 				// decompiler will be passed to the constructor of GenerateSourcesTask
 				GenerateSourcesTask generateSourcesTask = tasks.register(taskName, GenerateSourcesTask.class, decompiler).get();
 				generateSourcesTask.setInputJar(inputJar);
@@ -137,6 +140,13 @@ public final class LoomTasks {
 				if (mappingsProvider.hasUnpickDefinitions()) {
 					generateSourcesTask.dependsOn(tasks.getByName("unpickJar"));
 				}
+
+				GenerateIncrementalSourcesTask generateIncrementalSourcesTask = tasks.register(incrementalTaskName, GenerateIncrementalSourcesTask.class, decompiler).get();
+				generateIncrementalSourcesTask.setInputJar(inputJar);
+
+				if (mappingsProvider.hasUnpickDefinitions()) {
+					generateIncrementalSourcesTask.dependsOn(tasks.getByName("unpickJar"));
+				}
 			}
 		});
 	}