Add unpick support to layered file mappings, add direct url support to FileSpec. (#599)

* Add unpick support to layered file mappings, add direct url support to FileSpec.

* Disable unpick by default, fix unit tests.

* Fixes and improvements.

src/main/java/net/fabricmc/loom/api/mappings/layered/spec/FileMappingsSpecBuilder.java

@@ -71,6 +71,13 @@ public interface FileMappingsSpecBuilder {
 	 */
 	FileMappingsSpecBuilder enigmaMappings();
 
+	/**
+	 * Marks that the zip file contains unpick data.
+	 *
+	 * @return this builder
+	 */
+	FileMappingsSpecBuilder containsUnpick();
+
 	/**
 	 * Sets the merge namespace of this mappings spec.
 	 *

src/main/java/net/fabricmc/loom/api/mappings/layered/spec/FileSpec.java

@@ -25,6 +25,8 @@
 package net.fabricmc.loom.api.mappings.layered.spec;
 
 import java.io.File;
+import java.net.MalformedURLException;
+import java.net.URL;
 import java.nio.file.Path;
 import java.util.Objects;
 
@@ -36,6 +38,7 @@ import org.jetbrains.annotations.ApiStatus;
 
 import net.fabricmc.loom.api.mappings.layered.MappingContext;
 import net.fabricmc.loom.configuration.providers.mappings.utils.DependencyFileSpec;
+import net.fabricmc.loom.configuration.providers.mappings.utils.URLFileSpec;
 import net.fabricmc.loom.configuration.providers.mappings.utils.LocalFileSpec;
 import net.fabricmc.loom.configuration.providers.mappings.utils.MavenFileSpec;
 
@@ -63,6 +66,14 @@ public interface FileSpec {
 		Objects.requireNonNull(o, "Object cannot be null");
 
 		if (o instanceof CharSequence s) {
+			if (s.toString().startsWith("https://") || s.toString().startsWith("http://")) {
+				try {
+					return create(new URL(s.toString()));
+				} catch (MalformedURLException e) {
+					throw new RuntimeException("Failed to convert string to URL", e);
+				}
+			}
+
 			return createFromMavenDependency(s.toString());
 		} else if (o instanceof Dependency d) {
 			return createFromDependency(d);
@@ -74,6 +85,8 @@ public interface FileSpec {
 			return createFromFile(p);
 		} else if (o instanceof FileSystemLocation l) {
 			return createFromFile(l);
+		} else if (o instanceof URL url) {
+			return createFromUrl(url);
 		} else if (o instanceof FileSpec s) {
 			return s;
 		}
@@ -101,6 +114,10 @@ public interface FileSpec {
 		return createFromFile(path.toFile());
 	}
 
+	static FileSpec createFromUrl(URL url) {
+		return new URLFileSpec(url);
+	}
+
 	// Note resolved instantly, this is not lazy
 	static FileSpec createFromFile(RegularFileProperty regularFileProperty) {
 		return createFromFile(regularFileProperty.get());

src/main/java/net/fabricmc/loom/configuration/providers/mappings/LayeredMappingsDependency.java

@@ -48,6 +48,7 @@ import net.fabricmc.loom.LoomGradlePlugin;
 import net.fabricmc.loom.api.mappings.layered.MappingContext;
 import net.fabricmc.loom.api.mappings.layered.MappingLayer;
 import net.fabricmc.loom.api.mappings.layered.MappingsNamespace;
+import net.fabricmc.loom.configuration.providers.mappings.extras.unpick.UnpickLayer;
 import net.fabricmc.loom.util.ZipUtils;
 import net.fabricmc.mappingio.adapter.MappingDstNsReorder;
 import net.fabricmc.mappingio.adapter.MappingSourceNsSwitch;
@@ -84,6 +85,7 @@ public class LayeredMappingsDependency implements SelfResolvingDependency, FileC
 
 				writeMapping(processor, layers, mappingsFile);
 				writeSignatureFixes(processor, layers, mappingsFile);
+				writeUnpickData(processor, layers, mappingsFile);
 			} catch (IOException e) {
 				throw new RuntimeException("Failed to resolve layered mappings", e);
 			}
@@ -119,6 +121,17 @@ public class LayeredMappingsDependency implements SelfResolvingDependency, FileC
 		ZipUtils.add(mappingsFile, "extras/record_signatures.json", data);
 	}
 
+	private void writeUnpickData(LayeredMappingsProcessor processor, List<MappingLayer> layers, Path mappingsFile) throws IOException {
+		UnpickLayer.UnpickData unpickData = processor.getUnpickData(layers);
+
+		if (unpickData == null) {
+			return;
+		}
+
+		ZipUtils.add(mappingsFile, "extras/definitions.unpick", unpickData.definitions());
+		ZipUtils.add(mappingsFile, "extras/unpick.json", unpickData.metadata().asJson());
+	}
+
 	@Override
 	public Set<File> resolve(boolean transitive) {
 		return resolve();

src/main/java/net/fabricmc/loom/configuration/providers/mappings/LayeredMappingsProcessor.java

@@ -39,6 +39,7 @@ import net.fabricmc.loom.api.mappings.layered.MappingLayer;
 import net.fabricmc.loom.api.mappings.layered.MappingsNamespace;
 import net.fabricmc.loom.api.mappings.layered.spec.MappingsSpec;
 import net.fabricmc.loom.configuration.providers.mappings.extras.signatures.SignatureFixesLayer;
+import net.fabricmc.loom.configuration.providers.mappings.extras.unpick.UnpickLayer;
 import net.fabricmc.mappingio.adapter.MappingSourceNsSwitch;
 import net.fabricmc.mappingio.tree.MemoryMappingTree;
 
@@ -122,4 +123,29 @@ public class LayeredMappingsProcessor {
 
 		return Collections.unmodifiableMap(signatureFixes);
 	}
+
+	@Nullable
+	public UnpickLayer.UnpickData getUnpickData(List<MappingLayer> layers) throws IOException {
+		List<UnpickLayer.UnpickData> unpickDataList = new ArrayList<>();
+
+		for (MappingLayer layer : layers) {
+			if (layer instanceof UnpickLayer unpickLayer) {
+				UnpickLayer.UnpickData data = unpickLayer.getUnpickData();
+				if (data == null) continue;
+
+				unpickDataList.add(data);
+			}
+		}
+
+		if (unpickDataList.isEmpty()) {
+			return null;
+		}
+
+		if (unpickDataList.size() != 1) {
+			// TODO merge
+			throw new UnsupportedOperationException("Only one unpick layer is currently supported.");
+		}
+
+		return unpickDataList.get(0);
+	}
 }

src/main/java/net/fabricmc/loom/configuration/providers/mappings/extras/unpick/UnpickLayer.java

@@ -0,0 +1,61 @@
+/*
+ * This file is part of fabric-loom, licensed under the MIT License (MIT).
+ *
+ * Copyright (c) 2022 FabricMC
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package net.fabricmc.loom.configuration.providers.mappings.extras.unpick;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import org.jetbrains.annotations.ApiStatus;
+import org.jetbrains.annotations.Nullable;
+
+import net.fabricmc.loom.LoomGradlePlugin;
+
+@ApiStatus.Experimental
+public interface UnpickLayer {
+	@Nullable
+	UnpickData getUnpickData() throws IOException;
+
+	record UnpickData(Metadata metadata, byte[] definitions) {
+		public static UnpickData read(Path metadataPath, Path definitionPath) throws IOException {
+			final byte[] definitions = Files.readAllBytes(definitionPath);
+			final Metadata metadata;
+
+			try (Reader reader = Files.newBufferedReader(metadataPath, StandardCharsets.UTF_8)) {
+				metadata = LoomGradlePlugin.OBJECT_MAPPER.readValue(reader, Metadata.class);
+			}
+
+			return new UnpickData(metadata, definitions);
+		}
+
+		public record Metadata(int version, String unpickGroup, String unpickVersion) {
+			public String asJson() throws IOException {
+				return LoomGradlePlugin.OBJECT_MAPPER.writeValueAsString(this);
+			}
+		}
+	}
+}

src/main/java/net/fabricmc/loom/configuration/providers/mappings/file/FileMappingsLayer.java

@@ -25,12 +25,16 @@
 package net.fabricmc.loom.configuration.providers.mappings.file;
 
 import java.io.IOException;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.List;
 import java.util.Map;
 
+import org.jetbrains.annotations.Nullable;
+
 import net.fabricmc.loom.api.mappings.layered.MappingLayer;
 import net.fabricmc.loom.api.mappings.layered.MappingsNamespace;
+import net.fabricmc.loom.configuration.providers.mappings.extras.unpick.UnpickLayer;
 import net.fabricmc.loom.configuration.providers.mappings.intermediary.IntermediaryMappingLayer;
 import net.fabricmc.loom.util.FileSystemUtil;
 import net.fabricmc.loom.util.ZipUtils;
@@ -45,8 +49,12 @@ public record FileMappingsLayer(
 		Path path, String mappingPath,
 		String fallbackSourceNamespace, String fallbackTargetNamespace,
 		boolean enigma, // Enigma cannot be automatically detected since it's stored in a directory.
+		boolean unpick,
 		String mergeNamespace
-) implements MappingLayer {
+) implements MappingLayer, UnpickLayer {
+	private static final String UNPICK_METADATA_PATH = "extras/unpick.json";
+	private static final String UNPICK_DEFINITIONS_PATH = "extras/definitions.unpick";
+
 	@Override
 	public void visit(MappingVisitor mappingVisitor) throws IOException {
 		// Bare file
@@ -82,4 +90,27 @@ public record FileMappingsLayer(
 	public List<Class<? extends MappingLayer>> dependsOn() {
 		return List.of(IntermediaryMappingLayer.class);
 	}
+
+	@Override
+	public @Nullable UnpickData getUnpickData() throws IOException {
+		if (!unpick) {
+			return null;
+		}
+
+		if (!ZipUtils.isZip(path)) {
+			throw new UnsupportedOperationException("Unpick is only supported for zip file mapping layers.");
+		}
+
+		try (FileSystemUtil.Delegate fileSystem = FileSystemUtil.getJarFileSystem(path)) {
+			final Path unpickMetadata = fileSystem.get().getPath(UNPICK_METADATA_PATH);
+			final Path unpickDefinitions = fileSystem.get().getPath(UNPICK_DEFINITIONS_PATH);
+
+			if (!Files.exists(unpickMetadata)) {
+				// No unpick in this zip
+				return null;
+			}
+
+			return UnpickData.read(unpickMetadata, unpickDefinitions);
+		}
+	}
 }

src/main/java/net/fabricmc/loom/configuration/providers/mappings/file/FileMappingsSpec.java

@@ -31,11 +31,11 @@ import net.fabricmc.loom.api.mappings.layered.spec.MappingsSpec;
 public record FileMappingsSpec(
 		FileSpec fileSpec, String mappingPath,
 		String fallbackSourceNamespace, String fallbackTargetNamespace,
-		boolean enigma,
+		boolean enigma, boolean unpick,
 		String mergeNamespace
 ) implements MappingsSpec<FileMappingsLayer> {
 	@Override
 	public FileMappingsLayer createLayer(MappingContext context) {
-		return new FileMappingsLayer(fileSpec.get(context), mappingPath, fallbackSourceNamespace, fallbackTargetNamespace, enigma, mergeNamespace);
+		return new FileMappingsLayer(fileSpec.get(context), mappingPath, fallbackSourceNamespace, fallbackTargetNamespace, enigma, unpick, mergeNamespace);
 	}
 }

src/main/java/net/fabricmc/loom/configuration/providers/mappings/file/FileMappingsSpecBuilderImpl.java

@@ -41,6 +41,7 @@ public class FileMappingsSpecBuilderImpl implements FileMappingsSpecBuilder {
 	private String fallbackSourceNamespace = MappingsNamespace.INTERMEDIARY.toString();
 	private String fallbackTargetNamespace = MappingsNamespace.NAMED.toString();
 	private boolean enigma = false;
+	private boolean unpick = false;
 	private String mergeNamespace = MappingsNamespace.INTERMEDIARY.toString();
 
 	private FileMappingsSpecBuilderImpl(FileSpec fileSpec) {
@@ -70,6 +71,12 @@ public class FileMappingsSpecBuilderImpl implements FileMappingsSpecBuilder {
 		return this;
 	}
 
+	@Override
+	public FileMappingsSpecBuilderImpl containsUnpick() {
+		unpick = true;
+		return this;
+	}
+
 	@Override
 	public FileMappingsSpecBuilderImpl mergeNamespace(MappingsNamespace namespace) {
 		mergeNamespace = Objects.requireNonNull(namespace, "merge namespace cannot be null").toString();
@@ -89,6 +96,6 @@ public class FileMappingsSpecBuilderImpl implements FileMappingsSpecBuilder {
 	}
 
 	public FileMappingsSpec build() {
-		return new FileMappingsSpec(fileSpec, mappingPath, fallbackSourceNamespace, fallbackTargetNamespace, enigma, mergeNamespace);
+		return new FileMappingsSpec(fileSpec, mappingPath, fallbackSourceNamespace, fallbackTargetNamespace, enigma, unpick, mergeNamespace);
 	}
 }

src/main/java/net/fabricmc/loom/configuration/providers/mappings/utils/URLFileSpec.java

@@ -0,0 +1,59 @@
+/*
+ * This file is part of fabric-loom, licensed under the MIT License (MIT).
+ *
+ * Copyright (c) 2022 FabricMC
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package net.fabricmc.loom.configuration.providers.mappings.utils;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.URL;
+import java.nio.file.Path;
+import java.util.Objects;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import net.fabricmc.loom.api.mappings.layered.MappingContext;
+import net.fabricmc.loom.api.mappings.layered.spec.FileSpec;
+import net.fabricmc.loom.util.DownloadUtil;
+
+public record URLFileSpec(URL url) implements FileSpec {
+	private static final Logger LOGGER = LoggerFactory.getLogger(URLFileSpec.class);
+	@Override
+	public Path get(MappingContext context) {
+		try {
+			Path output = context.workingDirectory("%d.URLFileSpec".formatted(Objects.hash(url.toString())));
+			LOGGER.info("Downloading {} to {}", url, output);
+			DownloadUtil.downloadIfChanged(url, output.toFile(), LOGGER);
+			return output;
+		} catch (IOException e) {
+			throw new UncheckedIOException("Failed to download: " + url, e);
+		}
+	}
+
+	@Override
+	public int hashCode() {
+		// URL performs DNS requests if you .hashCode it (:
+		return Objects.hash(url.toString());
+	}
+}

src/main/java/net/fabricmc/loom/util/ZipUtils.java

@@ -131,6 +131,10 @@ public class ZipUtils {
 		}
 	}
 
+	public static void add(Path zip, String path, String str) throws IOException {
+		add(zip, path, str.getBytes(StandardCharsets.UTF_8));
+	}
+
 	public static void add(Path zip, String path, byte[] bytes) throws IOException {
 		add(zip, Collections.singleton(new Pair<>(path, bytes)));
 	}