patterns/magic: Add more Arma 3 files (#479)

* patterns/texheaders: Added pattern for Arma 3 texHeaders.bin

* magic/arma3: Added texHeaders.bin magic

* patterns/texheaders: Added test file

* patterns/paa: Small improvements

Added extra data description, and better indication of internally compressed data mipmap data.

* patterns/a3: Moved Arma 3 patterns into common folder

* patterns/a3: Added pattern for MLOD P3D

* patterns/a3: Added pattern for RAP

* magic/arma3: Added P3D and RAP to magic file

* patterns/a3: Added test files for P3D and RAP

* patterns/a3: Small correction to type names in TexHeaders format

README.md

@@ -36,9 +36,12 @@ Everything will immediately show up in ImHex's Content Store and gets bundled wi
 | ARC | | [`patterns/arc.hexpat`](patterns/arc.hexpat) | Minecraft Legacy Console Edition ARC files |
 | ARIA2 | | [`patterns/aria2.hexpat`](patterns/aria2.hexpat) | ARIA2 Download Manager Control files |
 | ARM VTOR | | [`patterns/arm_cm_vtor.hexpat`](patterns/arm_cm_vtor.hexpat) | ARM Cortex M Vector Table Layout |
-| Arma 3 PAA | `image/x.a3-paa` | [`patterns/a3_paa.hexpat`](patterns/a3_paa.hexpat) | Arma 3 PAA texture file |
-| Arma 3 RTM | `application/x.a3-rtm` | [`patterns/a3_rtm.hexpat`](patterns/a3_rtm.hexpat) | Arma 3 RTM animation file (plain) |
-| Arma 3 RTM (binarized) | `application/x.a3-bmtr` | [`patterns/a3_bmtr.hexpat`](patterns/a3_bmtr.hexpat) | Arma 3 RTM animation file (binarized) |
+| Arma 3 config | `application/x.a3-rap` | [`patterns/a3/a3_rap.hexpat`](patterns/a3/a3_rap.hexpat) | Arma 3 binary/rapified config |
+| Arma 3 P3D (MLOD) | `model/x.a3-p3d-mlod` | [`patterns/a3/a3_p3d_mlod.hexpat`](patterns/a3/a3_p3d_mlod.hexpat) | Arma 3 P3D model file (MLOD) |
+| Arma 3 PAA | `image/x.a3-paa` | [`patterns/a3/a3_paa.hexpat`](patterns/a3/a3_paa.hexpat) | Arma 3 PAA texture file |
+| Arma 3 RTM | `application/x.a3-rtm` | [`patterns/a3/a3_rtm.hexpat`](patterns/a3/a3_rtm.hexpat) | Arma 3 RTM animation file (plain) |
+| Arma 3 RTM (binarized) | `application/x.a3-bmtr` | [`patterns/a3/a3_bmtr.hexpat`](patterns/a3/a3_bmtr.hexpat) | Arma 3 RTM animation file (binarized) |
+| Arma 3 texHeaders.bin | `application/x.a3-texheaders` | [`patterns/a3/a3_texheaders.hexpat`](patterns/a3/a3_texheaders.hexpat) | Arma 3 texture index file |
 | Assassin's Creed: Unity | | [`patterns/AC Unity`](patterns/Assassin's Creed: Unity) | Assassin's Creed: Unity archive files -- .forge & .data (compressed and decompressed) -- |
 | Bastion | | [`patterns/bastion/*`](https://gitlab.com/EvelynTSMG/imhex-bastion-pats) | Various [Bastion](https://en.wikipedia.org/wiki/Bastion_(video_game)) files |
 | BeyondCompare BCSS | | [`patterns/bcss.hexpat`](patterns/bcss.hexpat) | BeyondCompare Snapshot (BCSS) file |

magic/arma3_magic

@@ -20,6 +20,22 @@
 !:ext rtm
 
 # Arma 3 binarized RTM animation
-0		string		BMTR	Arma 3 binarized RTM animation file
+0		string		BMTR	Arma 3 RTM animation file (binarized)
 !:mime application/x.a3-bmtr
 !:ext rtm
+
+# Arma 3 texture index
+0		string		0DHT	Arma 3 texture index file
+!:mime application/x.a3-texheaders
+!:ext bin
+
+# Arma 3 MLOD P3D model
+0		string		MLOD	Arma 3 P3D model file (MLOD)
+!:mime model/x.a3-p3d-mlod
+!:ext p3d
+>0x0c	string		P3DM	P3DM LOD type
+
+# Arma 3 binarized config
+0x01    string      raP     Arma 3 binary configuration file
+!:mime application/x.a3-rap
+!:ext bin

patterns/a3/a3_p3d_mlod.hexpat

@@ -0,0 +1,222 @@
+#pragma author MrClock
+#pragma description Arma 3 P3D model format (MLOD)
+
+#pragma endian little
+
+#pragma MIME model/x.a3-p3d-mlod
+
+fn get_data_description() {
+    return "MLOD type P3D files are used for authoring 3D models for Arma 3.\nThese files can be carated in and edited in the Object Builder application.\nAll data is stored uncompressed for ease of editing.\nDuring the PBO packing process they are further \"binarized\" into ODOL type P3D files, that are optimized for use by the game engine. (These are no longer editable, the conversion is irreversible.)\n\nP3D model files can by quite large, so by default only the 1st LOD is processed by this pattern. Processing of all LODs can be enabled in the pattern settings.";
+};
+
+import std.string;
+import std.core;
+
+bool process_all_lods in;
+
+using asciiz = std::string::NullString;
+
+enum FaceType: u32 {
+    TRIANGLE = 3,
+    QUAD = 4
+};
+
+struct Vector {
+    float x;
+    float y;
+    float z;
+} [[static,format("formatVector")]];
+
+struct UV {
+    float u;
+    float v;
+} [[static,format("formatUV")]];
+
+enum SurfaceFitting: u8 {
+    NORMAL = 0,
+    ON_SURFACE = 1,
+    ABOVE_SURFACE = 2,
+    UNDER_SURFACE = 4,
+    KEEP_HEIGHT = 8
+};
+
+enum Lighting: u8 {
+    NORMAL = 0,
+    SHINING = 1,
+    SHADOWED = 2,
+    FULL_LIT = 4,
+    HALF_LIT = 8
+};
+
+enum DecalMode: u8 {
+    NORMAL = 0,
+    DECAL = 1,
+    RADIO12 = 2
+};
+
+enum Fog: u8 {
+    NORMAL = 0,
+    NONE = 1,
+    SKY = 2
+};
+
+enum NormalCalculation: u8 {
+    FACE_AREA = 0,
+    HIDDE_VERTEX = 1,
+    FIXED = 2,
+    FACE_ANLGE = 4
+};
+
+bitfield VertexFlags {
+    padding : 5;
+    NormalCalculation normals : 3;
+    u8 user;
+    padding : 2;
+    Fog fog : 2;
+    padding : 2;
+    DecalMode decal : 2;
+    Lighting lighting : 4;
+    SurfaceFitting surface : 4;
+} [[bitfield_order(std::core::BitfieldOrder::MostToLeastSignificant, 32)]];
+
+struct Vertex {
+    Vector position;
+    VertexFlags flags;
+}[[static,format("formatVertex")]];
+
+struct FacePoint {
+    u32 vertex_index;
+    u32 normal_index;
+    UV uv;
+} [[static]];
+
+enum ZBiasFlag: u8 {
+    NONE = 0,
+    LOW = 1,
+    MIDDLE = 2,
+    HIGH = 3
+};
+
+bitfield FaceFlags {
+    user : 7;
+    disable_texture_merging : 1;
+    padding : 2;
+    flat_lighting : 1;
+    reversed_face : 1;
+    padding : 10;
+    ZBiasFlag zbias : 2;
+    position : 1;
+    padding : 1;
+    double_sided_face : 1;
+    disable_shadow : 1;
+    padding : 4;
+} [[bitfield_order(std::core::BitfieldOrder::MostToLeastSignificant, 32)]];
+
+struct Face {
+    FaceType type;
+    FacePoint points[4];
+    FaceFlags flags;
+    asciiz texture;
+    asciiz material;
+} [[format("formatFace")]];
+
+struct Edge {
+    u32 vertex_1;
+    u32 vertex_2;
+} [[sealed,static,format("formatEdge")]];
+
+struct Property {
+    char name[64] [[transform("std::string::to_string")]];
+    char value[64] [[transform("std::string::to_string")]];
+} [[static,sealed,format("formatProperty")]];
+
+struct Tagg {
+    bool active;
+    asciiz name;
+    u32 length;
+
+    if (name == "#EndOfFile#") {
+        break;
+    }
+
+    match (name) {
+        ("#SharpEdges#"): Edge edges[length/8];
+        ("#Property#"): Property property;
+        ("#Mass#"): float masses[parent.count_verticies];
+        ("#UVSet#"): {
+            u32 channel;
+            UV coordinates[(length - 4) / 8];
+        }
+        (_): if (std::string::starts_with(name, "#") && std::string::ends_with(name, "#")) {
+            u8 unknown_data[length];
+        } else {
+            u8 vertex_weights[parent.count_verticies];
+            u8 face_weights[parent.count_faces];
+        }
+    }
+} [[format("formatTagg")]];
+
+struct P3dmLod {
+    char signature[4];
+    u32 version_major;
+    u32 version_minor;
+    u32 count_verticies;
+    u32 count_normals;
+    u32 count_faces;
+    u32; // Unknown data (might be unused model flags)
+    Vertex verticies[count_verticies];
+    Vector normals[count_normals];
+    Face faces[count_faces];
+    char tagg_signature[4];
+    Tagg taggs[while(true)];
+    float resolution;
+} [[format("formatP3dmLod")]];
+
+struct P3D {
+    char signature[4];
+    u32 version;
+    u32 count_lods;
+    if (!process_all_lods) {
+        P3dmLod lod_0;
+    } else {
+        P3dmLod lods[count_lods];
+    }
+};
+
+fn formatVector(ref Vector pos) {
+    return std::format("[{0:.3f}, {1:.3f}, {1:.3f}]", pos.x, pos.y, pos.z);
+};
+
+fn formatUV(ref UV pos) {
+    return std::format("[{0:.3f}, {1:.3f}]", pos.u, pos.v);
+};
+
+fn formatVertex(ref Vertex vert) {
+    return formatVector(vert.position);
+};
+
+fn formatFace(ref Face face) {
+    return face.type == FaceType::TRIANGLE ? "triangle" : "quad";
+};
+
+fn formatEdge(ref Edge edge) {
+    return std::format("{0:d} <-> {1:d}", edge.vertex_1, edge.vertex_2);
+};
+
+fn formatProperty(ref Property prop) {
+    return std::format("\"{0:s}\" = \"{1:s}\"", prop.name, prop.value);
+};
+
+fn formatTagg(ref Tagg tagg) {
+    if (std::core::has_member(tagg, "vertex_weights")) {
+        return std::format("\"{0:s}\" selection", tagg.name);
+    } else {
+        return std::format("\"{0:s}\"", tagg.name);
+    }
+};
+
+fn formatP3dmLod(ref P3dmLod lod) {
+    return std::format("Resolution: {0}", lod.resolution);
+};
+
+P3D file @ 0x0000;

patterns/a3/a3_paa.hexpat

@@ -5,6 +5,10 @@
 
 #pragma MIME image/x.a3-paa
 
+fn get_data_description() {
+    return "PAA texture files are the proprietary image format used for textures in Arma 3.\nSimilar to most other formats used in game engines, the PAA stores not only a single resolution, but a series of precomputed mipmaps.\nPAA supports multiple pixel encoding formats, such as DXT1, DXT5, RGBA5551, grayscale, and others. Mipmap data in DXT encoded files is optionally compressed with the LZO1X algorithm. All other types are unconditionally LZSS compressed.";
+};
+
 import type.color;
 import std.mem;
 import std.sys;
@@ -50,6 +54,12 @@ enum Swizzle: u8 {
     BLANK_BLACK = 9
 };
 
+enum Compression: u8 {
+    NONE = 0,
+    LZO1X = 1,
+    LZSS = 2
+};
+
 struct Tagg {    
     char signature[8];
     u32 length;
@@ -73,14 +83,25 @@ struct Mipmap {
     u16 width_and_lzo [[format("format_width_lzo")]];
     u16 height;
     
-    u16 width = width_and_lzo & 0x7FFF;
+    u16 width = width_and_lzo;
+    Compression compression = Compression::NONE;
+    if ((u32(parent.format) & 0xFF00) == 0xFF00) {
+        width = width_and_lzo & 0x7FFF;
+        compression = width_and_lzo & 0x8000 ? Compression::LZO1X : Compression::NONE;
+    } else {
+        compression = Compression::LZSS;
+    }
     
     if (width == 0 && height == 0) {
         break;
     }
     
     u24 size;
-    u8 data[size];
+    match (compression) {
+        (Compression::NONE): u8 encoded_data[size];
+        (Compression::LZO1X): u8 lzo_compressed_data[size];
+        (Compression::LZSS): u8 lzss_compressed_data[size];
+    }
 } [[format("format_resolution")]];
 
 struct PAA {
@@ -100,7 +121,7 @@ fn format_resolution(ref auto mip) {
 fn format_width_lzo(u16 value) {
     u16 width = value & 0x7FFF;
     if (value & 0x8000) {
-        return std::format("{0:d} (+LZO)", width);
+        return std::format("{0:d} (+LZO flag)", width);
     } else {
         return std::format("{0:d}", width);
     }

patterns/a3/a3_rap.hexpat

@@ -0,0 +1,165 @@
+#pragma author MrClock
+#pragma description Arma 3 binary configuration format
+
+#pragma endian little
+
+#pragma MIME application/x.a3-rap
+
+fn get_data_description() {
+    return "The RAP format is the binarized/\"rapified\" version of configuration files for Arma 3. Plain text configuration, material definition, scenario description and other files using the configuration syntax are rapified during the PBO packing process. The game can work with the plain text versions (they are actually rapified during boot), but properly converting them into the binary format ahead of time makes the booting easier.";
+};
+
+import std.mem;
+import std.string;
+import std.core;
+import std.io;
+
+using asciiz = std::string::NullString [[format("formatAsciiz")]];
+
+/*
+Item counts are stored in 7-bit encoded integers. In each byte the top bit signals
+if the next byte belongs to the number as well.
+*/
+struct CompressedUint {
+    u8 extras[while(std::mem::read_unsigned($, 1) & 0x80)];
+    u8 last;
+} [[sealed,transform("transformCompressedUint"),format("formatCompressedUint")]];
+
+enum MemberType: u8 {
+    CLASS = 0,
+    LITERAL = 1,
+    ARRAY = 2,
+    EXTERNAL = 3,
+    DELETE = 4,
+    ARRAY_EXTENSION = 5
+};
+
+enum ValueType: u8 {
+    STRING = 0,
+    FLOAT = 1,
+    INTEGER = 2,
+    ARRAY = 3,
+    VARIABLE = 4
+};
+
+using Array;
+
+struct ArrayItem {
+    ValueType type;
+    match (type) {
+        (ValueType::STRING): asciiz value;
+        (ValueType::FLOAT): float value;
+        (ValueType::INTEGER): s32 value;
+        (ValueType::ARRAY): Array value;
+        (ValueType::VARIABLE): asciiz value;
+    }
+} [[format("formatArrayItem")]];
+
+struct Array {
+    CompressedUint count_items;
+    ArrayItem items[count_items];
+} [[format("formatArray")]];
+
+using ClassBody;
+
+struct Member {
+    MemberType type;
+
+    match(type) {
+        (MemberType::CLASS): {
+            asciiz name;
+            ClassBody *body : u32;
+        }
+        (MemberType::LITERAL): {
+            ValueType subtype;
+            asciiz name;
+            match (subtype) {
+                (ValueType::STRING): asciiz value;
+                (ValueType::FLOAT): float value;
+                (ValueType::INTEGER): s32 value;
+                (_): std::error(std::format("Unexpected subtype for literal: {}", subtype));
+            }
+        }
+        (MemberType::ARRAY | MemberType::ARRAY_EXTENSION): {
+            asciiz name;
+            Array value;
+        }
+        (MemberType::EXTERNAL): {
+            asciiz name;
+        }
+        (MemberType::DELETE): {
+            asciiz name;
+        }
+    }
+} [[format("formatMember")]];
+
+using Enums;
+
+struct ClassBody {
+    asciiz parent_name;
+    CompressedUint count_members;
+
+    Member members[count_members];
+    u8 *pointer : u32 [[comment("In the root body this points to the enum list.\nIn all others it points to the next body on the same level.")]];
+};
+
+struct EnumItem {
+    asciiz name;
+    s32 value;
+} [[format("formatEnumItem")]];
+
+struct Enums {
+    u32 count_items;
+    EnumItem items[count_items];
+};
+
+struct RAP {
+    char signature[4];
+    u32;
+    u32;
+    Enums *enums : u32;
+    ClassBody root;
+};
+
+fn formatAsciiz(ref asciiz value) {
+    return std::format("\"{0:s}\"", value);
+};
+
+fn transformCompressedUint(ref CompressedUint value) {
+    u64 result = 0;
+    for (u8 i = 0, i < sizeof(value.extras), i += 1) {
+        result += (value.extras[i] & 0x7F) << (7*i);
+    }
+
+    result += value.last << (sizeof(value.extras) * 7);
+
+    return result;
+};
+
+fn formatCompressedUint(ref CompressedUint value) {
+    return value;
+};
+
+fn formatArrayItem(ref ArrayItem value) {
+    return value.value;
+};
+
+fn formatArray(ref Array value) {
+    return "{...}";
+};
+
+fn formatMember(ref Member item) {
+    match (item.type) {
+        (MemberType::CLASS): return std::format("class {0:s} {{...}};", item.name);
+        (MemberType::LITERAL): return std::format("{0:s} = {1};", item.name, item.value);
+        (MemberType::ARRAY): return std::format("{0:s}[] = {{...}};", item.name);
+        (MemberType::EXTERNAL): return std::format("class {0:s};", item.name);
+        (MemberType::DELETE): return std::format("del {0:s};", item.name);
+    }
+};
+
+fn formatEnumItem(ref EnumItem value) {
+    return std::format("{0:s} = {1}", item.name, item.value);
+};
+
+RAP file @ 0x0000;

patterns/a3/a3_texheaders.hexpat

@@ -0,0 +1,131 @@
+#pragma author MrClock
+#pragma description Arma 3 texture index file format
+
+#pragma endian little
+
+#pragma MIME application/x.a3-texheaders
+
+fn get_data_description() {
+    return "TexHeaders.bin files are texture index files used in Arma 3 PBO archives.\nThe files are generated during the PBO packing process, and contain basic information about all the PAA texture files in the PBO.\nThe index for each texture includes the file paths relative to the PBO root, pixel format, suffix, number of mipmaps and their resolutions among other things.";
+};
+
+import std.string;
+import std.math;
+import type.color;
+
+using asciiz = std::string::NullString;
+
+struct BGRA8 {
+    u8 b;
+    u8 g;
+    u8 r;
+    u8 a;
+} [[
+    static,
+    sealed,
+    format("type::impl::format_color"),
+    color(std::format("{0:02X}{1:02X}{2:02X}", r, g, b))
+]];
+
+struct RGBAfloat {
+    float r [[transform("float2u8")]];
+    float g [[transform("float2u8")]];
+    float b [[transform("float2u8")]];
+    float a [[transform("float2u8")]];
+} [[
+    static,
+    sealed,
+    format("type::impl::format_color"),
+    color(std::format("{0:02X}{1:02X}{2:02X}", r, g, b))
+]];
+
+enum PixelFormat: u8 {
+    INDEXED = 0,
+    GRAY = 1,
+    RGB565 = 2,
+    RGBA5551 = 3,
+    RGBA4444 = 4,
+    RGBA8888 = 5,
+    DXT1 = 6,
+    DXT2 = 7,
+    DXT3 = 8,
+    DXT4 = 9,
+    DXT5 = 10
+};
+
+enum Suffix: u32 {
+    DIFFUSE = 0,
+    DIFFUSE_LINEAR = 1,
+    DETAIL = 2,
+    NORMAL = 3,
+    IRRADIANCE = 4,
+    RANDOM = 5,
+    TREECROWN = 6,
+    MACRO = 7,
+    SHADOW = 8,
+    SPECULAR = 9,
+    DITHERING = 10,
+    DETAIL_SPECULAR = 11,
+    MASK = 12,
+    THERMAL = 13
+};
+
+struct Mipmap {
+    u16 width;
+    u16 height;
+    padding[2];
+    PixelFormat format;
+    padding[1];
+    u32 offset [[comment("Byte offset in file")]];
+} [[static,format("formatMipmap")]];
+
+struct Texture {
+    u32 count_color_pallets [[comment("Always 1")]];
+    u32 pallet_pointer [[comment("Always 0")]];
+    RGBAfloat average_color_float;
+    BGRA8 average_color;
+    BGRA8 max_color;
+    u32 clamp_flags [[comment("Always 0")]];
+    u32 transparency [[comment("Always 0xffffffff")]];
+    bool maxcolor_defined [[comment("GGATCXAM was found in PAA")]];
+    bool alpha_interpolated;
+    bool alpha_binary;
+    bool non_opaque [[comment("Interpolated alpha and average alpha < 127")]];
+    u32 count_mipmaps;
+    /*
+    Technically the format is listed as u32 on the community wiki, but
+    for the sake of reusability in the Mipmap struct, it is broken up into
+    a u8 + 3 padding bytes here. (The enum values are in the u8 range
+    anyway.)
+    */
+    PixelFormat format;
+    padding[3];
+    bool little_endian [[comment("Always true")]];
+    bool is_paa [[comment("File is PAA not PAC")]];
+    asciiz path [[comment("Path relative to texHeaders.bin file")]];
+    Suffix suffix;
+    u32 count_mipmaps_again;
+    Mipmap mipmaps[count_mipmaps];
+    u32 filesize;
+}[[format("formatTexture")]];
+
+struct TexHeaders {
+    char signature[4];
+    u32 version [[comment("Always 1")]];
+    u32 count_textures;
+    Texture textures[count_textures];
+};
+
+fn float2u8(float value) {
+    return u8(std::math::round(value * 255));
+};
+
+fn formatMipmap(ref Mipmap mip) {
+    return std::format("{0:d} x {1:d}", mip.width, mip.height);
+};
+
+fn formatTexture(ref Texture tex) {
+    return std::format("{0:d} x {1:d} @ {2:s}", tex.mipmaps[0].width, tex.mipmaps[0].height, tex.path);
+};
+
+TexHeaders file @ 0x0000;