patterns/includes: Update standard library and patterns to support the new bitfields (#102)

* Add `current_bit_offset()` and `read_bits(...)` to `std::mem`

* Replace deprecated BitfieldOrder enum values with new clearer names

This adds new options named `MostToLeastSignificant` and `LeastToMostSignificant` to replace the old `LeftToRight` and `RightToLeft` names. These names should be much clearer about what they affect and how.

* Throw errors when `std::core::(get|set)_bitfield_order()` are called

* Update all patterns to work with the new bitfield behaviors

includes/std/core.pat

@@ -10,11 +10,21 @@
 namespace std::core {
 
     /**
-        The default ordering of bitfield members
+        The layout order of each field after byte-endianness has been handled.
+
+        `LeftToRight` and `RightToLeft` are deprecated in favor of the clearer `MostToLeastSignificant` and `LeastToMostSignificant` names.
     */
     enum BitfieldOrder : u8 {
+        /**
+            @warning deprecated
+        */
         LeftToRight = 0,
-        RightToLeft = 1
+        /**
+            @warning deprecated
+        */
+        RightToLeft = 1,
+        MostToLeastSignificant = 0,
+        LeastToMostSignificant = 1
     };
 
 
@@ -56,19 +66,17 @@ namespace std::core {
 
     
     /**
-        Sets the default bitfield order.
+        @warning Removed in 1.28.0
-        @param order The new default bitfield order
     */
     fn set_bitfield_order(BitfieldOrder order) {
-        builtin::std::core::set_bitfield_order(u32(order));
+        builtin::std::error("Runtime default bitfield order is no longer supported.\nConsider using `be` or `le` on your bitfield variables,\nor attach attribute `bitfield_order` to the bitfield.");
     };
 
     /**
-        Gets thee current default bitfield order
+        @warning Removed in 1.28.0
-        @return The currently set default bitfield order
     */
     fn get_bitfield_order() {
-        return builtin::std::core::get_bitfield_order();
+        builtin::std::error("Runtime default bitfield order is no longer supported.\nConsider using `be` or `le` on your bitfield variables,\nor attach attribute `bitfield_order` to the bitfield.");
     };
 
 

includes/std/mem.pat

@@ -113,6 +113,27 @@ namespace std::mem {
     };
 
 
+    /**
+        Gets the current bit offset within the current byte that a bitfield will read.
+    */
+    fn current_bit_offset() {
+        return builtin::std::mem::current_bit_offset();
+    };
+
+    /**
+        Reads a number of bits from the specified bit offset within the specified byte
+        @param byteOffset The byte offset within the data
+        @param bitOffset The bit offset to start the read at within the current byte
+        @param bitSize The total number of bits to read
+        @return A u128 containing the value read
+    */
+    fn read_bits(u128 byteOffset, u128 bitOffset, u64 bitSize) {
+        byteOffset += bitOffset >> 3;
+        bitOffset = bitOffset & 0x7;
+        return builtin::std::mem::read_bits(byteOffset, bitOffset, bitSize);
+    };
+
+
     /**
         Creates a new custom section with the given name
         @param name The name of the section

patterns/coff.hexpat

@@ -50,7 +50,7 @@ bitfield Characteristics {
     dll                     : 1;
     upSystemOnly            : 1;
     bytesReversedHi         : 1;
-} [[right_to_left]];
+};
 
 enum Type : u16 {
     Null    = 0,
@@ -158,7 +158,7 @@ bitfield SectionFlags {
     memExecute          : 1;
     memRead             : 1;
     memWrite            : 1;
-} [[right_to_left]];
+};
 
 fn format_alignment(u8 alignment) {
     return 1 << alignment;

patterns/cpio.pat

@@ -22,14 +22,14 @@ namespace old_binary {
     using SwappedU32 = u32 [[transform("old_binary::swap_32bit"), format("old_binary::swap_32bit")]];
 
     bitfield Mode {
-        file_type   : 4;
-        suid        : 1;
-        sgid        : 1;
-        sticky      : 1;
-        r           : 3;
-        w           : 3;
         x           : 3;
-    } [[left_to_right]];
+        w           : 3;
+        r           : 3;
+        sticky      : 1;
+        sgid        : 1;
+        suid        : 1;
+        file_type   : 4;
+    };
 
     struct CpioHeader {
         type::Oct<u16> magic;

patterns/elf.hexpat

@@ -4,10 +4,12 @@
 #pragma MIME application/x-object
 #pragma MIME application/x-sharedlib
 
-#include <std/io.pat>
 #include <std/core.pat>
+#include <std/io.pat>
 #include <std/mem.pat>
 
+using BitfieldOrder = std::core::BitfieldOrder;
+
 using EI_ABIVERSION     = u8;
 using Elf32_Addr        = u32;
 using Elf32_BaseAddr    = u32;
@@ -458,37 +460,37 @@ enum VER_NEED : Elf32_Half {
 };
 
 bitfield SYMINFO_FLG {
-    padding     : 10;
-    NOEXTDIRECT : 1;
-    DIRECTBIND  : 1;
-    LAZYLOAD    : 1;
-    COPY        : 1;
-    RESERVED    : 1;
     DIRECT      : 1;
-} [[left_to_right]];
+    RESERVED    : 1;
+    COPY        : 1;
+    LAZYLOAD    : 1;
+    DIRECTBIND  : 1;
+    NOEXTDIRECT : 1;
+    padding     : 10;
+};
 
 bitfield ST {
     ST_BIND : 4;
     ST_TYPE : 4;
-} [[left_to_right]];
+} [[bitfield_order(BitfieldOrder::MostToLeastSignificant, 8)]];
 
 bitfield SHF {
-    MASKPROC         : 4;
-    MASKOS           : 8;
-    UNKNOWN          : 8;
-    COMPRESSED       : 1;
-    TLS              : 1;
-    GROUP            : 1;
-    OS_NONCONFORMING : 1;
-    LINK_ORDER       : 1;
-    INFO_LINK        : 1;
-    STRINGS          : 1;
-    MERGE            : 1;
-    padding          : 1;
-    EXECINSTR        : 1;
-    ALLOC            : 1;
     WRITE            : 1;
-} [[left_to_right]];
+    ALLOC            : 1;
+    EXECINSTR        : 1;
+    padding          : 1;
+    MERGE            : 1;
+    STRINGS          : 1;
+    INFO_LINK        : 1;
+    LINK_ORDER       : 1;
+    OS_NONCONFORMING : 1;
+    GROUP            : 1;
+    TLS              : 1;
+    COMPRESSED       : 1;
+    UNKNOWN          : 8;
+    MASKOS           : 8;
+    MASKPROC         : 4;
+};
 
 bitfield ELF32_R_INFO {
     SYM  : 8;
@@ -501,13 +503,13 @@ bitfield ELF64_R_INFO {
 } [[left_to_right]];
 
 bitfield PF {
-    MASKPROC : 4;
-    MASKOS   : 4;
-    padding  : 17;
-    R        : 1;
-    W        : 1;
     X        : 1;
-} [[left_to_right]];
+    W        : 1;
+    R        : 1;
+    padding  : 17;
+    MASKOS   : 4;
+    MASKPROC : 4;
+};
 
 struct E_IDENT {
     char EI_MAG[4];

patterns/flac.hexpat

@@ -4,6 +4,8 @@
 
 #pragma endian big
 
+using BitfieldOrder = std::core::BitfieldOrder;
+
 u32 sampleSize;
 u32 bitsPerSample;
 
@@ -25,14 +27,14 @@ bitfield METADATA_BLOCK_HEADER {
 	lastMetadataBlock 	: 1;
 	blockType 			: 7;
 	length				: 24;
-} [[left_to_right]];
+};
 
 bitfield STREAMINFO_FLAGS {
 	sampleRate 			: 20;
 	numChannels 		: 3 [[format("format_channels")]];
 	bitsPerSample 		: 5;
 	numSamplesInStream	: 36;
-} [[inline, left_to_right]];
+} [[inline]];
 
 fn format_channels(u8 value) {
 	return value + 1;
@@ -82,8 +84,7 @@ struct METADATA_BLOCK_VORBIS_COMMENT {
 bitfield TRACK_FLAGS {
 	audioTrack 	: 1;
 	preEmphasis : 1;
-	padding 	: 6;
+} [[inline,bitfield_order(BitfieldOrder::LeastToMostSignificant, 8)]];
-} [[inline]];
 
 struct CUESHEET_TRACK_INDEX {
 	u64 sampleOffset;
@@ -160,8 +161,7 @@ bitfield FRAME_HEADER_FLAGS {
 	sampleRate			: 4;
 	channelAssignment	: 4;
 	sampleSize			: 3;
-	padding				: 1;
+} [[inline,bitfield_order(BitfieldOrder::LeastToMostSignificant, 32)]];
-} [[inline]];
 
 struct FRAME_HEADER {
 	FRAME_HEADER_FLAGS flags;
@@ -209,6 +209,8 @@ fn getBitsPerSample() {
 		return 20;
 	else if (sampleSize == 0b110)
 		return 24;
+    else
+		std::error(std::format("Invalid sample size {}.", sampleSize));
 };
 
 bitfield SUBFRAME_CONSTANT {
@@ -229,14 +231,14 @@ bitfield RESIDUAL_CODING_METHOD_PARTITIONED_RICE {
 	riceParameter : 4;
 	if (riceParameter == 0b1111)
 		bitsPerSample : 5;
-} [[right_to_left]];
+};
 
 bitfield RESIDUAL_CODING_METHOD_PARTITIONED_RICE2 {
 	partitionOrder : 4;
 	riceParameter  : 5;
 	if (riceParameter == 0b11111)
 		bitsPerSample : 5;
-} [[right_to_left]];
+};
 
 struct RESIDUAL {
 	if (parent.value.codingMethod == 0b00)

patterns/fs.hexpat

@@ -13,7 +13,7 @@ bitfield CHS {
 	h : 8;
 	s : 6;
 	c : 10;
-} [[right_to_left, format("chs_formatter")]];
+} [[format("chs_formatter")]];
 
 fn chs_formatter(CHS chs) {
 	return std::format("({:X}, {:X}, {:X}) | 0x{:X}", chs.c, chs.h, chs.s, (chs.c * 16 + chs.h) * 63 + (chs.s - 1));

patterns/gzip.hexpat

@@ -2,16 +2,18 @@
 
 #include <type/time.pat>
 #include <type/size.pat>
+#include <std/core.pat>
 #include <std/mem.pat>
 
+using BitfieldOrder = std::core::BitfieldOrder;
+
 bitfield Flags {
     FTEXT       : 1;
     FHCRC       : 1;
     FEXTRA      : 1;
     FNAME       : 1;
     FCOMMENT    : 1;
-    padding     : 3;
+} [[bitfield_order(BitfieldOrder::LeastToMostSignificant, 8)]];
-} [[right_to_left]];
 
 bitfield ExtraFlags {
     padding : 1;

patterns/ip.hexpat

@@ -1,5 +1,4 @@
  #pragma endian big
- #pragma bitfield_order left_to_right
 
 #include <std/sys.pat>
 #include <std/io.pat>

patterns/java_class.hexpat

@@ -1,9 +1,12 @@
 #pragma endian big
 #pragma MIME application/x-java-applet
 
+#include <std/core.pat>
 #include <std/io.pat>
 #include <std/sys.pat>
 
+using BitfieldOrder = std::core::BitfieldOrder;
+
 // The Java documentations use the number of bytes instead of the number of bits for its type names
 using u1 = u8;
 using u2 = u16;
@@ -206,7 +209,7 @@ bitfield access_flags_method  {
     padding         : 1;    // 0x2000
     padding         : 1;    // 0x4000
     padding         : 1;    // 0x8000
-};
+} [[bitfield_order(BitfieldOrder::LeastToMostSignificant, 16)]];
 
 bitfield access_flags_field  {
     ACC_PUBLIC      : 1;    // 0x0001
@@ -225,7 +228,7 @@ bitfield access_flags_field  {
     padding         : 1;    // 0x2000
     ACC_ENUM        : 1;    // 0x4000
     padding         : 1;    // 0x8000
-};
+} [[bitfield_order(BitfieldOrder::LeastToMostSignificant, 16)]];
 
 bitfield access_flags_class  {
     ACC_PUBLIC       : 1;    // 0x0001
@@ -244,7 +247,7 @@ bitfield access_flags_class  {
     ACC_ANNOTATION   : 1;    // 0x2000
     ACC_ENUM         : 1;    // 0x4000
     ACC_MODULE       : 1;    // 0x8000
-};
+} [[bitfield_order(BitfieldOrder::LeastToMostSignificant, 16)]];
 
 struct attribute_info {
     u2 attribute_name_info;

patterns/lnk.hexpat

@@ -1,8 +1,11 @@
 #pragma MIME application/x-ms-shortcut
 
+#include <std/core.pat>
 #include <type/guid.pat>
 #include <type/size.pat>
 
+using BitfieldOrder = std::core::BitfieldOrder;
+
 bitfield LinkFlags {
     HasLinkTargetIDList         : 1;
     HasLinkInfo                 : 1;
@@ -32,7 +35,7 @@ bitfield LinkFlags {
     PreferEnvironmentPath       : 1;
     KeepLocalIDListForUNCTarget : 1;
     padding                     : 5;
-} [[right_to_left]];
+};
 
 bitfield FileAttributesFlags {
     FILE_ATTRIBUTE_READONLY             : 1;
@@ -51,14 +54,14 @@ bitfield FileAttributesFlags {
     FILE_ATTRIBUTE_NOT_CONTENT_INDEXED  : 1;
     FILE_ATTRIBUTE_ENCRYPTED            : 1;
     padding                             : 17;
-} [[right_to_left]];
+};
 
 bitfield ModifierKeys {
     HOTKEYF_SHIFT   : 1;
     HOTKEYF_CONTROL : 1;
     HOTKEYF_ALT     : 1;
     padding         : 5;
-} [[right_to_left]];
+};
 
 enum Keys : u8 {
     KEY_NONE    = 0x00,
@@ -206,7 +209,7 @@ bitfield CommonNetworkRelativeLinkFlags {
     ValidDevice     : 1;
     ValidNetType    : 1;
     padding         : 30;
-} [[right_to_left]];
+};
 
 enum NetworkProviderType : u32 {
     WNNC_NET_AVID           = 0x001A0000,
@@ -334,7 +337,7 @@ bitfield FillAttributes {
     BACKGROUND_INTENSITY    : 1;
     
     padding                 : 8;
-} [[right_to_left]];
+};
 
 struct ConsoleDataBlock {
     FillAttributes FillAttributes;

patterns/macho.hexpat

@@ -169,7 +169,7 @@ enum SubCpuTypeVEO : u24 {
 bitfield Capabilities {
     padding : 7;
     lib64   : 1;
-} [[right_to_left]];
+};
 
 enum FileType : u32 {
     Object = 1,
@@ -214,7 +214,7 @@ bitfield Flags {
     appExtensionSafe : 1;
     nlistOutOfSyncWithDyldinof : 1;
     simSupport : 1;
-} [[right_to_left]];
+};
 
 struct Header {
     Magic magic;

patterns/minidump.hexpat

@@ -399,7 +399,7 @@ bitfield MINIDUMP_TYPE {
     MiniDumpWithIptTrace                    : 1;
     MiniDumpScanInaccessiblePartialPages    : 1;
     padding                                 : 40;
-} [[right_to_left]];
+};
 
 struct MINIDUMP_HEADER {
     char Signature[4];

patterns/ne.hexpat

@@ -1,7 +1,5 @@
 #include <std/mem.pat>
 
-#pragma bitfield_order right_to_left
-
 struct DOSHeader {
     char signature[2];
     u16 lastPageSize;
@@ -186,7 +184,7 @@ bitfield SegmentTableFlags {
 	containsRelocationInfo : 1;
 	padding : 1;
 	discardPriority : 4;
-} [[right_to_left]];
+};
 
 struct SegmentTable {
 	u16 segmentDataPointer;

patterns/ntag.hexpat

@@ -1,13 +1,17 @@
+#include <std/core.pat>
+
+using BitfieldOrder = std::core::BitfieldOrder;
+
 bitfield AccessCapability {
 	Read : 4;
 	Write : 4;
-} [[left_to_right]];
+} [[bitfield_order(BitfieldOrder::MostToLeastSignificant, 8)]];
 
 struct CapabilityContainer {
 	u8 magic;
 	u8 version;
 	u8 memorySize;
-	AccessCapability accessCapability;
+	be AccessCapability accessCapability;
 };
 
 bitfield NDEFFlags {
@@ -17,7 +21,7 @@ bitfield NDEFFlags {
 	SR	: 1;
 	IL	: 1;
 	TNF	: 3;
-} [[left_to_right]];
+} [[bitfield_order(BitfieldOrder::MostToLeastSignificant, 8)]];
 
 enum TNFType : u8 {
 	Empty 					= 0x00,
@@ -129,7 +133,7 @@ bitfield MIRROR {
 	padding     : 1;
 	STRG_MOD_EN	: 1;
 	padding     : 2;
-} [[left_to_right]];
+} [[bitfield_order(BitfieldOrder::MostToLeastSignificant, 8)]];
 
 bitfield ACCESS {
 	PROT				: 1;

patterns/pe.hexpat

@@ -1,6 +1,5 @@
 #pragma MIME application/x-dosexec
 #pragma MIME application/x-msdownload
-#pragma bitfield_order right_to_left
 #pragma pattern_limit 0x400000
 
 #include <std/string.pat>
@@ -378,7 +377,7 @@ fn architecture() {
 bitfield OrdinalFlagByte {
 	flag : 1;
 	padding : 7;
-} [[left_to_right]];
+};
 
 struct NameTableEntry {
 	u16 hint;

patterns/protobuf.hexpat

@@ -1,3 +1,4 @@
+#include <std/core.pat>
 #include <std/mem.pat>
 
 #include <type/leb128.pat>
@@ -30,7 +31,7 @@ enum WireType : u8 {
 bitfield Key {
 	field_number : 5;
 	wire_type    : 3;
-} [[left_to_right]];
+} [[bitfield_order(std::core::BitfieldOrder::MostToLeastSignificant, 8)]];
 
 union _64Bit {
 	u64 fixed64;

patterns/qoi.hexpat

@@ -1,6 +1,5 @@
 #pragma MIME image/qoi
 #pragma endian big
-#pragma bitfield_order left_to_right
 
 #include <std/mem.pat>
 

patterns/sit5.hexpat

@@ -13,13 +13,13 @@ namespace v5 {
         folder : 1;
         encrypted : 1;
         padding : 5;
-    } [[left_to_right]];
+    };
     
     bitfield Flags2 {
         padding : 7;
         resource_fork : 1;
         padding : 8;
-    } [[left_to_right]];
+    };
     
     using MacOSHFSPlusDate = u32 [[format("v5::format_macos_date")]];
     

patterns/spirv.hexpat

@@ -1,3 +1,5 @@
+#include <std/mem.pat>
+
 enum GeneratorID : u16 {
     Khronos = 0,
     LunarG = 1,
@@ -642,4 +644,4 @@ struct Instruction {
 Header header @ 0x00;
 
 // SPIR-V does not have any footer, increase number of instructions manually if you encounter a bigger shader
-Instruction instructions[1024] @ 0x14;
+Instruction instructions[while (!std::mem::eof())] @ 0x14;

patterns/usb.hexpat

@@ -1,7 +1,10 @@
+#include <std/core.pat>
 #include <std/io.pat>
 #include <std/mem.pat>
 #include <std/string.pat>
 
+using BitfieldOrder = std::core::BitfieldOrder;
+
 enum DescriptorType : u8 {
     DeviceDescriptor                    = 0x01,
     ConfigDescriptor                    = 0x02,
@@ -115,7 +118,7 @@ bitfield ConfigAttributes {
     SelfPowered : 1;
     RemoteWakeup : 1;
     padding : 5;
-} [[left_to_right]];
+} [[bitfield_order(BitfieldOrder::MostToLeastSignificant, 8)]];
 
 struct BCD<auto Size> {
     u8 bytes[Size];
@@ -194,16 +197,16 @@ fn format_direction(u8 value) {
 };
 
 bitfield EndpointAddress {
-    Direction      : 1 [[format("format_direction")]];
-    padding        : 3;
     EndpointNumber : 4;
-} [[left_to_right]];
+    padding        : 3;
+    Direction      : 1 [[format("format_direction")]];
+};
 
 bitfield EndpointAttributes {
     TransferType : 2;
     SynchronizationType : 2;
     UsageType : 2;
-} [[right_to_left]];
+};
 
 struct EndpointDescriptor {
     EndpointAddress bEndPointAddress;

patterns/xbeh.hexpat

@@ -22,7 +22,7 @@ bitfield AllowedMedia {
     padding             : 20;
     NonSecureHardDisk   : 1;
     NonSecureMode       : 1;
-} [[right_to_left]];
+};
 
 bitfield GameRegion {
     NorthAmerica    : 1;
@@ -30,7 +30,7 @@ bitfield GameRegion {
     RestOfTheWorld  : 1;
     padding         : 28;
     Manufacturing   : 1;
-} [[right_to_left]];
+};
 
 struct Certificate {
     type::Size<u32> certificateSize;
@@ -52,7 +52,7 @@ bitfield InitializationFlags {
     Limit64Megabytes    : 1;
     DontSetuptHarddisk  : 1;
     padding : 28;
-} [[right_to_left]];
+};
 
 union EntryPoint {
     u32 betaAddress [[format("format_beta_entrypoint")]];
@@ -333,7 +333,7 @@ bitfield LibraryFlags {
     QFEVersion  : 13;
     Approved    : 2;
     DebugBuild  : 1;
-} [[right_to_left]];
+};
 
 struct LibraryVersion {
     char libraryName[8];
@@ -349,7 +349,7 @@ bitfield SectionFlags {
     HeadPageReadOnly    : 1;
     TailPageReadOnly    : 1;
     padding : 26;
-} [[right_to_left]];
+};
 
 struct SectionHeader {
     SectionFlags sectionFlags;

patterns/zstd.hexpat

@@ -2,10 +2,13 @@
 
 #pragma MIME application/zstd
 
+#include <std/core.pat>
 #include <std/io.pat>
 #include <std/mem.pat>
 #include <std/sys.pat>
 
+using BitfieldOrder = std::core::BitfieldOrder;
+
 #define ZSTD_MAGIC_NUMBER 0xFD2FB528
 
 bitfield frame_header_descriptor_t {
@@ -15,12 +18,12 @@ bitfield frame_header_descriptor_t {
     reserved_bit : 1;
     content_checksum_flag : 1;
     dictionary_id_flag : 2;
-} [[left_to_right]];
+} [[bitfield_order(BitfieldOrder::MostToLeastSignificant, 8)]];
 
 bitfield window_descriptor_t {
     exponent : 5;
     mantissa : 3;
-} [[left_to_right]];
+} [[bitfield_order(BitfieldOrder::MostToLeastSignificant, 8)]];
 
 fn window_size(window_descriptor_t window_descriptor) {
     u64 window_log = 10 + window_descriptor.exponent;
@@ -67,7 +70,7 @@ bitfield block_header_t {
     last_block : 1;
     block_type : 2;
     block_size : 21;
-} [[right_to_left]];
+};
 
 enum block_type : u8 {
     raw_block = 0,