patterns: Add pattern for Vector BLF Frame Logging Format (#409)

* Add Vector Binary Logging Format pattern

* Combine object header extensions into single type

---------

Co-authored-by: Nik <werwolv98@gmail.com>

patterns/blf.hexpat

@@ -0,0 +1,533 @@
+
+
+// Pattern for binary logging files (Vector BLF)
+// References used for writing this:
+// https://python-can.readthedocs.io/en/stable/_modules/can/io/blf.html
+// https://bitbucket.org/tobylorenz/vector_blf
+
+#pragma magic [ 4C 4F 47 47 ] @ 0x00
+#pragma description Vector BLF Frame Logging Files
+#pragma array_limit 4194304
+#pragma pattern_limit 4294967296
+#pragma endian little
+
+import hex.dec;
+import std.io;
+import type.magic;
+
+enum object_type : u32 {
+    unknown = 0, //< unknown object
+    can_message = 1, //< CAN message object
+    can_error = 2, //< CAN error frame object
+    can_overload = 3, //< CAN overload frame object
+    can_statistic = 4, //< CAN driver statistics object
+    app_trigger = 5, //< application trigger object
+    env_integer = 6, //< environment integer object
+    env_double = 7, //< environment double object
+    env_string = 8, //< environment string object
+    env_data = 9, //< environment data object
+    log_container = 10, //< container object
+    lin_message = 11, //< LIN message object
+    lin_crc_error = 12, //< LIN CRC error object
+    lin_dlc_info = 13, //< LIN DLC info object
+    lin_rcv_error = 14, //< LIN receive error object
+    lin_snd_error = 15, //< LIN send error object
+    lin_slv_timeout = 16, //< LIN slave timeout object
+    lin_sched_modch = 17, //< LIN scheduler mode change object
+    lin_syn_error = 18, //< LIN sync error object
+    lin_baudrate = 19, //< LIN baudrate event object
+    lin_sleep = 20, //< LIN sleep mode event object
+    lin_wakeup = 21, //< LIN wakeup event object
+    most_spy = 22, //< MOST spy message object
+    most_ctrl = 23, //< MOST control message object
+    most_lightlock = 24, //< MOST light lock object
+    most_statistic = 25, //< MOST statistic object
+    
+    flexray_data = 29, //< FLEXRAY data object
+    flexray_sync = 30, //< FLEXRAY sync object
+    can_driver_error = 31, //< CAN driver error object
+    most_pkt = 32, //< MOST Packet
+    most_pkt2 = 33, //< MOST Packet including original timestamp
+    most_hwmode = 34, //< MOST hardware mode event
+    most_reg = 35, //< MOST register data (various chips)
+    most_genreg = 36, //< MOST register data (MOST register)
+    most_netstate = 37, //< MOST NetState event
+    most_datalost = 38, //< MOST data lost
+    most_trigger = 39, //< MOST trigger
+    flexray_cycle = 40, //< FLEXRAY V6 start cycle object
+    flexray_message = 41, //< FLEXRAY V6 message object
+    lin_checksum_info = 42, //< LIN checksum info event object
+    lin_spike_event = 43, //< LIN spike event object
+    can_driver_sync = 44, //< CAN driver hardware sync
+    flexray_status = 45, //< FLEXRAY status event object
+    gps_event = 46, //< GPS event object
+    flexray_error = 47, //< FLEXRAY error event object
+    flexray_status = 48, //< FLEXRAY status event object
+    flexray_startcycle = 49, //< FLEXRAY start cycle event object
+    flexray_rcv_message = 50, //< FLEXRAY receive message event object
+    realtime_clock = 51, //< Realtime clock object
+    
+    lin_statistic = 54, //< LIN statistic event object
+    j1708_message = 55, //< J1708 message object
+    j1708_virtual_message = 56, //< J1708 message object with more than 21 data bytes
+    lin_message2 = 57, //< LIN frame object - extended
+    lin_snd_error2 = 58, //< LIN transmission error object - extended
+    lin_syn_error2 = 59, //< LIN sync error object - extended
+    lin_crc_error2 = 60, //< LIN checksum error object - extended
+    lin_crv_error2 = 61, //< LIN receive error object
+    lin_wakeup2 = 62, //< LIN wakeup event object  - extended
+    lin_spike_event2 = 63, //< LIN spike event object - extended
+    lin_long_dom_sig = 64, //< LIN long dominant signal object
+    app_text = 65, //< text object
+    flexray_rcvmessage_ex = 66, //< FLEXRAY receive message ex event object
+    most_statistic_ex = 67, //< MOST extended statistic event
+    most_txlight = 68, //< MOST TxLight event
+    most_alloctab = 69, //< MOST Allocation table event
+    most_stress = 70, //< MOST Stress event
+    ethernet_frame = 71, //< Ethernet frame object
+    sys_variable = 72, //< system variable object
+    can_error_ext = 73, //< CAN error frame object (extended)
+    can_driver_error_ext = 74, //< CAN driver error object (extended)
+    lin_long_dom_sig2 = 75, //< LIN long dominant signal object - extended
+    most_150_message = 76, //< MOST150 Control channel message
+    most_150_pkt = 77, //< MOST150 Asynchronous channel message
+    most_ethernet_pkt = 78, //< MOST Ethernet channel message
+    most_150_message_fragment = 79, //< Partial transmitted MOST50/150 Control channel message
+    most_150_pkt_fragment = 80, //< Partial transmitted MOST50/150 data packet on asynchronous channel
+    most_ethernet_pkt_fragment = 81, //< Partial transmitted MOST Ethernet packet on asynchronous channel
+    most_system_event = 82, //< Event for various system states on MOST
+    most_150_alloctab = 83, //< MOST50/150 Allocation table event
+    most_50_message = 84, //< MOST50 Control channel message
+    most_50_pkg = 85, //< MOST50 Asynchronous channel message
+    can_message2 = 86, //< CAN message object - extended
+    lin_unexpected_wakeup = 87,
+    lin_short_or_slow_response = 88,
+    lin_disturbance_event = 89,
+    serial_event = 90,
+    overrun_error = 91, //< driver overrun event
+    event_comment = 92,
+    wlan_frame = 93,
+    wlan_statistic = 94,
+    most_ecl = 95, //< MOST Electrical Control Line event
+    global_marker = 96,
+    afdx_frame = 97,
+    afdx_statistic = 98,
+    kline_statusevent = 99, //< E.g. wake-up pattern
+    can_fd_message = 100, //< CAN FD message object
+    can_fd_message_64 = 101, //< CAN FD message object
+    ethernet_rx_error = 102, //< Ethernet RX error object
+    ethernet_status = 103, //< Ethernet status object
+    can_fd_error_64 = 104, //< CAN FD Error Frame object
+    lin_short_or_slow_response2 = 105,
+    afdx_status = 106, //< AFDX status object
+    afdx_bus_statistic = 107, //< AFDX line-dependent busstatistic object
+    
+    afdx_error_event = 109, //< AFDX asynchronous error event
+    a429_error = 110, //< A429 error object
+    a429_status = 111, //< A429 status object
+    a429_bus_statistic = 112, //< A429 busstatistic object
+    a429_message = 113, //< A429 Message
+    ethernet_statistic = 114, //< Ethernet statistic object
+    restore_point_container = 115, //< Restore point container, use unknown
+    
+    test_structure = 118, //< Event for test execution flow
+    diag_request_information = 119, //< Event for correct interpretation of diagnostic requests
+    ethernet_frame_ex = 120, //< Ethernet packet extended object
+    ethernet_frame_forwarded = 121, //< Ethernet packet forwarded object
+    ethernet_error_ex = 122, //< Ethernet error extended object
+    ethernet_error_forwarded = 123, //< Ethernet error forwarded object
+    function_bus = 124, //< FunctionBus object
+    data_lost_begin = 125, //< Data lost begin
+    data_lost_end = 126, //< Data lost end
+    water_mark_event = 127, //< Watermark event
+    trigger_condition = 128, //< Trigger Condition event
+    can_setting_changed = 129, //< CAN Settings Changed object
+    distributed_object_member = 130, //< Distributed object member (communication setup)
+    attribute_event = 131, //< ATTRIBUTE event (communication setup)
+};
+
+bitfield can_msg_flags {
+    is_tx : 1;
+    padding : 4;
+    nerr : 1;
+    wu : 1;
+    rtr : 1;
+};
+
+struct can_msg {
+    u16 channel;
+    can_msg_flags flags;
+    u8 dlc;
+    u32 id;
+    u8 data[8];
+};
+struct can_msg2 {
+    u16 channel;
+    can_msg_flags flags;
+    u8 dlc;
+    u32 id;
+
+    auto struct_len = 2 + 1 + 1 + 4 + 4 + 1 + 1 + 2; // TODO: Alternative way of doing this?
+    u8 data[parent.header.object_size - parent.header.header_size - struct_len];
+
+    // The frame length in nanoseconds
+    u32 frame_length;
+    // Total number of bits of the CAN frame
+    u8 bit_count;
+    padding[1];
+    padding[2];
+};
+
+bitfield can_fd_msg_flags {
+    edl : 1; //< Extended data length
+    brs : 1; //< Bit rate switch
+    esi : 1; //< Error state indicator
+    padding : 5;
+};
+
+fn format_can_fd_dlc(u8 dlc) {
+    if (dlc > 8)
+        return 8 + (dlc - 8) * 4;
+    return dlc;
+};
+
+struct can_fd_msg {
+    u16 channel;
+    u8 flags;
+    u8 dlc [[format("format_can_fd_dlc")]];
+    u32 id;
+    
+    // The frame length in nanoseconds
+    u32 frame_length;
+    u8 arbitration_bit_count;
+    can_fd_msg_flags fd_flags;
+    u8 valid_data_bytes;
+    padding[1];
+    padding[4];
+
+    u8 data[64];
+
+    padding[4];
+};
+
+bitfield can_fd_msg_64_flags {
+    padding : 2;
+    nerr : 1;
+    hv_wake_up : 1;
+    remote_frame : 1;
+    padding : 1;
+    tx_ack : 1;
+    tx_req : 1;
+    padding : 1;
+    srr : 1;
+    r0 : 1;
+    r1 : 1;
+    edl : 1; //< Extended data length
+    brs : 1; //< Bit rate switch
+    esi : 1; //< Error state indicator
+    padding : 2;
+    burst : 1;
+    padding : 13;
+};
+struct can_bitrate_cfg {
+    u8 quartz_frequency;
+    u8 prescaler;
+    u8 btl_cycles;
+    u8 sampling_point;
+};
+struct can_fd_msg_64 {
+    u8 channel;
+    u8 dlc [[format("format_can_fd_dlc")]];
+    u8 valid_data_bytes;
+    u8 tx_count;
+    u32 id;
+    u32 frame_length;
+    can_fd_msg_64_flags flags;
+    can_bitrate_cfg arbitration_bitrate;
+    can_bitrate_cfg data_bitrate;
+    u32 brs_time_offset;
+    u32 crc_time_offset;
+    u16 bit_length;
+    u8 direction;
+    u8 data_offset;
+    u32 crc;
+    
+    u8 data[valid_data_bytes];
+};
+
+fn format_bus_load(u16 bus_load) {
+    return std::format("{}%", float(bus_load) / 100.f);
+};
+struct can_statistic {
+    u16 channel;
+    // Bus load in 1/100 percent
+    u16 bus_load [[format("format_bus_load")]];
+    
+    u32 standard_data_frames;
+    u32 extended_data_frames;
+    
+    u32 standard_remote_frames;
+    u32 extended_remote_frames;
+    
+    u32 error_frames;
+    u32 overload_frames;
+    
+    u32 reserved;
+};
+
+struct can_driver_error {
+    u16 channel;
+    u8 tx_errors;
+    u8 rx_errors;
+    u32 error_code;
+};
+
+enum app_text_source : u32 {
+    comment = 0,
+    database_info = 1,
+    metadata = 2,
+};
+enum database_bus_type : u8 {
+    can = 1,
+    lin = 5,
+    most = 6,
+    flexray = 7,
+    j1708 = 9,
+    ethernet = 10,
+    wlan = 13,
+    afdx = 14,
+};
+bitfield app_text_database_info {
+    version : 8;
+    channel_num : 8;
+    database_bus_type bus_type : 8;
+    is_can_fd : 1;
+    padding : 7;
+};
+struct app_text {
+    app_text_source source;
+    if (source == 1)
+        app_text_database_info database_info;
+    else
+        padding[4]; // TODO: This is not necessarily padding, there's data here
+    u32 text_length;
+    padding[4];
+    char text[text_length];
+};
+
+// No idea what this is or does
+struct restore_point_container {
+    u8 rpc[14];
+    u16 data_len;
+    u8 data[data_len];
+};
+
+enum compression_method : u16 {
+    no_compression = 0,
+    zlib = 2,
+};
+
+// The following section contains all of the decompressed data at once
+std::mem::Section decompressed_data = std::mem::create_section("Decompressed data");
+// This section is used only for decompressing data
+std::mem::Section zlib_decompress_result = std::mem::create_section("zlib decompress result");
+
+struct log_container {
+    u64 container_begin = $;
+
+    compression_method compression_method;
+    padding[2];
+    padding[4];
+    u32 uncompressed_size;
+    padding[4];
+
+    if (compression_method == compression_method::zlib) {
+        std::mem::set_section_size(zlib_decompress_result, uncompressed_size);
+
+        // Create a pattern that defines the compressed array data
+        auto compressed_byte_len = parent.header.object_size - parent.header.header_size - ($ - container_begin);
+        u8 compressed[compressed_byte_len];
+        if (uncompressed_size != 0)
+            padding[parent.header.object_size % 4]; // Idk, the format wants this... for some reason
+
+        std::assert(hex::dec::zlib_decompress(compressed, zlib_decompress_result) == compressed_byte_len,
+            "zlib decompress needs to succeed");
+
+        // Copy the decompressed data to the end of the section
+        std::mem::copy_section_to_section(zlib_decompress_result, 0,
+            decompressed_data, std::mem::get_section_size(decompressed_data),
+            std::mem::get_section_size(zlib_decompress_result));
+    } else if (compression_method == compression_method::no_compression) {
+        u8 data[uncompressed_size];
+        std::mem::copy_value_to_section(data, decompressed_data, std::mem::get_section_size(decompressed_data));
+    } else {
+        std::assert(false, "Invalid/unknown compression method");
+    }
+};
+
+enum object_flags : u32 {
+    // Timestamps are stored with a unit of 10us (10 microseconds)
+    time_10_us = 1,
+    // Timestamps are stored with a unit of 1ns (1 nanosecond)
+    time_1_ns = 2,
+};
+
+enum timestamp_status : u8 {
+    // Means original timestamps are valid
+    orig = 0x01,
+    // Timestamp is generated by software (1) or by hardware (0)
+    swhw = 0x02,
+    user = 0x10,
+};
+
+struct obj_header_ext {
+    object_flags flags;
+    if (parent.header.header_version == 1) 
+        u16 client_index;
+    else if (parent.header.header_version == 2) {
+        timestamp_status timestamp_status;
+        padding[1];
+    }
+    u16 object_version;
+    u64 object_timestamp;
+    if (parent.header.header_version == 2)
+        u64 original_timestamp;
+};
+
+struct obj_header {
+    type::Magic<"LOBJ"> magic; // 4C 4F 42 4A
+    u16 header_size;
+    u16 header_version;
+    u32 object_size;
+    object_type object_type;
+
+    std::assert(header_version == 1 || header_version == 2, "Invalid/unknown header version");
+};
+
+struct obj_struct {
+    auto object_begin = $;
+    obj_header header [[inline]];
+
+    if (header.object_type == object_type::log_container) {
+        // Log containers seem to never include additional V1 or V2 headers
+        log_container log [[inline]];
+    } else {
+        obj_header_ext ext_header [[inline]];
+
+        match(header.object_type) {
+            (object_type::can_message): {
+                can_msg message [[inline]];
+            }
+            (object_type::can_statistic): {
+                can_statistic statistics [[inline]];
+            }
+            (object_type::can_driver_error): {
+                can_driver_error errors [[inline]];
+            }
+            (object_type::can_message2): {
+                can_msg2 message [[inline]];
+            }
+            (object_type::can_fd_message): {
+                can_fd_msg message [[inline]];
+            }
+            (object_type::can_fd_message_64): {
+                can_fd_msg_64 message [[inline]];
+                padding[header.object_size - ($ - object_begin)]; // TODO: This pattern doesn't support the extra data for this object
+            }
+            (object_type::app_text): {
+                app_text text [[inline]];
+                padding[header.object_size % 4];
+            }
+            (object_type::restore_point_container): {
+                restore_point_container rpc [[inline]];
+            }
+            (_): u8 bytes[header.object_size - header.header_size];
+        }
+    }
+};
+
+enum application_id : u8 {
+    unknown = 0,
+    canalyzer = 1,
+    canoe = 2,
+    canstress = 3,
+    canlog = 4,
+    canape = 5,
+    cancasexl = 6,
+    vlconfig = 7,
+    porsche_logger = 200,
+    caetec_logger = 201,
+    vector_net_sim = 202,
+    ipetronik_logger = 203,
+    rtpk = 204,
+    piketec = 205,
+    sparks = 206,
+};
+
+fn format_api_version(u32 api_version) {
+    return std::format("{}.{}.{}",
+        api_version / 1000000,
+        (api_version % 1000000) / 1000,
+        (api_version % 1000) / 100);
+};
+
+// Mostly just the zlib compression levels, but with some extras
+enum compression_level : u8 {
+    no_compression = 0,
+    best_speed = 1,
+    default_compression = 6,
+    best_compression = 9,
+    // This means that the file contains only log containers, usually compressed at level 6
+    default_container_compression = 10,
+};
+
+struct timestamp {
+    u16 year;
+    u16 month;
+    u16 day_of_week;
+    u16 day;
+    u16 hour;
+    u16 minute;
+    u16 second;
+    u16 millisecond;
+} [[format("format_timestamp")]];
+fn format_timestamp(timestamp ts) {
+    return std::format("{}-{}-{}_{}-{}-{}",
+        ts.year, ts.month, ts.day, ts.hour, ts.minute, ts.second);
+};
+
+struct file_header {
+    type::Magic<"LOGG"> magic; // 4C 4F 47 47
+    u32 header_length;
+
+    u32 api_version [[format("format_api_version")]];
+    application_id app_id;
+    compression_level compression_level;
+    u8 app_major;
+    u8 app_minor;
+
+    u64 file_length;
+    u64 uncompressed_length;
+    u32 object_count;
+    u32 application_build;
+
+    timestamp start_timestamp;
+    timestamp stop_timestamp;
+
+    u64 restore_point_offset; // ?
+} [[inline]];
+
+struct file_layout {
+    file_header header;
+    padding[header.header_length - sizeof(header)];
+    obj_struct objects[while($ < std::mem::size())];
+
+    // Decode all objects from the zlib compressed data
+    if (std::mem::get_section_size(decompressed_data) != 0)
+        obj_struct decompressed_objects[header.object_count] @ 0x00 in decompressed_data;
+} [[inline]];
+
+file_layout file @ 0x00;
+
+std::assert_warn(std::mem::size() == file.header.file_length, "file size mismatch");
+