Feature/parse empty pages

CHANGELOG.md

@@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+
+## [0.7.4-alpha.0 - 2021-12-08]
+### Added
+- Recover entries from empty pages 
+
 ## [0.7.3]
 ### Changed
 - Ignore invalid header flags - thanks @Oskar65536

Cargo.toml

@@ -28,6 +28,9 @@ winstructs = "0.3.0"
 # Optional for multithreading.
 rayon = { version = "1.5.0", optional = true }
 
+# Required for empty page recovery
+memchr = "2.4.1"
+
 # `evtx_dump` dependencies
 anyhow = { version = "1.0", optional = true }
 simplelog = { version = "^0.10", optional = true }

benches/test_bench.rs

@@ -0,0 +1,16 @@
+#![feature(test)]
+extern crate test;
+use memchr::memmem;
+use std::collections::VecDeque;
+use test::Bencher;
+
+
+#[bench]
+fn bench_regex_bytes(b: &mut Bencher) {
+    let blob = include_bytes!("../samples/security.evtx");
+    b.iter(|| {
+        memmem::find_iter(blob, &[0x2a, 0x2a, 0x00, 0x00])
+            .collect::<VecDeque<usize>>()
+    });
+}
+

src/bin/evtx_dump.rs

@@ -39,6 +39,7 @@ struct EvtxDump {
     output: Box<dyn Write>,
     verbosity_level: Option<Level>,
     stop_after_error: bool,
+    display_allocation: bool
 }
 
 impl EvtxDump {
@@ -72,7 +73,9 @@ impl EvtxDump {
             (v, None) => v,
         };
 
+        let display_allocation = matches.is_present("display-allocation");
         let separate_json_attrib_flag = matches.is_present("separate-json-attributes");
+        let parse_empty_chunks_flag = matches.is_present("parse-empty-chunks");
 
         let no_show_record_number = match (
             matches.is_present("no-show-record-number"),
@@ -139,6 +142,7 @@ impl EvtxDump {
                 .num_threads(num_threads)
                 .validate_checksums(validate_checksums)
                 .separate_json_attributes(separate_json_attrib_flag)
+                .parse_empty_chunks(parse_empty_chunks_flag)
                 .indent(!no_indent)
                 .ansi_codec(*ansi_codec),
             input,
@@ -147,6 +151,7 @@ impl EvtxDump {
             output,
             verbosity_level,
             stop_after_error,
+            display_allocation,
         })
     }
 
@@ -167,8 +172,8 @@ impl EvtxDump {
                 }
             }
             EvtxOutputFormat::JSON => {
-                for record in parser.records_json() {
-                    self.dump_record(record)?
+                for record in parser.records_json_value() {
+                    self.dump_json_record(record)?
                 }
             }
         };
@@ -221,11 +226,38 @@ impl EvtxDump {
         }
     }
 
+    fn dump_json_record(&mut self, record: EvtxResult<SerializedEvtxRecord<serde_json::Value>>) -> Result<()> {
+        match record.with_context(|| "Failed to dump the next record.") {
+            Ok(r) => {
+                let mut json_value = r.data;
+
+                if self.display_allocation {
+                    json_value["allocation"] = serde_json::Value::String(format!("{}", r.allocation));
+                }
+                writeln!(self.output, "{}", json_value)?;
+            }
+            // This error is non fatal.
+            Err(e) => {
+                eprintln!("{:?}", format_err!(e));
+
+                if self.stop_after_error {
+                    std::process::exit(1);
+                }
+            }
+        };
+
+        Ok(())
+    }
+
     fn dump_record(&mut self, record: EvtxResult<SerializedEvtxRecord<String>>) -> Result<()> {
         match record.with_context(|| "Failed to dump the next record.") {
             Ok(r) => {
                 if self.show_record_number {
-                    writeln!(self.output, "Record {}", r.event_record_id)?;
+                    let mut record_display = format!("Record {}", r.event_record_id);
+                    if self.display_allocation {
+                        record_display = format!("{} [{}]", record_display, r.allocation)
+                    }
+                    writeln!(self.output, "{}", record_display)?;
                 }
                 writeln!(self.output, "{}", r.data)?;
             }
@@ -327,6 +359,18 @@ fn main() -> Result<()> {
                 .takes_value(false)
                 .help("If outputting JSON, XML Element's attributes will be stored in a separate object named '<ELEMENTNAME>_attributes', with <ELEMENTNAME> containing the value of the node."),
         )
+        .arg(
+            Arg::with_name("parse-empty-chunks")
+                .long("--parse-empty-chunks")
+                .takes_value(false)
+                .help(indoc!("Attempt to recover records from empty chunks.")),
+        )
+        .arg(
+            Arg::with_name("display-allocation")
+                .long("--display-allocation")
+                .takes_value(false)
+                .help(indoc!("Display allocation status in output.")),
+        )
         .arg(
             Arg::with_name("no-show-record-number")
                 .long("--dont-show-record-number")

src/binxml/assemble.rs

@@ -249,7 +249,8 @@ fn expand_string_ref<'a>(
         None => {
             let mut cursor = Cursor::new(chunk.data);
             let cursor_ref = cursor.borrow_mut();
-            try_seek!(cursor_ref, string_ref.offset, "Cache missed string")?;
+            // The BinXmlName appears to start 6 bytes in from the offset
+            try_seek!(cursor_ref, string_ref.offset + 6, "Cache missed string")?;
 
             let string = BinXmlName::from_stream(cursor_ref)?;
             Ok(Cow::Owned(string))

src/evtx_chunk.rs

@@ -2,7 +2,7 @@ use crate::err::{
     ChunkError, DeserializationError, DeserializationResult, EvtxChunkResult, EvtxError,
 };
 
-use crate::evtx_record::{EvtxRecord, EvtxRecordHeader};
+use crate::evtx_record::{RecordAllocation, EvtxRecord, EvtxRecordHeader};
 
 use log::{debug, info, trace};
 use std::{
@@ -18,6 +18,9 @@ use crate::{ParserSettings, checksum_ieee};
 use byteorder::{LittleEndian, ReadBytesExt};
 use std::sync::Arc;
 
+use memchr::memmem;
+use std::collections::VecDeque;
+
 const EVTX_CHUNK_HEADER_SIZE: usize = 512;
 
 bitflags! {
@@ -194,11 +197,35 @@ impl<'chunk> EvtxChunk<'chunk> {
     /// See `IterChunkRecords` for a more detailed explanation regarding the lifetime scopes of the
     /// resulting records.
     pub fn iter(&mut self) -> IterChunkRecords {
+        let mut offset_array: Option<VecDeque<usize>> = None;
+        // Currently we only support recovering records in empty pages, but, it could be
+        // possible to recover records from chunk slack in the future
+        let recovery_type = RecordAllocation::EmptyPage;
+
+        if self.settings.should_parse_empty_chunks() {
+            // If the page is empty, we want to search for possible record signatures
+            if self.header.first_event_record_number == 1 && 
+            self.header.first_event_record_id == 0xffffffffffffffff &&
+            self.header.last_event_record_number == 0xffffffffffffffff &&
+            self.header.last_event_record_id == 0xffffffffffffffff {
+                // We know that this page is empty, so lets see if we can recover records
+                let found_signatures: VecDeque<usize> = memmem::find_iter(&self.data, &[0x2a, 0x2a, 0x00, 0x00])
+                    .collect();
+
+                // If signatures were found, add them to the offset array
+                if !found_signatures.is_empty() {
+                    offset_array = Some(found_signatures);
+                }
+            }
+        }
+
         IterChunkRecords {
             settings: Arc::clone(&self.settings),
             chunk: self,
             offset_from_chunk_start: EVTX_CHUNK_HEADER_SIZE as u64,
             exhausted: false,
+            offset_array,
+            recovery_type
         }
     }
 }
@@ -228,16 +255,32 @@ pub struct IterChunkRecords<'a> {
     offset_from_chunk_start: u64,
     exhausted: bool,
     settings: Arc<ParserSettings>,
+    /// offset_array is used for record recovery when the page is empty
+    offset_array: Option<VecDeque<usize>>,
+    /// The recovery method that represents the offset array
+    recovery_type: RecordAllocation
 }
 
 impl<'a> Iterator for IterChunkRecords<'a> {
     type Item = std::result::Result<EvtxRecord<'a>, EvtxError>;
 
     fn next(&mut self) -> Option<<Self as Iterator>::Item> {
-        if self.exhausted
-            || self.offset_from_chunk_start >= u64::from(self.chunk.header.free_space_offset)
-        {
-            return None;
+        let mut allocation = RecordAllocation::Allocated;
+
+        if let Some(offset_array) = self.offset_array.as_mut() {
+            // If we are using an offset_array, we want to set the offset_from_chunk_start with those values
+            if let Some(value) = offset_array.pop_back() {
+                self.offset_from_chunk_start = value as u64;
+                allocation = self.recovery_type.clone();
+            } else {
+                return None;
+            }
+        } else {
+            if self.exhausted
+                || self.offset_from_chunk_start >= u64::from(self.chunk.header.free_space_offset)
+            {
+                return None;
+            }
         }
 
         let mut cursor = Cursor::new(&self.chunk.data[self.offset_from_chunk_start as usize..]);
@@ -294,18 +337,22 @@ impl<'a> Iterator for IterChunkRecords<'a> {
             }
         }
 
-        self.offset_from_chunk_start += u64::from(record_header.data_size);
+        if self.offset_array.is_none() {
+            // Update the offset_from_chunk_start if we are not using an offset_array
+            self.offset_from_chunk_start += u64::from(record_header.data_size);
+        }
 
         if self.chunk.header.last_event_record_id == record_header.event_record_id {
             self.exhausted = true;
         }
 
         Some(Ok(EvtxRecord {
+            allocation,
             chunk: self.chunk,
             event_record_id: record_header.event_record_id,
             timestamp: record_header.timestamp,
             tokens,
-            settings: Arc::clone(&self.settings),
+            settings: Arc::clone(&self.settings)
         }))
     }
 }

src/evtx_parser.rs

@@ -136,6 +136,8 @@ pub struct ParserSettings {
     indent: bool,
     /// Controls the ansi codec used to deserialize ansi strings inside the xml document.
     ansi_codec: EncodingRef,
+    /// Flag to parse empty pages
+    parse_empty_chunks: bool
 }
 
 impl Debug for ParserSettings {
@@ -146,6 +148,7 @@ impl Debug for ParserSettings {
             .field("separate_json_attributes", &self.separate_json_attributes)
             .field("indent", &self.indent)
             .field("ansi_codec", &self.ansi_codec.name())
+            .field("parse_empty_chunks", &self.parse_empty_chunks)
             .finish()
     }
 }
@@ -157,6 +160,7 @@ impl PartialEq for ParserSettings {
             && self.validate_checksums == other.validate_checksums
             && self.separate_json_attributes == other.separate_json_attributes
             && self.indent == other.indent
+            && self.parse_empty_chunks == other.parse_empty_chunks
     }
 }
 
@@ -168,6 +172,7 @@ impl Default for ParserSettings {
             separate_json_attributes: false,
             indent: true,
             ansi_codec: WINDOWS_1252,
+            parse_empty_chunks: false
         }
     }
 }
@@ -218,6 +223,12 @@ impl ParserSettings {
         self
     }
 
+    pub fn parse_empty_chunks(mut self, parse_empty_chunks: bool) -> Self {
+        self.parse_empty_chunks = parse_empty_chunks;
+
+        self
+    }
+
     pub fn indent(mut self, pretty: bool) -> Self {
         self.indent = pretty;
 
@@ -233,6 +244,10 @@ impl ParserSettings {
         self.separate_json_attributes
     }
 
+    pub fn should_parse_empty_chunks(&self) -> bool {
+        self.parse_empty_chunks
+    }
+
     pub fn should_indent(&self) -> bool {
         self.indent
     }