Support for skipping various safety checks during traversal

Author: Byron

git-odb/src/pack/index/traverse/indexed.rs

@@ -1,4 +1,4 @@
-use super::{Error, Reducer};
+use super::{Error, Reducer, SafetyCheck};
 use crate::{
     pack,
     pack::index::access::PackOffset,
@@ -14,6 +14,7 @@ use std::collections::BTreeMap;
 impl index::File {
     pub(crate) fn traverse_with_index_lookup<P, Processor>(
         &self,
+        check: SafetyCheck,
         thread_limit: Option<usize>,
         new_processor: impl Fn() -> Processor + Send + Sync,
         mut root: P,
@@ -145,6 +146,7 @@ impl index::File {
 
                         let shared_cache = &mut SharedCache(&mut cache);
                         let mut stat = self.process_entry_dispatch(
+                            check,
                             pack,
                             shared_cache,
                             buf,
@@ -163,7 +165,7 @@ impl index::File {
 
                 Ok(stats)
             },
-            Reducer::from_progress(&reduce_progress, pack.data_len()),
+            Reducer::from_progress(&reduce_progress, pack.data_len(), check),
         )
     }
 }

git-odb/src/pack/index/traverse/lookup.rs

@@ -1,4 +1,4 @@
-use super::{Error, Reducer};
+use super::{Error, Reducer, SafetyCheck};
 use crate::pack::{self, data::decode, index, index::util};
 use git_features::{
     parallel::{self, in_parallel_if},
@@ -9,6 +9,7 @@ use git_features::{
 impl index::File {
     pub(crate) fn traverse_with_lookup<P, C, Processor>(
         &self,
+        check: SafetyCheck,
         thread_limit: Option<usize>,
         new_processor: impl Fn() -> Processor + Send + Sync,
         make_cache: impl Fn() -> C + Send + Sync,
@@ -61,6 +62,7 @@ impl index::File {
                 let mut header_buf = [0u8; 64];
                 for index_entry in entries.iter() {
                     stats.push(self.process_entry_dispatch(
+                        check,
                         pack,
                         cache,
                         buf,
@@ -73,7 +75,7 @@ impl index::File {
                 }
                 Ok(stats)
             },
-            Reducer::from_progress(&reduce_progress, pack.data_len()),
+            Reducer::from_progress(&reduce_progress, pack.data_len(), check),
         )
     }
 }

git-odb/src/pack/index/traverse/mod.rs

@@ -63,7 +63,7 @@ pub struct Outcome {
     pub pack_size: u64,
 }
 
-#[derive(Debug, PartialEq, Eq, Hash, Ord, PartialOrd, Clone)]
+#[derive(Debug, PartialEq, Eq, Hash, Ord, PartialOrd, Clone, Copy)]
 #[cfg_attr(feature = "serde1", derive(serde::Serialize, serde::Deserialize))]
 pub enum SafetyCheck {
     /// Don't verify the validity of the checksums stored in the index and pack file
@@ -75,13 +75,36 @@ pub enum SafetyCheck {
     /// All of the above, and only log object decode errors.
     ///
     /// Useful if there is a damaged pack and you would like to traverse as many objects as possible.
-    SkipFileAndObjectChecksumVerificationNoAbortOnDecodeError,
+    SkipFileAndObjectChecksumVerificationAndNoAbortOnDecodeError,
 
     /// Perform all available safety checks before operating on the pack and
     /// abort if any of them fails
     All,
 }
 
+impl SafetyCheck {
+    pub fn file_checksum(&self) -> bool {
+        match self {
+            SafetyCheck::All => true,
+            _ => false,
+        }
+    }
+    pub fn object_checksum(&self) -> bool {
+        match self {
+            SafetyCheck::All | SafetyCheck::SkipFileChecksumVerification => true,
+            _ => false,
+        }
+    }
+    pub fn fatal_decode_error(&self) -> bool {
+        match self {
+            SafetyCheck::All
+            | SafetyCheck::SkipFileChecksumVerification
+            | SafetyCheck::SkipFileAndObjectChecksumVerification => true,
+            SafetyCheck::SkipFileAndObjectChecksumVerificationAndNoAbortOnDecodeError => false,
+        }
+    }
+}
+
 impl Default for SafetyCheck {
     fn default() -> Self {
         SafetyCheck::All
@@ -136,7 +159,7 @@ impl index::File {
         Context {
             algorithm,
             thread_limit,
-            check: _,
+            check,
         }: Context,
         progress: Option<P>,
         new_processor: impl Fn() -> Processor + Send + Sync,
@@ -156,38 +179,45 @@ impl index::File {
     {
         let mut root = progress::DoOrDiscard::from(progress);
 
-        let progress = root.add_child("Sha1 of index");
-        let verify_self = move || self.verify_checksum(progress);
+        let id = if check.file_checksum() {
+            let progress = root.add_child("Sha1 of index");
+            let verify_self = move || self.verify_checksum(progress);
 
-        if self.pack_checksum() != pack.checksum() {
-            return Err(Error::PackMismatch {
-                actual: pack.checksum(),
-                expected: self.pack_checksum(),
-            });
-        }
-        let mut progress = root.add_child("Sha1 of pack");
-        let (pack_res, id) = parallel::join(
-            move || {
-                let throughput = TimeThroughput::new(pack.data_len());
-                let res = pack.verify_checksum();
-                progress.done(throughput);
-                res
-            },
-            verify_self,
-        );
-        pack_res?;
-        let id = id?;
+            if self.pack_checksum() != pack.checksum() {
+                return Err(Error::PackMismatch {
+                    actual: pack.checksum(),
+                    expected: self.pack_checksum(),
+                });
+            }
+            let mut progress = root.add_child("Sha1 of pack");
+            let (pack_res, id) = parallel::join(
+                move || {
+                    let throughput = TimeThroughput::new(pack.data_len());
+                    let res = pack.verify_checksum();
+                    progress.done(throughput);
+                    res
+                },
+                verify_self,
+            );
+            pack_res?;
+            id?
+        } else {
+            self.index_checksum()
+        };
 
         match algorithm {
-            Algorithm::Lookup => self.traverse_with_lookup(thread_limit, new_processor, make_cache, root, pack),
-            Algorithm::DeltaTreeLookup => self.traverse_with_index_lookup(thread_limit, new_processor, root, pack),
+            Algorithm::Lookup => self.traverse_with_lookup(check, thread_limit, new_processor, make_cache, root, pack),
+            Algorithm::DeltaTreeLookup => {
+                self.traverse_with_index_lookup(check, thread_limit, new_processor, root, pack)
+            }
         }
         .map(|stats| (id, stats))
     }
 
     #[allow(clippy::too_many_arguments)]
     pub(crate) fn process_entry_dispatch<C, P>(
         &self,
+        check: SafetyCheck,
         pack: &pack::data::File,
         cache: &mut C,
         buf: &mut Vec<u8>,
@@ -223,32 +253,34 @@ impl index::File {
         let object_kind = entry_stats.kind;
         let consumed_input = entry_stats.compressed_size;
 
-        let header_size = crate::loose::object::header::encode(object_kind, buf.len() as u64, &mut header_buf[..])
-            .expect("header buffer to be big enough");
-        let mut hasher = git_features::hash::Sha1::default();
-        hasher.update(&header_buf[..header_size]);
-        hasher.update(buf.as_slice());
+        if check.object_checksum() {
+            let header_size = crate::loose::object::header::encode(object_kind, buf.len() as u64, &mut header_buf[..])
+                .expect("header buffer to be big enough");
+            let mut hasher = git_features::hash::Sha1::default();
+            hasher.update(&header_buf[..header_size]);
+            hasher.update(buf.as_slice());
 
-        let actual_oid = owned::Id::new_sha1(hasher.digest());
-        if actual_oid != index_entry.oid {
-            return Err(Error::PackObjectMismatch {
-                actual: actual_oid,
-                expected: index_entry.oid,
-                offset: index_entry.pack_offset,
-                kind: object_kind,
-            });
-        }
-        if let Some(desired_crc32) = index_entry.crc32 {
-            let header_size = (pack_entry_data_offset - index_entry.pack_offset) as usize;
-            let actual_crc32 = pack.entry_crc32(index_entry.pack_offset, header_size + consumed_input);
-            if actual_crc32 != desired_crc32 {
-                return Err(Error::Crc32Mismatch {
-                    actual: actual_crc32,
-                    expected: desired_crc32,
+            let actual_oid = owned::Id::new_sha1(hasher.digest());
+            if actual_oid != index_entry.oid {
+                return Err(Error::PackObjectMismatch {
+                    actual: actual_oid,
+                    expected: index_entry.oid,
                     offset: index_entry.pack_offset,
                     kind: object_kind,
                 });
             }
+            if let Some(desired_crc32) = index_entry.crc32 {
+                let header_size = (pack_entry_data_offset - index_entry.pack_offset) as usize;
+                let actual_crc32 = pack.entry_crc32(index_entry.pack_offset, header_size + consumed_input);
+                if actual_crc32 != desired_crc32 {
+                    return Err(Error::Crc32Mismatch {
+                        actual: actual_crc32,
+                        expected: desired_crc32,
+                        offset: index_entry.pack_offset,
+                        kind: object_kind,
+                    });
+                }
+            }
         }
         processor(object_kind, buf.as_slice(), &index_entry, &entry_stats, progress)?;
         Ok(entry_stats)

git-odb/src/pack/index/traverse/reduce.rs

@@ -19,19 +19,25 @@ fn div_decode_result(lhs: &mut decode::Outcome, div: usize) {
 }
 
 pub struct Reducer<'a, P> {
-    pub progress: &'a std::sync::Mutex<P>,
-    pub then: Instant,
-    pub entries_seen: u32,
-    pub stats: traverse::Outcome,
+    progress: &'a std::sync::Mutex<P>,
+    check: traverse::SafetyCheck,
+    then: Instant,
+    entries_seen: u32,
+    stats: traverse::Outcome,
 }
 
 impl<'a, P> Reducer<'a, P>
 where
     P: Progress,
 {
-    pub fn from_progress(progress: &'a std::sync::Mutex<P>, pack_data_len_in_bytes: usize) -> Self {
+    pub fn from_progress(
+        progress: &'a std::sync::Mutex<P>,
+        pack_data_len_in_bytes: usize,
+        check: traverse::SafetyCheck,
+    ) -> Self {
         Reducer {
             progress: &progress,
+            check,
             then: Instant::now(),
             entries_seen: 0,
             stats: traverse::Outcome {
@@ -55,7 +61,16 @@ where
     type Error = traverse::Error;
 
     fn feed(&mut self, input: Self::Input) -> Result<(), Self::Error> {
-        let chunk_stats: Vec<_> = input?;
+        let chunk_stats: Vec<_> = match input {
+            Err(err @ traverse::Error::PackDecode(_, _, _)) if !self.check.fatal_decode_error() => {
+                self.progress
+                    .lock()
+                    .unwrap()
+                    .fail(format!("Ignoring decode error: {}", err));
+                return Ok(());
+            }
+            res => res,
+        }?;
         self.entries_seen += chunk_stats.len() as u32;
 
         let chunk_total = chunk_stats.into_iter().fold(