Skip to content

Commit 0d31ad1

Browse files
ByronByron
committed
The first 'explode' implementation…
…done entirely in gitoxide core based on quite powerful building blocks from the subcrates. Neat!
1 parent 1805d64 commit 0d31ad1

File tree

11 files changed

+85
-23
lines changed

11 files changed

+85
-23
lines changed

Diff for: ‎git-odb/src/pack/bundle.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -52,15 +52,15 @@ impl Bundle {
5252
out: &'a mut Vec<u8>,
5353
cache: &mut impl pack::cache::DecodeEntry,
5454
) -> Option<Result<Object<'a>, Error>> {
55-
let idx = self.index.lookup_index(id)?;
55+
let idx = self.index.lookup(id)?;
5656
let ofs = self.index.pack_offset_at_index(idx);
5757
let entry = self.pack.entry(ofs);
5858
self.pack
5959
.decode_entry(
6060
entry,
6161
out,
6262
|id, _out| {
63-
self.index.lookup_index(id).map(|idx| {
63+
self.index.lookup(id).map(|idx| {
6464
pack::data::decode::ResolvedBase::InPack(self.pack.entry(self.index.pack_offset_at_index(idx)))
6565
})
6666
},

Diff for: ‎git-odb/src/pack/index/access.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ impl index::File {
103103
}
104104

105105
/// Returns the offset of the given SHA1 for use with the `(oid|pack_offset|crc32)_at_index()`
106-
pub fn lookup_index(&self, id: borrowed::Id) -> Option<u32> {
106+
pub fn lookup(&self, id: borrowed::Id) -> Option<u32> {
107107
let first_byte = id.first_byte() as usize;
108108
let mut upper_bound = self.fan[first_byte];
109109
let mut lower_bound = if first_byte != 0 { self.fan[first_byte - 1] } else { 0 };

Diff for: ‎git-odb/src/pack/index/init.rs

+2-1
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ impl TryFrom<&Path> for index::File {
3636
type Error = Error;
3737

3838
fn try_from(path: &Path) -> Result<Self, Self::Error> {
39-
let data = FileBuffer::open(path).map_err(|e| Error::Io(e, path.to_owned()))?;
39+
let data = FileBuffer::open(&path).map_err(|e| Error::Io(e, path.to_owned()))?;
4040
let idx_len = data.len();
4141
if idx_len < FAN_LEN * N32_SIZE + FOOTER_SIZE {
4242
return Err(Error::Corrupt(format!(
@@ -73,6 +73,7 @@ impl TryFrom<&Path> for index::File {
7373
};
7474
Ok(index::File {
7575
data,
76+
path: path.to_owned(),
7677
kind,
7778
num_objects,
7879
version,

Diff for: ‎git-odb/src/pack/index/mod.rs

+4
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,7 @@ const FAN_LEN: usize = 256;
9090

9191
pub struct File {
9292
pub(crate) data: FileBuffer,
93+
path: std::path::PathBuf,
9394
kind: Kind,
9495
version: u32,
9596
num_objects: u32,
@@ -100,6 +101,9 @@ impl File {
100101
pub fn kind(&self) -> Kind {
101102
self.kind
102103
}
104+
pub fn path(&self) -> &std::path::Path {
105+
&self.path
106+
}
103107
pub fn num_objects(&self) -> u32 {
104108
self.num_objects
105109
}

Diff for: ‎git-odb/src/pack/index/traverse/indexed.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ impl index::File {
3737
sorted_entries.iter().map(|e| e.pack_offset),
3838
pack.path(),
3939
root.add_child("indexing"),
40-
|id| self.lookup_index(id).map(|idx| self.pack_offset_at_index(idx)),
40+
|id| self.lookup(id).map(|idx| self.pack_offset_at_index(idx)),
4141
)?;
4242
let if_there_are_enough_objects = || self.num_objects > 10_000;
4343

Diff for: ‎git-odb/src/pack/index/traverse/mod.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,7 @@ impl Default for Context {
153153

154154
/// Verify and validate the content of the index file
155155
impl index::File {
156-
pub fn traverse_index<P, C, Processor>(
156+
pub fn traverse<P, C, Processor>(
157157
&self,
158158
pack: &pack::data::File,
159159
Context {
@@ -243,7 +243,7 @@ impl index::File {
243243
pack_entry,
244244
buf,
245245
|id, _| {
246-
self.lookup_index(id).map(|index| {
246+
self.lookup(id).map(|index| {
247247
pack::data::decode::ResolvedBase::InPack(pack.entry(self.pack_offset_at_index(index)))
248248
})
249249
},

Diff for: ‎git-odb/src/pack/index/verify.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ impl index::File {
8888
match pack {
8989
None => verify_self().map_err(Into::into).map(|id| (id, None)),
9090
Some((pack, mode, algorithm)) => self
91-
.traverse_index(
91+
.traverse(
9292
pack,
9393
index::traverse::Context {
9494
algorithm,

Diff for: ‎git-odb/tests/pack/index.rs

+6-6
Original file line numberDiff line numberDiff line change
@@ -16,22 +16,22 @@ mod method {
1616
use git_odb::pack::index;
1717

1818
#[test]
19-
fn lookup_index() {
19+
fn lookup() {
2020
let idx = index::File::at(&fixture_path(INDEX_V1)).unwrap();
2121
for (id, desired_index, assertion) in &[
2222
(&b"036bd66fe9b6591e959e6df51160e636ab1a682e"[..], Some(0), "first"),
2323
(b"f7f791d96b9a34ef0f08db4b007c5309b9adc3d6", Some(65), "close to last"),
2424
(b"ffffffffffffffffffffffffffffffffffffffff", None, "not in pack"),
2525
] {
2626
assert_eq!(
27-
idx.lookup_index(owned::Id::from_40_bytes_in_hex(*id).unwrap().to_borrowed()),
27+
idx.lookup(owned::Id::from_40_bytes_in_hex(*id).unwrap().to_borrowed()),
2828
*desired_index,
2929
"{}",
3030
assertion
3131
);
3232
}
3333
for entry in idx.iter() {
34-
let index = idx.lookup_index(entry.oid.to_borrowed()).unwrap();
34+
let index = idx.lookup(entry.oid.to_borrowed()).unwrap();
3535
assert_eq!(entry.oid.to_borrowed(), idx.oid_at_index(index));
3636
assert_eq!(entry.pack_offset, idx.pack_offset_at_index(index));
3737
assert_eq!(entry.crc32, idx.crc32_at_index(index));
@@ -45,22 +45,22 @@ mod method {
4545
use git_odb::pack::index;
4646

4747
#[test]
48-
fn lookup_index() {
48+
fn lookup() {
4949
let idx = index::File::at(&fixture_path(INDEX_V2)).unwrap();
5050
for (id, desired_index, assertion) in &[
5151
(&b"0ead45fc727edcf5cadca25ef922284f32bb6fc1"[..], Some(0), "first"),
5252
(b"e800b9c207e17f9b11e321cc1fba5dfe08af4222", Some(29), "last"),
5353
(b"ffffffffffffffffffffffffffffffffffffffff", None, "not in pack"),
5454
] {
5555
assert_eq!(
56-
idx.lookup_index(owned::Id::from_40_bytes_in_hex(*id).unwrap().to_borrowed()),
56+
idx.lookup(owned::Id::from_40_bytes_in_hex(*id).unwrap().to_borrowed()),
5757
*desired_index,
5858
"{}",
5959
assertion
6060
);
6161
}
6262
for entry in idx.iter() {
63-
let index = idx.lookup_index(entry.oid.to_borrowed()).unwrap();
63+
let index = idx.lookup(entry.oid.to_borrowed()).unwrap();
6464
assert_eq!(entry.oid.to_borrowed(), idx.oid_at_index(index));
6565
assert_eq!(entry.pack_offset, idx.pack_offset_at_index(index));
6666
assert_eq!(entry.crc32, idx.crc32_at_index(index), "{} {:?}", index, entry);

Diff for: ‎gitoxide-core/src/pack/explode.rs

+57-8
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
11
use anyhow::{Context, Result};
22
use git_features::progress::Progress;
33
use git_object::{owned, HashKind};
4-
use git_odb::{loose, pack};
5-
use std::io::Read;
6-
use std::path::Path;
4+
use git_odb::{loose, pack, Write};
5+
use std::{
6+
fs,
7+
io::{self, Read},
8+
path::{Path, PathBuf},
9+
};
710

811
#[derive(PartialEq, Debug)]
912
pub enum SafetyCheck {
@@ -67,6 +70,17 @@ quick_error! {
6770
source(err)
6871
from()
6972
}
73+
Write(err: Box<dyn std::error::Error + Send + Sync>, kind: git_object::Kind, id: owned::Id) {
74+
display("Failed to write {} object {}", kind, id)
75+
source(&**err)
76+
}
77+
ObjectEncodeMismatch(kind: git_object::Kind, actual: owned::Id, expected: owned::Id) {
78+
display("{} object {} wasn't re-encoded without change - new hash is {}", kind, expected, actual)
79+
}
80+
RemoveFile(err: io::Error, index: PathBuf, data: PathBuf) {
81+
display("Failed to delete pack index file at '{} or data file at '{}'", index.display(), data.display())
82+
source(err)
83+
}
7084
}
7185
}
7286

@@ -98,21 +112,56 @@ impl git_odb::Write for OutputWriter {
98112
pub fn pack_or_pack_index<P>(
99113
pack_path: impl AsRef<Path>,
100114
object_path: Option<impl AsRef<Path>>,
101-
_check: SafetyCheck,
102-
_progress: Option<P>,
115+
check: SafetyCheck,
116+
thread_limit: Option<usize>,
117+
progress: Option<P>,
103118
_delete_pack: bool,
104119
) -> Result<()>
105120
where
106121
P: Progress,
122+
<P as Progress>::SubProgress: Send,
107123
{
108124
let path = pack_path.as_ref();
109-
let _bundle = pack::Bundle::at(path).with_context(|| {
125+
let bundle = pack::Bundle::at(path).with_context(|| {
110126
format!(
111127
"Could not find .idx or .pack file from given file at '{}'",
112128
path.display()
113129
)
114130
})?;
115131

116-
let _out = OutputWriter(object_path.map(|path| loose::Db::at(path.as_ref())));
117-
Ok(())
132+
let out = OutputWriter(object_path.map(|path| loose::Db::at(path.as_ref())));
133+
bundle.index.traverse(
134+
&bundle.pack,
135+
pack::index::traverse::Context {
136+
algorithm: pack::index::traverse::Algorithm::Lookup,
137+
thread_limit,
138+
check: check.into(),
139+
},
140+
progress,
141+
|| {
142+
|object_kind, buf, index_entry, _entry_stats, progress| {
143+
let written_id = out
144+
.write_buf(object_kind, buf, HashKind::Sha1)
145+
.map_err(|err| Error::Write(Box::new(err) as Box<dyn std::error::Error + Send + Sync>, object_kind, index_entry.oid))
146+
.map_err(|err| Box::new(err) as Box<dyn std::error::Error + Send + Sync>)?;
147+
if written_id != index_entry.oid {
148+
if let git_object::Kind::Tree = object_kind {
149+
progress.info(format!("The tree in pack named {} was written as {} due to modes 100664 and 100640 rewritten as 100644.", index_entry.oid, written_id));
150+
} else {
151+
return Err(Box::new(Error::ObjectEncodeMismatch(object_kind, index_entry.oid, written_id)))
152+
}
153+
}
154+
Ok(())
155+
}
156+
},
157+
pack::cache::DecodeEntryLRU::default,
158+
).with_context(|| "Some loose objects could not be extracted")?;
159+
160+
let (index_path, data_path) = (bundle.index.path().to_owned(), bundle.pack.path().to_owned());
161+
drop(bundle);
162+
163+
fs::remove_file(&index_path)
164+
.and_then(|_| fs::remove_file(&data_path))
165+
.map_err(|err| Error::RemoveFile(err, index_path, data_path))
166+
.map_err(Into::into)
118167
}

Diff for: ‎src/plumbing/lean.rs

+1
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,7 @@ pub fn main() -> Result<()> {
146146
pack_path,
147147
object_path,
148148
check.unwrap_or(core::pack::explode::SafetyCheck::All),
149+
thread_limit,
149150
progress,
150151
delete_pack,
151152
)

Diff for: ‎src/plumbing/pretty.rs

+8-1
Original file line numberDiff line numberDiff line change
@@ -233,7 +233,14 @@ pub fn main() -> Result<()> {
233233
progress,
234234
progress_keep_open,
235235
move |progress, _out, _err| {
236-
core::pack::explode::pack_or_pack_index(pack_path, object_path, check, progress, delete_pack)
236+
core::pack::explode::pack_or_pack_index(
237+
pack_path,
238+
object_path,
239+
check,
240+
thread_limit,
241+
progress,
242+
delete_pack,
243+
)
237244
},
238245
),
239246
Subcommands::PackVerify {

0 commit comments

Comments
 (0)