diff --git a/data/builder/dir_test.go b/data/builder/dir_test.go new file mode 100644 index 0000000..954d2dd --- /dev/null +++ b/data/builder/dir_test.go @@ -0,0 +1,72 @@ +package builder + +import ( + "bytes" + "fmt" + "testing" + + "github.com/ipfs/go-unixfsnode" + dagpb "github.com/ipld/go-codec-dagpb" + "github.com/ipld/go-ipld-prime" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" +) + +func mkEntries(cnt int, ls *ipld.LinkSystem) ([]dagpb.PBLink, error) { + entries := make([]dagpb.PBLink, 0, cnt) + for i := 0; i < cnt; i++ { + r := bytes.NewBufferString(fmt.Sprintf("%d", i)) + f, s, err := BuildUnixFSFile(r, "", ls) + if err != nil { + return nil, err + } + e, err := BuildUnixFSDirectoryEntry(fmt.Sprintf("file %d", i), int64(s), f) + if err != nil { + return nil, err + } + entries = append(entries, e) + } + return entries, nil +} + +func TestBuildUnixFSDirectory(t *testing.T) { + ls := cidlink.DefaultLinkSystem() + storage := cidlink.Memory{} + ls.StorageReadOpener = storage.OpenRead + ls.StorageWriteOpener = storage.OpenWrite + + testSizes := []int{100, 1000, 50000} + for _, cnt := range testSizes { + entries, err := mkEntries(cnt, &ls) + if err != nil { + t.Fatal(err) + } + + dl, err := BuildUnixFSDirectory(entries, &ls) + if err != nil { + t.Fatal(err) + } + + pbn, err := ls.Load(ipld.LinkContext{}, dl, dagpb.Type.PBNode) + if err != nil { + t.Fatal(err) + } + ufd, err := unixfsnode.Reify(ipld.LinkContext{}, pbn, &ls) + if err != nil { + t.Fatal(err) + } + observedCnt := 0 + + li := ufd.MapIterator() + for !li.Done() { + _, _, err := li.Next() + if err != nil { + t.Fatal(err) + } + observedCnt++ + } + if observedCnt != cnt { + fmt.Printf("%+v\n", ufd) + t.Fatalf("unexpected number of dir entries %d vs %d", observedCnt, cnt) + } + } +} diff --git a/data/builder/directory.go b/data/builder/directory.go new file mode 100644 index 0000000..c6b0549 --- /dev/null +++ b/data/builder/directory.go @@ -0,0 +1,130 @@ +package builder + +import ( + "fmt" + "io/fs" + "os" + "path" + + "github.com/ipfs/go-unixfsnode/data" + dagpb "github.com/ipld/go-codec-dagpb" + "github.com/ipld/go-ipld-prime" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/multiformats/go-multihash" +) + +// https://github.com/ipfs/go-ipfs/pull/8114/files#diff-eec963b47a6e1080d9d8023b4e438e6e3591b4154f7379a7e728401d2055374aR319 +const shardSplitThreshold = 262144 + +// https://github.com/ipfs/go-unixfs/blob/ec6bb5a4c5efdc3a5bce99151b294f663ee9c08d/io/directory.go#L29 +const defaultShardWidth = 256 + +// BuildUnixFSRecursive returns a link pointing to the UnixFS node representing +// the file or directory tree pointed to by `root` +func BuildUnixFSRecursive(root string, ls *ipld.LinkSystem) (ipld.Link, uint64, error) { + info, err := os.Lstat(root) + if err != nil { + return nil, 0, err + } + + m := info.Mode() + switch { + case m.IsDir(): + entries, err := os.ReadDir(root) + if err != nil { + return nil, 0, err + } + lnks := make([]dagpb.PBLink, 0, len(entries)) + for _, e := range entries { + lnk, sz, err := BuildUnixFSRecursive(path.Join(root, e.Name()), ls) + if err != nil { + return nil, 0, err + } + entry, err := BuildUnixFSDirectoryEntry(e.Name(), int64(sz), lnk) + if err != nil { + return nil, 0, err + } + lnks = append(lnks, entry) + } + outLnk, err := BuildUnixFSDirectory(lnks, ls) + return outLnk, 0, err + case m.Type() == fs.ModeSymlink: + content, err := os.Readlink(root) + if err != nil { + return nil, 0, err + } + return BuildUnixFSSymlink(content, ls) + case m.IsRegular(): + fp, err := os.Open(root) + if err != nil { + return nil, 0, err + } + defer fp.Close() + return BuildUnixFSFile(fp, "", ls) + default: + return nil, 0, fmt.Errorf("cannot encode non regular file: %s", root) + } +} + +// estimateDirSize estimates if a directory is big enough that it warrents sharding. +// The estimate is the sum over the len(linkName) + bytelen(linkHash) +// https://github.com/ipfs/go-unixfs/blob/master/io/directory.go#L152-L162 +func estimateDirSize(entries []dagpb.PBLink) int { + s := 0 + for _, e := range entries { + s += len(e.Name.Must().String()) + lnk := e.Hash.Link() + cl, ok := lnk.(cidlink.Link) + if ok { + s += cl.ByteLen() + } else { + s += len(lnk.Binary()) + } + } + return s +} + +// BuildUnixFSDirectory creates a directory link over a collection of entries. +func BuildUnixFSDirectory(entries []dagpb.PBLink, ls *ipld.LinkSystem) (ipld.Link, error) { + if estimateDirSize(entries) > shardSplitThreshold { + return BuildUnixFSShardedDirectory(defaultShardWidth, multihash.MURMUR3_128, entries, ls) + } + ufd, err := BuildUnixFS(func(b *Builder) { + DataType(b, data.Data_Directory) + }) + if err != nil { + return nil, err + } + pbb := dagpb.Type.PBNode.NewBuilder() + pbm, err := pbb.BeginMap(2) + if err != nil { + return nil, err + } + if err = pbm.AssembleKey().AssignString("Data"); err != nil { + return nil, err + } + if err = pbm.AssembleValue().AssignBytes(data.EncodeUnixFSData(ufd)); err != nil { + return nil, err + } + if err = pbm.AssembleKey().AssignString("Links"); err != nil { + return nil, err + } + lnks, err := pbm.AssembleValue().BeginList(int64(len(entries))) + if err != nil { + return nil, err + } + // sorting happens in codec-dagpb + for _, e := range entries { + if err := lnks.AssembleValue().AssignNode(e); err != nil { + return nil, err + } + } + if err := lnks.Finish(); err != nil { + return nil, err + } + if err := pbm.Finish(); err != nil { + return nil, err + } + node := pbb.Build() + return ls.Store(ipld.LinkContext{}, fileLinkProto, node) +} diff --git a/data/builder/dirshard.go b/data/builder/dirshard.go new file mode 100644 index 0000000..a25aa66 --- /dev/null +++ b/data/builder/dirshard.go @@ -0,0 +1,204 @@ +package builder + +import ( + "fmt" + "hash" + + bitfield "github.com/ipfs/go-bitfield" + "github.com/ipfs/go-unixfsnode/data" + "github.com/ipfs/go-unixfsnode/hamt" + dagpb "github.com/ipld/go-codec-dagpb" + "github.com/ipld/go-ipld-prime" + "github.com/multiformats/go-multihash" + "github.com/spaolacci/murmur3" +) + +type shard struct { + // metadata about the shard + hasher uint64 + size int + sizeLg2 int + width int + depth int + + children map[int]entry +} + +// a shard entry is either another shard, or a direct link. +type entry struct { + *shard + *hamtLink +} + +// a hamtLink is a member of the hamt - the file/directory pointed to, but +// stored with it's hashed key used for addressing. +type hamtLink struct { + hash hashBits + dagpb.PBLink +} + +// BuildUnixFSShardedDirectory will build a hamt of unixfs hamt shards encoing a directory with more entries +// than is typically allowed to fit in a standard IPFS single-block unixFS directory. +func BuildUnixFSShardedDirectory(size int, hasher uint64, entries []dagpb.PBLink, ls *ipld.LinkSystem) (ipld.Link, error) { + // hash the entries + var h hash.Hash + var err error + // TODO: use the multihash registry once murmur3 behavior is encoded there. + // https://github.com/multiformats/go-multihash/pull/150 + if hasher == hamt.HashMurmur3 { + h = murmur3.New64() + } else { + h, err = multihash.GetHasher(hasher) + if err != nil { + return nil, err + } + } + hamtEntries := make([]hamtLink, 0, len(entries)) + for _, e := range entries { + name := e.Name.Must().String() + sum := h.Sum([]byte(name)) + hamtEntries = append(hamtEntries, hamtLink{ + sum, + e, + }) + } + + sizeLg2, err := logtwo(size) + if err != nil { + return nil, err + } + + sharder := shard{ + hasher: hasher, + size: size, + sizeLg2: sizeLg2, + width: len(fmt.Sprintf("%X", size-1)), + depth: 0, + + children: make(map[int]entry), + } + + for _, entry := range hamtEntries { + err := sharder.add(entry) + if err != nil { + return nil, err + } + } + + return sharder.serialize(ls) +} + +func (s *shard) add(lnk hamtLink) error { + // get the bucket for lnk + bucket, err := lnk.hash.Slice(s.depth*s.sizeLg2, s.sizeLg2) + if err != nil { + return err + } + + current, ok := s.children[bucket] + if !ok { + s.children[bucket] = entry{nil, &lnk} + return nil + } else if current.shard != nil { + return current.shard.add(lnk) + } + // make a shard for current and lnk + newShard := entry{ + &shard{ + hasher: s.hasher, + size: s.size, + sizeLg2: s.sizeLg2, + width: s.width, + depth: s.depth + 1, + children: make(map[int]entry), + }, + nil, + } + if err := newShard.add(*current.hamtLink); err != nil { + return err + } + s.children[bucket] = newShard + return newShard.add(lnk) +} + +func (s *shard) formatLinkName(name string, idx int) string { + return fmt.Sprintf("%*X%s", s.width, idx, name) +} + +// bitmap calculates the bitmap of which links in the shard are set. +func (s *shard) bitmap() []byte { + bm := bitfield.NewBitfield(s.size) + for i := 0; i < s.size; i++ { + if _, ok := s.children[i]; ok { + bm.SetBit(i) + } + } + return bm.Bytes() +} + +// serialize stores the concrete representation of this shard in the link system and +// returns a link to it. +func (s *shard) serialize(ls *ipld.LinkSystem) (ipld.Link, error) { + ufd, err := BuildUnixFS(func(b *Builder) { + DataType(b, data.Data_HAMTShard) + HashType(b, s.hasher) + Data(b, s.bitmap()) + Fanout(b, uint64(s.size)) + }) + if err != nil { + return nil, err + } + pbb := dagpb.Type.PBNode.NewBuilder() + pbm, err := pbb.BeginMap(2) + if err != nil { + return nil, err + } + if err = pbm.AssembleKey().AssignString("Data"); err != nil { + return nil, err + } + if err = pbm.AssembleValue().AssignBytes(data.EncodeUnixFSData(ufd)); err != nil { + return nil, err + } + if err = pbm.AssembleKey().AssignString("Links"); err != nil { + return nil, err + } + + lnkBuilder := dagpb.Type.PBLinks.NewBuilder() + lnks, err := lnkBuilder.BeginList(int64(len(s.children))) + if err != nil { + return nil, err + } + // sorting happens in codec-dagpb + for idx, e := range s.children { + var lnk dagpb.PBLink + if e.shard != nil { + ipldLnk, err := e.shard.serialize(ls) + if err != nil { + return nil, err + } + fullName := s.formatLinkName("", idx) + lnk, err = BuildUnixFSDirectoryEntry(fullName, 0, ipldLnk) + if err != nil { + return nil, err + } + } else { + fullName := s.formatLinkName(e.Name.Must().String(), idx) + lnk, err = BuildUnixFSDirectoryEntry(fullName, e.Tsize.Must().Int(), e.Hash.Link()) + } + if err != nil { + return nil, err + } + if err := lnks.AssembleValue().AssignNode(lnk); err != nil { + return nil, err + } + } + if err := lnks.Finish(); err != nil { + return nil, err + } + pbm.AssembleValue().AssignNode(lnkBuilder.Build()) + if err := pbm.Finish(); err != nil { + return nil, err + } + node := pbb.Build() + return ls.Store(ipld.LinkContext{}, fileLinkProto, node) +} diff --git a/data/builder/file.go b/data/builder/file.go new file mode 100644 index 0000000..dcf3b81 --- /dev/null +++ b/data/builder/file.go @@ -0,0 +1,298 @@ +package builder + +import ( + "fmt" + "io" + + "github.com/ipfs/go-cid" + chunk "github.com/ipfs/go-ipfs-chunker" + "github.com/ipfs/go-unixfsnode/data" + dagpb "github.com/ipld/go-codec-dagpb" + "github.com/ipld/go-ipld-prime" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + basicnode "github.com/ipld/go-ipld-prime/node/basic" + "github.com/multiformats/go-multicodec" + multihash "github.com/multiformats/go-multihash/core" + + // raw needed for opening as bytes + _ "github.com/ipld/go-ipld-prime/codec/raw" +) + +// BuildUnixFSFile creates a dag of ipld Nodes representing file data. +// This recreates the functionality previously found in +// github.com/ipfs/go-unixfs/importer/balanced, but tailored to the +// go-unixfsnode & ipld-prime data layout of nodes. +// We make some assumptions in building files with this builder to reduce +// complexity, namely: +// * we assume we are using CIDv1, which has implied that the leaf +// data nodes are stored as raw bytes. +// ref: https://github.com/ipfs/go-mfs/blob/1b1fd06cff048caabeddb02d4dbf22d2274c7971/file.go#L50 +func BuildUnixFSFile(r io.Reader, chunker string, ls *ipld.LinkSystem) (ipld.Link, uint64, error) { + s, err := chunk.FromString(r, chunker) + if err != nil { + return nil, 0, err + } + + var prev []ipld.Link + var prevLen []uint64 + depth := 1 + for { + root, size, err := fileTreeRecursive(depth, prev, prevLen, s, ls) + if err != nil { + return nil, 0, err + } + + if prev != nil && prev[0] == root { + return root, size, nil + } + + prev = []ipld.Link{root} + prevLen = []uint64{size} + depth++ + } +} + +var fileLinkProto = cidlink.LinkPrototype{ + Prefix: cid.Prefix{ + Version: 1, + Codec: uint64(multicodec.DagPb), + MhType: multihash.SHA2_256, + MhLength: 32, + }, +} + +var leafLinkProto = cidlink.LinkPrototype{ + Prefix: cid.Prefix{ + Version: 1, + Codec: uint64(multicodec.Raw), + MhType: multihash.SHA2_256, + MhLength: 32, + }, +} + +func fileTreeRecursive(depth int, children []ipld.Link, childLen []uint64, src chunk.Splitter, ls *ipld.LinkSystem) (ipld.Link, uint64, error) { + if depth == 1 && len(children) > 0 { + return nil, 0, fmt.Errorf("leaf nodes cannot have children") + } else if depth == 1 { + leaf, err := src.NextBytes() + if err == io.EOF { + return nil, 0, nil + } else if err != nil { + return nil, 0, err + } + node := basicnode.NewBytes(leaf) + link, err := ls.Store(ipld.LinkContext{}, leafLinkProto, node) + return link, uint64(len(leaf)), err + } + // depth > 1. + totalSize := uint64(0) + blksizes := make([]uint64, 0, DefaultLinksPerBlock) + if children == nil { + children = make([]ipld.Link, 0) + } else { + for i := range children { + blksizes = append(blksizes, childLen[i]) + totalSize += childLen[i] + } + } + for len(children) < DefaultLinksPerBlock { + nxt, sz, err := fileTreeRecursive(depth-1, nil, nil, src, ls) + if err != nil { + return nil, 0, err + } else if nxt == nil { + // eof + break + } + totalSize += sz + children = append(children, nxt) + blksizes = append(blksizes, sz) + } + if len(children) == 0 { + // empty case. + return nil, 0, nil + } else if len(children) == 1 { + // degenerate case + return children[0], childLen[0], nil + } + + // make the unixfs node. + node, err := BuildUnixFS(func(b *Builder) { + FileSize(b, totalSize) + BlockSizes(b, blksizes) + }) + if err != nil { + return nil, 0, err + } + + // Pack into the dagpb node. + dpbb := dagpb.Type.PBNode.NewBuilder() + pbm, err := dpbb.BeginMap(2) + if err != nil { + return nil, 0, err + } + pblb, err := pbm.AssembleEntry("Links") + if err != nil { + return nil, 0, err + } + pbl, err := pblb.BeginList(int64(len(children))) + if err != nil { + return nil, 0, err + } + for i, c := range children { + pbln, err := BuildUnixFSDirectoryEntry("", int64(blksizes[i]), c) + if err != nil { + return nil, 0, err + } + if err = pbl.AssembleValue().AssignNode(pbln); err != nil { + return nil, 0, err + } + } + if err = pbl.Finish(); err != nil { + return nil, 0, err + } + if err = pbm.AssembleKey().AssignString("Data"); err != nil { + return nil, 0, err + } + if err = pbm.AssembleValue().AssignBytes(data.EncodeUnixFSData(node)); err != nil { + return nil, 0, err + } + if err = pbm.Finish(); err != nil { + return nil, 0, err + } + pbn := dpbb.Build() + + link, err := ls.Store(ipld.LinkContext{}, fileLinkProto, pbn) + if err != nil { + return nil, 0, err + } + // calculate the dagpb node's size and add as overhead. + cl, ok := link.(cidlink.Link) + if !ok { + return nil, 0, fmt.Errorf("unexpected non-cid linksystem") + } + rawlnk := cid.NewCidV1(uint64(multicodec.Raw), cl.Cid.Hash()) + rn, err := ls.Load(ipld.LinkContext{}, cidlink.Link{Cid: rawlnk}, basicnode.Prototype__Bytes{}) + if err != nil { + return nil, 0, fmt.Errorf("could not re-interpret dagpb node as bytes: %w", err) + } + rnb, err := rn.AsBytes() + if err != nil { + return nil, 0, fmt.Errorf("could not parse dagpb node as bytes: %w", err) + } + return link, totalSize + uint64(len(rnb)), nil +} + +// BuildUnixFSDirectoryEntry creates the link to a file or directory as it appears within a unixfs directory. +func BuildUnixFSDirectoryEntry(name string, size int64, hash ipld.Link) (dagpb.PBLink, error) { + dpbl := dagpb.Type.PBLink.NewBuilder() + lma, err := dpbl.BeginMap(3) + if err != nil { + return nil, err + } + if err = lma.AssembleKey().AssignString("Hash"); err != nil { + return nil, err + } + if err = lma.AssembleValue().AssignLink(hash); err != nil { + return nil, err + } + if err = lma.AssembleKey().AssignString("Name"); err != nil { + return nil, err + } + if err = lma.AssembleValue().AssignString(name); err != nil { + return nil, err + } + if err = lma.AssembleKey().AssignString("Tsize"); err != nil { + return nil, err + } + if err = lma.AssembleValue().AssignInt(size); err != nil { + return nil, err + } + if err = lma.Finish(); err != nil { + return nil, err + } + return dpbl.Build().(dagpb.PBLink), nil +} + +// BuildUnixFSSymlink builds a symlink entry in a unixfs tree +func BuildUnixFSSymlink(content string, ls *ipld.LinkSystem) (ipld.Link, uint64, error) { + // make the unixfs node. + node, err := BuildUnixFS(func(b *Builder) { + DataType(b, data.Data_Symlink) + Data(b, []byte(content)) + }) + if err != nil { + return nil, 0, err + } + + dpbb := dagpb.Type.PBNode.NewBuilder() + pbm, err := dpbb.BeginMap(2) + if err != nil { + return nil, 0, err + } + pblb, err := pbm.AssembleEntry("Links") + if err != nil { + return nil, 0, err + } + pbl, err := pblb.BeginList(0) + if err != nil { + return nil, 0, err + } + if err = pbl.Finish(); err != nil { + return nil, 0, err + } + if err = pbm.AssembleKey().AssignString("Data"); err != nil { + return nil, 0, err + } + if err = pbm.AssembleValue().AssignBytes(data.EncodeUnixFSData(node)); err != nil { + return nil, 0, err + } + if err = pbm.Finish(); err != nil { + return nil, 0, err + } + pbn := dpbb.Build() + + link, err := ls.Store(ipld.LinkContext{}, fileLinkProto, pbn) + if err != nil { + return nil, 0, err + } + // calculate the size and add as overhead. + cl, ok := link.(cidlink.Link) + if !ok { + return nil, 0, fmt.Errorf("unexpected non-cid linksystem") + } + rawlnk := cid.NewCidV1(uint64(multicodec.Raw), cl.Cid.Hash()) + rn, err := ls.Load(ipld.LinkContext{}, cidlink.Link{Cid: rawlnk}, basicnode.Prototype__Bytes{}) + if err != nil { + return nil, 0, fmt.Errorf("could not re-interpret dagpb node as bytes: %w", err) + } + rnb, err := rn.AsBytes() + if err != nil { + return nil, 0, fmt.Errorf("could not re-interpret dagpb node as bytes: %w", err) + } + return link, uint64(len(rnb)), nil +} + +// Constants below are from +// https://github.com/ipfs/go-unixfs/blob/ec6bb5a4c5efdc3a5bce99151b294f663ee9c08d/importer/helpers/helpers.go + +// BlockSizeLimit specifies the maximum size an imported block can have. +var BlockSizeLimit = 1048576 // 1 MB + +// rough estimates on expected sizes +var roughLinkBlockSize = 1 << 13 // 8KB +var roughLinkSize = 34 + 8 + 5 // sha256 multihash + size + no name + protobuf framing + +// DefaultLinksPerBlock governs how the importer decides how many links there +// will be per block. This calculation is based on expected distributions of: +// * the expected distribution of block sizes +// * the expected distribution of link sizes +// * desired access speed +// For now, we use: +// +// var roughLinkBlockSize = 1 << 13 // 8KB +// var roughLinkSize = 34 + 8 + 5 // sha256 multihash + size + no name +// // + protobuf framing +// var DefaultLinksPerBlock = (roughLinkBlockSize / roughLinkSize) +// = ( 8192 / 47 ) +// = (approximately) 174 +var DefaultLinksPerBlock = roughLinkBlockSize / roughLinkSize diff --git a/data/builder/file_test.go b/data/builder/file_test.go new file mode 100644 index 0000000..9fbe8e0 --- /dev/null +++ b/data/builder/file_test.go @@ -0,0 +1,41 @@ +package builder + +import ( + "bytes" + "testing" + + "github.com/ipfs/go-cid" + u "github.com/ipfs/go-ipfs-util" + "github.com/ipld/go-ipld-prime" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" +) + +func TestBuildUnixFSFile(t *testing.T) { + buf := make([]byte, 10*1024*1024) + u.NewSeededRand(0xdeadbeef).Read(buf) + r := bytes.NewReader(buf) + + ls := cidlink.DefaultLinkSystem() + storage := cidlink.Memory{} + ls.StorageReadOpener = storage.OpenRead + ls.StorageWriteOpener = storage.OpenWrite + + f, _, err := BuildUnixFSFile(r, "", &ls) + if err != nil { + t.Fatal(err) + } + + // Note: this differs from the previous + // go-unixfs version of this test (https://github.com/ipfs/go-unixfs/blob/master/importer/importer_test.go#L50) + // because this library enforces CidV1 encoding. + expected, err := cid.Decode("bafybeieyxejezqto5xwcxtvh5tskowwxrn3hmbk3hcgredji3g7abtnfkq") + if err != nil { + t.Fatal(err) + } + if !expected.Equals(f.(cidlink.Link).Cid) { + t.Fatalf("expected CID %s, got CID %s", expected, f) + } + if _, err := storage.OpenRead(ipld.LinkContext{}, f); err != nil { + t.Fatal("expected top of file to be in store.") + } +} diff --git a/data/builder/util.go b/data/builder/util.go new file mode 100644 index 0000000..8e5c0fb --- /dev/null +++ b/data/builder/util.go @@ -0,0 +1,56 @@ +package builder + +import ( + "fmt" + "math/bits" +) + +// Common code from go-unixfs/hamt/util.go + +// hashBits is a helper for pulling out sections of a hash +type hashBits []byte + +func mkmask(n int) byte { + return (1 << uint(n)) - 1 +} + +// Slice returns the 'width' bits of the hashBits value as an integer, or an +// error if there aren't enough bits. +func (hb hashBits) Slice(offset, width int) (int, error) { + if offset+width > len(hb)*8 { + return 0, fmt.Errorf("sharded directory too deep") + } + return hb.slice(offset, width), nil +} + +func (hb hashBits) slice(offset, width int) int { + curbi := offset / 8 + leftb := 8 - (offset % 8) + + curb := hb[curbi] + if width == leftb { + out := int(mkmask(width) & curb) + return out + } else if width < leftb { + a := curb & mkmask(leftb) // mask out the high bits we don't want + b := a & ^mkmask(leftb-width) // mask out the low bits we don't want + c := b >> uint(leftb-width) // shift whats left down + return int(c) + } else { + out := int(mkmask(leftb) & curb) + out <<= uint(width - leftb) + out += hb.slice(offset+leftb, width-leftb) + return out + } +} + +func logtwo(v int) (int, error) { + if v <= 0 { + return 0, fmt.Errorf("hamt size should be a power of two") + } + lg2 := bits.TrailingZeros(uint(v)) + if 1<