Merge pull request #56 from ipfs/fix/file-seek

fix: large files support io.SeekCurrent

Author: willscott

file/file_test.go

@@ -7,13 +7,16 @@ import (
 	"io"
 	"testing"
 
+	ipfsutil "github.com/ipfs/go-ipfs-util"
 	"github.com/ipfs/go-unixfsnode"
+	"github.com/ipfs/go-unixfsnode/data/builder"
 	"github.com/ipfs/go-unixfsnode/directory"
 	"github.com/ipfs/go-unixfsnode/file"
 	"github.com/ipld/go-car/v2/blockstore"
 	dagpb "github.com/ipld/go-codec-dagpb"
 	"github.com/ipld/go-ipld-prime"
 	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
+	"github.com/ipld/go-ipld-prime/node/basicnode"
 )
 
 func TestRootV0File(t *testing.T) {
@@ -61,6 +64,43 @@ func TestNamedV0File(t *testing.T) {
 	}
 }
 
+func TestFileSeeker(t *testing.T) {
+	ls := cidlink.DefaultLinkSystem()
+	storage := cidlink.Memory{}
+	ls.StorageReadOpener = storage.OpenRead
+	ls.StorageWriteOpener = storage.OpenWrite
+
+	// Make random file with 1024 bytes.
+	buf := make([]byte, 1024)
+	ipfsutil.NewSeededRand(0xdeadbeef).Read(buf)
+	r := bytes.NewReader(buf)
+
+	// Build UnixFS File as a single chunk
+	f, _, err := builder.BuildUnixFSFile(r, "size-1024", &ls)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Load the file.
+	fr, err := ls.Load(ipld.LinkContext{}, f, basicnode.Prototype.Bytes)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create it.
+	ufn, err := file.NewUnixFSFile(context.Background(), fr, &ls)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	rs, err := ufn.AsLargeBytes()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	testSeekIn1024ByteFile(t, rs)
+}
+
 func open(car string, t *testing.T) (ipld.Node, *ipld.LinkSystem) {
 	baseStore, err := blockstore.OpenReadOnly(car)
 	if err != nil {
@@ -88,3 +128,78 @@ func open(car string, t *testing.T) (ipld.Node, *ipld.LinkSystem) {
 	}
 	return root, &ls
 }
+
+func testSeekIn1024ByteFile(t *testing.T, rs io.ReadSeeker) {
+	// Seek from the start and try reading
+	offset, err := rs.Seek(128, io.SeekStart)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if offset != 128 {
+		t.Fatalf("expected offset %d, got %d", 484, offset)
+	}
+
+	readBuf := make([]byte, 256)
+	_, err = io.ReadFull(rs, readBuf)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Validate we can detect the offset with SeekCurrent
+	offset, err = rs.Seek(0, io.SeekCurrent)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if offset != 384 {
+		t.Fatalf("expected offset %d, got %d", 384, offset)
+	}
+
+	// Validate we can read after moving with SeekCurrent
+	offset, err = rs.Seek(100, io.SeekCurrent)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if offset != 484 {
+		t.Fatalf("expected offset %d, got %d", 484, offset)
+	}
+
+	_, err = io.ReadFull(rs, readBuf)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	offset, err = rs.Seek(0, io.SeekCurrent)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if offset != 740 {
+		t.Fatalf("expected offset %d, got %d", 740, offset)
+	}
+
+	// Validate we can read after moving with SeekEnd
+	offset, err = rs.Seek(-400, io.SeekEnd)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if offset != 624 {
+		t.Fatalf("expected offset %d, got %d", 624, offset)
+	}
+
+	_, err = io.ReadFull(rs, readBuf)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	offset, err = rs.Seek(0, io.SeekCurrent)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if offset != 880 {
+		t.Fatalf("expected offset %d, got %d", 880, offset)
+	}
+}

file/large_file_test.go

@@ -68,6 +68,43 @@ func TestLargeFileReader(t *testing.T) {
 	}
 }
 
+func TestLargeFileSeeker(t *testing.T) {
+	ls := cidlink.DefaultLinkSystem()
+	storage := cidlink.Memory{}
+	ls.StorageReadOpener = storage.OpenRead
+	ls.StorageWriteOpener = storage.OpenWrite
+
+	// Make random file with 1024 bytes.
+	buf := make([]byte, 1024)
+	ipfsutil.NewSeededRand(0xdeadbeef).Read(buf)
+	r := bytes.NewReader(buf)
+
+	// Build UnixFS File chunked in 256 byte parts.
+	f, _, err := builder.BuildUnixFSFile(r, "size-256", &ls)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Load the file.
+	fr, err := ls.Load(ipld.LinkContext{}, f, dagpb.Type.PBNode)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create it.
+	ufn, err := file.NewUnixFSFile(context.Background(), fr, &ls)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	rs, err := ufn.AsLargeBytes()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	testSeekIn1024ByteFile(t, rs)
+}
+
 func TestLargeFileReaderReadsOnlyNecessaryBlocks(t *testing.T) {
 	tracker, ls := mockTrackingLinkSystem()
 

file/shard.go

@@ -174,6 +174,7 @@ func (s *shardNodeReader) Read(p []byte) (int, error) {
 		s.rdr = rdr
 	}
 	n, err := s.rdr.Read(p)
+	s.offset += int64(n)
 	return n, err
 }