Merge pull request ipfs/kubo#6680 from ipfs/fix/symlink-size

improve gateway symlink handling

This commit was moved from ipfs/kubo@ca2767a

Author: Stebalien

gateway/core/corehttp/gateway_handler.go

@@ -289,7 +289,7 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
 		}
 
 		// write to request
-		http.ServeContent(w, r, "index.html", modtime, f)
+		i.serveFile(w, r, "index.html", modtime, f)
 		return
 	case resolver.ErrNoLink:
 		// no index.html; noop
@@ -306,14 +306,14 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
 	var dirListing []directoryItem
 	dirit := dir.Entries()
 	for dirit.Next() {
-		// See comment above where originalUrlPath is declared.
-		s, err := dirit.Node().Size()
-		if err != nil {
-			internalWebError(w, err)
-			return
+		size := "?"
+		if s, err := dirit.Node().Size(); err == nil {
+			// Size may not be defined/supported. Continue anyways.
+			size = humanize.Bytes(uint64(s))
 		}
 
-		di := directoryItem{humanize.Bytes(uint64(s)), dirit.Name(), gopath.Join(originalUrlPath, dirit.Name())}
+		// See comment above where originalUrlPath is declared.
+		di := directoryItem{size, dirit.Name(), gopath.Join(originalUrlPath, dirit.Name())}
 		dirListing = append(dirListing, di)
 	}
 	if dirit.Err() != nil {
@@ -372,48 +372,42 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
 	}
 }
 
-type sizeReadSeeker interface {
-	Size() (int64, error)
-
-	io.ReadSeeker
-}
-
-type sizeSeeker struct {
-	sizeReadSeeker
-}
-
-func (s *sizeSeeker) Seek(offset int64, whence int) (int64, error) {
-	if whence == io.SeekEnd && offset == 0 {
-		return s.Size()
+func (i *gatewayHandler) serveFile(w http.ResponseWriter, req *http.Request, name string, modtime time.Time, file files.File) {
+	size, err := file.Size()
+	if err != nil {
+		http.Error(w, "cannot serve files with unknown sizes", http.StatusBadGateway)
+		return
 	}
 
-	return s.sizeReadSeeker.Seek(offset, whence)
-}
-
-func (i *gatewayHandler) serveFile(w http.ResponseWriter, req *http.Request, name string, modtime time.Time, content io.ReadSeeker) {
-	if sp, ok := content.(sizeReadSeeker); ok {
-		content = &sizeSeeker{
-			sizeReadSeeker: sp,
-		}
+	content := &lazySeeker{
+		size:   size,
+		reader: file,
 	}
 
-	ctype := mime.TypeByExtension(gopath.Ext(name))
-	if ctype == "" {
-		buf := make([]byte, 512)
-		n, _ := io.ReadFull(content, buf[:])
-		ctype = http.DetectContentType(buf[:n])
-		_, err := content.Seek(0, io.SeekStart)
-		if err != nil {
-			http.Error(w, "seeker can't seek", http.StatusInternalServerError)
-			return
+	var ctype string
+	if _, isSymlink := file.(*files.Symlink); isSymlink {
+		// We should be smarter about resolving symlinks but this is the
+		// "most correct" we can be without doing that.
+		ctype = "inode/symlink"
+	} else {
+		ctype = mime.TypeByExtension(gopath.Ext(name))
+		if ctype == "" {
+			buf := make([]byte, 512)
+			n, _ := io.ReadFull(content, buf[:])
+			ctype = http.DetectContentType(buf[:n])
+			_, err := content.Seek(0, io.SeekStart)
+			if err != nil {
+				http.Error(w, "seeker can't seek", http.StatusInternalServerError)
+				return
+			}
+		}
+		// Strip the encoding from the HTML Content-Type header and let the
+		// browser figure it out.
+		//
+		// Fixes https://github.com/ipfs/go-ipfs/issues/2203
+		if strings.HasPrefix(ctype, "text/html;") {
+			ctype = "text/html"
 		}
-	}
-	// Strip the encoding from the HTML Content-Type header and let the
-	// browser figure it out.
-	//
-	// Fixes https://github.com/ipfs/go-ipfs/issues/2203
-	if strings.HasPrefix(ctype, "text/html;") {
-		ctype = "text/html"
 	}
 	w.Header().Set("Content-Type", ctype)
 

gateway/core/corehttp/lazyseek.go

@@ -0,0 +1,60 @@
+package corehttp
+
+import (
+	"fmt"
+	"io"
+)
+
+// The HTTP server uses seek to determine the file size. Actually _seeking_ can
+// be slow so we wrap the seeker in a _lazy_ seeker.
+type lazySeeker struct {
+	reader io.ReadSeeker
+
+	size       int64
+	offset     int64
+	realOffset int64
+}
+
+func (s *lazySeeker) Seek(offset int64, whence int) (int64, error) {
+	switch whence {
+	case io.SeekEnd:
+		return s.Seek(s.size+offset, io.SeekStart)
+	case io.SeekCurrent:
+		return s.Seek(s.offset+offset, io.SeekStart)
+	case io.SeekStart:
+		if offset < 0 {
+			return s.offset, fmt.Errorf("invalid seek offset")
+		}
+		s.offset = offset
+		return s.offset, nil
+	default:
+		return s.offset, fmt.Errorf("invalid whence: %d", whence)
+	}
+}
+
+func (s *lazySeeker) Read(b []byte) (int, error) {
+	// If we're past the end, EOF.
+	if s.offset >= s.size {
+		return 0, io.EOF
+	}
+
+	// actually seek
+	for s.offset != s.realOffset {
+		off, err := s.reader.Seek(s.offset, io.SeekStart)
+		if err != nil {
+			return 0, err
+		}
+		s.realOffset = off
+	}
+	off, err := s.reader.Read(b)
+	s.realOffset += int64(off)
+	s.offset += int64(off)
+	return off, err
+}
+
+func (s *lazySeeker) Close() error {
+	if closer, ok := s.reader.(io.Closer); ok {
+		return closer.Close()
+	}
+	return nil
+}

gateway/core/corehttp/lazyseek_test.go

@@ -0,0 +1,137 @@
+package corehttp
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"strings"
+	"testing"
+)
+
+type badSeeker struct {
+	io.ReadSeeker
+}
+
+var badSeekErr = fmt.Errorf("I'm a bad seeker")
+
+func (bs badSeeker) Seek(offset int64, whence int) (int64, error) {
+	off, err := bs.ReadSeeker.Seek(0, io.SeekCurrent)
+	if err != nil {
+		panic(err)
+	}
+	return off, badSeekErr
+}
+
+func TestLazySeekerError(t *testing.T) {
+	underlyingBuffer := strings.NewReader("fubar")
+	s := &lazySeeker{
+		reader: badSeeker{underlyingBuffer},
+		size:   underlyingBuffer.Size(),
+	}
+	off, err := s.Seek(0, io.SeekEnd)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if off != s.size {
+		t.Fatal("expected to seek to the end")
+	}
+
+	// shouldn't have actually seeked.
+	b, err := ioutil.ReadAll(s)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(b) != 0 {
+		t.Fatal("expected to read nothing")
+	}
+
+	// shouldn't need to actually seek.
+	off, err = s.Seek(0, io.SeekStart)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if off != 0 {
+		t.Fatal("expected to seek to the start")
+	}
+	b, err = ioutil.ReadAll(s)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(b) != "fubar" {
+		t.Fatal("expected to read string")
+	}
+
+	// should fail the second time.
+	off, err = s.Seek(0, io.SeekStart)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if off != 0 {
+		t.Fatal("expected to seek to the start")
+	}
+	// right here...
+	b, err = ioutil.ReadAll(s)
+	if err == nil {
+		t.Fatalf("expected an error, got output %s", string(b))
+	}
+	if err != badSeekErr {
+		t.Fatalf("expected a bad seek error, got %s", err)
+	}
+	if len(b) != 0 {
+		t.Fatalf("expected to read nothing")
+	}
+}
+
+func TestLazySeeker(t *testing.T) {
+	underlyingBuffer := strings.NewReader("fubar")
+	s := &lazySeeker{
+		reader: underlyingBuffer,
+		size:   underlyingBuffer.Size(),
+	}
+	expectByte := func(b byte) {
+		t.Helper()
+		var buf [1]byte
+		n, err := io.ReadFull(s, buf[:])
+		if err != nil {
+			t.Fatal(err)
+		}
+		if n != 1 {
+			t.Fatalf("expected to read one byte, read %d", n)
+		}
+		if buf[0] != b {
+			t.Fatalf("expected %b, got %b", b, buf[0])
+		}
+	}
+	expectSeek := func(whence int, off, expOff int64, expErr string) {
+		t.Helper()
+		n, err := s.Seek(off, whence)
+		if expErr == "" {
+			if err != nil {
+				t.Fatal("unexpected seek error: ", err)
+			}
+		} else {
+			if err == nil || err.Error() != expErr {
+				t.Fatalf("expected %s, got %s", err, expErr)
+			}
+		}
+		if n != expOff {
+			t.Fatalf("expected offset %d, got, %d", expOff, n)
+		}
+	}
+
+	expectSeek(io.SeekEnd, 0, s.size, "")
+	b, err := ioutil.ReadAll(s)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(b) != 0 {
+		t.Fatal("expected to read nothing")
+	}
+	expectSeek(io.SeekEnd, -1, s.size-1, "")
+	expectByte('r')
+	expectSeek(io.SeekStart, 0, 0, "")
+	expectByte('f')
+	expectSeek(io.SeekCurrent, 1, 2, "")
+	expectByte('b')
+	expectSeek(io.SeekCurrent, -100, 3, "invalid seek offset")
+}