Merge pull request #3608 from ipfs/feat/new-flatfs-sharding

Use newer flatfs sharding scheme

Author: whyrusleeping

package.json

@@ -213,9 +213,9 @@
     },
     {
       "author": "whyrusleeping",
-      "hash": "Qmbx2KUs8mUbDUiiESzC1ms7mdmh4pRu8X1V1tffC46M4n",
+      "hash": "Qmcdc2Str4Dkc8yVCzzUFpnxmkS9AZXnpfo6jbhjjtXXkB",
       "name": "go-ds-flatfs",
-      "version": "1.0.1"
+      "version": "1.1.0"
     },
     {
       "author": "whyrusleeping",

repo/fsrepo/defaultds.go

@@ -13,7 +13,7 @@ import (
 	levelds "gx/ipfs/QmaHHmfEozrrotyhyN44omJouyuEtx6ahddqV6W5yRaUSQ/go-ds-leveldb"
 	ldbopts "gx/ipfs/QmbBhyDKsY4mbY6xsKt3qu9Y7FPvMJ6qbD8AMjYYvPRw1g/goleveldb/leveldb/opt"
 	measure "gx/ipfs/QmbUSMTQtK9GRrUbD4ngqJwSzHsquUc8nyDubRWp4vPybH/go-ds-measure"
-	"gx/ipfs/Qmbx2KUs8mUbDUiiESzC1ms7mdmh4pRu8X1V1tffC46M4n/go-ds-flatfs"
+	"gx/ipfs/Qmcdc2Str4Dkc8yVCzzUFpnxmkS9AZXnpfo6jbhjjtXXkB/go-ds-flatfs"
 )
 
 const (
@@ -33,9 +33,10 @@ func openDefaultDatastore(r *FSRepo) (repo.Datastore, error) {
 	}
 
 	syncfs := !r.config.Datastore.NoSync
-	// 5 bytes of prefix gives us 25 bits of freedom, 16 of which are taken by
-	// by the Qm prefix. Leaving us with 9 bits, or 512 way sharding
-	blocksDS, err := flatfs.New(path.Join(r.path, flatfsDirectory), 5, syncfs)
+
+	// 2 characters of base32 suffix gives us 10 bits of freedom.
+	// Leaving us with 10 bits, or 1024 way sharding
+	blocksDS, err := flatfs.CreateOrOpen(path.Join(r.path, flatfsDirectory), flatfs.NextToLast(2), syncfs)
 	if err != nil {
 		return nil, fmt.Errorf("unable to open flatfs datastore: %v", err)
 	}

repo/fsrepo/fsrepo.go

@@ -29,7 +29,7 @@ import (
 var log = logging.Logger("fsrepo")
 
 // version number that we are currently expecting to see
-var RepoVersion = 4
+var RepoVersion = 5
 
 var migrationInstructions = `See https://github.com/ipfs/fs-repo-migrations/blob/master/run.md
 Sorry for the inconvenience. In the future, these will run automatically.`

repo/fsrepo/migrations/migrations.go

@@ -35,6 +35,9 @@ func migrationsBinName() string {
 }
 
 func RunMigration(newv int) error {
+	if newv == 5 {
+		return fmt.Errorf("No automatic migrations yet for repo version 5. You will have to build them from source.\nSee https://github.com/ipfs/fs-repo-migrations")
+	}
 	migrateBin := migrationsBinName()
 
 	fmt.Println("  => Looking for suitable fs-repo-migrations binary.")

test/sharness/t0066-migration.sh

@@ -35,8 +35,9 @@ test_expect_success "ipfs daemon --migrate=true runs migration" '
 '
 
 test_expect_success "output looks good" '
-	grep "Running: " true_out > /dev/null &&
-	grep "Success: fs-repo has been migrated to version 4." true_out > /dev/null
+	#grep "Running: " true_out > /dev/null &&
+	#grep "Success: fs-repo has been migrated to version 4." true_out > /dev/null
+	grep "No automatic migrations yet" true_out > /dev/null
 '
 
 test_expect_success "'ipfs daemon' prompts to auto migrate" '

test/sharness/t0084-repo-read-rehash.sh

@@ -16,8 +16,8 @@ test_init_ipfs
 H_BLOCK1=$(echo "Block 1" | ipfs add -q)
 H_BLOCK2=$(echo "Block 2" | ipfs add -q)
 
-BS_BLOCK1="CIQPD/CIQPDDQH5PDJTF4QSNMPFC45FQZH5MBSWCX2W254P7L7HGNHW5MQXZA.data"
-BS_BLOCK2="CIQNY/CIQNYWBOKHY7TCY7FUOBXKVJ66YRMARDT3KC7PPY6UWWPZR4YA67CKQ.data"
+BS_BLOCK1="XZ/CIQPDDQH5PDJTF4QSNMPFC45FQZH5MBSWCX2W254P7L7HGNHW5MQXZA.data"
+BS_BLOCK2="CK/CIQNYWBOKHY7TCY7FUOBXKVJ66YRMARDT3KC7PPY6UWWPZR4YA67CKQ.data"
 
 
 test_expect_success 'blocks are swapped' '