Skip to content

Commit ef2be20

Browse files
magik6kmagik6k
committed
Move datastores to plugins
License: MIT Signed-off-by: Łukasz Magiera <[email protected]>
1 parent bcb5fc2 commit ef2be20

File tree

7 files changed

+310
-188
lines changed

7 files changed

+310
-188
lines changed

‎plugin/loader/preload.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,10 @@ package loader
22

33
import (
44
"github.com/ipfs/go-ipfs/plugin"
5+
pluginbadgerds "github.com/ipfs/go-ipfs/plugin/plugins/badgerds"
6+
pluginflatfs "github.com/ipfs/go-ipfs/plugin/plugins/flatfs"
57
pluginipldgit "github.com/ipfs/go-ipfs/plugin/plugins/git"
8+
pluginlevelds "github.com/ipfs/go-ipfs/plugin/plugins/levelds"
69
)
710

811
// DO NOT EDIT THIS FILE
@@ -11,4 +14,7 @@ import (
1114

1215
var preloadPlugins = []plugin.Plugin{
1316
pluginipldgit.Plugins[0],
17+
pluginbadgerds.Plugins[0],
18+
pluginflatfs.Plugins[0],
19+
pluginlevelds.Plugins[0],
1420
}

‎plugin/loader/preload_list

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,7 @@
44
# name go-path number of the sub-plugin
55

66
ipldgit github.com/ipfs/go-ipfs/plugin/plugins/git 0
7+
8+
badgerds github.com/ipfs/go-ipfs/plugin/plugins/badgerds 0
9+
flatfs github.com/ipfs/go-ipfs/plugin/plugins/flatfs 0
10+
levelds github.com/ipfs/go-ipfs/plugin/plugins/levelds 0

‎plugin/plugins/Rules.mk

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
include mk/header.mk
22

3-
$(d)_plugins:=$(d)/git
3+
$(d)_plugins:=$(d)/git $(d)/badgerds $(d)/flatfs $(d)/levelds
44
$(d)_plugins_so:=$(addsuffix .so,$($(d)_plugins))
55
$(d)_plugins_main:=$(addsuffix /main/main.go,$($(d)_plugins))
66

‎plugin/plugins/badgerds/badgerds.go

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
package badgerds
2+
3+
import (
4+
"fmt"
5+
"os"
6+
"path/filepath"
7+
8+
"github.com/ipfs/go-ipfs/plugin"
9+
"github.com/ipfs/go-ipfs/repo"
10+
"github.com/ipfs/go-ipfs/repo/fsrepo"
11+
12+
humanize "gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize"
13+
badgerds "gx/ipfs/QmaixNkKwtinV3umL5VD1VDD5CQjnZhXY31awM2YHTzbui/go-ds-badger"
14+
)
15+
16+
// Plugins is exported list of plugins that will be loaded
17+
var Plugins = []plugin.Plugin{
18+
&badgerdsPlugin{},
19+
}
20+
21+
type badgerdsPlugin struct{}
22+
23+
var _ plugin.PluginDatastore = (*badgerdsPlugin)(nil)
24+
25+
func (*badgerdsPlugin) Name() string {
26+
return "ds-badgerds"
27+
}
28+
29+
func (*badgerdsPlugin) Version() string {
30+
return "0.1.0"
31+
}
32+
33+
func (*badgerdsPlugin) Init() error {
34+
return nil
35+
}
36+
37+
func (*badgerdsPlugin) DatastoreTypeName() string {
38+
return "badgerds"
39+
}
40+
41+
type datastoreConfig struct {
42+
path string
43+
syncWrites bool
44+
45+
vlogFileSize int64
46+
}
47+
48+
// BadgerdsDatastoreConfig returns a configuration stub for a badger datastore
49+
// from the given parameters
50+
func (*badgerdsPlugin) DatastoreConfigParser() fsrepo.ConfigFromMap {
51+
return func(params map[string]interface{}) (fsrepo.DatastoreConfig, error) {
52+
var c datastoreConfig
53+
var ok bool
54+
55+
c.path, ok = params["path"].(string)
56+
if !ok {
57+
return nil, fmt.Errorf("'path' field is missing or not string")
58+
}
59+
60+
sw, ok := params["syncWrites"]
61+
if !ok {
62+
c.syncWrites = true
63+
} else {
64+
if swb, ok := sw.(bool); ok {
65+
c.syncWrites = swb
66+
} else {
67+
return nil, fmt.Errorf("'syncWrites' field was not a boolean")
68+
}
69+
}
70+
71+
vls, ok := params["vlogFileSize"]
72+
if !ok {
73+
// default to 1GiB
74+
c.vlogFileSize = badgerds.DefaultOptions.ValueLogFileSize
75+
} else {
76+
if vlogSize, ok := vls.(string); ok {
77+
s, err := humanize.ParseBytes(vlogSize)
78+
if err != nil {
79+
return nil, err
80+
}
81+
c.vlogFileSize = int64(s)
82+
} else {
83+
return nil, fmt.Errorf("'vlogFileSize' field was not a string")
84+
}
85+
}
86+
87+
return &c, nil
88+
}
89+
}
90+
91+
func (c *datastoreConfig) DiskSpec() fsrepo.DiskSpec {
92+
return map[string]interface{}{
93+
"type": "badgerds",
94+
"path": c.path,
95+
}
96+
}
97+
98+
func (c *datastoreConfig) Create(path string) (repo.Datastore, error) {
99+
p := c.path
100+
if !filepath.IsAbs(p) {
101+
p = filepath.Join(path, p)
102+
}
103+
104+
err := os.MkdirAll(p, 0755)
105+
if err != nil {
106+
return nil, err
107+
}
108+
109+
defopts := badgerds.DefaultOptions
110+
defopts.SyncWrites = c.syncWrites
111+
defopts.ValueLogFileSize = c.vlogFileSize
112+
113+
return badgerds.NewDatastore(p, &defopts)
114+
}

‎plugin/plugins/flatfs/flatfs.go

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
package flatfs
2+
3+
import (
4+
"fmt"
5+
"path/filepath"
6+
7+
"github.com/ipfs/go-ipfs/plugin"
8+
"github.com/ipfs/go-ipfs/repo"
9+
"github.com/ipfs/go-ipfs/repo/fsrepo"
10+
11+
flatfs "gx/ipfs/QmbYQHTSz4Sg4Y2Vyd7hftGgM1yXwSUBKb47VpoPRuP5Nc/go-ds-flatfs"
12+
)
13+
14+
// Plugins is exported list of plugins that will be loaded
15+
var Plugins = []plugin.Plugin{
16+
&flatfsPlugin{},
17+
}
18+
19+
type flatfsPlugin struct{}
20+
21+
var _ plugin.PluginDatastore = (*flatfsPlugin)(nil)
22+
23+
func (*flatfsPlugin) Name() string {
24+
return "ds-flatfs"
25+
}
26+
27+
func (*flatfsPlugin) Version() string {
28+
return "0.1.0"
29+
}
30+
31+
func (*flatfsPlugin) Init() error {
32+
return nil
33+
}
34+
35+
func (*flatfsPlugin) DatastoreTypeName() string {
36+
return "flatfs"
37+
}
38+
39+
type datastoreConfig struct {
40+
path string
41+
shardFun *flatfs.ShardIdV1
42+
syncField bool
43+
}
44+
45+
// BadgerdsDatastoreConfig returns a configuration stub for a badger datastore
46+
// from the given parameters
47+
func (*flatfsPlugin) DatastoreConfigParser() fsrepo.ConfigFromMap {
48+
return func(params map[string]interface{}) (fsrepo.DatastoreConfig, error) {
49+
var c datastoreConfig
50+
var ok bool
51+
var err error
52+
53+
c.path, ok = params["path"].(string)
54+
if !ok {
55+
return nil, fmt.Errorf("'path' field is missing or not boolean")
56+
}
57+
58+
sshardFun, ok := params["shardFunc"].(string)
59+
if !ok {
60+
return nil, fmt.Errorf("'shardFunc' field is missing or not a string")
61+
}
62+
c.shardFun, err = flatfs.ParseShardFunc(sshardFun)
63+
if err != nil {
64+
return nil, err
65+
}
66+
67+
c.syncField, ok = params["sync"].(bool)
68+
if !ok {
69+
return nil, fmt.Errorf("'sync' field is missing or not boolean")
70+
}
71+
return &c, nil
72+
}
73+
}
74+
75+
func (c *datastoreConfig) DiskSpec() fsrepo.DiskSpec {
76+
return map[string]interface{}{
77+
"type": "flatfs",
78+
"path": c.path,
79+
"shardFunc": c.shardFun.String(),
80+
}
81+
}
82+
83+
func (c *datastoreConfig) Create(path string) (repo.Datastore, error) {
84+
p := c.path
85+
if !filepath.IsAbs(p) {
86+
p = filepath.Join(path, p)
87+
}
88+
89+
return flatfs.CreateOrOpen(p, c.shardFun, c.syncField)
90+
}

‎plugin/plugins/levelds/levelds.go

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
package levelds
2+
3+
import (
4+
"fmt"
5+
"path/filepath"
6+
7+
"github.com/ipfs/go-ipfs/plugin"
8+
"github.com/ipfs/go-ipfs/repo"
9+
"github.com/ipfs/go-ipfs/repo/fsrepo"
10+
11+
ldbopts "gx/ipfs/QmbBhyDKsY4mbY6xsKt3qu9Y7FPvMJ6qbD8AMjYYvPRw1g/goleveldb/leveldb/opt"
12+
levelds "gx/ipfs/QmccqjKZUTqp4ikWNyAbjBuP5HEdqSqRuAr9mcEhYab54a/go-ds-leveldb"
13+
)
14+
15+
// Plugins is exported list of plugins that will be loaded
16+
var Plugins = []plugin.Plugin{
17+
&leveldsPlugin{},
18+
}
19+
20+
type leveldsPlugin struct{}
21+
22+
var _ plugin.PluginDatastore = (*leveldsPlugin)(nil)
23+
24+
func (*leveldsPlugin) Name() string {
25+
return "ds-level"
26+
}
27+
28+
func (*leveldsPlugin) Version() string {
29+
return "0.1.0"
30+
}
31+
32+
func (*leveldsPlugin) Init() error {
33+
return nil
34+
}
35+
36+
func (*leveldsPlugin) DatastoreTypeName() string {
37+
return "levelds"
38+
}
39+
40+
type datastoreConfig struct {
41+
path string
42+
compression ldbopts.Compression
43+
}
44+
45+
// BadgerdsDatastoreConfig returns a configuration stub for a badger datastore
46+
// from the given parameters
47+
func (*leveldsPlugin) DatastoreConfigParser() fsrepo.ConfigFromMap {
48+
return func(params map[string]interface{}) (fsrepo.DatastoreConfig, error) {
49+
var c datastoreConfig
50+
var ok bool
51+
52+
c.path, ok = params["path"].(string)
53+
if !ok {
54+
return nil, fmt.Errorf("'path' field is missing or not string")
55+
}
56+
57+
switch cm := params["compression"].(string); cm {
58+
case "none":
59+
c.compression = ldbopts.NoCompression
60+
case "snappy":
61+
c.compression = ldbopts.SnappyCompression
62+
case "":
63+
c.compression = ldbopts.DefaultCompression
64+
default:
65+
return nil, fmt.Errorf("unrecognized value for compression: %s", cm)
66+
}
67+
68+
return &c, nil
69+
}
70+
}
71+
72+
func (c *datastoreConfig) DiskSpec() fsrepo.DiskSpec {
73+
return map[string]interface{}{
74+
"type": "levelds",
75+
"path": c.path,
76+
}
77+
}
78+
79+
func (c *datastoreConfig) Create(path string) (repo.Datastore, error) {
80+
p := c.path
81+
if !filepath.IsAbs(p) {
82+
p = filepath.Join(path, p)
83+
}
84+
85+
return levelds.NewDatastore(p, &levelds.Options{
86+
Compression: c.compression,
87+
})
88+
}

0 commit comments

Comments
 (0)