Skip to content

block cmd: Use coreapi #5331

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Sep 11, 2018
Merged

block cmd: Use coreapi #5331

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
daae93a
block cmd: use coreapi
magik6k Aug 2, 2018
c3e6907
block cmd: go-path update
magik6k Aug 2, 2018
9b925e0
coreapi: block: don't allow creation of invalid cidv0s
magik6k Aug 13, 2018
1f98f4b
coreapi: block: move option logic to options package
magik6k Aug 15, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
173 changes: 65 additions & 108 deletions core/commands/block.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,23 +1,19 @@
package commands

import (
"bytes"
"context"
"errors"
"fmt"
"io"
"io/ioutil"
"os"

util "github.com/ipfs/go-ipfs/blocks/blockstoreutil"
cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv"
e "github.com/ipfs/go-ipfs/core/commands/e"
coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface"
"github.com/ipfs/go-ipfs/core/coreapi/interface/options"

"gx/ipfs/QmPTfgFTo9PFr1PvPKyKoeMgBvYPh6cX3aDP7DHKVbnCbi/go-ipfs-cmds"
mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash"
"gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit"
blocks "gx/ipfs/QmWAzSEoqZ6xU6pu8yL8e5WaMb7wtbfbhhN4p1DknUPtr3/go-block-format"
cid "gx/ipfs/QmZFbDTY9jfSBms2MchvYM9oYRbAF19K7Pby47yDBfpPrb/go-cid"
)

type BlockStat struct {
Expand Down Expand Up @@ -64,15 +60,27 @@ on raw IPFS blocks. It outputs the following to stdout:
cmdkit.StringArg("key", true, false, "The base58 multihash of an existing block to stat.").EnableStdin(),
},
Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) {
b, err := getBlockForKey(req.Context, env, req.Arguments[0])
api, err := cmdenv.GetApi(env)
if err != nil {
res.SetError(err, cmdkit.ErrNormal)
return
}

p, err := coreiface.ParsePath(req.Arguments[0])
if err != nil {
res.SetError(err, cmdkit.ErrNormal)
return
}

b, err := api.Block().Stat(req.Context, p)
if err != nil {
res.SetError(err, cmdkit.ErrNormal)
return
}

err = cmds.EmitOnce(res, &BlockStat{
Key: b.Cid().String(),
Size: len(b.RawData()),
Key: b.Path().Cid().String(),
Size: b.Size(),
})
if err != nil {
log.Error(err)
Expand Down Expand Up @@ -104,13 +112,25 @@ It outputs to stdout, and <key> is a base58 encoded multihash.
cmdkit.StringArg("key", true, false, "The base58 multihash of an existing block to get.").EnableStdin(),
},
Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) {
b, err := getBlockForKey(req.Context, env, req.Arguments[0])
api, err := cmdenv.GetApi(env)
if err != nil {
res.SetError(err, cmdkit.ErrNormal)
return
}

err = res.Emit(bytes.NewReader(b.RawData()))
p, err := coreiface.ParsePath(req.Arguments[0])
if err != nil {
res.SetError(err, cmdkit.ErrNormal)
return
}

r, err := api.Block().Get(req.Context, p)
if err != nil {
res.SetError(err, cmdkit.ErrNormal)
return
}

err = res.Emit(r)
if err != nil {
log.Error(err)
}
Expand Down Expand Up @@ -138,7 +158,7 @@ than 'sha2-256' or format to anything other than 'v0' will result in CIDv1.
cmdkit.IntOption("mhlen", "multihash hash length").WithDefault(-1),
},
Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) {
n, err := cmdenv.GetNode(env)
api, err := cmdenv.GetApi(env)
if err != nil {
res.SetError(err, cmdkit.ErrNormal)
return
Expand All @@ -150,18 +170,6 @@ than 'sha2-256' or format to anything other than 'v0' will result in CIDv1.
return
}

data, err := ioutil.ReadAll(file)
if err != nil {
res.SetError(err, cmdkit.ErrNormal)
return
}

err = file.Close()
if err != nil {
res.SetError(err, cmdkit.ErrNormal)
return
}

mhtype, _ := req.Options["mhtype"].(string)
mhtval, ok := mh.Names[mhtype]
if !ok {
Expand All @@ -170,62 +178,30 @@ than 'sha2-256' or format to anything other than 'v0' will result in CIDv1.
return
}

var pref cid.Prefix
pref.Version = 1

format, formatSet := req.Options["format"].(string)
if !formatSet {
if mhtval == mh.SHA2_256 {
format = "v0"
} else {
format = "protobuf"
}
}

if format == "v0" {
pref.Version = 0
}
formatval, ok := cid.Codecs[format]
if !ok {
res.SetError(fmt.Errorf("unrecognized format: '%s'", format), cmdkit.ErrNormal)
return
}
if mhtval != mh.SHA2_256 && pref.Version == 0 {
res.SetError(errors.New("cannot generate CIDv0 with non-sha256 hash function"), cmdkit.ErrNormal)
return
}

pref.Codec = formatval
pref.MhType = mhtval

mhlen, ok := req.Options["mhlen"].(int)
if !ok {
res.SetError("missing option \"mhlen\"", cmdkit.ErrNormal)
return
}
pref.MhLength = mhlen

bcid, err := pref.Sum(data)
if err != nil {
res.SetError(err, cmdkit.ErrNormal)
return
}

b, err := blocks.NewBlockWithCid(data, bcid)
if err != nil {
res.SetError(err, cmdkit.ErrNormal)
return
format, formatSet := req.Options["format"].(string)
if !formatSet {
if mhtval != mh.SHA2_256 || (mhlen != -1 && mhlen != 32) {
format = "protobuf"
} else {
format = "v0"
}
}

err = n.Blocks.AddBlock(b)
p, err := api.Block().Put(req.Context, file, options.Block.Hash(mhtval, mhlen), options.Block.Format(format))
if err != nil {
res.SetError(err, cmdkit.ErrNormal)
return
}

err = cmds.EmitOnce(res, &BlockStat{
Key: b.Cid().String(),
Size: len(data),
Key: p.Path().Cid().String(),
Size: p.Size(),
})
if err != nil {
log.Error(err)
Expand All @@ -244,29 +220,6 @@ than 'sha2-256' or format to anything other than 'v0' will result in CIDv1.
Type: BlockStat{},
}

func getBlockForKey(ctx context.Context, env cmds.Environment, skey string) (blocks.Block, error) {
if len(skey) == 0 {
return nil, fmt.Errorf("zero length cid invalid")
}

n, err := cmdenv.GetNode(env)
if err != nil {
return nil, err
}

c, err := cid.Decode(skey)
if err != nil {
return nil, err
}

b, err := n.Blocks.GetBlock(ctx, c)
if err != nil {
return nil, err
}

return b, nil
}

var blockRmCmd = &cmds.Command{
Helptext: cmdkit.HelpText{
Tagline: "Remove IPFS block(s).",
Expand All @@ -283,38 +236,42 @@ It takes a list of base58 encoded multihashes to remove.
cmdkit.BoolOption("quiet", "q", "Write minimal output."),
},
Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) {
n, err := cmdenv.GetNode(env)
api, err := cmdenv.GetApi(env)
if err != nil {
res.SetError(err, cmdkit.ErrNormal)
return
}
hashes := req.Arguments

force, _ := req.Options["force"].(bool)
quiet, _ := req.Options["quiet"].(bool)
cids := make([]*cid.Cid, 0, len(hashes))
for _, hash := range hashes {
c, err := cid.Decode(hash)

// TODO: use batching coreapi when done
for _, b := range req.Arguments {
p, err := coreiface.ParsePath(b)
if err != nil {
err = fmt.Errorf("invalid content id: %s (%s)", hash, err)
res.SetError(err, cmdkit.ErrNormal)
return
}

cids = append(cids, c)
}
ch, err := util.RmBlocks(n.Blockstore, n.Pinning, cids, util.RmBlocksOpts{
Quiet: quiet,
Force: force,
})
rp, err := api.ResolvePath(req.Context, p)
if err != nil {
res.SetError(err, cmdkit.ErrNormal)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we emit error here too?
What do you think?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Reading original code we probably shouldn't unless ResolvePath does a lot more than just Decode.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It does handle ipns and path traversal too

return
}

if err != nil {
res.SetError(err, cmdkit.ErrNormal)
return
}
err = api.Block().Rm(req.Context, rp, options.Block.Force(force))
if err != nil {
res.Emit(&util.RemovedBlock{
Hash: rp.Cid().String(),
Error: err.Error(),
})
}

err = res.Emit(ch)
if err != nil {
log.Error(err)
if !quiet {
res.Emit(&util.RemovedBlock{
Hash: rp.Cid().String(),
})
}
}
},
PostRun: cmds.PostRunMap{
Expand Down
23 changes: 3 additions & 20 deletions core/coreapi/block.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,13 @@ import (
"bytes"
"context"
"errors"
"fmt"
"io"
"io/ioutil"

util "github.com/ipfs/go-ipfs/blocks/blockstoreutil"
coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface"
caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options"

mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash"
blocks "gx/ipfs/QmWAzSEoqZ6xU6pu8yL8e5WaMb7wtbfbhhN4p1DknUPtr3/go-block-format"
cid "gx/ipfs/QmZFbDTY9jfSBms2MchvYM9oYRbAF19K7Pby47yDBfpPrb/go-cid"
)
Expand All @@ -24,8 +22,8 @@ type BlockStat struct {
size int
}

func (api *BlockAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.BlockPutOption) (coreiface.ResolvedPath, error) {
settings, err := caopts.BlockPutOptions(opts...)
func (api *BlockAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.BlockPutOption) (coreiface.BlockStat, error) {
_, pref, err := caopts.BlockPutOptions(opts...)
if err != nil {
return nil, err
}
Expand All @@ -35,21 +33,6 @@ func (api *BlockAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.Bloc
return nil, err
}

var pref cid.Prefix
pref.Version = 1

formatval, ok := cid.Codecs[settings.Codec]
if !ok {
return nil, fmt.Errorf("unrecognized format: %s", settings.Codec)
}
if settings.Codec == "v0" && settings.MhType == mh.SHA2_256 {
pref.Version = 0
}
pref.Codec = formatval

pref.MhType = settings.MhType
pref.MhLength = settings.MhLength

bcid, err := pref.Sum(data)
if err != nil {
return nil, err
Expand All @@ -65,7 +48,7 @@ func (api *BlockAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.Bloc
return nil, err
}

return coreiface.IpldPath(b.Cid()), nil
return &BlockStat{path: coreiface.IpldPath(b.Cid()), size: len(data)}, nil
}

func (api *BlockAPI) Get(ctx context.Context, p coreiface.Path) (io.Reader, error) {
Expand Down
Loading