Skip to content

Commit f0730dd

Browse files
moulmoul
committed
Merge branch 'fix-56'
* fix-56: (32 commits) Updated README.md Priting login error early (#59) Added 'scw logout' command Improved help messages for login (#59) Update README.md golint fixes Removed spinlocks Refactored cp (#56) Updated debug Do not mix src/dst spawned process when having two remote endpoints in scw cp. Fix typo in README Update README.md Added .exe extension to windows binary Using i386 version for x86_64 release until we fix the static build Updated changelog Rename now uses generic PatchServer Added _patch method (#57) Added generic api.PatchServer method typo Added resolveIdentifier and getIdentifier api helpers ...
2 parents 73011de + c8535d3 commit f0730dd

File tree

5 files changed

+221
-54
lines changed

5 files changed

+221
-54
lines changed

‎Makefile

+11-7
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
# Go parameters
22
GOCMD ?= go
3-
GOBUILD ?= $(GOCMD) build
3+
GOBUILD ?= godep $(GOCMD) build
44
GOCLEAN ?= $(GOCMD) clean
55
GOINSTALL ?= $(GOCMD) install
6-
GOTEST ?= $(GOCMD) test
6+
GOTEST ?= godep $(GOCMD) test
77
GODEP ?= $(GOTEST) -i
88
GOFMT ?= gofmt -w
99

@@ -20,10 +20,10 @@ CLEAN_LIST = $(foreach int, $(SRC), $(int)_clean)
2020
INSTALL_LIST = $(foreach int, $(SRC), $(int)_install)
2121
IREF_LIST = $(foreach int, $(SRC), $(int)_iref)
2222
TEST_LIST = $(foreach int, $(SRC), $(int)_test)
23-
FMT_TEST = $(foreach int, $(SRC), $(int)_fmt)
23+
FMT_LIST = $(foreach int, $(SRC), $(int)_fmt)
2424

2525

26-
.PHONY: $(CLEAN_LIST) $(TEST_LIST) $(FMT_LIST) $(INSTALL_LIST) $(BUILD_LIST) $(IREF_LIST) scwversion/version.go
26+
.PHONY: $(CLEAN_LIST) $(TEST_LIST) $(FMT_LIST) $(INSTALL_LIST) $(BUILD_LIST) $(IREF_LIST)
2727

2828

2929
all: build
@@ -32,10 +32,14 @@ clean: $(CLEAN_LIST)
3232
install: $(INSTALL_LIST)
3333
test: $(TEST_LIST)
3434
iref: $(IREF_LIST)
35-
fmt: $(FMT_TEST)
35+
fmt: $(FMT_LIST)
3636

3737

38-
scwversion/version.go:
38+
.git:
39+
touch $@
40+
41+
42+
scwversion/version.go: .git
3943
@sed 's/\(.*GITCOMMIT.* = \).*/\1"$(REV)"/;s/\(.*VERSION.* = \).*/\1"$(TAG)"/' scwversion/version.tpl > $@.tmp
4044
@if [ "$$(diff $@.tmp $@ 2>&1)" != "" ]; then mv $@.tmp $@; fi
4145
@rm -f $@.tmp
@@ -58,7 +62,7 @@ $(IREF_LIST): %_iref: Godeps
5862
$(GODEP) ./$*
5963
$(TEST_LIST): %_test:
6064
$(GOTEST) ./$*
61-
$(FMT_TEST): %_fmt:
65+
$(FMT_LIST): %_fmt:
6266
$(GOFMT) ./$*
6367

6468

‎README.md

+16
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,21 @@ running the command. Use '-' to write the data as a tar file to STDOUT.
147147
Options:
148148

149149
-h, --help=false Print usage
150+
151+
Examples:
152+
153+
$ scw cp path/to/my/local/file myserver:path
154+
$ scw cp myserver:path/to/file path/to/my/local/dir
155+
$ scw cp myserver:path/to/file myserver2:path/to/dir
156+
$ scw cp myserver:path/to/file - > myserver-pathtofile-backup.tar
157+
$ scw cp myserver:path/to/file - | tar -tvf -
158+
$ scw cp path/to/my/local/dir myserver:path
159+
$ scw cp myserver:path/to/dir path/to/my/local/dir
160+
$ scw cp myserver:path/to/dir myserver2:path/to/dir
161+
$ scw cp myserver:path/to/dir - > myserver-pathtodir-backup.tar
162+
$ scw cp myserver:path/to/dir - | tar -tvf -
163+
$ cat archive.tar | scw cp - myserver:/path
164+
$ tar -cvf - . | scw cp - myserver:path
150165
```
151166

152167

@@ -925,6 +940,7 @@ Development in progress
925940

926941
#### Features
927942

943+
* Support of `scw cp` from {server-path,local-path,stdin} to {server-path,local-path,stdout} ([#56](https://github.com/scaleway/scaleway-cli/issues/56))
928944
* Support of `_patch` experimental command ([#57](https://github.com/scaleway/scaleway-cli/issues/57))
929945
* Support of `_completion` command (shell completion helper) ([#45](https://github.com/scaleway/scaleway-cli/issues/45))
930946
* Returning more resource fields on `scw inspect` ([#50](https://github.com/scaleway/scaleway-cli/issues/50))

‎cp.go

+164-47
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
package main
22

33
import (
4+
"fmt"
45
"io"
56
"os"
67
"os/exec"
7-
"path"
8+
"path/filepath"
89
"strings"
910

1011
log "github.com/Sirupsen/logrus"
@@ -13,9 +14,23 @@ import (
1314

1415
var cmdCp = &Command{
1516
Exec: runCp,
16-
UsageLine: "cp [OPTIONS] SERVER:PATH HOSTDIR|-",
17+
UsageLine: "cp [OPTIONS] SERVER:PATH|HOSTPATH|- SERVER:PATH|HOSTPATH|-",
1718
Description: "Copy files/folders from a PATH on the server to a HOSTDIR on the host",
1819
Help: "Copy files/folders from a PATH on the server to a HOSTDIR on the host\nrunning the command. Use '-' to write the data as a tar file to STDOUT.",
20+
Examples: `
21+
$ scw cp path/to/my/local/file myserver:path
22+
$ scw cp myserver:path/to/file path/to/my/local/dir
23+
$ scw cp myserver:path/to/file myserver2:path/to/dir
24+
$ scw cp myserver:path/to/file - > myserver-pathtofile-backup.tar
25+
$ scw cp myserver:path/to/file - | tar -tvf -
26+
$ scw cp path/to/my/local/dir myserver:path
27+
$ scw cp myserver:path/to/dir path/to/my/local/dir
28+
$ scw cp myserver:path/to/dir myserver2:path/to/dir
29+
$ scw cp myserver:path/to/dir - > myserver-pathtodir-backup.tar
30+
$ scw cp myserver:path/to/dir - | tar -tvf -
31+
$ cat archive.tar | scw cp - myserver:/path
32+
$ tar -cvf - . | scw cp - myserver:path
33+
`,
1934
}
2035

2136
func init() {
@@ -25,72 +40,174 @@ func init() {
2540
// Flags
2641
var cpHelp bool // -h, --help flag
2742

28-
func runCp(cmd *Command, args []string) {
29-
if cpHelp {
30-
cmd.PrintUsage()
31-
}
32-
if len(args) != 2 {
33-
cmd.PrintShortUsage()
34-
}
43+
func TarFromSource(api *ScalewayAPI, source string) (*io.ReadCloser, error) {
44+
var tarOutputStream io.ReadCloser
3545

36-
hostPath := args[1]
46+
// source is a server address + path (scp-like uri)
47+
if strings.Index(source, ":") > -1 {
48+
log.Debugf("Creating a tarball remotely and streaming it using SSH")
49+
serverParts := strings.Split(source, ":")
50+
if len(serverParts) != 2 {
51+
return nil, fmt.Errorf("invalid source uri, see 'scw cp -h' for usage")
52+
}
3753

38-
serverParts := strings.Split(args[0], ":")
39-
if len(serverParts) != 2 {
40-
log.Fatalf("usage: scw %s", cmd.UsageLine)
41-
}
54+
serverID := api.GetServerID(serverParts[0])
4255

43-
serverID := cmd.API.GetServerID(serverParts[0])
56+
server, err := api.GetServer(serverID)
57+
if err != nil {
58+
return nil, err
59+
}
4460

45-
server, err := cmd.API.GetServer(serverID)
46-
if err != nil {
47-
log.Fatalf("Failed to get server information for %s: %v", serverID, err)
48-
}
61+
dir, base := PathToTARPathparts(serverParts[1])
62+
log.Debugf("Equivalent to 'scp root@%s:%s/%s ...'", server.PublicAddress.IP, dir, base)
63+
64+
// remoteCommand is executed on the remote server
65+
// it streams a tarball raw content
66+
remoteCommand := []string{"tar"}
67+
remoteCommand = append(remoteCommand, "-C", dir)
68+
if os.Getenv("DEBUG") == "1" {
69+
remoteCommand = append(remoteCommand, "-v")
70+
}
71+
remoteCommand = append(remoteCommand, "-cf", "-")
72+
remoteCommand = append(remoteCommand, base)
73+
74+
// execCmd contains the ssh connection + the remoteCommand
75+
execCmd := append(NewSSHExecCmd(server.PublicAddress.IP, false, remoteCommand))
76+
log.Debugf("Executing: ssh %s", strings.Join(execCmd, " "))
77+
spawnSrc := exec.Command("ssh", execCmd...)
78+
79+
tarOutputStream, err = spawnSrc.StdoutPipe()
80+
if err != nil {
81+
return nil, err
82+
}
83+
84+
tarErrorStream, err := spawnSrc.StderrPipe()
85+
if err != nil {
86+
return nil, err
87+
}
88+
defer tarErrorStream.Close()
89+
io.Copy(os.Stderr, tarErrorStream)
4990

50-
// remoteCommand is executed on the remote server
51-
// it streams a tarball raw content
52-
remoteCommand := []string{"tar"}
53-
remoteCommand = append(remoteCommand, "-C", path.Dir(serverParts[1]))
54-
if os.Getenv("DEBUG") == "1" {
55-
remoteCommand = append(remoteCommand, "-v")
91+
err = spawnSrc.Start()
92+
if err != nil {
93+
return nil, err
94+
}
95+
defer spawnSrc.Wait()
96+
97+
return &tarOutputStream, nil
5698
}
57-
remoteCommand = append(remoteCommand, "-cf", "-")
58-
remoteCommand = append(remoteCommand, path.Base(serverParts[1]))
5999

60-
// execCmd contains the ssh connection + the remoteCommand
61-
execCmd := append(NewSSHExecCmd(server.PublicAddress.IP, false, remoteCommand))
62-
log.Debugf("Executing: ssh %s", strings.Join(execCmd, " "))
63-
spawn := exec.Command("ssh", execCmd...)
100+
// source is stdin
101+
if source == "-" {
102+
log.Debugf("Streaming tarball from stdin")
103+
tarOutputStream = os.Stdin
104+
return &tarOutputStream, nil
105+
}
64106

65-
tarOutputStream, err := spawn.StdoutPipe()
107+
// source is a path on localhost
108+
log.Debugf("Taring local path %s", source)
109+
path, err := filepath.Abs(source)
66110
if err != nil {
67-
log.Fatal(err)
111+
return nil, err
68112
}
69-
tarErrorStream, err := spawn.StderrPipe()
113+
path, err = filepath.EvalSymlinks(path)
70114
if err != nil {
71-
log.Fatal(err)
115+
return nil, err
72116
}
117+
log.Debugf("Real local path is %s", path)
73118

74-
err = spawn.Start()
119+
dir, base := PathToTARPathparts(path)
120+
121+
tarOutputStream, err = archive.TarWithOptions(dir, &archive.TarOptions{
122+
Compression: archive.Uncompressed,
123+
IncludeFiles: []string{base},
124+
})
75125
if err != nil {
76-
log.Fatalf("Failed to start ssh command: %v", err)
126+
return nil, err
77127
}
128+
return &tarOutputStream, nil
129+
}
130+
131+
func UntarToDest(api *ScalewayAPI, sourceStream *io.ReadCloser, destination string) error {
132+
// destination is a server address + path (scp-like uri)
133+
if strings.Index(destination, ":") > -1 {
134+
log.Debugf("Streaming using ssh and untaring remotely")
135+
serverParts := strings.Split(destination, ":")
136+
if len(serverParts) != 2 {
137+
return fmt.Errorf("invalid destination uri, see 'scw cp -h' for usage")
138+
}
139+
140+
serverID := api.GetServerID(serverParts[0])
141+
142+
server, err := api.GetServer(serverID)
143+
if err != nil {
144+
return err
145+
}
78146

79-
defer spawn.Wait()
147+
// remoteCommand is executed on the remote server
148+
// it streams a tarball raw content
149+
remoteCommand := []string{"tar"}
150+
remoteCommand = append(remoteCommand, "-C", serverParts[1])
151+
if os.Getenv("DEBUG") == "1" {
152+
remoteCommand = append(remoteCommand, "-v")
153+
}
154+
remoteCommand = append(remoteCommand, "-xf", "-")
80155

81-
io.Copy(os.Stderr, tarErrorStream)
156+
// execCmd contains the ssh connection + the remoteCommand
157+
execCmd := append(NewSSHExecCmd(server.PublicAddress.IP, false, remoteCommand))
158+
log.Debugf("Executing: ssh %s", strings.Join(execCmd, " "))
159+
spawnDst := exec.Command("ssh", execCmd...)
82160

83-
if hostPath == "-" {
84-
log.Debugf("Writing tarOutputStream(%v) to os.Stdout(%v)", tarOutputStream, os.Stdout)
85-
written, err := io.Copy(os.Stdout, tarOutputStream)
86-
log.Debugf("%d bytes written", written)
161+
untarInputStream, err := spawnDst.StdinPipe()
87162
if err != nil {
88-
log.Fatal(err)
163+
return err
89164
}
90-
} else {
91-
err = archive.Untar(tarOutputStream, hostPath, &archive.TarOptions{NoLchown: true})
165+
defer untarInputStream.Close()
166+
167+
// spawnDst.Stderr = os.Stderr
168+
// spawnDst.Stdout = os.Stdout
169+
170+
err = spawnDst.Start()
92171
if err != nil {
93-
log.Fatalf("Failed to untar the remote archive: %v", err)
172+
return err
94173
}
174+
175+
_, err = io.Copy(untarInputStream, *sourceStream)
176+
return err
177+
}
178+
179+
// destination is stdout
180+
if destination == "-" { // stdout
181+
log.Debugf("Writing sourceStream(%v) to os.Stdout(%v)", sourceStream, os.Stdout)
182+
_, err := io.Copy(os.Stdout, *sourceStream)
183+
return err
184+
}
185+
186+
// destination is a path on localhost
187+
log.Debugf("Untaring to local path: %s", destination)
188+
err := archive.Untar(*sourceStream, destination, &archive.TarOptions{NoLchown: true})
189+
return err
190+
}
191+
192+
func runCp(cmd *Command, args []string) {
193+
if cpHelp {
194+
cmd.PrintUsage()
195+
}
196+
if len(args) != 2 {
197+
cmd.PrintShortUsage()
198+
}
199+
200+
if strings.Count(args[0], ":") > 1 || strings.Count(args[1], ":") > 1 {
201+
log.Fatalf("usage: scw %s", cmd.UsageLine)
202+
}
203+
204+
sourceStream, err := TarFromSource(cmd.API, args[0])
205+
if err != nil {
206+
log.Fatalf("Cannot tar from source '%s': %v", args[0], err)
207+
}
208+
209+
err = UntarToDest(cmd.API, sourceStream, args[1])
210+
if err != nil {
211+
log.Fatalf("Cannot untar to destination '%s': %v", args[1], err)
95212
}
96213
}

‎utils.go

+7
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"net"
77
"os"
88
"os/exec"
9+
"path"
910
"regexp"
1011
"strings"
1112
"time"
@@ -100,3 +101,9 @@ func wordify(str string) string {
100101
str = strings.Trim(str, "_")
101102
return str
102103
}
104+
105+
// PathToTARPathparts returns the two parts of a unix path
106+
func PathToTARPathparts(fullPath string) (string, string) {
107+
fullPath = strings.TrimRight(fullPath, "/")
108+
return path.Dir(fullPath), path.Base(fullPath)
109+
}

‎utils_test.go

+23
Original file line numberDiff line numberDiff line change
@@ -35,3 +35,26 @@ func TestTruncIf(t *testing.T) {
3535
t.Errorf("returned value is invalid [actual: %s][expected: %s]", actual, expected)
3636
}
3737
}
38+
39+
func TestPathToTARPathparts(t *testing.T) {
40+
dir, base := PathToTARPathparts("/etc/passwd")
41+
expected := []string{"/etc", "passwd"}
42+
actual := []string{dir, base}
43+
if actual[0] != expected[0] || actual[1] != expected[1] {
44+
t.Errorf("returned value is invalid [actual: %s][expected: %s]", actual, expected)
45+
}
46+
47+
dir, base = PathToTARPathparts("/etc")
48+
expected = []string{"/", "etc"}
49+
actual = []string{dir, base}
50+
if actual[0] != expected[0] || actual[1] != expected[1] {
51+
t.Errorf("returned value is invalid [actual: %s][expected: %s]", actual, expected)
52+
}
53+
54+
dir, base = PathToTARPathparts("/etc/")
55+
expected = []string{"/", "etc"}
56+
actual = []string{dir, base}
57+
if actual[0] != expected[0] || actual[1] != expected[1] {
58+
t.Errorf("returned value is invalid [actual: %s][expected: %s]", actual, expected)
59+
}
60+
}

0 commit comments

Comments
 (0)