Check if a signed URL is specified and use it download tools

tools/download.go

@@ -16,43 +16,18 @@
 package tools
 
 import (
-	"bytes"
 	"context"
-	"crypto/sha256"
-	"encoding/hex"
-	"encoding/json"
 	"errors"
-	"fmt"
-	"io"
-	"net/http"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"runtime"
 
+	"github.com/arduino/arduino-create-agent/gen/tools"
+	"github.com/arduino/arduino-create-agent/utilities"
 	"github.com/arduino/arduino-create-agent/v2/pkgs"
-	"github.com/arduino/go-paths-helper"
-	"github.com/blang/semver"
-	"github.com/codeclysm/extract/v3"
 )
 
-// public vars to allow override in the tests
-var (
-	OS   = runtime.GOOS
-	Arch = runtime.GOARCH
-)
-
-func pathExists(path string) bool {
-	_, err := os.Stat(path)
-	if err == nil {
-		return true
-	}
-	if os.IsNotExist(err) {
-		return false
-	}
-	return true
-}
-
 // Download will parse the index at the indexURL for the tool to download.
 // It will extract it in a folder in .arduino-create, and it will update the
 // Installed map.
@@ -70,97 +45,21 @@ func pathExists(path string) bool {
 // if it already exists.
 func (t *Tools) Download(pack, name, version, behaviour string) error {
 
-	body, err := t.index.Read()
-	if err != nil {
-		return err
-	}
-
-	var data pkgs.Index
-	json.Unmarshal(body, &data)
-
-	// Find the tool by name
-	correctTool, correctSystem := findTool(pack, name, version, data)
-
-	if correctTool.Name == "" || correctSystem.URL == "" {
-		t.logger("We couldn't find a tool with the name " + name + " and version " + version + " packaged by " + pack)
-		return nil
-	}
-
-	key := correctTool.Name + "-" + correctTool.Version
-
-	// Check if it already exists
-	if behaviour == "keep" {
-		location, ok := t.getMapValue(key)
-		if ok && pathExists(location) {
-			// overwrite the default tool with this one
-			t.setMapValue(correctTool.Name, location)
-			t.logger("The tool is already present on the system")
-			return t.writeMap()
-		}
-	}
-
-	// Download the tool
-	t.logger("Downloading tool " + name + " from " + correctSystem.URL)
-	resp, err := http.Get(correctSystem.URL)
+	tool := pkgs.New(t.index, t.directory.String(), behaviour)
+	_, err := tool.Install(context.Background(), &tools.ToolPayload{Name: name, Version: version, Packager: pack})
 	if err != nil {
 		return err
 	}
-	defer resp.Body.Close()
-
-	// Read the body
-	body, err = io.ReadAll(resp.Body)
-	if err != nil {
-		return err
-	}
-
-	// Checksum
-	checksum := sha256.Sum256(body)
-	checkSumString := "SHA-256:" + hex.EncodeToString(checksum[:sha256.Size])
-
-	if checkSumString != correctSystem.Checksum {
-		return errors.New("checksum doesn't match")
-	}
-
-	tempPath := paths.TempDir()
-	// Create a temporary dir to extract package
-	if err := tempPath.MkdirAll(); err != nil {
-		return fmt.Errorf("creating temp dir for extraction: %s", err)
-	}
-	tempDir, err := tempPath.MkTempDir("package-")
-	if err != nil {
-		return fmt.Errorf("creating temp dir for extraction: %s", err)
-	}
-	defer tempDir.RemoveAll()
 
-	t.logger("Unpacking tool " + name)
-	ctx := context.Background()
-	reader := bytes.NewReader(body)
-	// Extract into temp directory
-	if err := extract.Archive(ctx, reader, tempDir.String(), nil); err != nil {
-		return fmt.Errorf("extracting archive: %s", err)
-	}
-
-	location := t.directory.Join(pack, correctTool.Name, correctTool.Version)
-	err = location.RemoveAll()
+	path := filepath.Join(pack, name, version)
+	safePath, err := utilities.SafeJoin(t.directory.String(), path)
 	if err != nil {
 		return err
 	}
 
-	// Check package content and find package root dir
-	root, err := findPackageRoot(tempDir)
-	if err != nil {
-		return fmt.Errorf("searching package root dir: %s", err)
-	}
-
-	if err := root.Rename(location); err != nil {
-		if err := root.CopyDirTo(location); err != nil {
-			return fmt.Errorf("moving extracted archive to destination dir: %s", err)
-		}
-	}
-
 	// if the tool contains a post_install script, run it: it means it is a tool that needs to install drivers
 	// AFAIK this is only the case for the windows-driver tool
-	err = t.installDrivers(location.String())
+	err = t.installDrivers(safePath)
 	if err != nil {
 		return err
 	}
@@ -169,63 +68,20 @@ func (t *Tools) Download(pack, name, version, behaviour string) error {
 	t.logger("Ensure that the files are executable")
 
 	// Update the tool map
-	t.logger("Updating map with location " + location.String())
-
-	t.setMapValue(name, location.String())
-	t.setMapValue(name+"-"+correctTool.Version, location.String())
-	return t.writeMap()
-}
+	t.logger("Updating map with location " + safePath)
 
-func findPackageRoot(parent *paths.Path) (*paths.Path, error) {
-	files, err := parent.ReadDir()
-	if err != nil {
-		return nil, fmt.Errorf("reading package root dir: %s", err)
-	}
-	files.FilterOutPrefix("__MACOSX")
+	t.setMapValue(name, safePath)
+	t.setMapValue(name+"-"+version, safePath)
 
-	// if there is only one dir, it is the root dir
-	if len(files) == 1 && files[0].IsDir() {
-		return files[0], nil
-	}
-	return parent, nil
-}
-
-func findTool(pack, name, version string, data pkgs.Index) (pkgs.Tool, pkgs.System) {
-	var correctTool pkgs.Tool
-	correctTool.Version = "0.0"
-
-	for _, p := range data.Packages {
-		if p.Name != pack {
-			continue
-		}
-		for _, t := range p.Tools {
-			if version != "latest" {
-				if t.Name == name && t.Version == version {
-					correctTool = t
-				}
-			} else {
-				// Find latest
-				v1, _ := semver.Make(t.Version)
-				v2, _ := semver.Make(correctTool.Version)
-				if t.Name == name && v1.Compare(v2) > 0 {
-					correctTool = t
-				}
-			}
-		}
-	}
-
-	// Find the url based on system
-	correctSystem := correctTool.GetFlavourCompatibleWith(OS, Arch)
-
-	return correctTool, correctSystem
+	return nil
 }
 
 func (t *Tools) installDrivers(location string) error {
 	OkPressed := 6
 	extension := ".bat"
 	// add .\ to force locality
 	preamble := ".\\"
-	if OS != "windows" {
+	if runtime.GOOS != "windows" {
 		extension = ".sh"
 		// add ./ to force locality
 		preamble = "./"
@@ -237,7 +93,7 @@ func (t *Tools) installDrivers(location string) error {
 			os.Chdir(location)
 			t.logger(preamble + "post_install" + extension)
 			oscmd := exec.Command(preamble + "post_install" + extension)
-			if OS != "linux" {
+			if runtime.GOOS != "linux" {
 				// spawning a shell could be the only way to let the user type his password
 				TellCommandNotToSpawnShell(oscmd)
 			}

tools/download_test.go

@@ -42,8 +42,8 @@ func TestDownloadCorrectPlatform(t *testing.T) {
 		{"linux", "arm", "arm-linux-gnueabihf"},
 	}
 	defer func() {
-		OS = runtime.GOOS     // restore `runtime.OS`
-		Arch = runtime.GOARCH // restore `runtime.ARCH`
+		pkgs.OS = runtime.GOOS     // restore `runtime.OS`
+		pkgs.Arch = runtime.GOARCH // restore `runtime.ARCH`
 	}()
 	testIndex := paths.New("testdata", "test_tool_index.json")
 	buf, err := testIndex.ReadFile()
@@ -54,10 +54,11 @@ func TestDownloadCorrectPlatform(t *testing.T) {
 	require.NoError(t, err)
 	for _, tc := range testCases {
 		t.Run(tc.hostOS+tc.hostArch, func(t *testing.T) {
-			OS = tc.hostOS     // override `runtime.OS` for testing purposes
-			Arch = tc.hostArch // override `runtime.ARCH` for testing purposes
+			pkgs.OS = tc.hostOS     // override `runtime.OS` for testing purposes
+			pkgs.Arch = tc.hostArch // override `runtime.ARCH` for testing purposes
 			// Find the tool by name
-			correctTool, correctSystem := findTool("arduino-test", "arduino-fwuploader", "2.2.2", data)
+			correctTool, correctSystem, found := pkgs.FindTool("arduino-test", "arduino-fwuploader", "2.2.2", data)
+			require.True(t, found)
 			require.NotNil(t, correctTool)
 			require.NotNil(t, correctSystem)
 			require.Equal(t, correctTool.Name, "arduino-fwuploader")
@@ -78,8 +79,8 @@ func TestDownloadFallbackPlatform(t *testing.T) {
 		{"windows", "amd64", "i686-mingw32"},
 	}
 	defer func() {
-		OS = runtime.GOOS     // restore `runtime.OS`
-		Arch = runtime.GOARCH // restore `runtime.ARCH`
+		pkgs.OS = runtime.GOOS     // restore `runtime.OS`
+		pkgs.Arch = runtime.GOARCH // restore `runtime.ARCH`
 	}()
 	testIndex := paths.New("testdata", "test_tool_index.json")
 	buf, err := testIndex.ReadFile()
@@ -90,10 +91,11 @@ func TestDownloadFallbackPlatform(t *testing.T) {
 	require.NoError(t, err)
 	for _, tc := range testCases {
 		t.Run(tc.hostOS+tc.hostArch, func(t *testing.T) {
-			OS = tc.hostOS     // override `runtime.OS` for testing purposes
-			Arch = tc.hostArch // override `runtime.ARCH` for testing purposes
+			pkgs.OS = tc.hostOS     // override `runtime.OS` for testing purposes
+			pkgs.Arch = tc.hostArch // override `runtime.ARCH` for testing purposes
 			// Find the tool by name
-			correctTool, correctSystem := findTool("arduino-test", "arduino-fwuploader", "2.2.0", data)
+			correctTool, correctSystem, found := pkgs.FindTool("arduino-test", "arduino-fwuploader", "2.2.0", data)
+			require.True(t, found)
 			require.NotNil(t, correctTool)
 			require.NotNil(t, correctSystem)
 			require.Equal(t, correctTool.Name, "arduino-fwuploader")
@@ -145,7 +147,7 @@ func TestDownload(t *testing.T) {
 				if filePath.IsDir() {
 					require.DirExists(t, filePath.String())
 				} else {
-					if OS == "windows" {
+					if runtime.GOOS == "windows" {
 						require.FileExists(t, filePath.String()+".exe")
 					} else {
 						require.FileExists(t, filePath.String())

tools/tools.go

@@ -78,18 +78,6 @@ func (t *Tools) getMapValue(key string) (string, bool) {
 	return value, ok
 }
 
-// writeMap() writes installed map to the json file "installed.json"
-func (t *Tools) writeMap() error {
-	t.mutex.RLock()
-	defer t.mutex.RUnlock()
-	b, err := json.Marshal(t.installed)
-	if err != nil {
-		return err
-	}
-	filePath := t.directory.Join("installed.json")
-	return filePath.WriteFile(b)
-}
-
 // readMap() reads the installed map from json file "installed.json"
 func (t *Tools) readMap() error {
 	t.mutex.Lock()

v2/http.go

@@ -40,7 +40,7 @@ func Server(directory string, index *index.Resource) http.Handler {
 	logAdapter := LogAdapter{Logger: logger}
 
 	// Mount tools
-	toolsSvc := pkgs.New(index, directory)
+	toolsSvc := pkgs.New(index, directory, "replace")
 	toolsEndpoints := toolssvc.NewEndpoints(toolsSvc)
 	toolsServer := toolssvr.New(toolsEndpoints, mux, CustomRequestDecoder, goahttp.ResponseEncoder, errorHandler(logger), nil)
 	toolssvr.Mount(mux, toolsServer)

v2/pkgs/tools.go

@@ -33,9 +33,16 @@ import (
 	"github.com/arduino/arduino-create-agent/gen/tools"
 	"github.com/arduino/arduino-create-agent/index"
 	"github.com/arduino/arduino-create-agent/utilities"
+	"github.com/blang/semver"
 	"github.com/codeclysm/extract/v3"
 )
 
+// public vars to allow override in the tests
+var (
+	OS   = runtime.GOOS
+	Arch = runtime.GOARCH
+)
+
 // Tools is a client that implements github.com/arduino/arduino-create-agent/gen/tools.Service interface.
 // It saves tools in a specified folder with this structure: packager/name/version
 // For example:
@@ -50,17 +57,19 @@ import (
 //
 // It requires an Index Resource to search for tools
 type Tools struct {
-	index  *index.Resource
-	folder string
+	index     *index.Resource
+	folder    string
+	behaviour string
 }
 
 // New will return a Tool object, allowing the caller to execute operations on it.
 // The New function will accept an index as parameter (used to download the indexes)
 // and a folder used to download the indexes
-func New(index *index.Resource, folder string) *Tools {
+func New(index *index.Resource, folder, behaviour string) *Tools {
 	return &Tools{
-		index:  index,
-		folder: folder,
+		index:     index,
+		folder:    folder,
+		behaviour: behaviour,
 	}
 }
 
@@ -166,21 +175,28 @@ func (t *Tools) Install(ctx context.Context, payload *tools.ToolPayload) (*tools
 	var index Index
 	json.Unmarshal(body, &index)
 
-	for _, packager := range index.Packages {
-		if packager.Name != payload.Packager {
-			continue
-		}
-
-		for _, tool := range packager.Tools {
-			if tool.Name == payload.Name &&
-				tool.Version == payload.Version {
-
-				sys := tool.GetFlavourCompatibleWith(runtime.GOOS, runtime.GOARCH)
+	correctTool, correctSystem, found := FindTool(payload.Packager, payload.Name, payload.Version, index)
+	path = filepath.Join(payload.Packager, correctTool.Name, correctTool.Version)
 
-				return t.install(ctx, path, sys.URL, sys.Checksum)
+	key := correctTool.Name + "-" + correctTool.Version
+	// Check if it already exists
+	if t.behaviour == "keep" && pathExists(t.folder) {
+		location, ok, err := checkInstalled(t.folder, key)
+		if err != nil {
+			return nil, err
+		}
+		if ok && pathExists(location) {
+			// overwrite the default tool with this one
+			err := writeInstalled(t.folder, path)
+			if err != nil {
+				return nil, err
 			}
+			return &tools.Operation{Status: "ok"}, nil
 		}
 	}
+	if found {
+		return t.install(ctx, path, correctSystem.URL, correctSystem.Checksum)
+	}
 
 	return nil, tools.MakeNotFound(
 		fmt.Errorf("tool not found with packager '%s', name '%s', version '%s'",
@@ -256,27 +272,51 @@ func (t *Tools) Remove(ctx context.Context, payload *tools.ToolPayload) (*tools.
 func rename(base string) extract.Renamer {
 	return func(path string) string {
 		parts := strings.Split(filepath.ToSlash(path), "/")
-		path = strings.Join(parts[1:], "/")
-		path = filepath.Join(base, path)
+		newPath := strings.Join(parts[1:], "/")
+		if newPath == "" {
+			newPath = filepath.Join(newPath, path)
+		}
+		path = filepath.Join(base, newPath)
 		return path
 	}
 }
 
-func writeInstalled(folder, path string) error {
+func readInstalled(installedFile string) (map[string]string, error) {
 	// read installed.json
 	installed := map[string]string{}
-
-	installedFile, err := utilities.SafeJoin(folder, "installed.json")
-	if err != nil {
-		return err
-	}
 	data, err := os.ReadFile(installedFile)
 	if err == nil {
 		err = json.Unmarshal(data, &installed)
 		if err != nil {
-			return err
+			return nil, err
 		}
 	}
+	return installed, nil
+}
+
+func checkInstalled(folder, key string) (string, bool, error) {
+	installedFile, err := utilities.SafeJoin(folder, "installed.json")
+	if err != nil {
+		return "", false, err
+	}
+	installed, err := readInstalled(installedFile)
+	if err != nil {
+		return "", false, err
+	}
+	location, ok := installed[key]
+	return location, ok, err
+}
+
+func writeInstalled(folder, path string) error {
+	// read installed.json
+	installedFile, err := utilities.SafeJoin(folder, "installed.json")
+	if err != nil {
+		return err
+	}
+	installed, err := readInstalled(installedFile)
+	if err != nil {
+		return err
+	}
 
 	parts := strings.Split(path, string(filepath.Separator))
 	tool := parts[len(parts)-2]
@@ -288,10 +328,55 @@ func writeInstalled(folder, path string) error {
 	installed[tool] = toolFile
 	installed[toolWithVersion] = toolFile
 
-	data, err = json.Marshal(installed)
+	data, err := json.Marshal(installed)
 	if err != nil {
 		return err
 	}
 
 	return os.WriteFile(installedFile, data, 0644)
 }
+
+func pathExists(path string) bool {
+	_, err := os.Stat(path)
+	if err == nil {
+		return true
+	}
+	if os.IsNotExist(err) {
+		return false
+	}
+	return true
+}
+
+// FindTool searches the index for the correct tool and system that match the specified tool name and version
+func FindTool(pack, name, version string, data Index) (Tool, System, bool) {
+	var correctTool Tool
+	correctTool.Version = "0.0"
+	found := false
+
+	for _, p := range data.Packages {
+		if p.Name != pack {
+			continue
+		}
+		for _, t := range p.Tools {
+			if version != "latest" {
+				if t.Name == name && t.Version == version {
+					correctTool = t
+					found = true
+				}
+			} else {
+				// Find latest
+				v1, _ := semver.Make(t.Version)
+				v2, _ := semver.Make(correctTool.Version)
+				if t.Name == name && v1.Compare(v2) > 0 {
+					correctTool = t
+					found = true
+				}
+			}
+		}
+	}
+
+	// Find the url based on system
+	correctSystem := correctTool.GetFlavourCompatibleWith(OS, Arch)
+
+	return correctTool, correctSystem, found
+}

v2/pkgs/tools_test.go

@@ -45,7 +45,7 @@ func TestTools(t *testing.T) {
 	// Instantiate Index
 	Index := index.Init(indexURL, config.GetDataDir())
 
-	service := pkgs.New(Index, tmp)
+	service := pkgs.New(Index, tmp, "replace")
 
 	ctx := context.Background()
 
@@ -126,7 +126,7 @@ func TestEvilFilename(t *testing.T) {
 	// Instantiate Index
 	Index := index.Init(indexURL, config.GetDataDir())
 
-	service := pkgs.New(Index, tmp)
+	service := pkgs.New(Index, tmp, "replace")
 
 	ctx := context.Background()
 
@@ -195,7 +195,7 @@ func TestInstalledHead(t *testing.T) {
 	// Instantiate Index
 	Index := index.Init(indexURL, config.GetDataDir())
 
-	service := pkgs.New(Index, tmp)
+	service := pkgs.New(Index, tmp, "replace")
 
 	ctx := context.Background()
 
@@ -216,7 +216,7 @@ func TestInstall(t *testing.T) {
 		LastRefresh: time.Now(),
 	}
 
-	tool := pkgs.New(testIndex, tmp)
+	tool := pkgs.New(testIndex, tmp, "replace")
 
 	ctx := context.Background()