oneclickexport: add core export functionality (#39813)

This includes ExportRequest processing, site config fetching and redacting and creating a zip archive.

Author: sashaostrikov

cmd/frontend/oneclickexport/export.go

@@ -0,0 +1,95 @@
+package oneclickexport
+
+import (
+	"archive/zip"
+	"bytes"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/sourcegraph/log"
+)
+
+type Exporter interface {
+	// Export accepts an ExportRequest and returns bytes of a zip archive
+	// with requested data.
+	Export(request ExportRequest) ([]byte, error)
+}
+
+var _ Exporter = &DataExporter{}
+
+type DataExporter struct {
+	logger           log.Logger
+	configProcessors map[string]Processor[ConfigRequest]
+}
+
+type ExportRequest struct {
+	IncludeSiteConfig bool `json:"includeSiteConfig"`
+}
+
+// Export generates and returns a ZIP archive with the data, specified in request.
+// It works like this:
+// 1) tmp directory is created (exported files will end up in this directory and
+// this directory is zipped in the end)
+// 2) ExportRequest is read and each corresponding processor is invoked
+// 3) Tmp directory is zipped after all the Processors finished their job
+func (e *DataExporter) Export(request ExportRequest) ([]byte, error) {
+	// 1) creating a tmp dir
+	dir, err := os.MkdirTemp(os.TempDir(), "export-*")
+	if err != nil {
+		e.logger.Fatal("Error during code tmp dir creation", log.Error(err))
+	}
+	defer os.RemoveAll(dir)
+
+	// 2) tmp dir is passed to every processor
+	if request.IncludeSiteConfig {
+		e.configProcessors["siteConfig"].Process(ConfigRequest{}, dir)
+	}
+
+	// 3) after all request parts are processed, zip the tmp dir and return its bytes
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// currently, all the directories are skipped because only files are added to the
+		// archive
+		if info.IsDir() {
+			return nil
+		}
+
+		// create file header
+		header, err := zip.FileInfoHeader(info)
+		if err != nil {
+			return err
+		}
+
+		header.Method = zip.Deflate
+		header.Name = filepath.Base(path)
+
+		headerWriter, err := zw.CreateHeader(header)
+		if err != nil {
+			return err
+		}
+
+		file, err := os.Open(path)
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+
+		_, err = io.Copy(headerWriter, file)
+		return err
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	if err := zw.Close(); err != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}

cmd/frontend/oneclickexport/export_test.go

@@ -0,0 +1,112 @@
+package oneclickexport
+
+import (
+	"archive/zip"
+	"bytes"
+	"io"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/sourcegraph/log/logtest"
+	"github.com/sourcegraph/sourcegraph/internal/conf"
+	"github.com/sourcegraph/sourcegraph/internal/extsvc"
+	"github.com/sourcegraph/sourcegraph/lib/errors"
+	"github.com/sourcegraph/sourcegraph/schema"
+)
+
+func TestExport(t *testing.T) {
+	logger := logtest.Scoped(t)
+
+	conf.Mock(&conf.Unified{
+		SiteConfiguration: schema.SiteConfiguration{
+			AuthProviders: []schema.AuthProviders{{
+				Github: &schema.GitHubAuthProvider{
+					ClientID:     "myclientid",
+					ClientSecret: "myclientsecret",
+					DisplayName:  "GitHub",
+					Type:         extsvc.TypeGitHub,
+					Url:          "https://github.com",
+					AllowOrgs:    []string{"myorg"},
+				},
+			}},
+			PermissionsUserMapping: &schema.PermissionsUserMapping{
+				BindID:  "username",
+				Enabled: true,
+			},
+			ExperimentalFeatures: &schema.ExperimentalFeatures{
+				SearchIndexQueryContexts: true,
+			},
+		},
+	})
+	t.Cleanup(func() { conf.Mock(nil) })
+
+	exporter := &DataExporter{
+		logger: logger,
+		configProcessors: map[string]Processor[ConfigRequest]{
+			"siteConfig": &SiteConfigProcessor{
+				logger: logger,
+				Type:   "siteConfig",
+			},
+		},
+	}
+
+	archive, err := exporter.Export(ExportRequest{IncludeSiteConfig: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	zr, err := zip.NewReader(bytes.NewReader(archive), int64(len(archive)))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	found := false
+
+	want := `{
+  "auth.providers": [
+    {
+      "allowOrgs": [
+        "myorg"
+      ],
+      "clientID": "myclientid",
+      "clientSecret": "REDACTED",
+      "displayName": "GitHub",
+      "type": "github",
+      "url": "https://github.com"
+    }
+  ],
+  "experimentalFeatures": {
+    "search.index.query.contexts": true
+  },
+  "permissions.userMapping": {
+    "bindID": "username",
+    "enabled": true
+  }
+}`
+
+	for _, f := range zr.File {
+		if f.Name != "site-config.json" {
+			continue
+		}
+		found = true
+		rc, err := f.Open()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		haveBytes, err := io.ReadAll(rc)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		have := string(haveBytes)
+
+		if diff := cmp.Diff(want, have); diff != "" {
+			t.Fatalf("Exported site config is different. (-want +got):\n%s", diff)
+		}
+	}
+
+	if !found {
+		t.Fatal(errors.New("site config file not found in exported zip archive"))
+	}
+}

cmd/frontend/oneclickexport/processor.go

@@ -0,0 +1,48 @@
+package oneclickexport
+
+import (
+	"io/ioutil"
+
+	"github.com/sourcegraph/log"
+	"github.com/sourcegraph/sourcegraph/internal/conf"
+)
+
+// Processor is a generic interface for any data export processor.
+//
+// Processors are called from DataExporter and store exported data in provided
+// directory which is zipped after all the processors finished their job.
+type Processor[T any] interface {
+	Process(payload T, dir string)
+	ProcessorType() string
+}
+
+var _ Processor[ConfigRequest] = &SiteConfigProcessor{}
+
+type SiteConfigProcessor struct {
+	logger log.Logger
+	Type   string
+}
+
+type ConfigRequest struct {
+}
+
+// Process function of SiteConfigProcessor loads site config, redacts the secrets
+// and stores it in a provided tmp directory dir.
+func (g SiteConfigProcessor) Process(_ ConfigRequest, dir string) {
+	siteConfig, err := conf.RedactSecrets(conf.Raw())
+	if err != nil {
+		g.logger.Error("Error during site config redacting", log.Error(err))
+	}
+
+	configBytes := []byte(siteConfig.Site)
+
+	err = ioutil.WriteFile(dir+"/site-config.json", configBytes, 0644)
+
+	if err != nil {
+		g.logger.Error("Error during site config export", log.Error(err))
+	}
+}
+
+func (g SiteConfigProcessor) ProcessorType() string {
+	return g.Type
+}

internal/conf/store.go

@@ -1,6 +1,7 @@
 package conf
 
 import (
+	"encoding/json"
 	"fmt"
 	"os"
 	"runtime/debug"
@@ -55,6 +56,17 @@ func (s *store) Raw() conftypes.RawUnified {
 
 	s.rawMu.RLock()
 	defer s.rawMu.RUnlock()
+
+	if s.mock != nil {
+		raw, err := json.Marshal(s.mock.SiteConfig())
+		if err != nil {
+			return conftypes.RawUnified{}
+		}
+		return conftypes.RawUnified{
+			Site:               string(raw),
+			ServiceConnections: s.mock.ServiceConnectionConfig,
+		}
+	}
 	return s.raw
 }