Support pre-compressed uploads in code-intel upload (#1146)

Author: antonsviridov-src

.github/workflows/scip.yml

@@ -11,24 +11,46 @@ jobs:
     container: sourcegraph/scip-go
     steps:
       - uses: actions/checkout@v4
-      - name: Get src-cli
-        run: curl -L https://sourcegraph.com/.api/src-cli/src_linux_amd64 -o /usr/local/bin/src;
-          chmod +x /usr/local/bin/src
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 1.23.2
+
       - name: Set directory to safe for git
         run: git config --global --add safe.directory $GITHUB_WORKSPACE
+
+      - name: Build src-cli
+        run: go build -o ./src-cli ./cmd/src
+
       - name: Enable pulling Go modules from private sourcegraph/sourcegraph
         run: git config --global url."https://${PRIVATE_TOKEN}@github.com/sourcegraph/".insteadOf "https://github.com/sourcegraph/"
+
       - name: Generate SCIP data
         run: scip-go
 
       - name: Upload SCIP to Cloud
-        run: src code-intel upload -github-token='${{ secrets.GITHUB_TOKEN }}' -no-progress
+        run: ./src-cli code-intel upload -github-token='${{ secrets.GITHUB_TOKEN }}' -no-progress
         env:
           SRC_ENDPOINT: https://sourcegraph.com/
           SRC_ACCESS_TOKEN: ${{ secrets.SRC_ACCESS_TOKEN_DOTCOM }}
 
       - name: Upload SCIP to S2
-        run: src code-intel upload -github-token='${{ secrets.GITHUB_TOKEN }}' -no-progress
+        run: ./src-cli code-intel upload -github-token='${{ secrets.GITHUB_TOKEN }}' -no-progress
+        env:
+          SRC_ENDPOINT: https://sourcegraph.sourcegraph.com/
+          SRC_ACCESS_TOKEN: ${{ secrets.SRC_ACCESS_TOKEN_S2 }}
+
+      - name: Compress SCIP file
+        run: gzip index.scip
+
+      - name: Upload compressed SCIP to Cloud
+        run: ./src-cli code-intel upload -github-token='${{ secrets.GITHUB_TOKEN }}' -no-progress
+        env:
+          SRC_ENDPOINT: https://sourcegraph.com/
+          SRC_ACCESS_TOKEN: ${{ secrets.SRC_ACCESS_TOKEN_DOTCOM }}
+
+      - name: Upload compressed SCIP to S2
+        run: ./src-cli code-intel upload -github-token='${{ secrets.GITHUB_TOKEN }}' -no-progress
         env:
           SRC_ENDPOINT: https://sourcegraph.sourcegraph.com/
           SRC_ACCESS_TOKEN: ${{ secrets.SRC_ACCESS_TOKEN_S2 }}

CHANGELOG.md

@@ -11,6 +11,8 @@ All notable changes to `src-cli` are documented in this file.
 
 ## Unreleased
 
+- Support uploading GZIP compressed SCIP indexes [1146](https://github.com/sourcegraph/src-cli/pull/1146)
+
 ## 6.0.1
 
 - Container signature verification support: Container signatures can now be verified for Sourcegraph releases after 5.11.4013 using `src signature verify -v <release>` [#1143](https://github.com/sourcegraph/src-cli/pull/1143)
@@ -153,7 +155,7 @@ All notable changes to `src-cli` are documented in this file.
 
 ### Fixed
 
-- Fixed `src validate install` requiring `SRC_GITHUB_TOKEN` in all cases. [958](https://github.com/sourcegraph/src-cli/pull/958) 
+- Fixed `src validate install` requiring `SRC_GITHUB_TOKEN` in all cases. [958](https://github.com/sourcegraph/src-cli/pull/958)
 
 ## 5.0.1
 

cmd/src/code_intel_upload.go

@@ -82,7 +82,12 @@ func handleCodeIntelUpload(args []string) error {
 	})
 
 	uploadOptions := codeintelUploadOptions(out)
-	uploadID, err := upload.UploadIndex(ctx, codeintelUploadFlags.file, client, uploadOptions)
+	var uploadID int
+	if codeintelUploadFlags.gzipCompressed {
+		uploadID, err = UploadCompressedIndex(ctx, codeintelUploadFlags.file, client, uploadOptions, 0)
+	} else {
+		uploadID, err = UploadUncompressedIndex(ctx, codeintelUploadFlags.file, client, uploadOptions)
+	}
 	if err != nil {
 		return handleUploadError(uploadOptions.SourcegraphInstanceOptions.AccessToken, err)
 	}

cmd/src/code_intel_upload_flags.go

@@ -1,9 +1,12 @@
 package main
 
 import (
+	"compress/gzip"
 	"flag"
 	"fmt"
+	"io"
 	"os"
+	"path"
 	"strings"
 
 	"github.com/sourcegraph/scip/bindings/go/scip"
@@ -15,7 +18,8 @@ import (
 )
 
 var codeintelUploadFlags struct {
-	file string
+	file           string
+	gzipCompressed bool
 
 	// UploadRecordOptions
 	repo              string
@@ -110,7 +114,7 @@ func parseAndValidateCodeIntelUploadFlags(args []string) (*output.Output, error)
 	}
 
 	if !isFlagSet(codeintelUploadFlagSet, "file") {
-		defaultFile, err := inferDefaultFile()
+		defaultFile, err := inferDefaultFile(out)
 		if err != nil {
 			return nil, err
 		}
@@ -131,6 +135,10 @@ func parseAndValidateCodeIntelUploadFlags(args []string) (*output.Output, error)
 		return nil, errors.Newf("file %q does not exist", codeintelUploadFlags.file)
 	}
 
+	if err := inferGzipFlag(); err != nil {
+		return nil, err
+	}
+
 	// Infer the remaining default arguments (may require reading from new file)
 	if inferenceErrors := inferMissingCodeIntelUploadFlags(); len(inferenceErrors) > 0 {
 		return nil, formatInferenceError(inferenceErrors[0])
@@ -143,6 +151,22 @@ func parseAndValidateCodeIntelUploadFlags(args []string) (*output.Output, error)
 	return out, nil
 }
 
+func inferGzipFlag() error {
+	if codeintelUploadFlags.gzipCompressed || path.Ext(codeintelUploadFlags.file) == ".gz" {
+		file, err := os.Open(codeintelUploadFlags.file)
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+		if err := checkGzipHeader(file); err != nil {
+			return errors.Wrapf(err, "could not verify that %s is a valid gzip file", codeintelUploadFlags.file)
+		}
+		codeintelUploadFlags.gzipCompressed = true
+	}
+
+	return nil
+}
+
 // codeintelUploadOutput returns an output object that should be used to print the progres
 // of requests made during this upload. If -json, -no-progress, or -trace>0 is given,
 // then no output object is defined.
@@ -164,19 +188,40 @@ type argumentInferenceError struct {
 	err      error
 }
 
-func inferDefaultFile() (string, error) {
+func inferDefaultFile(out *output.Output) (string, error) {
 	const scipFilename = "index.scip"
-	_, err := os.Stat(scipFilename)
+	const scipCompressedFilename = "index.scip.gz"
 
-	if err == nil {
+	hasSCIP, err := doesFileExist(scipFilename)
+	if err != nil {
+		return "", err
+	}
+	hasCompressedSCIP, err := doesFileExist(scipCompressedFilename)
+	if err != nil {
+		return "", err
+	}
+
+	if !hasSCIP && !hasCompressedSCIP {
+		return "", formatInferenceError(argumentInferenceError{"file", errors.Newf("Unable to locate SCIP index. Checked paths: %q and %q.", scipFilename, scipCompressedFilename)})
+	} else if hasCompressedSCIP {
+		if hasSCIP {
+			out.WriteLine(output.Linef(output.EmojiInfo, output.StyleBold, "Both %s and %s exist, choosing %s", scipFilename, scipCompressedFilename, scipCompressedFilename))
+		}
+		return scipCompressedFilename, nil
+	} else {
 		return scipFilename, nil
 	}
+}
 
+func doesFileExist(filename string) (bool, error) {
+	info, err := os.Stat(filename)
 	if os.IsNotExist(err) {
-		return "", formatInferenceError(argumentInferenceError{"file", errors.Newf("%s does not exist", scipFilename)})
+		return false, nil
+	} else if err != nil {
+		return false, err
+	} else {
+		return !info.IsDir(), nil
 	}
-
-	return "", err
 }
 
 func formatInferenceError(inferenceErr argumentInferenceError) error {
@@ -271,6 +316,17 @@ func readIndexerNameAndVersion(indexFile string) (string, string, error) {
 	}
 	defer file.Close()
 
+	var indexReader io.Reader = file
+
+	if codeintelUploadFlags.gzipCompressed {
+		gzipReader, err := gzip.NewReader(file)
+		if err != nil {
+			return "", "", err
+		}
+		indexReader = gzipReader
+		defer gzipReader.Close()
+	}
+
 	var metadata *scip.Metadata
 
 	visitor := scip.IndexVisitor{
@@ -280,7 +336,7 @@ func readIndexerNameAndVersion(indexFile string) (string, string, error) {
 	}
 
 	// convert file to io.Reader
-	if err := visitor.ParseStreaming(file); err != nil {
+	if err := visitor.ParseStreaming(indexReader); err != nil {
 		return "", "", err
 	}
 
@@ -307,3 +363,12 @@ func validateCodeIntelUploadFlags() error {
 
 	return nil
 }
+
+func checkGzipHeader(r io.Reader) error {
+	gz, err := gzip.NewReader(r)
+	if err != nil {
+		return err
+	}
+	defer gz.Close()
+	return nil
+}

cmd/src/code_intel_upload_transfer.go

@@ -0,0 +1,82 @@
+package main
+
+import (
+	"context"
+	"os"
+
+	"github.com/sourcegraph/sourcegraph/lib/codeintel/upload"
+	"github.com/sourcegraph/sourcegraph/lib/output"
+)
+
+func UploadUncompressedIndex(ctx context.Context, filename string, httpClient upload.Client, opts upload.UploadOptions) (int, error) {
+	originalReader, originalSize, err := openFileAndGetSize(filename)
+	if err != nil {
+		return 0, err
+	}
+	defer func() {
+		_ = originalReader.Close()
+	}()
+
+	bars := []output.ProgressBar{{Label: "Compressing", Max: 1.0}}
+	progress, _, cleanup := logProgress(
+		opts.Output,
+		bars,
+		"Index compressed",
+		"Failed to compress index",
+	)
+
+	compressedFile, err := compressReaderToDisk(originalReader, originalSize, progress)
+	if err != nil {
+		cleanup(err)
+		return 0, err
+	}
+	defer func() {
+		_ = os.Remove(compressedFile)
+	}()
+
+	compressedSize, err := getFileSize(compressedFile)
+	if err != nil {
+		cleanup(err)
+		return 0, err
+	}
+
+	cleanup(nil)
+
+	if opts.Output != nil {
+		opts.Output.WriteLine(output.Linef(
+			output.EmojiLightbulb,
+			output.StyleItalic,
+			"Indexed compressed (%.2fMB -> %.2fMB).",
+			float64(originalSize)/1000/1000,
+			float64(compressedSize)/1000/1000,
+		))
+	}
+
+	return UploadCompressedIndex(ctx, compressedFile, httpClient, opts, originalSize)
+}
+
+func UploadCompressedIndex(ctx context.Context, compressedFile string, httpClient upload.Client, opts upload.UploadOptions, uncompressedSize int64) (int, error) {
+	compressedReader, compressedSize, err := openFileAndGetSize(compressedFile)
+	if err != nil {
+		// cleanup(err)
+		return 0, err
+	}
+	defer func() {
+		_ = compressedReader.Close()
+	}()
+
+	if compressedSize <= opts.MaxPayloadSizeBytes {
+		return uploadIndex(ctx, httpClient, opts, compressedReader, compressedSize, uncompressedSize)
+	}
+
+	return uploadMultipartIndex(ctx, httpClient, opts, compressedReader, compressedSize, uncompressedSize)
+}
+
+func getFileSize(filename string) (int64, error) {
+	fileInfo, err := os.Stat(filename)
+	if err != nil {
+		return 1, err
+	}
+
+	return fileInfo.Size(), nil
+}