From 6fceb981262a308772b3f8b5ecb0003b248dc580 Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Fri, 21 Feb 2025 09:50:20 +0000 Subject: [PATCH 1/4] Remove LSIF support from src-cli --- cmd/src/code_intel.go | 4 +- cmd/src/code_intel_upload.go | 36 +--- cmd/src/code_intel_upload_flags.go | 254 ++++-------------------- cmd/src/code_intel_upload_flags_test.go | 71 +------ cmd/src/lsif.go | 30 --- cmd/src/main.go | 3 +- internal/codeintel/gitutil_test.go | 6 +- 7 files changed, 68 insertions(+), 336 deletions(-) delete mode 100644 cmd/src/lsif.go diff --git a/cmd/src/code_intel.go b/cmd/src/code_intel.go index af9f48795e..3dc3f28d8c 100644 --- a/cmd/src/code_intel.go +++ b/cmd/src/code_intel.go @@ -16,13 +16,13 @@ Usage: The commands are: - upload uploads a SCIP or LSIF index + upload uploads a SCIP index Use "src code-intel [command] -h" for more information about a command. ` flagSet := flag.NewFlagSet("code-intel", flag.ExitOnError) handler := func(args []string) error { - lsifCommands.run(flagSet, "src code-intel", usage, args) + codeintelCommands.run(flagSet, "src code-intel", usage, args) return nil } diff --git a/cmd/src/code_intel_upload.go b/cmd/src/code_intel_upload.go index 71d62116c6..dfe761939e 100644 --- a/cmd/src/code_intel_upload.go +++ b/cmd/src/code_intel_upload.go @@ -9,7 +9,6 @@ import ( "io" "net/url" "os" - "path/filepath" "strings" "time" @@ -28,7 +27,7 @@ func init() { Examples: Before running any of these, first use src auth to authenticate. Alternately, use the SRC_ACCESS_TOKEN environment variable for - individual src-cli invocations. + individual src-cli invocations. If run from within the project itself, src-cli will infer various flags based on git metadata. @@ -48,9 +47,6 @@ Examples: $ src code-intel upload -github-token=BAZ, or $ src code-intel upload -gitlab-token=BAZ - - For any of these commands, an LSIF index (default name: dump.lsif) can be - used instead of a SCIP index (default name: index.scip). ` codeintelCommands = append(codeintelCommands, &command{ flagSet: codeintelUploadFlagSet, @@ -61,24 +57,13 @@ Examples: fmt.Println(usage) }, }) - - // Make 'upload' available under 'src lsif' for backwards compatibility. - lsifCommands = append(lsifCommands, &command{ - flagSet: codeintelUploadFlagSet, - handler: handleCodeIntelUpload, - usageFunc: func() { - fmt.Fprintf(flag.CommandLine.Output(), "Usage of 'src lsif %s':\n", codeintelUploadFlagSet.Name()) - codeintelUploadFlagSet.PrintDefaults() - fmt.Println(usage) - }, - }) } // handleCodeIntelUpload is the handler for `src code-intel upload`. func handleCodeIntelUpload(args []string) error { ctx := context.Background() - out, isSCIPAvailable, err := parseAndValidateCodeIntelUploadFlags(args) + out, err := parseAndValidateCodeIntelUploadFlags(args) if !codeintelUploadFlags.json { if out != nil { printInferredArguments(out) @@ -96,7 +81,7 @@ func handleCodeIntelUpload(args []string) error { Flags: codeintelUploadFlags.apiFlags, }) - uploadOptions := codeintelUploadOptions(out, isSCIPAvailable) + uploadOptions := codeintelUploadOptions(out) uploadID, err := upload.UploadIndex(ctx, codeintelUploadFlags.file, client, uploadOptions) if err != nil { return handleUploadError(uploadOptions.SourcegraphInstanceOptions.AccessToken, err) @@ -141,18 +126,13 @@ func handleCodeIntelUpload(args []string) error { } // codeintelUploadOptions creates a set of upload options given the values in the flags. -func codeintelUploadOptions(out *output.Output, isSCIPAvailable bool) upload.UploadOptions { +func codeintelUploadOptions(out *output.Output) upload.UploadOptions { var associatedIndexID *int if codeintelUploadFlags.associatedIndexID != -1 { associatedIndexID = &codeintelUploadFlags.associatedIndexID } - cfg.AdditionalHeaders["Content-Type"] = "application/x-ndjson+lsif" - path := codeintelUploadFlags.uploadRoute - if isSCIPAvailable && filepath.Ext(codeintelUploadFlags.file) == ".scip" { - cfg.AdditionalHeaders["Content-Type"] = "application/x-protobuf+scip" - path = strings.ReplaceAll(path, "lsif", "scip") - } + cfg.AdditionalHeaders["Content-Type"] = "application/x-protobuf+scip" logger := upload.NewRequestLogger( os.Stdout, @@ -178,7 +158,7 @@ func codeintelUploadOptions(out *output.Output, isSCIPAvailable bool) upload.Upl AdditionalHeaders: cfg.AdditionalHeaders, MaxRetries: 5, RetryInterval: time.Second, - Path: path, + Path: codeintelUploadFlags.uploadRoute, MaxPayloadSizeBytes: codeintelUploadFlags.maxPayloadSizeMb * 1000 * 1000, MaxConcurrency: codeintelUploadFlags.maxConcurrency, GitHubToken: codeintelUploadFlags.gitHubToken, @@ -263,7 +243,7 @@ func attachHintsForAuthorizationError(accessToken string, originalError error) e if likelyTokenError { return errorWithHint{err: originalError, hint: strings.Join(mergeStringSlices( - []string{"A Sourcegraph access token must be provided via SRC_ACCESS_TOKEN for uploading SCIP/LSIF data."}, + []string{"A Sourcegraph access token must be provided via SRC_ACCESS_TOKEN for uploading SCIP data."}, actionableHints, []string{"For more details, see https://sourcegraph.com/docs/cli/how-tos/creating_an_access_token."}, ), "\n")} @@ -304,7 +284,7 @@ func attachHintsForAuthorizationError(accessToken string, originalError error) e } return errorWithHint{err: originalError, hint: strings.Join(mergeStringSlices( - []string{"This Sourcegraph instance has enforced auth for SCIP/LSIF uploads."}, + []string{"This Sourcegraph instance has enforced auth for SCIP uploads."}, actionableHints, []string{"For more details, see https://docs.sourcegraph.com/cli/references/code-intel/upload."}, ), "\n")} diff --git a/cmd/src/code_intel_upload_flags.go b/cmd/src/code_intel_upload_flags.go index a724806a90..01148d9197 100644 --- a/cmd/src/code_intel_upload_flags.go +++ b/cmd/src/code_intel_upload_flags.go @@ -6,17 +6,12 @@ import ( "fmt" "net/http" "os" - "path" - "path/filepath" "strings" - "time" "github.com/sourcegraph/scip/bindings/go/scip" - libscip "github.com/sourcegraph/sourcegraph/lib/codeintel/lsif/scip" "github.com/sourcegraph/sourcegraph/lib/codeintel/upload" "github.com/sourcegraph/sourcegraph/lib/errors" "github.com/sourcegraph/sourcegraph/lib/output" - "google.golang.org/protobuf/proto" "github.com/sourcegraph/src-cli/internal/api" "github.com/sourcegraph/src-cli/internal/codeintel" @@ -57,26 +52,21 @@ var ( // Used to include the insecure-skip-verify flag in the help output, as we don't use any of the // other api.Client methods, so only the insecureSkipVerify flag is relevant here. dummyflag bool - - // Used to skip the LSIF -> SCIP conversion during the migration. Not expected to be used outside - // of codeintel-qa pipelines and not expected to last much longer than a few releases while we - // deprecate all LSIF code paths. - skipConversionToSCIP bool ) func init() { - codeintelUploadFlagSet.StringVar(&codeintelUploadFlags.file, "file", "", `The path to the LSIF dump file.`) + codeintelUploadFlagSet.StringVar(&codeintelUploadFlags.file, "file", "", `The path to the SCIP index file.`) // UploadRecordOptions codeintelUploadFlagSet.StringVar(&codeintelUploadFlags.repo, "repo", "", `The name of the repository (e.g. github.com/gorilla/mux). By default, derived from the origin remote.`) codeintelUploadFlagSet.StringVar(&codeintelUploadFlags.commit, "commit", "", `The 40-character hash of the commit. Defaults to the currently checked-out commit.`) - codeintelUploadFlagSet.StringVar(&codeintelUploadFlags.root, "root", "", `The path in the repository that matches the LSIF projectRoot (e.g. cmd/project1). Defaults to the directory where the dump file is located.`) - codeintelUploadFlagSet.StringVar(&codeintelUploadFlags.indexer, "indexer", "", `The name of the indexer that generated the dump. This will override the 'toolInfo.name' field in the metadata vertex of the LSIF dump file. This must be supplied if the indexer does not set this field (in which case the upload will fail with an explicit message).`) - codeintelUploadFlagSet.StringVar(&codeintelUploadFlags.indexerVersion, "indexerVersion", "", `The version of the indexer that generated the dump. This will override the 'toolInfo.version' field in the metadata vertex of the LSIF dump file. This must be supplied if the indexer does not set this field (in which case the upload will fail with an explicit message).`) + codeintelUploadFlagSet.StringVar(&codeintelUploadFlags.root, "root", "", `The path in the repository that matches the SCIP projectRoot (e.g. cmd/project1). Defaults to the directory where the scip index file is located.`) + codeintelUploadFlagSet.StringVar(&codeintelUploadFlags.indexer, "indexer", "", `The name of the indexer that generated the dump. This will override the 'toolInfo.name' field in the metadata section of SCIP index. This must be supplied if the indexer does not set this field (in which case the upload will fail with an explicit message).`) + codeintelUploadFlagSet.StringVar(&codeintelUploadFlags.indexerVersion, "indexerVersion", "", `The version of the indexer that generated the dump. This will override the 'toolInfo.version' field in the metadata section of SCIP index. This must be supplied if the indexer does not set this field (in which case the upload will fail with an explicit message).`) codeintelUploadFlagSet.IntVar(&codeintelUploadFlags.associatedIndexID, "associated-index-id", -1, "ID of the associated index record for this upload. For internal use only.") // SourcegraphInstanceOptions - codeintelUploadFlagSet.StringVar(&codeintelUploadFlags.uploadRoute, "upload-route", "/.api/lsif/upload", "The path of the upload route. For internal use only.") + codeintelUploadFlagSet.StringVar(&codeintelUploadFlags.uploadRoute, "upload-route", "/.api/scip/upload", "The path of the upload route. For internal use only.") codeintelUploadFlagSet.Int64Var(&codeintelUploadFlags.maxPayloadSizeMb, "max-payload-size", 100, `The maximum upload size (in megabytes). Indexes exceeding this limit will be uploaded over multiple HTTP requests.`) codeintelUploadFlagSet.IntVar(&codeintelUploadFlags.maxConcurrency, "max-concurrency", -1, "The maximum number of concurrent uploads. Only relevant for multipart uploads. Defaults to all parts concurrently.") @@ -89,11 +79,8 @@ func init() { codeintelUploadFlagSet.BoolVar(&codeintelUploadFlags.noProgress, "no-progress", false, `Do not display progress updates.`) codeintelUploadFlagSet.IntVar(&codeintelUploadFlags.verbosity, "trace", 0, "-trace=0 shows no logs; -trace=1 shows requests and response metadata; -trace=2 shows headers, -trace=3 shows response body") codeintelUploadFlagSet.BoolVar(&codeintelUploadFlags.json, "json", false, `Output relevant state in JSON on success.`) - codeintelUploadFlagSet.BoolVar(&codeintelUploadFlags.open, "open", false, `Open the LSIF upload page in your browser.`) + codeintelUploadFlagSet.BoolVar(&codeintelUploadFlags.open, "open", false, `Open the SCIP upload page in your browser.`) codeintelUploadFlagSet.BoolVar(&dummyflag, "insecure-skip-verify", false, "Skip validation of TLS certificates against trusted chains") - - // Testing flags - codeintelUploadFlagSet.BoolVar(&skipConversionToSCIP, "skip-scip", false, "Skip converting LSIF index to SCIP if the instance supports it; this option should only used for debugging") } // parseAndValidateCodeIntelUploadFlags calls codeintelUploadFlagset.Parse, then infers values for @@ -102,9 +89,9 @@ func init() { // // On success, the global codeintelUploadFlags object will be populated with valid values. An // error is returned on failure. -func parseAndValidateCodeIntelUploadFlags(args []string) (*output.Output, bool, error) { +func parseAndValidateCodeIntelUploadFlags(args []string) (*output.Output, error) { if err := codeintelUploadFlagSet.Parse(args); err != nil { - return nil, false, err + return nil, err } out := codeintelUploadOutput() @@ -122,13 +109,13 @@ func parseAndValidateCodeIntelUploadFlags(args []string) (*output.Output, bool, // and maybe we'll use some in the future codeintelUploadFlags.apiFlags = api.NewFlags(apiClientFlagSet) if err := apiClientFlagSet.Parse(insecureSkipVerifyFlag); err != nil { - return nil, false, err + return nil, err } if !isFlagSet(codeintelUploadFlagSet, "file") { defaultFile, err := inferDefaultFile() if err != nil { - return nil, false, err + return nil, err } codeintelUploadFlags.file = defaultFile } @@ -136,42 +123,38 @@ func parseAndValidateCodeIntelUploadFlags(args []string) (*output.Output, bool, // Check to see if input file exists if _, err := os.Stat(codeintelUploadFlags.file); os.IsNotExist(err) { if !isFlagSet(codeintelUploadFlagSet, "file") { - return nil, false, formatInferenceError(argumentInferenceError{"file", err}) + return nil, formatInferenceError(argumentInferenceError{"file", err}) } - return nil, false, errors.Newf("file %q does not exist", codeintelUploadFlags.file) + return nil, errors.Newf("file %q does not exist", codeintelUploadFlags.file) } isSCIPAvailable, err := isSCIPAvailable() if err != nil { - return nil, false, err + return nil, err } if !isSCIPAvailable { - if err := handleSCIP(out); err != nil { - return nil, false, err - } - } else { - if err := handleLSIF(out); err != nil { - return nil, false, err - } + return nil, errors.New( + "src-cli no longer supports LSIF->SCIP conversion, and the Sourcegraph " + + "instance you are uploading to does not support SCIP") } // Check for new file existence after transformation if _, err := os.Stat(codeintelUploadFlags.file); os.IsNotExist(err) { - return nil, false, errors.Newf("file %q does not exist", codeintelUploadFlags.file) + return nil, errors.Newf("file %q does not exist", codeintelUploadFlags.file) } // Infer the remaining default arguments (may require reading from new file) if inferenceErrors := inferMissingCodeIntelUploadFlags(); len(inferenceErrors) > 0 { - return nil, false, formatInferenceError(inferenceErrors[0]) + return nil, formatInferenceError(inferenceErrors[0]) } if err := validateCodeIntelUploadFlags(); err != nil { - return nil, false, err + return nil, err } - return out, isSCIPAvailable, nil + return out, nil } // codeintelUploadOutput returns an output object that should be used to print the progres @@ -214,101 +197,6 @@ type argumentInferenceError struct { err error } -func replaceExtension(oldPath string, newExtension string) string { - oldExtLen := len(path.Ext(oldPath)) - if oldExtLen == 0 { - panic(fmt.Sprintf("Expected path %s to have an extension", oldPath)) - } - return oldPath[:len(oldPath)-oldExtLen] + newExtension -} - -func replaceBaseName(oldPath string, newBaseName string) string { - if filepath.Dir(newBaseName) != "." { - panic(fmt.Sprintf("Expected bare file name but found %s", newBaseName)) - } - return filepath.Join(filepath.Dir(oldPath), newBaseName) -} - -func handleSCIP(out *output.Output) error { - fileExt := path.Ext(codeintelUploadFlags.file) - if len(fileExt) == 0 { - return errors.Newf("missing file extension for %s; expected .scip or .lsif", codeintelUploadFlags.file) - } - inputFile := codeintelUploadFlags.file - if fileExt == ".scip" || fileExt == ".lsif-typed" { - // The user explicitly passed in a -file flag that points to an SCIP index. - outputFile := replaceExtension(inputFile, ".lsif") - if filepath.Base(inputFile) == "index.scip" { - outputFile = replaceBaseName(inputFile, "dump.lsif") - } - // HACK: Modify the flags to point to the output file, because - // that field of the flags is read when performing the upload. - codeintelUploadFlags.file = outputFile - return convertSCIPToLSIFGraph(out, inputFile, outputFile) - } - - if _, err := os.Stat(inputFile); err == nil { - // Do nothing, the provided -flag flag points to an existing - // file that does not have the file extension `.lsif-typed` or `.scip`. - return nil - } - - scipFile := replaceExtension(inputFile, ".scip") - if _, err := os.Stat(scipFile); os.IsNotExist(err) { - // The input may be named 'dump.lsif', but the default name for SCIP - // indexes is 'index.scip', not 'dump.scip'. - scipFile = replaceBaseName(inputFile, "index.scip") - if _, err := os.Stat(scipFile); os.IsNotExist(err) { - lsifTypedFile := replaceExtension(inputFile, ".lsif-typed") - if _, err := os.Stat(lsifTypedFile); os.IsNotExist(err) { - // There is no `*.scip` or `*.lsif-typed` file for the inferred path. - return nil - } - scipFile = lsifTypedFile - } - } - - // The provided -file flag points to an `*.lsif` file that doesn't exist - // so we convert the sibling file (which we confirmed exists). - return convertSCIPToLSIFGraph(out, scipFile, codeintelUploadFlags.file) -} - -// Reads the SCIP encoded input file and writes the corresponding LSIF -// Graph encoded output file. -func convertSCIPToLSIFGraph(out *output.Output, inputFile, outputFile string) error { - if out != nil { - out.Writef("%s Converting %s into %s", output.EmojiInfo, inputFile, outputFile) - } - tmp, err := os.Create(outputFile) - if err != nil { - return err - } - defer tmp.Close() - - data, err := os.ReadFile(inputFile) - if err != nil { - return errors.Wrapf(err, "failed to read SCIP index '%s'", inputFile) - } - index := scip.Index{} - err = proto.Unmarshal(data, &index) - if err != nil { - return errors.Wrapf(err, "failed to parse protobuf file '%s'", inputFile) - } - els, err := scip.ConvertSCIPToLSIF(&index) - if err != nil { - return errors.Wrapf(err, "failed to convert SCIP index at '%s' to LSIF", inputFile) - } - err = scip.WriteNDJSON(scip.ElementsToJsonElements(els), tmp) - if err != nil { - return errors.Wrapf(err, "failed to write LSIF JSON output to '%s'", tmp.Name()) - } - err = tmp.Close() - if err != nil { - return err - } - return nil -} - func inferDefaultFile() (string, error) { hasSCIP := true const scipFilename = "index.scip" @@ -320,28 +208,12 @@ func inferDefaultFile() (string, error) { } } - hasLSIF := true - const lsifFilename = "dump.lsif" - if _, err := os.Stat(lsifFilename); err != nil { - if os.IsNotExist(err) { - hasLSIF = false - } else { - return "", err - } - } - - if hasSCIP && hasLSIF { - return "", errors.Newf("both %s and %s exists - cannot determine unambiguous choice", scipFilename, lsifFilename) - } - if !(hasSCIP || hasLSIF) { - return "", formatInferenceError(argumentInferenceError{"file", errors.Newf("neither %s nor %s exists", scipFilename, lsifFilename)}) - } - - if hasSCIP { + if !hasSCIP { + return "", formatInferenceError(argumentInferenceError{"file", errors.Newf("%s does not exist", scipFilename)}) + } else { return scipFilename, nil } - return lsifFilename, nil } func formatInferenceError(inferenceErr argumentInferenceError) error { @@ -356,68 +228,13 @@ func formatInferenceError(inferenceErr argumentInferenceError) error { } } -func handleLSIF(out *output.Output) error { - if skipConversionToSCIP { - return nil - } - - fileExt := path.Ext(codeintelUploadFlags.file) - if len(fileExt) == 0 { - return errors.Newf("missing file extension for %s; expected .scip or .lsif", codeintelUploadFlags.file) - } - - if fileExt == ".lsif" || fileExt == ".dump" { - inputFile := codeintelUploadFlags.file - outputFile := replaceExtension(inputFile, ".scip") - codeintelUploadFlags.file = outputFile - return convertLSIFToSCIP(out, inputFile, outputFile) - } - - return nil -} - -// Reads the LSIF encoded input file and writes the corresponding SCIP encoded output file. -func convertLSIFToSCIP(out *output.Output, inputFile, outputFile string) error { - if out != nil { - out.Writef("%s Converting %s into %s", output.EmojiInfo, inputFile, outputFile) - } - - ctx := context.Background() - uploadID := -time.Now().Nanosecond() - root := codeintelUploadFlags.root - - if !isFlagSet(codeintelUploadFlagSet, "root") { - // Best-effort infer the root; we have a strange cyclic init order where we're - // currently trying to determine the filename that determines the root, but we - // need the root when converting from LSIF to SCIP. - root, _ = inferIndexRoot() - } - - rc, err := os.Open(inputFile) - if err != nil { - return err - } - defer rc.Close() - - index, err := libscip.ConvertLSIF(ctx, uploadID, rc, root) - if err != nil { - return err - } - serialized, err := proto.Marshal(index) - if err != nil { - return err - } - - return os.WriteFile(outputFile, serialized, os.ModePerm) -} - // inferMissingCodeIntelUploadFlags updates the flags values which were not explicitly // supplied by the user with default values inferred from the current git state and // filesystem. // // Note: This function must not be called before codeintelUploadFlagset.Parse. func inferMissingCodeIntelUploadFlags() (inferErrors []argumentInferenceError) { - indexerName, indexerVersion, readIndexerNameAndVersionErr := readIndexerNameAndVersion() + indexerName, indexerVersion, readIndexerNameAndVersionErr := readIndexerNameAndVersion(codeintelUploadFlags.file) getIndexerName := func() (string, error) { return indexerName, readIndexerNameAndVersionErr } getIndexerVersion := func() (string, error) { return indexerVersion, readIndexerNameAndVersionErr } @@ -484,14 +301,31 @@ func inferIndexRoot() (string, error) { // toolInfo value in the configured index file. // // Note: This function must not be called before codeintelUploadFlagset.Parse. -func readIndexerNameAndVersion() (string, string, error) { - file, err := os.Open(codeintelUploadFlags.file) +func readIndexerNameAndVersion(indexFile string) (string, string, error) { + file, err := os.Open(indexFile) if err != nil { return "", "", err } defer file.Close() - return upload.ReadIndexerNameAndVersion(file) + var metadata *scip.Metadata + + visitor := scip.IndexVisitor{ + VisitMetadata: func(m *scip.Metadata) { + metadata = m + }, + } + + // convert file to io.Reader + if err := visitor.ParseStreaming(file); err != nil { + return "", "", err + } + + if metadata == nil { + return "", "", errors.New("index file does not contain metadata") + } + + return metadata.ToolInfo.Name, metadata.ToolInfo.Version, nil } // validateCodeIntelUploadFlags returns an error if any of the parsed flag values are illegal. diff --git a/cmd/src/code_intel_upload_flags_test.go b/cmd/src/code_intel_upload_flags_test.go index 9725a45f42..cf675ddeb2 100644 --- a/cmd/src/code_intel_upload_flags_test.go +++ b/cmd/src/code_intel_upload_flags_test.go @@ -21,9 +21,6 @@ var exampleSCIPIndex = scip.Index{ }, } -var exampleLSIFString = `{"id":1,"version":"0.4.3","positionEncoding":"utf-8","toolInfo":{"name":"hello","version":"1.0.0"},"type":"vertex","label":"metaData"} -` - func exampleSCIPBytes(t *testing.T) []byte { bytes, err := proto.Marshal(&exampleSCIPIndex) if err != nil { @@ -32,67 +29,19 @@ func exampleSCIPBytes(t *testing.T) []byte { return bytes } -func createTempSCIPFile(t *testing.T, scipFileName string) (scipFilePath string, lsifFilePath string) { +func createTempSCIPFile(t *testing.T, scipFileName string) string { + t.Helper() dir := t.TempDir() require.NotEqual(t, "", scipFileName) - scipFilePath = filepath.Join(dir, scipFileName) - lsifFilePath = filepath.Join(dir, "dump.lsif") + scipFilePath := filepath.Join(dir, scipFileName) err := os.WriteFile(scipFilePath, exampleSCIPBytes(t), 0755) - if err != nil { - t.Fatal(err) - } - return scipFilePath, lsifFilePath -} - -func assertLSIFOutput(t *testing.T, lsifFile, expectedLSIFString string) { - out := codeintelUploadOutput() - handleSCIP(out) - lsif, err := os.ReadFile(lsifFile) - if err != nil { - t.Fatal(err) - } - obtained := string(lsif) - if obtained != expectedLSIFString { - t.Fatalf("unexpected LSIF output %s", obtained) - } - if lsifFile != codeintelUploadFlags.file { - t.Fatalf("unexpected codeintelUploadFlag.file value %s, expected %s", codeintelUploadFlags.file, lsifFile) - } + require.NoError(t, err) + return scipFilePath } -func TestImplicitlyConvertSCIPIntoLSIF(t *testing.T) { - for _, filename := range []string{"index.scip", "dump.scip", "dump.lsif-typed"} { - _, lsifFile := createTempSCIPFile(t, filename) - codeintelUploadFlags.file = lsifFile - assertLSIFOutput(t, lsifFile, exampleLSIFString) - } -} - -func TestImplicitlyIgnoreSCIP(t *testing.T) { - for _, filename := range []string{"index.scip", "dump.scip", "dump.lsif-typed"} { - _, lsifFile := createTempSCIPFile(t, filename) - codeintelUploadFlags.file = lsifFile - os.WriteFile(lsifFile, []byte("hello world"), 0755) - assertLSIFOutput(t, lsifFile, "hello world") - } -} - -func TestExplicitlyConvertSCIPIntoGraph(t *testing.T) { - for _, filename := range []string{"index.scip", "dump.scip", "dump.lsif-typed"} { - scipFile, lsifFile := createTempSCIPFile(t, filename) - codeintelUploadFlags.file = scipFile - assertLSIFOutput(t, lsifFile, exampleLSIFString) - } -} - -func TestReplaceExtension(t *testing.T) { - require.Panics(t, func() { replaceExtension("foo", ".xyz") }) - require.Equal(t, "foo.xyz", replaceExtension("foo.abc", ".xyz")) -} - -func TestReplaceBaseName(t *testing.T) { - require.Panics(t, func() { replaceBaseName("mydir", filepath.Join("dir", "file")) }) - - require.Equal(t, filepath.Join("a", "d.e"), - replaceBaseName(filepath.Join("a", "b.c"), "d.e")) +func TestInferIndexerNameAndVersion(t *testing.T) { + name, version, err := readIndexerNameAndVersion(createTempSCIPFile(t, "index.scip")) + require.NoError(t, err) + require.Equal(t, "hello", name) + require.Equal(t, "1.0.0", version) } diff --git a/cmd/src/lsif.go b/cmd/src/lsif.go deleted file mode 100644 index 7b5939569c..0000000000 --- a/cmd/src/lsif.go +++ /dev/null @@ -1,30 +0,0 @@ -package main - -import ( - "flag" - "fmt" -) - -var lsifCommands commander - -func init() { - usage := `[DEPRECATED] 'src lsif' is a tool that manages LSIF data on a Sourcegraph instance. - -Use 'src code-intel' instead. -` - flagSet := flag.NewFlagSet("lsif", flag.ExitOnError) - handler := func(args []string) error { - lsifCommands.run(flagSet, "src lsif", usage, args) - return nil - } - - // Register the command. - commands = append(commands, &command{ - flagSet: flagSet, - aliases: []string{"lsif"}, - handler: handler, - usageFunc: func() { - fmt.Println(usage) - }, - }) -} diff --git a/cmd/src/main.go b/cmd/src/main.go index d8d32b7983..06feaa5ac0 100644 --- a/cmd/src/main.go +++ b/cmd/src/main.go @@ -28,7 +28,7 @@ Environment variables SRC_ENDPOINT endpoint to use, if unset will default to "https://sourcegraph.com" SRC_PROXY A proxy to use for proxying requests to the Sourcegraph endpoint. Supports HTTP(S), SOCKS5/5h, and UNIX Domain Socket proxies. - If a UNIX Domain Socket, the path can be either an absolute path, + If a UNIX Domain Socket, the path can be either an absolute path, or can start with ~/ or %USERPROFILE%\ for a path in the user's home directory. Examples: - https://localhost:3080 @@ -55,7 +55,6 @@ The commands are: extsvc manages external services gateway interacts with Cody Gateway login authenticate to a Sourcegraph instance with your user credentials - lsif manages LSIF data (deprecated: use 'code-intel') orgs,org manages organizations teams,team manages teams repos,repo manages repositories diff --git a/internal/codeintel/gitutil_test.go b/internal/codeintel/gitutil_test.go index 893f790a2a..29dde13d85 100644 --- a/internal/codeintel/gitutil_test.go +++ b/internal/codeintel/gitutil_test.go @@ -63,9 +63,9 @@ func TestParseRemote(t *testing.T) { func TestInferRoot(t *testing.T) { testCases := map[string]string{ - "gitutil.go": filepath.Join("internal", "codeintel"), - "../../cmd/src/lsif.go": filepath.Join("cmd", "src"), - "../../README.md": ".", + "gitutil.go": filepath.Join("internal", "codeintel"), + "../../cmd/src/code_intel_upload.go": filepath.Join("cmd", "src"), + "../../README.md": ".", } for input, expectedOutput := range testCases { From 7d9f9c7913873854dbd4cdb55d9f6d436f61f13f Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Fri, 21 Feb 2025 10:08:18 +0000 Subject: [PATCH 2/4] consistent casing --- cmd/src/code_intel_upload_flags.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/src/code_intel_upload_flags.go b/cmd/src/code_intel_upload_flags.go index 01148d9197..aae24865e2 100644 --- a/cmd/src/code_intel_upload_flags.go +++ b/cmd/src/code_intel_upload_flags.go @@ -60,7 +60,7 @@ func init() { // UploadRecordOptions codeintelUploadFlagSet.StringVar(&codeintelUploadFlags.repo, "repo", "", `The name of the repository (e.g. github.com/gorilla/mux). By default, derived from the origin remote.`) codeintelUploadFlagSet.StringVar(&codeintelUploadFlags.commit, "commit", "", `The 40-character hash of the commit. Defaults to the currently checked-out commit.`) - codeintelUploadFlagSet.StringVar(&codeintelUploadFlags.root, "root", "", `The path in the repository that matches the SCIP projectRoot (e.g. cmd/project1). Defaults to the directory where the scip index file is located.`) + codeintelUploadFlagSet.StringVar(&codeintelUploadFlags.root, "root", "", `The path in the repository that matches the SCIP projectRoot (e.g. cmd/project1). Defaults to the directory where the SCIP index file is located.`) codeintelUploadFlagSet.StringVar(&codeintelUploadFlags.indexer, "indexer", "", `The name of the indexer that generated the dump. This will override the 'toolInfo.name' field in the metadata section of SCIP index. This must be supplied if the indexer does not set this field (in which case the upload will fail with an explicit message).`) codeintelUploadFlagSet.StringVar(&codeintelUploadFlags.indexerVersion, "indexerVersion", "", `The version of the indexer that generated the dump. This will override the 'toolInfo.version' field in the metadata section of SCIP index. This must be supplied if the indexer does not set this field (in which case the upload will fail with an explicit message).`) codeintelUploadFlagSet.IntVar(&codeintelUploadFlags.associatedIndexID, "associated-index-id", -1, "ID of the associated index record for this upload. For internal use only.") From c636e434676a45e92cf940a2e2a75424b9c3458b Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Fri, 21 Feb 2025 14:24:57 +0000 Subject: [PATCH 3/4] Remove isSCIPAvailable check --- cmd/src/code_intel_upload_flags.go | 33 ------------------------------ 1 file changed, 33 deletions(-) diff --git a/cmd/src/code_intel_upload_flags.go b/cmd/src/code_intel_upload_flags.go index aae24865e2..f2eaa27203 100644 --- a/cmd/src/code_intel_upload_flags.go +++ b/cmd/src/code_intel_upload_flags.go @@ -1,15 +1,12 @@ package main import ( - "context" "flag" "fmt" - "net/http" "os" "strings" "github.com/sourcegraph/scip/bindings/go/scip" - "github.com/sourcegraph/sourcegraph/lib/codeintel/upload" "github.com/sourcegraph/sourcegraph/lib/errors" "github.com/sourcegraph/sourcegraph/lib/output" @@ -129,17 +126,6 @@ func parseAndValidateCodeIntelUploadFlags(args []string) (*output.Output, error) return nil, errors.Newf("file %q does not exist", codeintelUploadFlags.file) } - isSCIPAvailable, err := isSCIPAvailable() - if err != nil { - return nil, err - } - - if !isSCIPAvailable { - return nil, errors.New( - "src-cli no longer supports LSIF->SCIP conversion, and the Sourcegraph " + - "instance you are uploading to does not support SCIP") - } - // Check for new file existence after transformation if _, err := os.Stat(codeintelUploadFlags.file); os.IsNotExist(err) { return nil, errors.Newf("file %q does not exist", codeintelUploadFlags.file) @@ -173,25 +159,6 @@ func codeintelUploadOutput() (out *output.Output) { }) } -func isSCIPAvailable() (bool, error) { - client := cfg.apiClient(codeintelUploadFlags.apiFlags, codeintelUploadFlagSet.Output()) - req, err := client.NewHTTPRequest(context.Background(), "HEAD", strings.ReplaceAll(codeintelUploadFlags.uploadRoute, "lsif", "scip"), nil) - if err != nil { - return false, err - } - - resp, err := client.Do(req) - if err != nil { - return false, err - } - defer resp.Body.Close() - if resp.StatusCode == http.StatusUnauthorized { - return false, upload.ErrUnauthorized - } - - return resp.StatusCode == http.StatusOK, nil -} - type argumentInferenceError struct { argument string err error From dd04f2e72e50f40742621ecf7082739df8f225f4 Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Fri, 21 Feb 2025 14:46:05 +0000 Subject: [PATCH 4/4] Simplify file check logic --- cmd/src/code_intel_upload_flags.go | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/cmd/src/code_intel_upload_flags.go b/cmd/src/code_intel_upload_flags.go index f2eaa27203..b63735944a 100644 --- a/cmd/src/code_intel_upload_flags.go +++ b/cmd/src/code_intel_upload_flags.go @@ -165,22 +165,18 @@ type argumentInferenceError struct { } func inferDefaultFile() (string, error) { - hasSCIP := true const scipFilename = "index.scip" - if _, err := os.Stat(scipFilename); err != nil { - if os.IsNotExist(err) { - hasSCIP = false - } else { - return "", err - } + _, err := os.Stat(scipFilename) + + if err == nil { + return scipFilename, nil } - if !hasSCIP { + if os.IsNotExist(err) { return "", formatInferenceError(argumentInferenceError{"file", errors.Newf("%s does not exist", scipFilename)}) - } else { - return scipFilename, nil } + return "", err } func formatInferenceError(inferenceErr argumentInferenceError) error { @@ -288,8 +284,8 @@ func readIndexerNameAndVersion(indexFile string) (string, string, error) { return "", "", err } - if metadata == nil { - return "", "", errors.New("index file does not contain metadata") + if metadata == nil || metadata.ToolInfo == nil { + return "", "", errors.New("index file does not contain valid metadata") } return metadata.ToolInfo.Name, metadata.ToolInfo.Version, nil