From 43d24761b51f134acccbe6521fb6bd40afc995b5 Mon Sep 17 00:00:00 2001 From: Patrick Date: Wed, 6 Mar 2024 01:11:50 +0800 Subject: [PATCH 01/13] enable 9.6 9.8 for bench pipeline --- .github/workflows/bench.yml | 6 ++++-- bench/config.yaml | 6 +++--- haskell-language-server.cabal | 2 -- shake-bench/shake-bench.cabal | 2 -- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 768c79e47e..5a6a8b617f 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -49,6 +49,8 @@ jobs: ghc: - '9.2' - '9.4' + - '9.6' + - '9.8' os: - ubuntu-latest @@ -115,13 +117,13 @@ jobs: strategy: fail-fast: false matrix: - ghc: ['9.2', '9.4'] + ghc: ['9.2', '9.4', '9.6', '9.8'] os: [ubuntu-latest] cabal: ['3.10'] example: ['cabal', 'lsp-types'] steps: - - uses: haskell-actions/setup@v2.6.2 + - uses: haskell-actions/setup@v2.6.1 with: ghc-version : ${{ matrix.ghc }} cabal-version: ${{ matrix.cabal }} diff --git a/bench/config.yaml b/bench/config.yaml index f8a062dc3d..a66a9f5204 100644 --- a/bench/config.yaml +++ b/bench/config.yaml @@ -21,10 +21,10 @@ examples: # Medium-sized project without TH - name: cabal package: Cabal - version: 3.6.3.0 + version: 3.10.2.1 modules: - src/Distribution/Simple.hs - - src/Distribution/Types/Module.hs + - src/Distribution/Types/ComponentLocalBuildInfo.hs extra-args: [] # extra HLS command line args # Small-sized project with TH - name: lsp-types @@ -129,7 +129,7 @@ versions: # WARNING: Currently bench versions later than e4234a3a5e347db249fccefb8e3fb36f89e8eafb # will be unable to send plugin configurations to earlier HLS versions. This causes # all plugins in those versions to always be enabled. -# In addition bench proactively disables all plugins it knows about besides the +# In addition bench proactively disables all plugins it knows about besides the # ones in the following list. However because it can only disable plugins it # knows about, any plugins that are in old versions but were removed from HLS # before the current bench will not be disabled. diff --git a/haskell-language-server.cabal b/haskell-language-server.cabal index a65398308d..900185d8a8 100644 --- a/haskell-language-server.cabal +++ b/haskell-language-server.cabal @@ -1835,8 +1835,6 @@ test-suite wrapper-test benchmark benchmark import: defaults, warnings -- Depends on shake-bench which is unbuildable after this point - if impl(ghc >= 9.5) - buildable: False type: exitcode-stdio-1.0 ghc-options: -threaded main-is: Main.hs diff --git a/shake-bench/shake-bench.cabal b/shake-bench/shake-bench.cabal index 540b05d81c..08f54bc56d 100644 --- a/shake-bench/shake-bench.cabal +++ b/shake-bench/shake-bench.cabal @@ -17,8 +17,6 @@ source-repository head library -- Depends on Chart which is unbuildable after this point - if impl(ghc >= 9.5) - buildable: False exposed-modules: Development.Benchmark.Rules hs-source-dirs: src build-depends: From c8ff6124577e54af0449fafa7e1e6e761c0f7649 Mon Sep 17 00:00:00 2001 From: Patrick Date: Wed, 6 Mar 2024 02:59:41 +0800 Subject: [PATCH 02/13] update bench example --- bench/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/config.yaml b/bench/config.yaml index a66a9f5204..f5d459f8c0 100644 --- a/bench/config.yaml +++ b/bench/config.yaml @@ -29,7 +29,7 @@ examples: # Small-sized project with TH - name: lsp-types package: lsp-types - version: 1.5.0.0 + version: 2.1.1.0 modules: - src/Language/LSP/Types/WatchedFiles.hs - src/Language/LSP/Types/CallHierarchy.hs From eba48a33fff9c654f76269985fe1a97bf3868a1c Mon Sep 17 00:00:00 2001 From: Patrick Date: Wed, 6 Mar 2024 04:51:31 +0800 Subject: [PATCH 03/13] update lsp-types example --- bench/config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bench/config.yaml b/bench/config.yaml index f5d459f8c0..0492e04eac 100644 --- a/bench/config.yaml +++ b/bench/config.yaml @@ -31,8 +31,8 @@ examples: package: lsp-types version: 2.1.1.0 modules: - - src/Language/LSP/Types/WatchedFiles.hs - - src/Language/LSP/Types/CallHierarchy.hs + - generated/Language/LSP/Protocol/Internal/Types/DidChangeWatchedFilesParams.hs + - src/Language/LSP/Protocol/Types/SemanticTokens.hs - name: MultiLayerModules path: bench/MultiLayerModules.sh From 2d25b07ca2cfdf04ea171bc955abb3c97cd67c3e Mon Sep 17 00:00:00 2001 From: Patrick Date: Wed, 6 Mar 2024 07:42:05 +0800 Subject: [PATCH 04/13] fix bench --- shake-bench/src/Development/Benchmark/Rules.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shake-bench/src/Development/Benchmark/Rules.hs b/shake-bench/src/Development/Benchmark/Rules.hs index 5993229217..9c8675d03c 100644 --- a/shake-bench/src/Development/Benchmark/Rules.hs +++ b/shake-bench/src/Development/Benchmark/Rules.hs @@ -333,7 +333,7 @@ benchRules build MkBenchRules{..} = do ++ concat [[ "-h" , "-i" <> show i - , "-po" <> outHp + , "-po" <> dropExtension outHp , "-qg"] | CheapHeapProfiling i <- [prof]] ++ ["-RTS"] From 24f86ed05f4f73a6452552e8dcbfe33f52b1a909 Mon Sep 17 00:00:00 2001 From: Patrick Date: Thu, 7 Mar 2024 00:03:11 +0800 Subject: [PATCH 05/13] use known component for bench --- bench/config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bench/config.yaml b/bench/config.yaml index 0492e04eac..29adbb57ef 100644 --- a/bench/config.yaml +++ b/bench/config.yaml @@ -31,8 +31,8 @@ examples: package: lsp-types version: 2.1.1.0 modules: - - generated/Language/LSP/Protocol/Internal/Types/DidChangeWatchedFilesParams.hs - - src/Language/LSP/Protocol/Types/SemanticTokens.hs + - metamodel/Language/LSP/MetaModel/Types.hs + - generator/CodeGen.hs - name: MultiLayerModules path: bench/MultiLayerModules.sh @@ -102,7 +102,7 @@ experiments: - "completions after edit" - "code actions" - "code actions after edit" - - "code actions after cradle edit" + # - "code actions after cradle edit" - "documentSymbols after edit" - "hole fit suggestions" From 1ff52c1cd80bc77571aa7b052b44e7ec8413f1b7 Mon Sep 17 00:00:00 2001 From: Patrick Date: Thu, 7 Mar 2024 02:59:04 +0800 Subject: [PATCH 06/13] remove usage of implicit-hie:gen-hie in bench, Due to its error behaviour on lsp-types-3.1.1.0, In favor of using basic configuration for hie.yaml. --- ghcide-bench/ghcide-bench.cabal | 1 - ghcide-bench/src/Experiments.hs | 26 ++++++++++++++++++-------- haskell-language-server.cabal | 1 - 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/ghcide-bench/ghcide-bench.cabal b/ghcide-bench/ghcide-bench.cabal index 071cb00947..794a551c7c 100644 --- a/ghcide-bench/ghcide-bench.cabal +++ b/ghcide-bench/ghcide-bench.cabal @@ -89,7 +89,6 @@ test-suite test default-language: GHC2021 build-tool-depends: ghcide:ghcide, - implicit-hie:gen-hie main-is: Main.hs hs-source-dirs: test ghc-options: -Wunused-packages diff --git a/ghcide-bench/src/Experiments.hs b/ghcide-bench/src/Experiments.hs index 587f27781b..9670f883e6 100644 --- a/ghcide-bench/src/Experiments.hs +++ b/ghcide-bench/src/Experiments.hs @@ -594,15 +594,25 @@ callCommandLogging cmd = do output cmd callCommand cmd +simpleCabalCradleContent :: String +simpleCabalCradleContent = "cradle:\n cabal:" + +simpleStackCradleContent :: String +simpleStackCradleContent = "cradle:\n stack:" + +-- | Setup the benchmark +-- we need to create a hie.yaml file for the examples +-- or the hie.yaml file would be searched in the parent directories recursively +-- implicit-hie is error prone for the example test `lsp-types-2.1.1.0` +-- we are using the simpleCabalCradleContent for the hie.yaml file instead. +-- it works if we have cabal > 3.2. setup :: HasConfig => IO SetupResult setup = do --- when alreadyExists $ removeDirectoryRecursive examplesPath benchDir <- case exampleDetails(example ?config) of ExamplePath examplePath -> do let hieYamlPath = examplePath "hie.yaml" alreadyExists <- doesFileExist hieYamlPath - unless alreadyExists $ - cmd_ (Cwd examplePath) (FileStdout hieYamlPath) ("gen-hie"::String) + unless alreadyExists $ writeFile hieYamlPath simpleCabalCradleContent return examplePath ExampleScript examplePath' scriptArgs -> do let exampleDir = examplesPath exampleName (example ?config) @@ -613,13 +623,14 @@ setup = do cmd_ (Cwd exampleDir) examplePath scriptArgs let hieYamlPath = exampleDir "hie.yaml" alreadyExists <- doesFileExist hieYamlPath - unless alreadyExists $ - cmd_ (Cwd exampleDir) (FileStdout hieYamlPath) ("gen-hie"::String) + unless alreadyExists $ writeFile hieYamlPath simpleCabalCradleContent + return exampleDir ExampleHackage ExamplePackage{..} -> do let path = examplesPath package package = packageName <> "-" <> showVersion packageVersion hieYamlPath = path "hie.yaml" + print hieYamlPath alreadySetup <- doesDirectoryExist path unless alreadySetup $ case buildTool ?config of @@ -627,7 +638,7 @@ setup = do let cabalVerbosity = "-v" ++ show (fromEnum (verbose ?config)) callCommandLogging $ "cabal get " <> cabalVerbosity <> " " <> package <> " -d " <> examplesPath let hieYamlPath = path "hie.yaml" - cmd_ (Cwd path) (FileStdout hieYamlPath) ("gen-hie"::String) + writeFile hieYamlPath simpleCabalCradleContent -- Need this in case there is a parent cabal.project somewhere writeFile (path "cabal.project") @@ -655,8 +666,7 @@ setup = do ,"compiler"] ] ) - - cmd_ (Cwd path) (FileStdout hieYamlPath) ("gen-hie"::String) ["--stack"::String] + writeFile hieYamlPath simpleStackCradleContent return path whenJust (shakeProfiling ?config) $ createDirectoryIfMissing True diff --git a/haskell-language-server.cabal b/haskell-language-server.cabal index 900185d8a8..fe9b16a063 100644 --- a/haskell-language-server.cabal +++ b/haskell-language-server.cabal @@ -1842,7 +1842,6 @@ benchmark benchmark build-tool-depends: ghcide-bench:ghcide-bench, hp2pretty:hp2pretty, - implicit-hie:gen-hie default-extensions: LambdaCase RecordWildCards From b5604d94329c528da6fa939a038fc5f6f7f2c57a Mon Sep 17 00:00:00 2001 From: Patrick Date: Thu, 7 Mar 2024 04:13:04 +0800 Subject: [PATCH 07/13] update bench config --- bench/config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bench/config.yaml b/bench/config.yaml index 29adbb57ef..818ed6b9db 100644 --- a/bench/config.yaml +++ b/bench/config.yaml @@ -31,8 +31,8 @@ examples: package: lsp-types version: 2.1.1.0 modules: - - metamodel/Language/LSP/MetaModel/Types.hs - - generator/CodeGen.hs + - src/Language/LSP/Protocol/Types/SemanticTokens.hs + - generated/Language/LSP/Protocol/Internal/Types/NotebookDocumentChangeEvent.hs - name: MultiLayerModules path: bench/MultiLayerModules.sh From 6bfe01ad48cee74b92335aa7976a4563099f8ef6 Mon Sep 17 00:00:00 2001 From: Patrick Date: Thu, 7 Mar 2024 04:16:35 +0800 Subject: [PATCH 08/13] update --- .github/workflows/bench.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 5a6a8b617f..8352449832 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -123,7 +123,7 @@ jobs: example: ['cabal', 'lsp-types'] steps: - - uses: haskell-actions/setup@v2.6.1 + - uses: haskell-actions/setup@v2.6.2 with: ghc-version : ${{ matrix.ghc }} cabal-version: ${{ matrix.cabal }} From e26b2b7af3531cbaefb2dc3ef5b951928a66ae38 Mon Sep 17 00:00:00 2001 From: Patrick Date: Thu, 7 Mar 2024 04:29:23 +0800 Subject: [PATCH 09/13] fix cradle edit --- bench/config.yaml | 2 +- ghcide-bench/src/Experiments.hs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bench/config.yaml b/bench/config.yaml index 818ed6b9db..76fbfc3617 100644 --- a/bench/config.yaml +++ b/bench/config.yaml @@ -102,7 +102,7 @@ experiments: - "completions after edit" - "code actions" - "code actions after edit" - # - "code actions after cradle edit" + - "code actions after cradle edit" - "documentSymbols after edit" - "hole fit suggestions" diff --git a/ghcide-bench/src/Experiments.hs b/ghcide-bench/src/Experiments.hs index 9670f883e6..e280f5f8ac 100644 --- a/ghcide-bench/src/Experiments.hs +++ b/ghcide-bench/src/Experiments.hs @@ -595,10 +595,10 @@ callCommandLogging cmd = do callCommand cmd simpleCabalCradleContent :: String -simpleCabalCradleContent = "cradle:\n cabal:" +simpleCabalCradleContent = "cradle:\n cabal:\n" simpleStackCradleContent :: String -simpleStackCradleContent = "cradle:\n stack:" +simpleStackCradleContent = "cradle:\n stack:\n" -- | Setup the benchmark -- we need to create a hie.yaml file for the examples From b3fbab5f5ea59e57264496917f254b9d1f211b6a Mon Sep 17 00:00:00 2001 From: soulomoon Date: Thu, 7 Mar 2024 16:57:08 +0800 Subject: [PATCH 10/13] Update Experiments.hs Co-authored-by: fendor --- ghcide-bench/src/Experiments.hs | 1 - 1 file changed, 1 deletion(-) diff --git a/ghcide-bench/src/Experiments.hs b/ghcide-bench/src/Experiments.hs index e280f5f8ac..8805b05434 100644 --- a/ghcide-bench/src/Experiments.hs +++ b/ghcide-bench/src/Experiments.hs @@ -630,7 +630,6 @@ setup = do let path = examplesPath package package = packageName <> "-" <> showVersion packageVersion hieYamlPath = path "hie.yaml" - print hieYamlPath alreadySetup <- doesDirectoryExist path unless alreadySetup $ case buildTool ?config of From 90c115d68c2ed97ec0c1bada559f3ab7cce9e210 Mon Sep 17 00:00:00 2001 From: Patrick Date: Thu, 7 Mar 2024 18:26:23 +0800 Subject: [PATCH 11/13] cleanup --- shake-bench/shake-bench.cabal | 1 - 1 file changed, 1 deletion(-) diff --git a/shake-bench/shake-bench.cabal b/shake-bench/shake-bench.cabal index 08f54bc56d..eccd84edeb 100644 --- a/shake-bench/shake-bench.cabal +++ b/shake-bench/shake-bench.cabal @@ -16,7 +16,6 @@ source-repository head location: https://github.com/haskell/haskell-language-server.git library - -- Depends on Chart which is unbuildable after this point exposed-modules: Development.Benchmark.Rules hs-source-dirs: src build-depends: From 081a3351cde7f1dd5d58cb2d5d4ec668dfa5babd Mon Sep 17 00:00:00 2001 From: Patrick Date: Thu, 7 Mar 2024 18:30:54 +0800 Subject: [PATCH 12/13] pick two latest --- .github/workflows/bench.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 8352449832..bd8fadee68 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -46,9 +46,10 @@ jobs: strategy: fail-fast: false matrix: + # benching the two latest GHCs we support now + # since benchmark are expansive. + # choosing the two latest are easier to maintain and more forward looking ghc: - - '9.2' - - '9.4' - '9.6' - '9.8' os: @@ -117,7 +118,7 @@ jobs: strategy: fail-fast: false matrix: - ghc: ['9.2', '9.4', '9.6', '9.8'] + ghc: ['9.6', '9.8'] os: [ubuntu-latest] cabal: ['3.10'] example: ['cabal', 'lsp-types'] From 30e51a1c03563d273b6dfc1540a349fb8d06cd29 Mon Sep 17 00:00:00 2001 From: Patrick Date: Thu, 7 Mar 2024 18:43:53 +0800 Subject: [PATCH 13/13] update doc --- .github/workflows/bench.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index bd8fadee68..73fae005ab 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -49,6 +49,8 @@ jobs: # benching the two latest GHCs we support now # since benchmark are expansive. # choosing the two latest are easier to maintain and more forward looking + # see discussion https://github.com/haskell/haskell-language-server/pull/4118 + # also possible to add more GHCs if we performs better in the future. ghc: - '9.6' - '9.8'