Update to Rust 1.39

appveyor.yml → .appveyor.yml

@@ -4,14 +4,15 @@ clone_folder: c:\language-rust
 # Add new environments to the build here:
 environment:
   matrix:
-    # - resolver: lts-6.35 # ghc-7.10.3
-    # - resolver: lts-7.24 # ghc-8.0.1
-    # - resolver: lts-9.21 # ghc-8.0.2
     - resolver: lts-11.22 # ghc-8.2.2
     - resolver: lts-12.14 # ghc-8.4.4
     - resolver: lts-14.4  # ghc-8.6.5
     - resolver: nightly
 
+matrix:
+  allow_failures:
+    - resolver: nightly
+
 # Manually fetch stack
 install:
   - set PATH=C:\Program Files\Git\mingw64\bin;%PATH%

.gitignore

@@ -13,8 +13,9 @@ bench/allocations/
 bench/timings/
 
 # Sample source files
-sample-sources/
+sample-sources/*
 !sample-sources/attributes.rs
+!sample-sources/difficult.rs
 !sample-sources/empty.rs
 !sample-sources/expressions.rs
 !sample-sources/items.rs
@@ -26,3 +27,4 @@ sample-sources/
 !sample-sources/statement-expressions.rs
 !sample-sources/statements.rs
 !sample-sources/types.rs
+!sample-sources/visibility.rs

ChangeLog.md

@@ -1,5 +1,24 @@
 # Revision history for language-rust
 
+## 0.3.0.39  -- 2019-11-01
+
+* Bump Rust version
+    - async blocks, async closures, async functions
+    - try blocks
+    - await
+    - or-patterns
+    - associated type constraints
+    - macro definitions (fixes #31)
+    - optional discriminator (::) in type paths
+    - foreign macros
+
+* Fix a couple old issues
+    - parsing of expressions which conflict with statements is much more robust
+    - parsing of expressions in the no-struct context
+    - handle semicolons more like rustc
+
+* Test harnesses have more options for skipping, removing, etc. tests
+
 ## 0.2.0.27  -- 2018-04-22
 
 * Bump Rust version

README.md

@@ -28,12 +28,13 @@ it :: Doc b
 
 ## Building
 
+You'll need a supported version of GHC (check supported versions by searching for `tested-with` in
+`language-rust.cabal`). Odds are the version you want to use is supported.
+
 ### Cabal
 
-With Cabal and GHC, run
+With a recent version of Cabal, run
 
-    cabal install happy --constraint 'happy >= 1.19.8'
-    cabal install alex
     cabal configure
     cabal build
 
@@ -44,10 +45,8 @@ With the [Stack][1] tool installed, run
     stack init
     stack build
 
-The second command is responsible for pulling in all of the dependencies (including executable
-tools like [Alex][2], [Happy][3], and GHC itself) and then compiling everything. If Stack complains
-about the version of Happy installed, you can explicitly install a recent one with `stack install
-happy-1.19.8`.
+If Stack complains about the version of Happy installed, you can explicitly install a recent one
+with `stack install happy-1.19.12`.
 
 ## Evolution of Rust
 

bench/README.md

@@ -21,11 +21,11 @@ directory).
 # Tools
 
 Since some of these tests take a while, you can add a `.benchignore` file in `sample-sources` which
-lists files to skip for benchmarking (one file name per line). 
+lists files to skip for benchmarking (one file name per line).
 
 There is also a `bench.py` utility in this directory which lets you compare benchmarks across
 different commits. It relies on the JSON files in `allocations` and `timings`, so you will have to
-checkout and run the benchmarks on commits you want to compare against (to generate the 
+checkout and run the benchmarks on commits you want to compare against (to generate the
 corresponding JSON file).
 
 ```

bench/allocation-benchmarks/Main.hs

@@ -2,9 +2,9 @@
 
 import Weigh
 
+import Control.DeepSeq (rnf)
 import Control.Monad (filterM)
-import Control.Exception (catch, throwIO)
-import Data.Foldable (for_)
+import Control.Exception (catch, throwIO, evaluate)
 import Data.Traversable (for)
 import GHC.Exts (fromString)
 
@@ -18,7 +18,6 @@ import System.Process (proc, readCreateProcess)
 import System.IO.Error (isDoesNotExistError)
 
 import Data.Aeson
-import qualified Data.ByteString.Lazy as BL
 
 -- TODO: only allocation and GCs seem to be really reproducible. Live and max sometimes are 0.
 
@@ -29,7 +28,7 @@ main = do
   logFileName <- case status of
                    "" -> init <$> readCreateProcess (proc "git" ["rev-parse", "HEAD"]) ""
                    _ -> pure "WIP"
-  
+
   -- Get the test cases
   workingDirectory <- getCurrentDirectory
   let sampleSources = workingDirectory </> "sample-sources"
@@ -46,25 +45,21 @@ main = do
         (\e -> if isDoesNotExistError e then pure () else throwIO e)
 
   -- Run 'weigh' tests
-  fileStreams <- for files $ \file -> do { is <- readInputStream file; pure (takeFileName file, is) }
-  let weigh = do setColumns [ Case, Max, Allocated, GCs, Live ]
-                 for_ fileStreams $ \(file,is) -> func file (parse' :: InputStream -> SourceFile Span) is
-  mainWith weigh
-  (wr, _) <- weighResults weigh
-  let results = object [ case maybeErr of
-                           Nothing -> key .= object [ "allocated" .= weightAllocatedBytes weight
---                                                    , "max"       .= weightMaxBytes w
---                                                    , "live"      .= weightLiveBytes w
---                                                    , "GCs"       .= weightGCs w
-                                                    ]
-                           Just err -> key .= String (fromString err)
-                       | (weight, maybeErr) <- wr
-                       , let key = fromString (weightLabel weight)
-                       ]
+  weighResultsJson <- fmap object . for files $ \file -> do
+    is <- readInputStream file
+    evaluate (rnf is)
+    let testName = fromString (takeFileName file)
+    (allocatedBytes, _gcs, _liveBytes, _maxBytes) <-
+      weighFunc (parse' :: InputStream -> SourceFile Span) is
+    pure $ testName .= object [ "allocated" .= allocatedBytes
+--                               , "max"       .= maxBytes
+--                               , "live"      .= liveBytes
+--                               , "GCs"       .= gcs
+                              ]
 
   -- Save the output to JSON
   createDirectoryIfMissing False (workingDirectory </> "bench" </> "allocations")
   let logFile = workingDirectory </> "bench" </> "allocations" </> logFileName <.> "json"
   putStrLn $ "writing results to: " ++ logFile
-  logFile `BL.writeFile` encode results
+  encodeFile logFile weighResultsJson
 

bench/timing-benchmarks/Main.hs

@@ -5,8 +5,9 @@ import Criterion.Main (defaultConfig)
 import Criterion.Types (anMean, reportAnalysis, timeLimit, anOutlierVar, ovEffect, OutlierEffect(Moderate))
 import Statistics.Types (Estimate(..), ConfInt(..))
 
+import Control.DeepSeq (rnf)
 import Control.Monad (filterM)
-import Control.Exception (catch, throwIO)
+import Control.Exception (catch, evaluate, throwIO)
 import Data.Foldable (for_)
 import Data.Traversable (for)
 import GHC.Exts (fromString)
@@ -21,7 +22,6 @@ import System.Process (proc, readCreateProcess)
 import System.IO.Error (isDoesNotExistError)
 
 import Data.Aeson
-import qualified Data.ByteString.Lazy as BL
 
 main :: IO ()
 main = do
@@ -30,7 +30,7 @@ main = do
   logFileName <- case status of
                    "" -> init <$> readCreateProcess (proc "git" ["rev-parse", "HEAD"]) ""
                    _ -> pure "WIP"
-  
+
   -- Get the test cases
   workingDirectory <- getCurrentDirectory
   let sampleSources = workingDirectory </> "sample-sources"
@@ -51,6 +51,7 @@ main = do
     let name = takeFileName f
     putStrLn name
     is <- readInputStream f
+    evaluate (rnf is)
     bnch <- benchmarkWith' defaultConfig{ timeLimit = 20 } (nf (parse' :: InputStream -> SourceFile Span) is)
     pure (name, bnch)
   let results = object [ fromString name .= object [ "mean" .= m
@@ -68,5 +69,5 @@ main = do
   createDirectoryIfMissing False (workingDirectory </> "bench" </> "timings")
   let logFile = workingDirectory </> "bench" </> "timings" </> logFileName <.> "json"
   putStrLn $ "writing results to: " ++ logFile
-  logFile `BL.writeFile` encode results
+  encodeFile logFile results
 

get-rust-sources.sh

@@ -31,7 +31,7 @@ curl https://api.github.com/orgs/rust-lang-nursery/repos > rust-lang-nursery.jso
 # Make one big JSON array of repos and extract the name and clone url
 (jq -rs '.[0] + .[1] | .[] | (.name, .clone_url)' rust-lang.json rust-lang-nursery.json \
 ) | while read -r REPO_NAME; read -r REPO_CLONE; do
- 
+
   # Skip 'multirust-rs-binaries' and 'rustc-timing-archive' in particular
   if [ $REPO_NAME = "multirust-rs-binaries" ] || [ $REPO_NAME = "rustc-timing-archive" ]
   then
@@ -48,7 +48,7 @@ curl https://api.github.com/orgs/rust-lang-nursery/repos > rust-lang-nursery.jso
   # compile.
   echo "Finding rust files in $REPO_NAME"
   find $REPO_NAME -type f -name '*.rs' | while read -r FILE; do
-    
+
     # Escaped file name
     DEST_FILE="../$DEST/${FILE//\//|}"
 

language-rust.cabal

@@ -1,22 +1,23 @@
 name:                language-rust
-version:             0.2.0.27
+version:             0.3.0.39
 
 synopsis:            Parsing and pretty printing of Rust code
 description:         Language Rust is a library for the analysis of Rust code. It includes a
-                     complete, well tested parser and pretty printer. 
+                     complete, well tested parser and pretty printer.
 
 homepage:            https://github.com/harpocrates/language-rust
 license:             BSD3
 license-file:        LICENSE
 author:              Alec Theriault
 maintainer:          [email protected]
-copyright:           (c) 2017-2018 Alec Theriault
+copyright:           (c) 2017-2019 Alec Theriault
 stability:           provisional
 bug-reports:         https://github.com/harpocrates/language-rust/issues
 category:            Language
 build-type:          Simple
 extra-source-files:  ChangeLog.md README.md
 cabal-version:       >=1.10
+tested-with:         GHC==8.0.2, GHC==8.2.2, GHC==8.4.4, GHC==8.6.5, GHC==8.8.1
 
 source-repository head
   type:              git
@@ -34,12 +35,13 @@ library
   hs-source-dirs:      src
 
   ghc-options:         -Wall
-
-  if impl(ghc >= 8)
-    ghc-options:
+                       -Wcompat
+                       -Widentities
                        -Wincomplete-patterns
+                       -Wincomplete-record-updates
                        -Wincomplete-uni-patterns
                        -Wmissing-signatures
+                       -Wredundant-constraints
 
   build-tools:         alex >=3.1, happy >=1.19.8
   default-language:    Haskell2010
@@ -130,6 +132,7 @@ test-suite rustc-tests
   main-is:             Main.hs
   other-modules:       Diff
                        DiffUtils
+                       Options
 
   other-extensions:    InstanceSigs
                      , OverloadedStrings
@@ -142,7 +145,7 @@ test-suite rustc-tests
 
   build-depends:       process              >=1.3
                      , bytestring           >=0.10
-                     , aeson                >=0.11.0.0
+                     , aeson                >=1.2.4.0
                      , directory            >=1.2.5.0
                      , filepath             >=1.4.0.0
                      , test-framework       >=0.8.0
@@ -166,18 +169,19 @@ benchmark timing-benchmarks
   default-language:    Haskell2010
   build-depends:       process              >=1.3
                      , bytestring           >=0.10
-                     , aeson                >=0.11.0.0
+                     , aeson                >=1.2.4.0
                      , directory            >=1.2.5.0
                      , filepath             >=1.4.0.0
                      , criterion            >=1.1.1.0
                      , statistics           >=0.14.0
 
                      , language-rust
                      , base
+                     , deepseq
 
 benchmark allocation-benchmarks
   hs-source-dirs:      bench/allocation-benchmarks
-  ghc-options:         -Wall
+  ghc-options:         -Wall -with-rtsopts=-T
   main-is:             Main.hs
 
 
@@ -186,12 +190,12 @@ benchmark allocation-benchmarks
   type:                exitcode-stdio-1.0
   default-language:    Haskell2010
   build-depends:       process              >=1.3
-                     , bytestring           >=0.10
-                     , aeson                >=0.11.0.0
+                     , aeson                >=1.2.4.0
                      , directory            >=1.2.5.0
                      , filepath             >=1.4.0.0
-                     , weigh                >=0.0.6
+                     , weigh                >=0.0.8 && <=0.0.14
 
                      , language-rust
                      , base
+                     , deepseq