Test that all.css contains a valid CSS selector (#199)

MaxDesiatov · mattt · web-flow · commit 006dddcdccf0 · 2020-10-06T18:14:47.000+01:00
Verifies that generated `all.css` contains the `:root` string on CI. It doesn't do any actual CSS parsing to verify its full validity, only does a simple `grep` check, but I hope this will be enough to prevent issues like #198 in the future. * Test that `all.css` contains a valid CSS selector * Update `CHANGELOG.md` * Install graphviz to avoid warnings on macOS * Add EndToEndTests subclassing XCTest * Apply suggestions from code review Co-authored-by: Mattt <mattt@me.com> * Remove stray comment leftover from review * Set macOS 10.13 as minimum platform target * Refactor EndToEndTests Create helper methods on Process and Bundle * Test whether generated HTML output contains subdirectories * Add test for CommonMark output * Add tests for coverage subcommand * Add tests for diagram subcommand * Update Changelog entry for #199 * Work around unavailability of hasDirectoryPath * Update Changelog.md Co-authored-by: Max Desiatov <max@desiatov.com> * Update Changelog entry for #199 Co-authored-by: Mattt <mattt@me.com>
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -25,8 +25,12 @@ jobs:
           key: ${{ runner.os }}-spm-xcode-${{ matrix.xcode }}-${{ hashFiles('**/Package.resolved') }}
           restore-keys: |
             ${{ runner.os }}-spm-xcode-${{ matrix.xcode }}-
+      - name: Install System Dependencies
+        run: |
+          brew install graphviz
       - name: Build and Test
-        run: swift test -c release
+        run: |
+          swift test -c release
         env:
           DEVELOPER_DIR: /Applications/Xcode_${{ matrix.xcode }}.app/Contents/Developer
 
@@ -52,6 +56,6 @@ jobs:
       - name: Install System Dependencies
         run: |
           apt-get update
-          apt-get install -y libxml2-dev
+          apt-get install -y libxml2-dev graphviz
       - name: Build and Test
         run: swift test -c release --enable-test-discovery
diff --git a/Changelog.md b/Changelog.md
@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Added end-to-end tests for command-line interface.
+  #199 by @MaxDesiatov and @mattt.
+
 ## [1.0.0-beta.5] - 2020-09-29
 
 ### Added
diff --git a/Package.swift b/Package.swift
@@ -5,6 +5,9 @@ import PackageDescription
 
 let package = Package(
     name: "swift-doc",
+    platforms: [
+        .macOS(.v10_13)
+    ],
     products: [
         .executable(name: "swift-doc", targets: ["swift-doc"]),
         .library(name: "SwiftDoc", targets: ["SwiftDoc"])
@@ -63,5 +66,11 @@ let package = Package(
                 .product(name: "SwiftMarkup", package: "SwiftMarkup")
             ]
         ),
+        .testTarget(
+            name: "EndToEndTests",
+            dependencies: [
+                .target(name: "swift-doc"),
+            ]
+        ),
     ]
 )
diff --git a/Package@swift-5.2.swift b/Package@swift-5.2.swift
@@ -41,5 +41,11 @@ let package = Package(
             name: "SwiftDocTests",
             dependencies: ["SwiftDoc", "SwiftSyntax", "SwiftSemantics", "SwiftMarkup"]
         ),
+        .testTarget(
+            name: "EndToEndTests",
+            dependencies: [
+                .target(name: "swift-doc"),
+            ]
+        ),
     ]
 )
diff --git a/Tests/EndToEndTests/CoverageSubcommandTests.swift b/Tests/EndToEndTests/CoverageSubcommandTests.swift
@@ -0,0 +1,47 @@
+import XCTest
+
+final class CoverageSubcommandTests: XCTestCase {
+    func testStandardOutput() throws {
+        let command = Bundle.productsDirectory.appendingPathComponent("swift-doc")
+
+        let outputDirectory = try temporaryDirectory()
+        defer { try? FileManager.default.removeItem(at: outputDirectory) }
+
+        try Process.run(command: command,
+                        arguments: [
+                            "coverage",
+                            "Sources"
+                        ]
+        ) { result in
+            XCTAssertEqual(result.terminationStatus, EXIT_SUCCESS)
+            XCTAssertEqual(result.output?.starts(with: "Total"), true)
+            XCTAssertEqual(result.error, "")
+        }
+    }
+
+    func testFileOutput() throws {
+        let command = Bundle.productsDirectory.appendingPathComponent("swift-doc")
+        
+        let outputDirectory = try temporaryDirectory()
+        let outputFile = outputDirectory.appendingPathComponent("report.json")
+        defer { try? FileManager.default.removeItem(at: outputDirectory) }
+
+        try Process.run(command: command,
+                        arguments: [
+                            "coverage",
+                            "--output", outputFile.path,
+                            "Sources"
+                        ]
+        ) { result in
+            XCTAssertEqual(result.terminationStatus, EXIT_SUCCESS)
+            XCTAssertEqual(result.output, "")
+            XCTAssertEqual(result.error, "")
+
+            do {
+                let data = try Data(contentsOf: outputFile)
+                let json = try JSONSerialization.jsonObject(with: data, options: []) as? [String: Any]
+                XCTAssertEqual(json?["type"] as? String, "org.dcov.report.json.export")
+            }
+        }
+    }
+}
diff --git a/Tests/EndToEndTests/DiagramSubcommandTests.swift b/Tests/EndToEndTests/DiagramSubcommandTests.swift
@@ -0,0 +1,21 @@
+import XCTest
+
+final class DiagramSubcommandTests: XCTestCase {
+    func testStandardOutput() throws {
+        let command = Bundle.productsDirectory.appendingPathComponent("swift-doc")
+
+        let outputDirectory = try temporaryDirectory()
+        defer { try? FileManager.default.removeItem(at: outputDirectory) }
+
+        try Process.run(command: command,
+                        arguments: [
+                            "diagram",
+                            "Sources"
+                        ]
+        ) { result in
+            XCTAssertEqual(result.terminationStatus, EXIT_SUCCESS)
+            XCTAssertEqual(result.output?.starts(with: "digraph {"), true)
+            XCTAssertEqual(result.error, "")
+        }
+    }
+}
diff --git a/Tests/EndToEndTests/GenerateSubcommandTests.swift b/Tests/EndToEndTests/GenerateSubcommandTests.swift
@@ -0,0 +1,82 @@
+import XCTest
+
+final class GenerateSubcommandTests: XCTestCase {
+    func testCommonMark() throws {
+        let command = Bundle.productsDirectory.appendingPathComponent("swift-doc")
+
+        let outputDirectory = try temporaryDirectory()
+        defer { try? FileManager.default.removeItem(at: outputDirectory) }
+
+        try Process.run(command: command,
+                        arguments: [
+                            "generate",
+                            "--module-name", "SwiftDoc",
+                            "--format", "commonmark",
+                            "--output", outputDirectory.path,
+                            "Sources"
+                        ]
+        ) { result in
+            XCTAssertEqual(result.terminationStatus, EXIT_SUCCESS)
+            XCTAssertEqual(result.output, "")
+            XCTAssertEqual(result.error, "")
+
+            do {
+                let commonmark = try String(contentsOf: outputDirectory.appendingPathComponent("Home.md"))
+                XCTAssertTrue(commonmark.contains("# Types"))
+            }
+
+            do {
+                let commonmark = try String(contentsOf: outputDirectory.appendingPathComponent("_Sidebar.md"))
+                XCTAssertTrue(commonmark.contains("<summary>Types</summary>"))
+            }
+
+            do {
+                let commonmark = try String(contentsOf: outputDirectory.appendingPathComponent("_Footer.md"))
+                XCTAssertTrue(commonmark.contains("[swift-doc](https://github.com/SwiftDocOrg/swift-doc)"))
+            }
+
+            do {
+                let contents = try FileManager.default.contentsOfDirectory(at: outputDirectory, includingPropertiesForKeys: [.isDirectoryKey], options: [.skipsHiddenFiles])
+                let subdirectories = try contents.filter { try $0.resourceValues(forKeys: [.isDirectoryKey]).isDirectory == true }
+                XCTAssertEqual(subdirectories.count, 0, "output should not contain any subdirectories")
+            }
+        }
+    }
+
+    func testHTML() throws {
+        let command = Bundle.productsDirectory.appendingPathComponent("swift-doc")
+        let outputDirectory = try temporaryDirectory()
+
+        defer { try? FileManager.default.removeItem(at: outputDirectory) }
+        try Process.run(command: command,
+                        arguments: [
+                            "generate",
+                            "--module-name", "SwiftDoc",
+                            "--format", "html",
+                            "--output", outputDirectory.path,
+                            "Sources"
+                        ]
+        ) { result in
+            XCTAssertEqual(result.terminationStatus, EXIT_SUCCESS)
+            XCTAssertEqual(result.output, "")
+            XCTAssertEqual(result.error, "")
+
+            do {
+                let html = try String(contentsOf: outputDirectory.appendingPathComponent("index.html"))
+                XCTAssertTrue(html.contains("<!DOCTYPE html>"))
+            }
+
+            do {
+                let css = try String(contentsOf: outputDirectory.appendingPathComponent("all.css"))
+                XCTAssertTrue(css.contains(":root"))
+            }
+
+            do {
+                let contents = try FileManager.default.contentsOfDirectory(at: outputDirectory, includingPropertiesForKeys: [.isDirectoryKey], options: [.skipsHiddenFiles])
+                let subdirectories = try contents.filter { try $0.resourceValues(forKeys: [.isDirectoryKey]).isDirectory == true }
+                                                 .filter { FileManager.default.fileExists(atPath: $0.appendingPathComponent("index.html").path) }
+                XCTAssertGreaterThanOrEqual(subdirectories.count, 1, "output should contain one or more subdirectories containing index.html")
+            }
+        }
+    }
+}
diff --git a/Tests/EndToEndTests/Helpers/Bundle+Extensions.swift b/Tests/EndToEndTests/Helpers/Bundle+Extensions.swift
@@ -0,0 +1,14 @@
+import Foundation
+
+extension Bundle {
+    static var productsDirectory: URL = {
+        #if os(macOS)
+        for bundle in Bundle.allBundles where bundle.bundlePath.hasSuffix(".xctest") {
+            return bundle.bundleURL.deletingLastPathComponent()
+        }
+        fatalError("couldn't find the products directory")
+        #else
+        return Bundle.main.bundleURL
+        #endif
+    }()
+}
diff --git a/Tests/EndToEndTests/Helpers/Process+Extensions.swift b/Tests/EndToEndTests/Helpers/Process+Extensions.swift
@@ -0,0 +1,34 @@
+import Foundation
+
+extension Process {
+    typealias Result = (terminationStatus: Int32, output: String?, error: String?)
+
+    static func run(command executableURL: URL, arguments: [String] = [], completion: (Result) throws -> Void) throws {
+        let process = Process()
+        if #available(OSX 10.13, *) {
+            process.executableURL = executableURL
+        } else {
+            process.launchPath = executableURL.path
+        }
+        process.arguments = arguments
+
+        let standardOutput = Pipe()
+        process.standardOutput = standardOutput
+
+        let standardError = Pipe()
+        process.standardError = standardError
+
+        if #available(OSX 10.13, *) {
+          try process.run()
+        } else {
+          process.launch()
+        }
+        process.waitUntilExit()
+
+        let output = String(data: standardOutput.fileHandleForReading.readDataToEndOfFile(), encoding: .utf8)
+        let error = String(data: standardError.fileHandleForReading.readDataToEndOfFile(), encoding: .utf8)
+
+        try completion((numericCast(process.terminationStatus), output, error))
+    }
+
+}
diff --git a/Tests/EndToEndTests/Helpers/temporaryDirectory.swift b/Tests/EndToEndTests/Helpers/temporaryDirectory.swift
@@ -0,0 +1,9 @@
+import Foundation
+
+func temporaryDirectory() throws -> URL {
+    let temporaryDirectoryURL = URL(fileURLWithPath: NSTemporaryDirectory()).appendingPathComponent(ProcessInfo.processInfo.globallyUniqueString)
+    try FileManager.default.createDirectory(at: temporaryDirectoryURL, withIntermediateDirectories: true)
+
+    return temporaryDirectoryURL
+}
+

Original file line number	Diff line number	Diff line change
`@@ -41,5 +41,11 @@ let package = Package(`
`41`	`41`	`name: "SwiftDocTests",`
`42`	`42`	`dependencies: ["SwiftDoc", "SwiftSyntax", "SwiftSemantics", "SwiftMarkup"]`
`43`	`43`	`),`
	`44`	`+ .testTarget(`
	`45`	`+ name: "EndToEndTests",`
	`46`	`+ dependencies: [`
	`47`	`+ .target(name: "swift-doc"),`
	`48`	`+ ]`
	`49`	`+ ),`
`44`	`50`	`]`
`45`	`51`	`)`