From b95d3b6b3a94144388e2dfe1d93c3d6a921d8784 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Thu, 20 Apr 2023 13:01:17 -0700 Subject: [PATCH 1/4] [CAS] Swift CAS dependency scanning for explicit build --- .../swift-c/DependencyScan/DependencyScan.h | 52 +++ include/swift/AST/ModuleDependencies.h | 278 +++++++++++--- include/swift/Basic/LangOptions.h | 3 + .../swift/DependencyScan/DependencyScanImpl.h | 15 + .../SerializedModuleDependencyCacheFormat.h | 15 +- include/swift/Frontend/CachingUtils.h | 22 ++ .../swift/Frontend/ModuleInterfaceLoader.h | 101 ++++- include/swift/Option/Options.td | 5 + .../Serialization/ModuleDependencyScanner.h | 27 +- lib/AST/ModuleDependencies.cpp | 150 +++++++- lib/ClangImporter/ClangImporter.cpp | 9 + .../ClangModuleDependencyScanner.cpp | 78 +++- lib/DependencyScan/DependencyScanningTool.cpp | 4 + .../ModuleDependencyCacheSerialization.cpp | 123 ++++-- lib/DependencyScan/ScanDependencies.cpp | 283 ++++++++++++-- lib/Frontend/CachingUtils.cpp | 47 +++ lib/Frontend/CompilerInvocation.cpp | 2 + lib/Frontend/Frontend.cpp | 38 +- lib/Frontend/ModuleInterfaceLoader.cpp | 356 +++++++++++++++++- lib/Serialization/ModuleDependencyScanner.cpp | 30 +- lib/Serialization/SerializedModuleLoader.cpp | 3 +- test/CAS/Inputs/SwiftDepsExtractor.py | 32 ++ test/CAS/binary_module_deps.swift | 41 ++ test/CAS/cas-explicit-module-map.swift | 103 +++++ test/CAS/cas_fs.swift | 1 - test/CAS/module_deps.swift | 195 ++++++++++ tools/libSwiftScan/libSwiftScan.cpp | 92 +++++ tools/libSwiftScan/libSwiftScan.exports | 9 + 28 files changed, 1925 insertions(+), 189 deletions(-) create mode 100755 test/CAS/Inputs/SwiftDepsExtractor.py create mode 100644 test/CAS/binary_module_deps.swift create mode 100644 test/CAS/cas-explicit-module-map.swift create mode 100644 test/CAS/module_deps.swift diff --git a/include/swift-c/DependencyScan/DependencyScan.h b/include/swift-c/DependencyScan/DependencyScan.h index c08a6fb7482c3..5749b6cc9e9a2 100644 --- a/include/swift-c/DependencyScan/DependencyScan.h +++ b/include/swift-c/DependencyScan/DependencyScan.h @@ -154,6 +154,14 @@ SWIFTSCAN_PUBLIC swiftscan_string_set_t * swiftscan_swift_textual_detail_get_swift_overlay_dependencies( swiftscan_module_details_t details); +SWIFTSCAN_PUBLIC swiftscan_string_ref_t +swiftscan_swift_textual_detail_get_cas_fs_root_id( + swiftscan_module_details_t details); + +SWIFTSCAN_PUBLIC swiftscan_string_ref_t +swiftscan_swift_textual_detail_get_module_cache_key( + swiftscan_module_details_t details); + //=== Swift Binary Module Details query APIs ------------------------------===// SWIFTSCAN_PUBLIC swiftscan_string_ref_t @@ -172,6 +180,10 @@ SWIFTSCAN_PUBLIC bool swiftscan_swift_binary_detail_get_is_framework( swiftscan_module_details_t details); +SWIFTSCAN_PUBLIC swiftscan_string_ref_t +swiftscan_swift_binary_detail_get_module_cache_key( + swiftscan_module_details_t details); + //=== Swift Placeholder Module Details query APIs -------------------------===// SWIFTSCAN_PUBLIC swiftscan_string_ref_t @@ -200,6 +212,12 @@ swiftscan_clang_detail_get_command_line(swiftscan_module_details_t details); SWIFTSCAN_PUBLIC swiftscan_string_set_t * swiftscan_clang_detail_get_captured_pcm_args(swiftscan_module_details_t details); +SWIFTSCAN_PUBLIC swiftscan_string_ref_t +swiftscan_clang_detail_get_cas_fs_root_id(swiftscan_module_details_t details); + +SWIFTSCAN_PUBLIC swiftscan_string_ref_t +swiftscan_clang_detail_get_module_cache_key(swiftscan_module_details_t details); + //=== Batch Scan Input Functions ------------------------------------------===// /// Create an \c swiftscan_batch_scan_input_t instance. @@ -402,6 +420,40 @@ swiftscan_scanner_cache_reset(swiftscan_scanner_t scanner); /// An entry point to invoke the compiler via a library call. SWIFTSCAN_PUBLIC int invoke_swift_compiler(int argc, const char **argv); +//=== Scanner CAS Operations ----------------------------------------------===// + +/// Opaque container for a CAS instance that includes both ObjectStore and +/// ActionCache. +typedef struct swiftscan_cas_s *swiftscan_cas_t; + +/// Enum types for output types for cache key computation. +/// TODO: complete the list. +typedef enum { + SWIFTSCAN_OUTPUT_TYPE_OBJECT = 0, + SWIFTSCAN_OUTPUT_TYPE_SWIFTMODULE = 1, + SWIFTSCAN_OUTPUT_TYPE_SWIFTINTERFACE = 2, + SWIFTSCAN_OUTPUT_TYPE_SWIFTPRIAVEINTERFACE = 3, + SWIFTSCAN_OUTPUT_TYPE_CLANG_MODULE = 4, + SWIFTSCAN_OUTPUT_TYPE_CLANG_PCH = 5 +} swiftscan_output_kind_t; + +/// Create a \c cas instance that points to path. +SWIFTSCAN_PUBLIC swiftscan_cas_t swiftscan_cas_create(const char *path); + +/// Dispose the \c cas instance. +SWIFTSCAN_PUBLIC void swiftscan_cas_dispose(swiftscan_cas_t cas); + +/// Store content into CAS. Return \c CASID as string. +SWIFTSCAN_PUBLIC swiftscan_string_ref_t swiftscan_cas_store(swiftscan_cas_t cas, + uint8_t *data, + unsigned size); + +/// Compute \c CacheKey for output of \c kind from the compiler invocation \c +/// argc and \c argv with \c input. Return \c CacheKey as string. +SWIFTSCAN_PUBLIC swiftscan_string_ref_t +swiftscan_compute_cache_key(swiftscan_cas_t cas, int argc, const char **argv, + const char *input, swiftscan_output_kind_t kind); + //===----------------------------------------------------------------------===// SWIFTSCAN_END_DECLS diff --git a/include/swift/AST/ModuleDependencies.h b/include/swift/AST/ModuleDependencies.h index 8fb3c39faa1da..c872b7e8216e7 100644 --- a/include/swift/AST/ModuleDependencies.h +++ b/include/swift/AST/ModuleDependencies.h @@ -23,9 +23,14 @@ #include "clang/Tooling/DependencyScanning/DependencyScanningService.h" #include "clang/Tooling/DependencyScanning/DependencyScanningTool.h" #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/StringSet.h" +#include "llvm/CAS/CASReference.h" +#include "llvm/CAS/CachingOnDiskFileSystem.h" #include "llvm/Support/Mutex.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/VirtualFileSystem.h" #include #include #include @@ -121,8 +126,12 @@ class ModuleDependencyInfoStorageBase { }; struct CommonSwiftTextualModuleDependencyDetails { - CommonSwiftTextualModuleDependencyDetails(ArrayRef extraPCMArgs) - : extraPCMArgs(extraPCMArgs.begin(), extraPCMArgs.end()) {} + CommonSwiftTextualModuleDependencyDetails( + ArrayRef extraPCMArgs, ArrayRef buildCommandLine, + const std::string &CASFileSystemRootID) + : extraPCMArgs(extraPCMArgs.begin(), extraPCMArgs.end()), + buildCommandLine(buildCommandLine.begin(), buildCommandLine.end()), + CASFileSystemRootID(CASFileSystemRootID) {} /// To build a PCM to be used by this Swift module, we need to append these /// arguments to the generic PCM build arguments reported from the dependency @@ -141,6 +150,13 @@ struct CommonSwiftTextualModuleDependencyDetails { /// Dependencies comprised of Swift overlay modules of direct and /// transitive Clang dependencies. std::vector swiftOverlayDependencies; + + /// The Swift frontend invocation arguments to build the Swift module from the + /// interface. + std::vector buildCommandLine; + + /// CASID for the Root of CASFS. Empty if CAS is not used. + std::string CASFileSystemRootID; }; /// Describes the dependencies of a Swift module described by an Swift interface file. @@ -158,10 +174,6 @@ class SwiftInterfaceModuleDependenciesStorage : /// Potentially ready-to-use compiled modules for the interface file. const std::vector compiledModuleCandidates; - /// The Swift frontend invocation arguments to build the Swift module from the - /// interface. - std::vector buildCommandLine; - /// The hash value that will be used for the generated module const std::string contextHash; @@ -171,22 +183,27 @@ class SwiftInterfaceModuleDependenciesStorage : /// Details common to Swift textual (interface or source) modules CommonSwiftTextualModuleDependencyDetails textualModuleDetails; + /// The cache key for the produced module. + std::string moduleCacheKey; + SwiftInterfaceModuleDependenciesStorage( - const std::string moduleOutputPath, - const std::string swiftInterfaceFile, + const std::string &moduleOutputPath, + const std::string &swiftInterfaceFile, ArrayRef compiledModuleCandidates, ArrayRef buildCommandLine, ArrayRef extraPCMArgs, StringRef contextHash, - bool isFramework + bool isFramework, + const std::string &RootID, + const std::string &ModuleCacheKey ) : ModuleDependencyInfoStorageBase(ModuleDependencyKind::SwiftInterface), moduleOutputPath(moduleOutputPath), swiftInterfaceFile(swiftInterfaceFile), compiledModuleCandidates(compiledModuleCandidates.begin(), compiledModuleCandidates.end()), - buildCommandLine(buildCommandLine.begin(), buildCommandLine.end()), contextHash(contextHash), isFramework(isFramework), - textualModuleDetails(extraPCMArgs) + textualModuleDetails(extraPCMArgs, buildCommandLine, RootID), + moduleCacheKey(ModuleCacheKey) {} ModuleDependencyInfoStorageBase *clone() const override { @@ -198,7 +215,15 @@ class SwiftInterfaceModuleDependenciesStorage : } void updateCommandLine(const std::vector &newCommandLine) { - buildCommandLine = newCommandLine; + textualModuleDetails.buildCommandLine = newCommandLine; + } + + void updateCASFileSystemID(const std::string &ID) { + textualModuleDetails.CASFileSystemRootID = ID; + } + + void updateModuleCacheKey(const std::string &Key) { + moduleCacheKey = Key; } }; @@ -218,11 +243,12 @@ class SwiftSourceModuleDependenciesStorage : /// Collection of module imports that were detected to be `@Testable` llvm::StringSet<> testableImports; - SwiftSourceModuleDependenciesStorage( - ArrayRef extraPCMArgs - ) : ModuleDependencyInfoStorageBase(ModuleDependencyKind::SwiftSource), - textualModuleDetails(extraPCMArgs), - testableImports(llvm::StringSet<>()) {} + SwiftSourceModuleDependenciesStorage(const std::string &RootID, + ArrayRef buildCommandLine, + ArrayRef extraPCMArgs) + : ModuleDependencyInfoStorageBase(ModuleDependencyKind::SwiftSource), + textualModuleDetails(extraPCMArgs, buildCommandLine, RootID), + testableImports(llvm::StringSet<>()) {} ModuleDependencyInfoStorageBase *clone() const override { return new SwiftSourceModuleDependenciesStorage(*this); @@ -232,6 +258,14 @@ class SwiftSourceModuleDependenciesStorage : return base->dependencyKind == ModuleDependencyKind::SwiftSource; } + void updateCommandLine(const std::vector &newCommandLine) { + textualModuleDetails.buildCommandLine = newCommandLine; + } + + void updateCASFileSystemID(const std::string &ID) { + textualModuleDetails.CASFileSystemRootID = ID; + } + void addTestableImport(ImportPath::Module module) { testableImports.insert(module.front().Item.str()); } @@ -245,12 +279,12 @@ class SwiftBinaryModuleDependencyStorage : public ModuleDependencyInfoStorageBas SwiftBinaryModuleDependencyStorage(const std::string &compiledModulePath, const std::string &moduleDocPath, const std::string &sourceInfoPath, - const bool isFramework) + const bool isFramework, + const std::string ModuleCacheKey) : ModuleDependencyInfoStorageBase(ModuleDependencyKind::SwiftBinary), - compiledModulePath(compiledModulePath), - moduleDocPath(moduleDocPath), - sourceInfoPath(sourceInfoPath), - isFramework(isFramework) {} + compiledModulePath(compiledModulePath), moduleDocPath(moduleDocPath), + sourceInfoPath(sourceInfoPath), isFramework(isFramework), + moduleCacheKey(ModuleCacheKey) {} ModuleDependencyInfoStorageBase *clone() const override { return new SwiftBinaryModuleDependencyStorage(*this); @@ -268,9 +302,16 @@ class SwiftBinaryModuleDependencyStorage : public ModuleDependencyInfoStorageBas /// A flag that indicates this dependency is a framework const bool isFramework; + /// The cache key for the produced module. + std::string moduleCacheKey; + static bool classof(const ModuleDependencyInfoStorageBase *base) { return base->dependencyKind == ModuleDependencyKind::SwiftBinary; } + + void updateModuleCacheKey(const std::string &Key) { + moduleCacheKey = Key; + } }; /// Describes the dependencies of a Clang module. @@ -288,7 +329,7 @@ class ClangModuleDependencyStorage : public ModuleDependencyInfoStorageBase { const std::string contextHash; /// Partial (Clang) command line that can be used to build this module. - const std::vector nonPathCommandLine; + std::vector nonPathCommandLine; /// The file dependencies const std::vector fileDependencies; @@ -297,20 +338,30 @@ class ClangModuleDependencyStorage : public ModuleDependencyInfoStorageBase { /// as found by the scanning action that discovered it const std::vector capturedPCMArgs; + /// CASID for the Root of CASFS. Empty if CAS is not used. + std::string CASFileSystemRootID; + + /// The cache key for the produced module. + std::string moduleCacheKey; + ClangModuleDependencyStorage( const std::string &pcmOutputPath, const std::string &moduleMapFile, const std::string &contextHash, const std::vector &nonPathCommandLine, const std::vector &fileDependencies, - const std::vector &capturedPCMArgs + const std::vector &capturedPCMArgs, + const std::string &CASFileSystemRootID, + const std::string &ModuleCacheKey ) : ModuleDependencyInfoStorageBase(ModuleDependencyKind::Clang), pcmOutputPath(pcmOutputPath), moduleMapFile(moduleMapFile), contextHash(contextHash), nonPathCommandLine(nonPathCommandLine), fileDependencies(fileDependencies), - capturedPCMArgs(capturedPCMArgs) {} + capturedPCMArgs(capturedPCMArgs), + CASFileSystemRootID(CASFileSystemRootID), + moduleCacheKey(ModuleCacheKey) {} ModuleDependencyInfoStorageBase *clone() const override { return new ClangModuleDependencyStorage(*this); @@ -319,6 +370,15 @@ class ClangModuleDependencyStorage : public ModuleDependencyInfoStorageBase { static bool classof(const ModuleDependencyInfoStorageBase *base) { return base->dependencyKind == ModuleDependencyKind::Clang; } + + void updateCommandLine(const std::vector &newCommandLine) { + nonPathCommandLine = newCommandLine; + } + + void updateModuleCacheKey(const std::string &Key) { + assert(moduleCacheKey.empty()); + moduleCacheKey = Key; + } }; /// Describes an placeholder Swift module dependency module stub. @@ -381,35 +441,42 @@ class ModuleDependencyInfo { /// Describe the module dependencies for a Swift module that can be /// built from a Swift interface file (\c .swiftinterface). - static ModuleDependencyInfo forSwiftInterfaceModule( - const std::string &moduleOutputPath, - const std::string &swiftInterfaceFile, - ArrayRef compiledCandidates, - ArrayRef buildCommands, - ArrayRef extraPCMArgs, - StringRef contextHash, - bool isFramework) { + static ModuleDependencyInfo + forSwiftInterfaceModule(const std::string &moduleOutputPath, + const std::string &swiftInterfaceFile, + ArrayRef compiledCandidates, + ArrayRef buildCommands, + ArrayRef extraPCMArgs, + StringRef contextHash, bool isFramework, + const std::string &CASFileSystemRootID, + const std::string &moduleCacheKey) { return ModuleDependencyInfo( std::make_unique( - moduleOutputPath, swiftInterfaceFile, compiledCandidates, buildCommands, - extraPCMArgs, contextHash, isFramework)); + moduleOutputPath, swiftInterfaceFile, compiledCandidates, + buildCommands, extraPCMArgs, contextHash, isFramework, + CASFileSystemRootID, moduleCacheKey)); } /// Describe the module dependencies for a serialized or parsed Swift module. - static ModuleDependencyInfo forSwiftBinaryModule( - const std::string &compiledModulePath, - const std::string &moduleDocPath, - const std::string &sourceInfoPath, - bool isFramework) { + static ModuleDependencyInfo + forSwiftBinaryModule(const std::string &compiledModulePath, + const std::string &moduleDocPath, + const std::string &sourceInfoPath, bool isFramework, + const std::string &moduleCacheKey) { return ModuleDependencyInfo( std::make_unique( - compiledModulePath, moduleDocPath, sourceInfoPath, isFramework)); + compiledModulePath, moduleDocPath, sourceInfoPath, isFramework, + moduleCacheKey)); } /// Describe the main Swift module. - static ModuleDependencyInfo forSwiftSourceModule(ArrayRef extraPCMArgs) { + static ModuleDependencyInfo + forSwiftSourceModule(const std::string &CASFileSystemRootID, + ArrayRef buildCommands, + ArrayRef extraPCMArgs) { return ModuleDependencyInfo( - std::make_unique(extraPCMArgs)); + std::make_unique( + CASFileSystemRootID, buildCommands, extraPCMArgs)); } /// Describe the module dependencies for a Clang module that can be @@ -420,11 +487,14 @@ class ModuleDependencyInfo { const std::string &contextHash, const std::vector &nonPathCommandLine, const std::vector &fileDependencies, - const std::vector &capturedPCMArgs) { + const std::vector &capturedPCMArgs, + const std::string &CASFileSystemRootID, + const std::string &moduleCacheKey) { return ModuleDependencyInfo( std::make_unique( pcmOutputPath, moduleMapFile, contextHash, - nonPathCommandLine, fileDependencies, capturedPCMArgs)); + nonPathCommandLine, fileDependencies, capturedPCMArgs, + CASFileSystemRootID, moduleCacheKey)); } /// Describe a placeholder dependency swift module. @@ -453,6 +523,32 @@ class ModuleDependencyInfo { return storage->resolvedModuleDependencies; } + std::string getModuleCacheKey() const { + assert(storage->resolved); + if (auto *dep = getAsSwiftInterfaceModule()) + return dep->moduleCacheKey; + else if (auto *dep = getAsSwiftBinaryModule()) + return dep->moduleCacheKey; + else if (auto *dep = getAsClangModule()) + return dep->moduleCacheKey; + + llvm_unreachable("Unexpected type"); + } + + void updateModuleCacheKey(const std::string &key) { + if (isSwiftInterfaceModule()) + return cast(storage.get()) + ->updateModuleCacheKey(key); + else if (isSwiftBinaryModule()) + return cast(storage.get()) + ->updateModuleCacheKey(key); + else if (isClangModule()) + return cast(storage.get()) + ->updateModuleCacheKey(key); + + llvm_unreachable("Unexpected type"); + } + /// Resolve a dependency's set of `imports` with qualified Module IDs void resolveDependencies(const ArrayRef dependencyIDs) { assert(!storage->resolved && "Resolving an already-resolved dependency"); @@ -474,9 +570,37 @@ class ModuleDependencyInfo { textualModuleDetails->swiftOverlayDependencies.assign(dependencyIDs.begin(), dependencyIDs.end()); } + std::vector getCommandline() const { + if (auto *detail = getAsClangModule()) + return detail->nonPathCommandLine; + else if (auto *detail = getAsSwiftInterfaceModule()) + return detail->textualModuleDetails.buildCommandLine; + else if (auto *detail = getAsSwiftSourceModule()) + return detail->textualModuleDetails.buildCommandLine; + return {}; + } + void updateCommandLine(const std::vector &newCommandLine) { - assert(isSwiftInterfaceModule() && "Can only update command line on Swift interface dependency"); - cast(storage.get())->updateCommandLine(newCommandLine); + if (isSwiftInterfaceModule()) + return cast(storage.get()) + ->updateCommandLine(newCommandLine); + else if (isSwiftSourceModule()) + return cast(storage.get()) + ->updateCommandLine(newCommandLine); + else if (isClangModule()) + return cast(storage.get()) + ->updateCommandLine(newCommandLine); + llvm_unreachable("Unexpected type"); + } + + void updateCASFileSystemID(const std::string &ID) { + if (isSwiftInterfaceModule()) + return cast(storage.get()) + ->updateCASFileSystemID(ID); + else if (isSwiftSourceModule()) + return cast(storage.get()) + ->updateCASFileSystemID(ID); + llvm_unreachable("Unexpected type"); } bool isResolved() const { @@ -568,6 +692,12 @@ class ModuleDependencyInfo { /// Get the bridging header. Optional getBridgingHeader() const; + /// Get CAS Filesystem RootID. + Optional getCASFSRootID() const; + + /// Get module output path. + std::string getModuleOutputPath() const; + /// Add a bridging header to a Swift module's dependencies. void addBridgingHeader(StringRef bridgingHeader); @@ -597,6 +727,23 @@ using ModuleDependenciesKindRefMap = llvm::StringMap, ModuleDependencyKindHash>; +// MARK: SwiftDependencyTracker +/// Track swift dependency +class SwiftDependencyTracker { +public: + SwiftDependencyTracker(llvm::cas::CachingOnDiskFileSystem &FS, + const std::vector &CommonFiles) + : FS(FS.createProxyFS()), Files(CommonFiles) {} + + void startTracking(); + void trackFile(const Twine &path) { (void)FS->status(path); } + llvm::Expected createTreeFromDependencies(); + +private: + llvm::IntrusiveRefCntPtr FS; + const std::vector &Files; +}; + // MARK: SwiftDependencyScanningService /// A carrier of state shared among possibly multiple invocations of the dependency /// scanner. Acts as a global cache of discovered module dependencies and @@ -620,7 +767,15 @@ class SwiftDependencyScanningService { }; /// The persistent Clang dependency scanner service - clang::tooling::dependencies::DependencyScanningService ClangScanningService; + Optional + ClangScanningService; + + /// CachingOnDiskFileSystem for dependency tracking. + llvm::IntrusiveRefCntPtr CacheFS; + + /// The common dependencies that is needed for every swift compiler instance. + std::vector CommonDependencyFiles; + /// The global file system cache. Optional< clang::tooling::dependencies::DependencyScanningFilesystemSharedCache> @@ -671,9 +826,33 @@ class SwiftDependencyScanningService { return getCacheForScanningContextHash(scanningContextHash)->alreadySeenClangModules; } + bool usingCachingFS() const { return (bool)CacheFS; } + + llvm::cas::CachingOnDiskFileSystem &getSharedCachingFS() const { + assert(CacheFS && "Expect CachingOnDiskFileSystem"); + return *CacheFS; + } + + Optional createSwiftDependencyTracker() const { + if (!CacheFS) + return None; + + return SwiftDependencyTracker(*CacheFS, CommonDependencyFiles); + } + + llvm::IntrusiveRefCntPtr getClangScanningFS() const { + if (CacheFS) + return CacheFS->createProxyFS(); + + return llvm::vfs::createPhysicalFileSystem(); + } + /// Wrap the filesystem on the specified `CompilerInstance` with a /// caching `DependencyScanningWorkerFilesystem` void overlaySharedFilesystemCacheForCompilation(CompilerInstance &Instance); + + /// Setup caching service. + void setupCachingDependencyScanningService(CompilerInstance &Instance); private: /// Enforce clients not being allowed to query this cache directly, it must be /// wrapped in an instance of `ModuleDependenciesCache`. @@ -769,6 +948,9 @@ class ModuleDependenciesCache { clang::tooling::dependencies::DependencyScanningTool& getClangScannerTool() { return clangScanningTool; } + SwiftDependencyScanningService &getScanService() { + return globalScanningService; + } llvm::StringSet<>& getAlreadySeenClangModules() { return globalScanningService.getAlreadySeenClangModules(scannerContextHash); } diff --git a/include/swift/Basic/LangOptions.h b/include/swift/Basic/LangOptions.h index 782803d85d6fc..3e4766a69cd96 100644 --- a/include/swift/Basic/LangOptions.h +++ b/include/swift/Basic/LangOptions.h @@ -818,6 +818,9 @@ namespace swift { /// clang CASOptions. std::string CASPath; + /// Cache key for imported bridging header. + std::string BridgingHeaderPCHCacheKey; + /// Disable validating the persistent PCH. bool PCHDisableValidation = false; diff --git a/include/swift/DependencyScan/DependencyScanImpl.h b/include/swift/DependencyScan/DependencyScanImpl.h index dd0463e502e44..d335055648c17 100644 --- a/include/swift/DependencyScan/DependencyScanImpl.h +++ b/include/swift/DependencyScan/DependencyScanImpl.h @@ -96,6 +96,12 @@ typedef struct { /// A flag to indicate whether or not this module is a framework. bool is_framework; + + /// The CASID for CASFileSystemRoot + swiftscan_string_ref_t cas_fs_root_id; + + /// ModuleCacheKey + swiftscan_string_ref_t module_cache_key; } swiftscan_swift_textual_details_t; /// Swift modules with only a binary module file. @@ -111,6 +117,9 @@ typedef struct { /// A flag to indicate whether or not this module is a framework. bool is_framework; + + /// ModuleCacheKey + swiftscan_string_ref_t module_cache_key; } swiftscan_swift_binary_details_t; /// Swift placeholder modules carry additional details that specify their @@ -139,6 +148,12 @@ typedef struct { /// The swift-specific PCM arguments captured by this dependencies object swiftscan_string_set_t *captured_pcm_args; + + /// The CASID for CASFileSystemRoot + swiftscan_string_ref_t cas_fs_root_id; + + /// ModuleCacheKey + swiftscan_string_ref_t module_cache_key; } swiftscan_clang_details_t; struct swiftscan_module_details_s { diff --git a/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h b/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h index bad67c1e43bf1..e2d547fcfe870 100644 --- a/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h +++ b/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h @@ -140,7 +140,9 @@ using SwiftInterfaceModuleDetailsLayout = FileIDArrayIDField, // sourceFiles FileIDArrayIDField, // bridgingSourceFiles FileIDArrayIDField, // bridgingModuleDependencies - DependencyIDArrayIDField // swiftOverlayDependencies + DependencyIDArrayIDField, // swiftOverlayDependencies + IdentifierIDField, // CASFileSystemRootID + IdentifierIDField // moduleCacheKey >; using SwiftSourceModuleDetailsLayout = @@ -150,7 +152,9 @@ using SwiftSourceModuleDetailsLayout = FileIDArrayIDField, // sourceFiles FileIDArrayIDField, // bridgingSourceFiles FileIDArrayIDField, // bridgingModuleDependencies - DependencyIDArrayIDField // swiftOverlayDependencies + DependencyIDArrayIDField, // swiftOverlayDependencies + IdentifierIDField, // CASFileSystemRootID + FlagIDArrayIDField // buildCommandLine >; using SwiftBinaryModuleDetailsLayout = @@ -158,7 +162,8 @@ using SwiftBinaryModuleDetailsLayout = FileIDField, // compiledModulePath FileIDField, // moduleDocPath FileIDField, // moduleSourceInfoPath - IsFrameworkField // isFramework + IsFrameworkField, // isFramework + IdentifierIDField // moduleCacheKey >; using SwiftPlaceholderModuleDetailsLayout = @@ -175,7 +180,9 @@ using ClangModuleDetailsLayout = ContextHashIDField, // contextHash FlagIDArrayIDField, // commandLine FileIDArrayIDField, // fileDependencies - FlagIDArrayIDField // capturedPCMArgs + FlagIDArrayIDField, // capturedPCMArgs + IdentifierIDField, // CASFileSystemRootID + IdentifierIDField // moduleCacheKey >; } // namespace graph_block diff --git a/include/swift/Frontend/CachingUtils.h b/include/swift/Frontend/CachingUtils.h index 36709d8c3a38e..97d92de02f8ef 100644 --- a/include/swift/Frontend/CachingUtils.h +++ b/include/swift/Frontend/CachingUtils.h @@ -54,6 +54,28 @@ llvm::Error storeCachedCompilerOutput(llvm::cas::ObjectStore &CAS, StringRef CorrespondingInput, file_types::ID OutputKind); +namespace cas { +/// Helper class to manage CAS/Caching from libSwiftScan C APIs. +class CachingTool { +public: + // Create the tool with a list of arguments from compiler invocation. + CachingTool(llvm::StringRef Path); + + // Compute the CASID for PCH output from invocation. + std::string computeCacheKey(llvm::ArrayRef Args, + StringRef InputPath, file_types::ID OutputKind); + + // Store content into CAS. + std::string storeContent(llvm::StringRef Content); + + // Check if the tool is correctly initialized. + bool isValid() const { return CAS && Cache; } + +private: + std::unique_ptr CAS; + std::unique_ptr Cache; +}; +} // namespace cas } #endif diff --git a/include/swift/Frontend/ModuleInterfaceLoader.h b/include/swift/Frontend/ModuleInterfaceLoader.h index d5533b10ec14e..3c0d148ba62e0 100644 --- a/include/swift/Frontend/ModuleInterfaceLoader.h +++ b/include/swift/Frontend/ModuleInterfaceLoader.h @@ -111,9 +111,17 @@ #include "swift/Frontend/Frontend.h" #include "swift/Frontend/ModuleInterfaceSupport.h" #include "swift/Serialization/SerializedModuleLoader.h" +#include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/StringSaver.h" #include "llvm/Support/YAMLTraits.h" +namespace llvm { +namespace cas { +class ObjectStore; +class ActionCache; +} // namespace cas +} // namespace llvm + namespace clang { class CompilerInstance; } @@ -178,6 +186,55 @@ class ExplicitSwiftModuleLoader: public SerializedModuleLoaderBase { ~ExplicitSwiftModuleLoader(); }; +class ExplicitCASModuleLoader : public SerializedModuleLoaderBase { + explicit ExplicitCASModuleLoader(ASTContext &ctx, llvm::cas::ObjectStore &CAS, + llvm::cas::ActionCache &cache, + DependencyTracker *tracker, + ModuleLoadingMode loadMode, + bool IgnoreSwiftSourceInfoFile); + + bool findModule(ImportPath::Element moduleID, + SmallVectorImpl *moduleInterfacePath, + SmallVectorImpl *moduleInterfaceSourcePath, + std::unique_ptr *moduleBuffer, + std::unique_ptr *moduleDocBuffer, + std::unique_ptr *moduleSourceInfoBuffer, + bool skipBuildingInterface, bool isTestableDependencyLookup, + bool &isFramework, bool &isSystemModule) override; + + std::error_code findModuleFilesInDirectory( + ImportPath::Element ModuleID, const SerializedModuleBaseName &BaseName, + SmallVectorImpl *ModuleInterfacePath, + SmallVectorImpl *ModuleInterfaceSourcePath, + std::unique_ptr *ModuleBuffer, + std::unique_ptr *ModuleDocBuffer, + std::unique_ptr *ModuleSourceInfoBuffer, + bool SkipBuildingInterface, bool IsFramework, + bool IsTestableDependencyLookup = false) override; + + bool canImportModule(ImportPath::Module named, + ModuleVersionInfo *versionInfo, + bool isTestableDependencyLookup = false) override; + + struct Implementation; + Implementation &Impl; + +public: + static std::unique_ptr + create(ASTContext &ctx, llvm::cas::ObjectStore &CAS, + llvm::cas::ActionCache &cache, DependencyTracker *tracker, + ModuleLoadingMode loadMode, StringRef ExplicitSwiftModuleMap, + const std::vector> + &ExplicitSwiftModuleInputs, + bool IgnoreSwiftSourceInfoFile); + + /// Append visible module names to \p names. Note that names are possibly + /// duplicated, and not guaranteed to be ordered in any way. + void collectVisibleTopLevelModuleNames( + SmallVectorImpl &names) const override; + + ~ExplicitCASModuleLoader(); +}; // Explicitly-specified Swift module inputs struct ExplicitSwiftModuleInputInfo { @@ -185,12 +242,14 @@ struct ExplicitSwiftModuleInputInfo { llvm::Optional moduleDocPath, llvm::Optional moduleSourceInfoPath, bool isFramework = false, - bool isSystem = false) + bool isSystem = false, + llvm::Optional moduleCacheKey = None) : modulePath(modulePath), moduleDocPath(moduleDocPath), moduleSourceInfoPath(moduleSourceInfoPath), isFramework(isFramework), - isSystem(isSystem) {} + isSystem(isSystem), + moduleCacheKey(moduleCacheKey) {} // Path of the .swiftmodule file. std::string modulePath; // Path of the .swiftmoduledoc file. @@ -201,6 +260,8 @@ struct ExplicitSwiftModuleInputInfo { bool isFramework = false; // A flag that indicates whether this module is a system module bool isSystem = false; + // The cache key for clang module. + llvm::Optional moduleCacheKey; }; // Explicitly-specified Clang module inputs @@ -208,11 +269,13 @@ struct ExplicitClangModuleInputInfo { ExplicitClangModuleInputInfo(std::string moduleMapPath, std::string modulePath, bool isFramework = false, - bool isSystem = false) + bool isSystem = false, + llvm::Optional moduleCacheKey = None) : moduleMapPath(moduleMapPath), modulePath(modulePath), isFramework(isFramework), - isSystem(isSystem) {} + isSystem(isSystem), + moduleCacheKey(moduleCacheKey) {} // Path of the Clang module map file. std::string moduleMapPath; // Path of a compiled Clang explicit module file (pcm). @@ -221,6 +284,8 @@ struct ExplicitClangModuleInputInfo { bool isFramework = false; // A flag that indicates whether this module is a system module bool isSystem = false; + // The cache key for clang module. + llvm::Optional moduleCacheKey; }; /// Parser of explicit module maps passed into the compiler. @@ -233,6 +298,8 @@ struct ExplicitClangModuleInputInfo { // "isFramework": false, // "clangModuleMapPath": "A/module.modulemap", // "clangModulePath": "A.pcm", +// "moduleCacheKey": "llvmcas://", +// "clangModuleCacheKey": "llvmcas://", // }, // { // "moduleName": "B", @@ -242,6 +309,8 @@ struct ExplicitClangModuleInputInfo { // "isFramework": false, // "clangModuleMapPath": "B/module.modulemap", // "clangModulePath": "B.pcm", +// "moduleCacheKey": "llvmcas://", +// "clangModuleCacheKey": "llvmcas://", // } // ] class ExplicitModuleMapParser { @@ -249,21 +318,14 @@ class ExplicitModuleMapParser { ExplicitModuleMapParser(llvm::BumpPtrAllocator &Allocator) : Saver(Allocator) {} std::error_code - parseSwiftExplicitModuleMap(llvm::StringRef fileName, + parseSwiftExplicitModuleMap(llvm::MemoryBufferRef BufferRef, llvm::StringMap &swiftModuleMap, llvm::StringMap &clangModuleMap) { using namespace llvm::yaml; - // Load the input file. - llvm::ErrorOr> fileBufOrErr = - llvm::MemoryBuffer::getFile(fileName); - if (!fileBufOrErr) { - return std::make_error_code(std::errc::no_such_file_or_directory); - } - StringRef Buffer = fileBufOrErr->get()->getBuffer(); // Use a new source manager instead of the one from ASTContext because we // don't want the JSON file to be persistent. llvm::SourceMgr SM; - Stream Stream(llvm::MemoryBufferRef(Buffer, fileName), SM); + Stream Stream(BufferRef, SM); for (auto DI = Stream.begin(); DI != Stream.end(); ++DI) { assert(DI != Stream.end() && "Failed to read a document"); if (auto *MN = dyn_cast_or_null(DI->getRoot())) { @@ -305,7 +367,8 @@ class ExplicitModuleMapParser { return true; StringRef moduleName; llvm::Optional swiftModulePath, swiftModuleDocPath, - swiftModuleSourceInfoPath; + swiftModuleSourceInfoPath, swiftModuleCacheKey, + clangModuleCacheKey; std::string clangModuleMapPath = "", clangModulePath = ""; bool isFramework = false, isSystem = false; for (auto &entry : *mapNode) { @@ -327,6 +390,10 @@ class ExplicitModuleMapParser { clangModuleMapPath = val.str(); } else if (key == "clangModulePath") { clangModulePath = val.str(); + } else if (key == "moduleCacheKey") { + swiftModuleCacheKey = val.str(); + } else if (key == "clangModuleCacheKey") { + clangModuleCacheKey = val.str(); } else { // Being forgiving for future fields. continue; @@ -343,7 +410,8 @@ class ExplicitModuleMapParser { swiftModuleDocPath, swiftModuleSourceInfoPath, isFramework, - isSystem); + isSystem, + swiftModuleCacheKey); swiftModuleMap.try_emplace(moduleName, std::move(entry)); } else { assert((!clangModuleMapPath.empty() || @@ -352,7 +420,8 @@ class ExplicitModuleMapParser { ExplicitClangModuleInputInfo entry(clangModuleMapPath, clangModulePath, isFramework, - isSystem); + isSystem, + clangModuleCacheKey); clangModuleMap.try_emplace(moduleName, std::move(entry)); } diff --git a/include/swift/Option/Options.td b/include/swift/Option/Options.td index 984d69949697f..c5854c65e8d54 100644 --- a/include/swift/Option/Options.td +++ b/include/swift/Option/Options.td @@ -1829,6 +1829,11 @@ def allow_unstable_cache_key_for_testing: Flag<["-"], "allow-unstable-cache-key- Flags<[FrontendOption, HelpHidden, NoDriverOption]>, HelpText<"Allow compilation caching with unstable inputs for testing purpose">; +def bridging_header_pch_key : Separate<["-"], "bridging-header-pch-key">, + Flags<[FrontendOption, HelpHidden, ArgumentIsPath]>, + HelpText<"Cache Key for bridging header pch">; + + // END ONLY SUPPORTED IN NEW DRIVER def enable_cas: Flag<["-"], "enable-cas">, diff --git a/include/swift/Serialization/ModuleDependencyScanner.h b/include/swift/Serialization/ModuleDependencyScanner.h index 0132c7ce58920..1567d63995281 100644 --- a/include/swift/Serialization/ModuleDependencyScanner.h +++ b/include/swift/Serialization/ModuleDependencyScanner.h @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// #include "swift/AST/ASTContext.h" +#include "swift/AST/ModuleDependencies.h" #include "swift/Frontend/ModuleInterfaceLoader.h" #include "swift/Serialization/SerializedModuleLoader.h" @@ -41,18 +42,22 @@ namespace swift { /// Location where pre-built moduels are to be built into. std::string moduleCachePath; + + Optional dependencyTracker; public: Optional dependencies; ModuleDependencyScanner(ASTContext &ctx, ModuleLoadingMode LoadMode, Identifier moduleName, InterfaceSubContextDelegate &astDelegate, - ScannerKind kind = MDS_plain) + ScannerKind kind = MDS_plain, + Optional tracker = None) : SerializedModuleLoaderBase(ctx, nullptr, LoadMode, /*IgnoreSwiftSourceInfoFile=*/true), kind(kind), moduleName(moduleName), astDelegate(astDelegate), moduleCachePath(getModuleCachePathFromClang( - ctx.getClangModuleLoader()->getClangInstance())) {} + ctx.getClangModuleLoader()->getClangInstance())), + dependencyTracker(tracker) {} std::error_code findModuleFilesInDirectory( ImportPath::Element ModuleID, @@ -87,9 +92,16 @@ namespace swift { void parsePlaceholderModuleMap(StringRef fileName) { ExplicitModuleMapParser parser(Allocator); llvm::StringMap ClangDependencyModuleMap; - auto result = - parser.parseSwiftExplicitModuleMap(fileName, PlaceholderDependencyModuleMap, - ClangDependencyModuleMap); + llvm::ErrorOr> fileBufOrErr = + llvm::MemoryBuffer::getFile(fileName); + if (!fileBufOrErr) { + Ctx.Diags.diagnose(SourceLoc(), + diag::explicit_swift_module_map_missing, fileName); + return; + } + auto result = parser.parseSwiftExplicitModuleMap( + (*fileBufOrErr)->getMemBufferRef(), PlaceholderDependencyModuleMap, + ClangDependencyModuleMap); if (result == std::errc::invalid_argument) { Ctx.Diags.diagnose(SourceLoc(), diag::placeholder_dependency_module_map_corrupted, @@ -109,9 +121,10 @@ namespace swift { PlaceholderSwiftModuleScanner(ASTContext &ctx, ModuleLoadingMode LoadMode, Identifier moduleName, StringRef PlaceholderDependencyModuleMap, - InterfaceSubContextDelegate &astDelegate) + InterfaceSubContextDelegate &astDelegate, + Optional tracker = None) : ModuleDependencyScanner(ctx, LoadMode, moduleName, astDelegate, - MDS_placeholder) { + MDS_placeholder, tracker) { // FIXME: Find a better place for this map to live, to avoid // doing the parsing on every module. diff --git a/lib/AST/ModuleDependencies.cpp b/lib/AST/ModuleDependencies.cpp index a9f5435d05d34..a9660e6f96979 100644 --- a/lib/AST/ModuleDependencies.cpp +++ b/lib/AST/ModuleDependencies.cpp @@ -15,8 +15,14 @@ //===----------------------------------------------------------------------===// #include "swift/AST/ModuleDependencies.h" #include "swift/AST/Decl.h" +#include "swift/AST/DiagnosticsFrontend.h" #include "swift/AST/SourceFile.h" #include "swift/Frontend/Frontend.h" +#include "llvm/CAS/CASProvidingFileSystem.h" +#include "llvm/CAS/CachingOnDiskFileSystem.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Path.h" +#include using namespace swift; ModuleDependencyInfoStorageBase::~ModuleDependencyInfoStorageBase() {} @@ -175,6 +181,64 @@ Optional ModuleDependencyInfo::getBridgingHeader() const { } } +Optional ModuleDependencyInfo::getCASFSRootID() const { + std::string Root; + switch (getKind()) { + case swift::ModuleDependencyKind::SwiftInterface: { + auto swiftInterfaceStorage = + cast(storage.get()); + Root = swiftInterfaceStorage->textualModuleDetails.CASFileSystemRootID; + break; + } + case swift::ModuleDependencyKind::SwiftSource: { + auto swiftSourceStorage = + cast(storage.get()); + Root = swiftSourceStorage->textualModuleDetails.CASFileSystemRootID; + break; + } + case swift::ModuleDependencyKind::Clang: { + auto clangModuleStorage = cast(storage.get()); + Root = clangModuleStorage->CASFileSystemRootID; + break; + } + default: + return None; + } + if (Root.empty()) + return None; + + return Root; +} + +std::string ModuleDependencyInfo::getModuleOutputPath() const { + switch (getKind()) { + case swift::ModuleDependencyKind::SwiftInterface: { + auto swiftInterfaceStorage = + cast(storage.get()); + return swiftInterfaceStorage->moduleOutputPath; + } + case swift::ModuleDependencyKind::SwiftSource: { + return ""; + } + case swift::ModuleDependencyKind::Clang: { + auto clangModuleStorage = cast(storage.get()); + return clangModuleStorage->pcmOutputPath; + } + case swift::ModuleDependencyKind::SwiftBinary: { + auto swiftBinaryStorage = + cast(storage.get()); + return swiftBinaryStorage->compiledModulePath; + } + case swift::ModuleDependencyKind::SwiftPlaceholder: { + auto swiftPlaceholderStorage = + cast(storage.get()); + return swiftPlaceholderStorage->compiledModulePath; + } + default: + llvm_unreachable("Unexpected dependency kind"); + } +} + void ModuleDependencyInfo::addBridgingHeader(StringRef bridgingHeader) { switch (getKind()) { case swift::ModuleDependencyKind::SwiftInterface: { @@ -253,15 +317,28 @@ void ModuleDependencyInfo::addBridgingModuleDependency( } } -SwiftDependencyScanningService::SwiftDependencyScanningService() - : ClangScanningService(clang::tooling::dependencies::ScanningMode::DependencyDirectivesScan, - clang::tooling::dependencies::ScanningOutputFormat::Full, - clang::CASOptions(), - /* CAS (llvm::cas::ObjectStore) */ nullptr, - /* Cache (llvm::cas::ActionCache) */ nullptr, - /* SharedFS */ nullptr, - /* OptimizeArgs */ true) { - SharedFilesystemCache.emplace(); +SwiftDependencyScanningService::SwiftDependencyScanningService() { + ClangScanningService.emplace( + clang::tooling::dependencies::ScanningMode::DependencyDirectivesScan, + clang::tooling::dependencies::ScanningOutputFormat::Full, + clang::CASOptions(), + /* CAS (llvm::cas::ObjectStore) */ nullptr, + /* Cache (llvm::cas::ActionCache) */ nullptr, + /* SharedFS */ nullptr, + /* OptimizeArgs */ true); + SharedFilesystemCache.emplace(); +} + +void SwiftDependencyTracker::startTracking() { + FS->trackNewAccesses(); + + for (auto &file : Files) + (void)FS->status(file); +} + +llvm::Expected +SwiftDependencyTracker::createTreeFromDependencies() { + return FS->createTreeFromNewAccesses(); } void SwiftDependencyScanningService::overlaySharedFilesystemCacheForCompilation(CompilerInstance &Instance) { @@ -274,6 +351,52 @@ void SwiftDependencyScanningService::overlaySharedFilesystemCacheForCompilation( Instance.getSourceMgr().setFileSystem(depFS); } +void SwiftDependencyScanningService::setupCachingDependencyScanningService( + CompilerInstance &Instance) { + if (!Instance.getInvocation().getFrontendOptions().EnableCAS) + return; + + // Add SDKSetting file. + SmallString<256> SDKSettingPath; + llvm::sys::path::append( + SDKSettingPath, + Instance.getInvocation().getSearchPathOptions().getSDKPath(), + "SDKSettings.json"); + CommonDependencyFiles.emplace_back(SDKSettingPath.data(), + SDKSettingPath.size()); + + // Add Legacy layout file (maybe just hard code instead of searching). + StringRef RuntimeLibPath = + Instance.getInvocation().getSearchPathOptions().RuntimeLibraryPaths[0]; + auto &FS = Instance.getFileSystem(); + std::error_code EC; + for (auto F = FS.dir_begin(RuntimeLibPath, EC); + !EC && F != llvm::vfs::directory_iterator(); F.increment(EC)) { + if (F->path().endswith(".yaml")) + CommonDependencyFiles.emplace_back(F->path().str()); + } + + auto CachingFS = llvm::cas::createCachingOnDiskFileSystem(Instance.getObjectStore()); + if (!CachingFS) { + Instance.getDiags().diagnose(SourceLoc(), diag::error_create_cas, + "CachingOnDiskFS", + toString(CachingFS.takeError())); + return; + } + CacheFS = std::move(*CachingFS); + + clang::CASOptions CASOpts; + CASOpts.CASPath = Instance.getInvocation().getFrontendOptions().CASPath; + CASOpts.ensurePersistentCAS(); + + ClangScanningService.emplace( + clang::tooling::dependencies::ScanningMode::DependencyDirectivesScan, + clang::tooling::dependencies::ScanningOutputFormat::FullTree, CASOpts, + Instance.getSharedCASInstance(), Instance.getSharedCacheInstance(), + CacheFS, + /* ReuseFileManager */ false, /* OptimizeArgs */ false); +} + SwiftDependencyScanningService::ContextSpecificGlobalCacheState * SwiftDependencyScanningService::getCacheForScanningContextHash(StringRef scanningContextHash) const { auto contextSpecificCache = ContextSpecificCacheMap.find(scanningContextHash); @@ -385,12 +508,12 @@ ModuleDependenciesCache::getDependencyReferencesMap( ModuleDependenciesCache::ModuleDependenciesCache( SwiftDependencyScanningService &globalScanningService, - std::string mainScanModuleName, - std::string scannerContextHash) + std::string mainScanModuleName, std::string scannerContextHash) : globalScanningService(globalScanningService), mainScanModuleName(mainScanModuleName), scannerContextHash(scannerContextHash), - clangScanningTool(globalScanningService.ClangScanningService) { + clangScanningTool(*globalScanningService.ClangScanningService, + globalScanningService.getClangScanningFS()) { globalScanningService.configureForContextHash(scannerContextHash); for (auto kind = ModuleDependencyKind::FirstKind; kind != ModuleDependencyKind::LastKind; ++kind) { @@ -399,8 +522,7 @@ ModuleDependenciesCache::ModuleDependenciesCache( } } -Optional -ModuleDependenciesCache::findDependency( +Optional ModuleDependenciesCache::findDependency( StringRef moduleName, Optional kind) const { auto optionalDep = globalScanningService.findDependency(moduleName, kind, scannerContextHash); diff --git a/lib/ClangImporter/ClangImporter.cpp b/lib/ClangImporter/ClangImporter.cpp index 8b9e4a2663b82..3688e347e8c7c 100644 --- a/lib/ClangImporter/ClangImporter.cpp +++ b/lib/ClangImporter/ClangImporter.cpp @@ -707,6 +707,15 @@ importer::getNormalInvocationArguments( invocationArgStrs.push_back((llvm::Twine(searchPathOpts.RuntimeResourcePath) + llvm::sys::path::get_separator() + "apinotes").str()); + + if (!importerOpts.CASPath.empty()) { + invocationArgStrs.push_back("-Xclang"); + invocationArgStrs.push_back("-fcas-path"); + invocationArgStrs.push_back("-Xclang"); + invocationArgStrs.push_back(importerOpts.CASPath); + invocationArgStrs.push_back("-Xclang"); + invocationArgStrs.push_back("-fno-pch-timestamp"); + } } static void diff --git a/lib/ClangImporter/ClangModuleDependencyScanner.cpp b/lib/ClangImporter/ClangModuleDependencyScanner.cpp index 4633a050c18d5..dc86bc5808692 100644 --- a/lib/ClangImporter/ClangModuleDependencyScanner.cpp +++ b/lib/ClangImporter/ClangModuleDependencyScanner.cpp @@ -18,11 +18,16 @@ #include "swift/AST/ModuleDependencies.h" #include "swift/Basic/SourceManager.h" #include "swift/ClangImporter/ClangImporter.h" +#include "clang/Basic/Diagnostic.h" +#include "clang/Frontend/CompilerInvocation.h" #include "clang/Tooling/DependencyScanning/DependencyScanningService.h" #include "clang/Tooling/DependencyScanning/DependencyScanningTool.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/Support/Allocator.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Signals.h" #include "llvm/Support/Path.h" +#include "llvm/Support/StringSaver.h" using namespace swift; @@ -175,18 +180,77 @@ void ClangImporter::recordModuleDependencies( // Swift frontend option for input file path (Foo.modulemap). swiftArgs.push_back(clangModuleDep.ClangModuleMapFile); + // Handle VFSOverlay. + if (!ctx.SearchPathOpts.VFSOverlayFiles.empty()) { + for (auto &overlay : ctx.SearchPathOpts.VFSOverlayFiles) { + swiftArgs.push_back("-vfsoverlay"); + swiftArgs.push_back(overlay); + } + } else { + // HACK: find the -ivfsoverlay option from clang scanner and pass to + // swift. + bool addOption = false; + for (auto &arg : clangModuleDep.BuildArguments) { + if (addOption) { + swiftArgs.push_back("-vfsoverlay"); + swiftArgs.push_back(arg); + addOption = false; + } else if (arg == "-ivfsoverlay") + addOption = true; + } + } + // Add args reported by the scanner. - llvm::for_each(clangModuleDep.BuildArguments, addClangArg); + + // Round-trip clang args to canonicalize and clear the options that swift + // compiler doesn't need. + clang::CompilerInvocation depsInvocation; + clang::DiagnosticsEngine clangDiags(new clang::DiagnosticIDs(), + new clang::DiagnosticOptions(), + new clang::IgnoringDiagConsumer()); + + llvm::SmallVector clangArgs; + llvm::for_each(clangModuleDep.BuildArguments, [&](const std::string &Arg) { + clangArgs.push_back(Arg.c_str()); + }); + + bool success = clang::CompilerInvocation::CreateFromArgs( + depsInvocation, clangArgs, clangDiags); + (void)success; + assert(success && "clang option from dep scanner round trip failed"); + + // Clear the cache key for module. The module key is computed from clang + // invocation, not swift invocation. + depsInvocation.getFrontendOpts().ModuleCacheKeys.clear(); + + llvm::BumpPtrAllocator allocator; + llvm::StringSaver saver(allocator); + clangArgs.clear(); + depsInvocation.generateCC1CommandLine( + clangArgs, + [&saver](const llvm::Twine &T) { return saver.save(T).data(); }); + + llvm::for_each(clangArgs, addClangArg); + + // CASFileSystemRootID. + std::string RootID = clangModuleDep.CASFileSystemRootID + ? clangModuleDep.CASFileSystemRootID->toString() + : ""; + + if (!RootID.empty()) { + swiftArgs.push_back("-enable-cas"); + swiftArgs.push_back("-cas-path"); + swiftArgs.push_back(ctx.ClangImporterOpts.CASPath); + swiftArgs.push_back("-cas-fs"); + swiftArgs.push_back(RootID); + } // Module-level dependencies. llvm::StringSet<> alreadyAddedModules; auto dependencies = ModuleDependencyInfo::forClangModule( - pcmPath, - clangModuleDep.ClangModuleMapFile, - clangModuleDep.ID.ContextHash, - swiftArgs, - fileDeps, - capturedPCMArgs); + pcmPath, clangModuleDep.ClangModuleMapFile, + clangModuleDep.ID.ContextHash, swiftArgs, fileDeps, capturedPCMArgs, + RootID, /*module-cache-key*/ ""); for (const auto &moduleName : clangModuleDep.ClangModuleDeps) { dependencies.addModuleImport(moduleName.ModuleName, &alreadyAddedModules); // It is safe to assume that all dependencies of a Clang module are Clang modules. diff --git a/lib/DependencyScan/DependencyScanningTool.cpp b/lib/DependencyScan/DependencyScanningTool.cpp index 084922efbcf64..0c150f93ad6e1 100644 --- a/lib/DependencyScan/DependencyScanningTool.cpp +++ b/lib/DependencyScan/DependencyScanningTool.cpp @@ -280,6 +280,10 @@ DependencyScanningTool::initCompilerInstanceForScan( if (Instance->setup(Invocation, InstanceSetupError)) { return std::make_error_code(std::errc::not_supported); } + + // Setup the caching service after the instance finishes setup. + ScanningService->setupCachingDependencyScanningService(*Instance); + (void)Instance->getMainModule(); return Instance; diff --git a/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp b/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp index 6cf39b6fb96b9..99759a11653b2 100644 --- a/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp +++ b/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp @@ -235,17 +235,19 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi llvm::report_fatal_error( "Unexpected SWIFT_TEXTUAL_MODULE_DETAILS_NODE record"); cache.configureForContextHash(getContextHash()); - unsigned outputPathFileID, interfaceFileID, compiledModuleCandidatesArrayID, - buildCommandLineArrayID, extraPCMArgsArrayID, contextHashID, - isFramework, bridgingHeaderFileID, sourceFilesArrayID, - bridgingSourceFilesArrayID, bridgingModuleDependenciesArrayID, - overlayDependencyIDArrayID; + unsigned outputPathFileID, interfaceFileID, + compiledModuleCandidatesArrayID, buildCommandLineArrayID, + extraPCMArgsArrayID, contextHashID, isFramework, bridgingHeaderFileID, + sourceFilesArrayID, bridgingSourceFilesArrayID, + bridgingModuleDependenciesArrayID, overlayDependencyIDArrayID, + CASFileSystemRootID, moduleCacheKeyID; SwiftInterfaceModuleDetailsLayout::readRecord( - Scratch, outputPathFileID, interfaceFileID, compiledModuleCandidatesArrayID, - buildCommandLineArrayID, extraPCMArgsArrayID, contextHashID, - isFramework, bridgingHeaderFileID, sourceFilesArrayID, - bridgingSourceFilesArrayID, bridgingModuleDependenciesArrayID, - overlayDependencyIDArrayID); + Scratch, outputPathFileID, interfaceFileID, + compiledModuleCandidatesArrayID, buildCommandLineArrayID, + extraPCMArgsArrayID, contextHashID, isFramework, bridgingHeaderFileID, + sourceFilesArrayID, bridgingSourceFilesArrayID, + bridgingModuleDependenciesArrayID, overlayDependencyIDArrayID, + CASFileSystemRootID, moduleCacheKeyID); auto outputModulePath = getIdentifier(outputPathFileID); if (!outputModulePath) @@ -278,11 +280,18 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi for (auto &arg : *extraPCMArgs) extraPCMRefs.push_back(arg); + auto rootFileSystemID = getIdentifier(CASFileSystemRootID); + if (!rootFileSystemID) + llvm::report_fatal_error("Bad CASFileSystem RootID"); + auto moduleCacheKey = getIdentifier(moduleCacheKeyID); + if (!moduleCacheKeyID) + llvm::report_fatal_error("Bad moduleCacheKey"); + // Form the dependencies storage object auto moduleDep = ModuleDependencyInfo::forSwiftInterfaceModule( - outputModulePath.value(), - optionalSwiftInterfaceFile.value(), *compiledModuleCandidates, - buildCommandRefs, extraPCMRefs, *contextHash, isFramework); + outputModulePath.value(), optionalSwiftInterfaceFile.value(), + *compiledModuleCandidates, buildCommandRefs, extraPCMRefs, + *contextHash, isFramework, *rootFileSystemID, *moduleCacheKey); // Add imports of this module for (const auto &moduleName : *currentModuleImports) @@ -345,11 +354,13 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi "SWIFT_SOURCE_MODULE_DETAILS_NODE record"); unsigned extraPCMArgsArrayID, bridgingHeaderFileID, sourceFilesArrayID, bridgingSourceFilesArrayID, bridgingModuleDependenciesArrayID, - overlayDependencyIDArrayID; + overlayDependencyIDArrayID, CASFileSystemRootID, + buildCommandLineArrayID; SwiftSourceModuleDetailsLayout::readRecord( Scratch, extraPCMArgsArrayID, bridgingHeaderFileID, sourceFilesArrayID, bridgingSourceFilesArrayID, - bridgingModuleDependenciesArrayID, overlayDependencyIDArrayID); + bridgingModuleDependenciesArrayID, overlayDependencyIDArrayID, + CASFileSystemRootID, buildCommandLineArrayID); auto extraPCMArgs = getStringArray(extraPCMArgsArrayID); if (!extraPCMArgs) @@ -358,8 +369,19 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi for (auto &arg : *extraPCMArgs) extraPCMRefs.push_back(arg); + auto rootFileSystemID = getIdentifier(CASFileSystemRootID); + if (!rootFileSystemID) + llvm::report_fatal_error("Bad CASFileSystem RootID"); + auto commandLine = getStringArray(buildCommandLineArrayID); + if (!commandLine) + llvm::report_fatal_error("Bad command line"); + std::vector buildCommandRefs; + for (auto &arg : *commandLine) + buildCommandRefs.push_back(arg); + // Form the dependencies storage object - auto moduleDep = ModuleDependencyInfo::forSwiftSourceModule(extraPCMRefs); + auto moduleDep = ModuleDependencyInfo::forSwiftSourceModule( + *rootFileSystemID, buildCommandRefs, extraPCMRefs); // Add dependencies of this module for (const auto &moduleName : *currentModuleImports) @@ -414,10 +436,10 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi "Unexpected SWIFT_BINARY_MODULE_DETAILS_NODE record"); cache.configureForContextHash(getContextHash()); unsigned compiledModulePathID, moduleDocPathID, moduleSourceInfoPathID, - isFramework; + isFramework, moduleCacheKeyID; SwiftBinaryModuleDetailsLayout::readRecord( Scratch, compiledModulePathID, moduleDocPathID, - moduleSourceInfoPathID, isFramework); + moduleSourceInfoPathID, isFramework, moduleCacheKeyID); auto compiledModulePath = getIdentifier(compiledModulePathID); if (!compiledModulePath) @@ -428,11 +450,14 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi auto moduleSourceInfoPath = getIdentifier(moduleSourceInfoPathID); if (!moduleSourceInfoPath) llvm::report_fatal_error("Bad module source info path"); + auto moduleCacheKey = getIdentifier(moduleCacheKeyID); + if (!moduleCacheKeyID) + llvm::report_fatal_error("Bad moduleCacheKey"); // Form the dependencies storage object auto moduleDep = ModuleDependencyInfo::forSwiftBinaryModule( *compiledModulePath, *moduleDocPath, *moduleSourceInfoPath, - isFramework); + isFramework, *moduleCacheKey); // Add dependencies of this module for (const auto &moduleName : *currentModuleImports) moduleDep.addModuleImport(moduleName); @@ -481,11 +506,14 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi llvm::report_fatal_error("Unexpected CLANG_MODULE_DETAILS_NODE record"); cache.configureForContextHash(getContextHash()); unsigned pcmOutputPathID, moduleMapPathID, contextHashID, commandLineArrayID, - fileDependenciesArrayID, capturedPCMArgsArrayID; + fileDependenciesArrayID, capturedPCMArgsArrayID, CASFileSystemRootID, + moduleCacheKeyID; ClangModuleDetailsLayout::readRecord(Scratch, pcmOutputPathID, moduleMapPathID, contextHashID, commandLineArrayID, fileDependenciesArrayID, - capturedPCMArgsArrayID); + capturedPCMArgsArrayID, + CASFileSystemRootID, + moduleCacheKeyID); auto pcmOutputPath = getIdentifier(pcmOutputPathID); if (!pcmOutputPath) llvm::report_fatal_error("Bad pcm output path"); @@ -504,11 +532,17 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi auto capturedPCMArgs = getStringArray(capturedPCMArgsArrayID); if (!capturedPCMArgs) llvm::report_fatal_error("Bad captured PCM Args"); + auto rootFileSystemID = getIdentifier(CASFileSystemRootID); + if (!rootFileSystemID) + llvm::report_fatal_error("Bad CASFileSystem RootID"); + auto moduleCacheKey = getIdentifier(moduleCacheKeyID); + if (!moduleCacheKeyID) + llvm::report_fatal_error("Bad moduleCacheKey"); // Form the dependencies storage object auto moduleDep = ModuleDependencyInfo::forClangModule(*pcmOutputPath, *moduleMapPath, *contextHash, *commandLineArgs, *fileDependencies, - *capturedPCMArgs); + *capturedPCMArgs, *rootFileSystemID, *moduleCacheKey); // Add dependencies of this module for (const auto &moduleName : *currentModuleImports) @@ -863,7 +897,9 @@ void ModuleDependenciesCacheSerializer::writeModuleInfo(ModuleDependencyID modul getArrayID(moduleID, ModuleIdentifierArrayKind::SourceFiles), getArrayID(moduleID, ModuleIdentifierArrayKind::BridgingSourceFiles), getArrayID(moduleID, ModuleIdentifierArrayKind::BridgingModuleDependencies), - getArrayID(moduleID, ModuleIdentifierArrayKind::SwiftOverlayDependencyIDs)); + getArrayID(moduleID, ModuleIdentifierArrayKind::SwiftOverlayDependencyIDs), + getIdentifier(swiftTextDeps->textualModuleDetails.CASFileSystemRootID), + getIdentifier(swiftTextDeps->moduleCacheKey)); break; } case swift::ModuleDependencyKind::SwiftSource: { @@ -883,7 +919,9 @@ void ModuleDependenciesCacheSerializer::writeModuleInfo(ModuleDependencyID modul getArrayID(moduleID, ModuleIdentifierArrayKind::SourceFiles), getArrayID(moduleID, ModuleIdentifierArrayKind::BridgingSourceFiles), getArrayID(moduleID, ModuleIdentifierArrayKind::BridgingModuleDependencies), - getArrayID(moduleID, ModuleIdentifierArrayKind::SwiftOverlayDependencyIDs)); + getArrayID(moduleID, ModuleIdentifierArrayKind::SwiftOverlayDependencyIDs), + getIdentifier(swiftSourceDeps->textualModuleDetails.CASFileSystemRootID), + getArrayID(moduleID, ModuleIdentifierArrayKind::BuildCommandLine)); break; } case swift::ModuleDependencyKind::SwiftBinary: { @@ -895,7 +933,8 @@ void ModuleDependenciesCacheSerializer::writeModuleInfo(ModuleDependencyID modul getIdentifier(swiftBinDeps->compiledModulePath), getIdentifier(swiftBinDeps->moduleDocPath), getIdentifier(swiftBinDeps->sourceInfoPath), - swiftBinDeps->isFramework); + swiftBinDeps->isFramework, + getIdentifier(swiftBinDeps->moduleCacheKey)); break; } @@ -922,7 +961,9 @@ void ModuleDependenciesCacheSerializer::writeModuleInfo(ModuleDependencyID modul getIdentifier(clangDeps->contextHash), getArrayID(moduleID, ModuleIdentifierArrayKind::NonPathCommandLine), getArrayID(moduleID, ModuleIdentifierArrayKind::FileDependencies), - getArrayID(moduleID, ModuleIdentifierArrayKind::CapturedPCMArgs)); + getArrayID(moduleID, ModuleIdentifierArrayKind::CapturedPCMArgs), + getIdentifier(clangDeps->CASFileSystemRootID), + getIdentifier(clangDeps->moduleCacheKey)); break; } @@ -1035,26 +1076,28 @@ void ModuleDependenciesCacheSerializer::collectStringsAndArrays( addIdentifier(swiftTextDeps->moduleOutputPath); addIdentifier(swiftTextDeps->swiftInterfaceFile); addStringArray(moduleID, - ModuleIdentifierArrayKind::CompiledModuleCandidates, - swiftTextDeps->compiledModuleCandidates); + ModuleIdentifierArrayKind::CompiledModuleCandidates, + swiftTextDeps->compiledModuleCandidates); addStringArray(moduleID, ModuleIdentifierArrayKind::BuildCommandLine, - swiftTextDeps->buildCommandLine); + swiftTextDeps->textualModuleDetails.buildCommandLine); addStringArray(moduleID, ModuleIdentifierArrayKind::ExtraPCMArgs, - swiftTextDeps->textualModuleDetails.extraPCMArgs); + swiftTextDeps->textualModuleDetails.extraPCMArgs); addIdentifier(swiftTextDeps->contextHash); if (swiftTextDeps->textualModuleDetails.bridgingHeaderFile.has_value()) - addIdentifier( - swiftTextDeps->textualModuleDetails.bridgingHeaderFile.value()); + addIdentifier(swiftTextDeps->textualModuleDetails.bridgingHeaderFile + .value()); addStringArray(moduleID, ModuleIdentifierArrayKind::SourceFiles, - std::vector()); + std::vector()); addStringArray(moduleID, ModuleIdentifierArrayKind::BridgingSourceFiles, - swiftTextDeps->textualModuleDetails.bridgingSourceFiles); + swiftTextDeps->textualModuleDetails.bridgingSourceFiles); addStringArray( - moduleID, ModuleIdentifierArrayKind::BridgingModuleDependencies, - swiftTextDeps->textualModuleDetails.bridgingModuleDependencies); + moduleID, ModuleIdentifierArrayKind::BridgingModuleDependencies, + swiftTextDeps->textualModuleDetails.bridgingModuleDependencies); addDependencyIDArray( moduleID, ModuleIdentifierArrayKind::SwiftOverlayDependencyIDs, swiftTextDeps->textualModuleDetails.swiftOverlayDependencies); + addIdentifier(swiftTextDeps->textualModuleDetails.CASFileSystemRootID); + addIdentifier(swiftTextDeps->moduleCacheKey); break; } case swift::ModuleDependencyKind::SwiftBinary: { @@ -1063,6 +1106,7 @@ void ModuleDependenciesCacheSerializer::collectStringsAndArrays( addIdentifier(swiftBinDeps->compiledModulePath); addIdentifier(swiftBinDeps->moduleDocPath); addIdentifier(swiftBinDeps->sourceInfoPath); + addIdentifier(swiftBinDeps->moduleCacheKey); break; } case swift::ModuleDependencyKind::SwiftPlaceholder: { @@ -1093,6 +1137,11 @@ void ModuleDependenciesCacheSerializer::collectStringsAndArrays( addDependencyIDArray( moduleID, ModuleIdentifierArrayKind::SwiftOverlayDependencyIDs, swiftSourceDeps->textualModuleDetails.swiftOverlayDependencies); + addStringArray( + moduleID, ModuleIdentifierArrayKind::BuildCommandLine, + swiftSourceDeps->textualModuleDetails.buildCommandLine); + addIdentifier( + swiftSourceDeps->textualModuleDetails.CASFileSystemRootID); break; } case swift::ModuleDependencyKind::Clang: { @@ -1107,6 +1156,8 @@ void ModuleDependenciesCacheSerializer::collectStringsAndArrays( clangDeps->fileDependencies); addStringArray(moduleID, ModuleIdentifierArrayKind::CapturedPCMArgs, clangDeps->capturedPCMArgs); + addIdentifier(clangDeps->CASFileSystemRootID); + addIdentifier(clangDeps->moduleCacheKey); break; } default: diff --git a/lib/DependencyScan/ScanDependencies.cpp b/lib/DependencyScan/ScanDependencies.cpp index 9d94da4e784dd..1dae3fef61877 100644 --- a/lib/DependencyScan/ScanDependencies.cpp +++ b/lib/DependencyScan/ScanDependencies.cpp @@ -12,14 +12,12 @@ #include "swift/Basic/PrettyStackTrace.h" -#include "swift/DependencyScan/ScanDependencies.h" -#include "swift/DependencyScan/SerializedModuleDependencyCacheFormat.h" #include "swift/AST/ASTContext.h" #include "swift/AST/Decl.h" #include "swift/AST/DiagnosticEngine.h" +#include "swift/AST/DiagnosticsDriver.h" #include "swift/AST/DiagnosticsFrontend.h" #include "swift/AST/DiagnosticsSema.h" -#include "swift/AST/DiagnosticsDriver.h" #include "swift/AST/FileSystem.h" #include "swift/AST/Module.h" #include "swift/AST/ModuleDependencies.h" @@ -30,28 +28,38 @@ #include "swift/Basic/STLExtras.h" #include "swift/ClangImporter/ClangImporter.h" #include "swift/DependencyScan/DependencyScanImpl.h" +#include "swift/DependencyScan/ScanDependencies.h" +#include "swift/DependencyScan/SerializedModuleDependencyCacheFormat.h" #include "swift/DependencyScan/StringUtils.h" +#include "swift/Frontend/CompileJobCacheKey.h" #include "swift/Frontend/Frontend.h" #include "swift/Frontend/FrontendOptions.h" #include "swift/Frontend/ModuleInterfaceLoader.h" #include "swift/Strings.h" #include "clang/Basic/Module.h" -#include "llvm/ADT/SetVector.h" +#include "clang/Frontend/CompileJobCacheResult.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SetOperations.h" +#include "llvm/ADT/SetVector.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/StringSet.h" +#include "llvm/CAS/ActionCache.h" +#include "llvm/CAS/CASReference.h" +#include "llvm/CAS/HierarchicalTreeBuilder.h" +#include "llvm/CAS/ObjectStore.h" #include "llvm/Support/CommandLine.h" +#include "llvm/Support/Error.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/StringSaver.h" #include "llvm/Support/VirtualOutputBackend.h" #include "llvm/Support/YAMLParser.h" #include "llvm/Support/YAMLTraits.h" #include "llvm/Support/raw_ostream.h" +#include #include -#include #include -#include +#include using namespace swift; using namespace swift::dependencies; @@ -249,17 +257,74 @@ computeTransitiveClosureOfExplicitDependencies( return result; } -static void -resolveExplicitModuleInputs(ModuleDependencyID moduleID, - const ModuleDependencyInfo &resolvingDepInfo, - const std::set &dependencies, - ModuleDependenciesCache &cache) { - auto resolvingInterfaceDepDetails = - resolvingDepInfo.getAsSwiftInterfaceModule(); - assert(resolvingInterfaceDepDetails && - "Expected Swift Interface dependency."); +static llvm::Expected +mergeCASFileSystem(llvm::cas::ObjectStore &CAS, ArrayRef FSRoots) { + assert(!FSRoots.empty() && "no root ID provided"); + + llvm::cas::HierarchicalTreeBuilder Builder; + for (auto &Root : FSRoots) { + auto ID = CAS.parseID(Root); + if (!ID) + return ID.takeError(); - auto commandLine = resolvingInterfaceDepDetails->buildCommandLine; + auto Ref = CAS.getReference(*ID); + assert(Ref && "CASFSRootID is missing from the ObjectStore instance"); + Builder.pushTreeContent(*Ref, ""); + } + + auto NewRoot = Builder.create(CAS); + if (!NewRoot) + return NewRoot.takeError(); + + return NewRoot->getRef(); +} + +static llvm::Expected +updateModuleCacheKey(ModuleDependencyInfo &depInfo, + llvm::cas::ObjectStore &CAS) { + auto commandLine = depInfo.getCommandline(); + std::vector Args; + if (commandLine.size() > 1) + for (auto &c : ArrayRef(commandLine).drop_front(1)) + Args.push_back(c.c_str()); + + auto base = createCompileJobBaseCacheKey(CAS, Args); + if (!base) + return base.takeError(); + + StringRef InputPath; + file_types::ID OutputType = file_types::ID::TY_INVALID; + if (auto *dep = depInfo.getAsClangModule()) { + OutputType = file_types::ID::TY_ClangModuleFile; + InputPath = dep->moduleMapFile; + } else if (auto *dep = depInfo.getAsSwiftInterfaceModule()) { + OutputType = file_types::ID::TY_SwiftModuleFile; + InputPath = dep->swiftInterfaceFile; + } else + llvm_unreachable("Unhandled dependency kind"); + + auto key = + createCompileJobCacheKeyForOutput(CAS, *base, InputPath, OutputType); + if (!key) + return key.takeError(); + + depInfo.updateModuleCacheKey(CAS.getID(*key).toString()); + return *key; +} + +static llvm::Error resolveExplicitModuleInputs( + ModuleDependencyID moduleID, const ModuleDependencyInfo &resolvingDepInfo, + const std::set &dependencies, + ModuleDependenciesCache &cache, CompilerInstance &instance) { + // Only need to resolve dependency for following dependencies. + if (moduleID.second == ModuleDependencyKind::SwiftPlaceholder) + return llvm::Error::success(); + + std::vector rootIDs; + if (auto ID = resolvingDepInfo.getCASFSRootID()) + rootIDs.push_back(*ID); + + std::vector commandLine = resolvingDepInfo.getCommandline(); for (const auto &depModuleID : dependencies) { const auto optionalDepInfo = cache.findDependency(depModuleID.first, depModuleID.second); @@ -269,14 +334,20 @@ resolveExplicitModuleInputs(ModuleDependencyID moduleID, case swift::ModuleDependencyKind::SwiftInterface: { auto interfaceDepDetails = depInfo->getAsSwiftInterfaceModule(); assert(interfaceDepDetails && "Expected Swift Interface dependency."); + auto &path = interfaceDepDetails->moduleCacheKey.empty() + ? interfaceDepDetails->moduleOutputPath + : interfaceDepDetails->moduleCacheKey; commandLine.push_back("-swift-module-file=" + depModuleID.first + "=" + - interfaceDepDetails->moduleOutputPath); + path); } break; case swift::ModuleDependencyKind::SwiftBinary: { auto binaryDepDetails = depInfo->getAsSwiftBinaryModule(); assert(binaryDepDetails && "Expected Swift Binary Module dependency."); + auto &path = binaryDepDetails->moduleCacheKey.empty() + ? binaryDepDetails->compiledModulePath + : binaryDepDetails->moduleCacheKey; commandLine.push_back("-swift-module-file=" + depModuleID.first + "=" + - binaryDepDetails->compiledModulePath); + path); } break; case swift::ModuleDependencyKind::SwiftPlaceholder: { auto placeholderDetails = depInfo->getAsPlaceholderDependencyModule(); @@ -293,7 +364,44 @@ resolveExplicitModuleInputs(ModuleDependencyID moduleID, commandLine.push_back("-Xcc"); commandLine.push_back("-fmodule-map-file=" + clangDepDetails->moduleMapFile); + if (!clangDepDetails->moduleCacheKey.empty()) { + auto appendXclang = [&]() { + if (!resolvingDepInfo.isClangModule()) { + // clang module build using cc1 arg so this is not needed. + commandLine.push_back("-Xcc"); + commandLine.push_back("-Xclang"); + } + commandLine.push_back("-Xcc"); + }; + appendXclang(); + commandLine.push_back("-fmodule-file-cache-key"); + appendXclang(); + commandLine.push_back(clangDepDetails->pcmOutputPath); + appendXclang(); + commandLine.push_back(clangDepDetails->moduleCacheKey); + } + + // Only need to merge the CASFS from clang importer. + if (auto ID = depInfo->getCASFSRootID()) + rootIDs.push_back(*ID); } break; + case swift::ModuleDependencyKind::SwiftSource: { + auto sourceDepDetails = depInfo->getAsSwiftSourceModule(); + assert(sourceDepDetails && "Expected source dependency"); + if (!sourceDepDetails->textualModuleDetails.bridgingSourceFiles.empty()) { + if (auto tracker = + cache.getScanService().createSwiftDependencyTracker()) { + tracker->startTracking(); + for (auto &file : sourceDepDetails->textualModuleDetails.bridgingSourceFiles) + tracker->trackFile(file); + auto bridgeRoot = tracker->createTreeFromDependencies(); + if (!bridgeRoot) + return bridgeRoot.takeError(); + rootIDs.push_back(bridgeRoot->getID().toString()); + } + } + break; + } default: llvm_unreachable("Unhandled dependency kind."); } @@ -301,8 +409,65 @@ resolveExplicitModuleInputs(ModuleDependencyID moduleID, // Update the dependency in the cache with the modified command-line. auto dependencyInfoCopy = resolvingDepInfo; - dependencyInfoCopy.updateCommandLine(commandLine); + if (resolvingDepInfo.isSwiftInterfaceModule() || + resolvingDepInfo.isClangModule()) + dependencyInfoCopy.updateCommandLine(commandLine); + + // Handle CAS options. + if (instance.getInvocation().getFrontendOptions().EnableCAS) { + // Merge CASFS from clang dependency. + auto &CASFS = cache.getScanService().getSharedCachingFS(); + auto &CAS = CASFS.getCAS(); + + // Update CASFS RootID. + if (resolvingDepInfo.isSwiftInterfaceModule() || + resolvingDepInfo.isSwiftSourceModule()) { + auto NewRoot = mergeCASFileSystem(CAS, rootIDs); + if (!NewRoot) + return NewRoot.takeError(); + auto NewID = CAS.getID(*NewRoot).toString(); + dependencyInfoCopy.updateCASFileSystemID(NewID); + + // Update with casfs option. + std::vector newCommandLine = + dependencyInfoCopy.getCommandline(); + newCommandLine.push_back("-cas-fs"); + newCommandLine.push_back(NewID); + dependencyInfoCopy.updateCommandLine(newCommandLine); + } + + if (resolvingDepInfo.isClangModule() || + resolvingDepInfo.isSwiftInterfaceModule()) { + // Compute and update module cache key. + auto Key = updateModuleCacheKey(dependencyInfoCopy, CAS); + if (!Key) + return Key.takeError(); + } + + // For binary module, we need to make sure the lookup key is setup here in + // action cache. We just use the CASID of the binary module itself as key. + if (auto *binaryDep = dependencyInfoCopy.getAsSwiftBinaryModule()) { + auto Ref = + CASFS.getObjectRefForFileContent(binaryDep->compiledModulePath); + if (!Ref) + return llvm::errorCodeToError(Ref.getError()); + assert(*Ref && "Binary module should be loaded into CASFS already"); + dependencyInfoCopy.updateModuleCacheKey(CAS.getID(**Ref).toString()); + + clang::cas::CompileJobCacheResult::Builder Builder; + Builder.addOutput( + clang::cas::CompileJobCacheResult::OutputKind::MainOutput, **Ref); + auto Result = Builder.build(CAS); + if (!Result) + return Result.takeError(); + if (auto E = instance.getActionCache().put(CAS.getID(**Ref), + CAS.getID(*Result))) + return E; + } + } cache.updateDependency(moduleID, dependencyInfoCopy); + + return llvm::Error::success(); } /// Resolve the direct dependencies of the given module. @@ -477,7 +642,8 @@ static void discoverCrossImportOverlayDependencies( // Construct a dummy main to resolve the newly discovered cross import // overlays. StringRef dummyMainName = "DummyMainModuleForResolvingCrossImportOverlays"; - auto dummyMainDependencies = ModuleDependencyInfo::forSwiftSourceModule({}); + auto dummyMainDependencies = + ModuleDependencyInfo::forSwiftSourceModule({}, {}, {}); std::for_each(newOverlays.begin(), newOverlays.end(), [&](Identifier modName) { dummyMainDependencies.addModuleImport(modName.str()); @@ -858,6 +1024,16 @@ static void writeJSON(llvm::raw_ostream &out, bool commaAfterFramework = swiftTextualDeps->extra_pcm_args->count != 0 || hasBridgingHeaderPath; + if (swiftTextualDeps->cas_fs_root_id.length != 0) { + writeJSONSingleField(out, "casFSRootID", + swiftTextualDeps->cas_fs_root_id, 5, + /*trailingComma=*/true); + } + if (swiftTextualDeps->module_cache_key.length != 0) { + writeJSONSingleField(out, "moduleCacheKey", + swiftTextualDeps->module_cache_key, 5, + /*trailingComma=*/true); + } writeJSONSingleField(out, "isFramework", swiftTextualDeps->is_framework, 5, commaAfterFramework); if (swiftTextualDeps->extra_pcm_args->count != 0) { @@ -942,6 +1118,11 @@ static void writeJSON(llvm::raw_ostream &out, swiftBinaryDeps->module_source_info_path, /*indentLevel=*/5, /*trailingComma=*/true); + if (swiftBinaryDeps->module_cache_key.length != 0) { + writeJSONSingleField(out, "moduleCacheKey", + swiftBinaryDeps->module_cache_key, 5, + /*trailingComma=*/true); + } writeJSONSingleField(out, "isFramework", swiftBinaryDeps->is_framework, 5, /*trailingComma=*/false); } else { @@ -959,9 +1140,18 @@ static void writeJSON(llvm::raw_ostream &out, writeJSONSingleField(out, "commandLine", clangDeps->command_line, 5, /*trailingComma=*/true); + if (clangDeps->cas_fs_root_id.length != 0) + writeJSONSingleField(out, "casFSRootID", clangDeps->cas_fs_root_id, 5, + /*trailingComma=*/true); + if (clangDeps->module_cache_key.length != 0) + writeJSONSingleField(out, "moduleCacheKey", clangDeps->module_cache_key, + 5, + /*trailingComma=*/true); + // Captured PCM arguments. writeJSONSingleField(out, "capturedPCMArgs", clangDeps->captured_pcm_args, 5, /*trailingComma=*/false, /*nested=*/true); + } out.indent(4 * 2); @@ -1107,10 +1297,12 @@ generateFullDependencyGraph(CompilerInstance &instance, create_set(swiftTextualDeps->textualModuleDetails.bridgingSourceFiles), create_set(swiftTextualDeps->textualModuleDetails.bridgingModuleDependencies), create_set(bridgedOverlayDependencyNames), - create_set(swiftTextualDeps->buildCommandLine), + create_set(swiftTextualDeps->textualModuleDetails.buildCommandLine), create_set(swiftTextualDeps->textualModuleDetails.extraPCMArgs), create_clone(swiftTextualDeps->contextHash.c_str()), - swiftTextualDeps->isFramework}; + swiftTextualDeps->isFramework, + create_clone(swiftTextualDeps->textualModuleDetails.CASFileSystemRootID.c_str()), + create_clone(swiftTextualDeps->moduleCacheKey.c_str())}; } else if (swiftSourceDeps) { swiftscan_string_ref_t moduleInterfacePath = create_null(); swiftscan_string_ref_t bridgingHeaderPath = @@ -1131,10 +1323,12 @@ generateFullDependencyGraph(CompilerInstance &instance, create_set(swiftSourceDeps->textualModuleDetails.bridgingSourceFiles), create_set(swiftSourceDeps->textualModuleDetails.bridgingModuleDependencies), create_set(bridgedOverlayDependencyNames), - create_empty_set(), + create_set(swiftSourceDeps->textualModuleDetails.buildCommandLine), create_set(swiftSourceDeps->textualModuleDetails.extraPCMArgs), /*contextHash*/create_null(), - /*isFramework*/false}; + /*isFramework*/false, + /*CASFS*/create_clone(swiftSourceDeps->textualModuleDetails.CASFileSystemRootID.c_str()), + /*CacheKey*/create_clone("")}; } else if (swiftPlaceholderDeps) { details->kind = SWIFTSCAN_DEPENDENCY_INFO_SWIFT_PLACEHOLDER; details->swift_placeholder_details = { @@ -1147,7 +1341,8 @@ generateFullDependencyGraph(CompilerInstance &instance, create_clone(swiftBinaryDeps->compiledModulePath.c_str()), create_clone(swiftBinaryDeps->moduleDocPath.c_str()), create_clone(swiftBinaryDeps->sourceInfoPath.c_str()), - swiftBinaryDeps->isFramework}; + swiftBinaryDeps->isFramework, + create_clone(swiftBinaryDeps->moduleCacheKey.c_str())}; } else { // Clang module details details->kind = SWIFTSCAN_DEPENDENCY_INFO_CLANG; @@ -1155,7 +1350,9 @@ generateFullDependencyGraph(CompilerInstance &instance, create_clone(clangDeps->moduleMapFile.c_str()), create_clone(clangDeps->contextHash.c_str()), create_set(clangDeps->nonPathCommandLine), - create_set(clangDeps->capturedPCMArgs) + create_set(clangDeps->capturedPCMArgs), + create_clone(clangDeps->CASFileSystemRootID.c_str()), + create_clone(clangDeps->moduleCacheKey.c_str()) }; } return details; @@ -1353,8 +1550,9 @@ forEachBatchEntry(CompilerInstance &invocationInstance, return false; } -static ModuleDependencyInfo -identifyMainModuleDependencies(CompilerInstance &instance) { +static ModuleDependencyInfo identifyMainModuleDependencies( + CompilerInstance &instance, + Optional tracker = None) { ModuleDecl *mainModule = instance.getMainModule(); // Main module file name. auto newExt = file_types::getExtension(file_types::TY_SwiftModuleFile); @@ -1375,7 +1573,11 @@ identifyMainModuleDependencies(CompilerInstance &instance) { ExtraPCMArgs.insert(ExtraPCMArgs.begin(), {"-Xcc", "-target", "-Xcc", instance.getASTContext().LangOpts.Target.str()}); - auto mainDependencies = ModuleDependencyInfo::forSwiftSourceModule(ExtraPCMArgs); + auto mainDependencies = + ModuleDependencyInfo::forSwiftSourceModule({}, {}, ExtraPCMArgs); + + if (tracker) + tracker->startTracking(); // Compute Implicit dependencies of the main module { @@ -1386,6 +1588,8 @@ identifyMainModuleDependencies(CompilerInstance &instance) { continue; mainDependencies.addModuleImport(*sf, alreadyAddedModules); + if (tracker) + tracker->trackFile(sf->getFilename()); } const auto &importInfo = mainModule->getImplicitImportInfo(); @@ -1433,6 +1637,11 @@ identifyMainModuleDependencies(CompilerInstance &instance) { } } + if (tracker) { + auto rootID = cantFail(tracker->createTreeFromDependencies()); + mainDependencies.updateCASFileSystemID(rootID.getID().toString()); + } + return mainDependencies; } @@ -1475,6 +1684,7 @@ bool swift::dependencies::scanDependencies(CompilerInstance &instance) { deserializeDependencyCache(instance, service); // Wrap the filesystem with a caching `DependencyScanningWorkerFilesystem` service.overlaySharedFilesystemCacheForCompilation(instance); + service.setupCachingDependencyScanningService(instance); ModuleDependenciesCache cache(service, instance.getMainModule()->getNameStr().str(), instance.getInvocation().getModuleScanningHash()); @@ -1542,6 +1752,7 @@ bool swift::dependencies::batchScanDependencies( SwiftDependencyScanningService singleUseService; singleUseService.overlaySharedFilesystemCacheForCompilation(instance); + singleUseService.setupCachingDependencyScanningService(instance); ModuleDependenciesCache cache(singleUseService, instance.getMainModule()->getNameStr().str(), instance.getInvocation().getModuleScanningHash()); @@ -1630,7 +1841,8 @@ swift::dependencies::performModuleScan(CompilerInstance &instance, ModuleDependenciesCache &cache) { ModuleDecl *mainModule = instance.getMainModule(); // First, identify the dependencies of the main module - auto mainDependencies = identifyMainModuleDependencies(instance); + auto mainDependencies = identifyMainModuleDependencies( + instance, cache.getScanService().createSwiftDependencyTracker()); auto &ctx = instance.getASTContext(); // Add the main module. @@ -1702,17 +1914,18 @@ swift::dependencies::performModuleScan(CompilerInstance &instance, auto moduleTransitiveClosures = computeTransitiveClosureOfExplicitDependencies(topoSortedModuleList, cache); - for (const auto &dependencyClosure : moduleTransitiveClosures) { - auto &modID = dependencyClosure.first; + for (const auto &modID : llvm::reverse(topoSortedModuleList)) { + auto dependencyClosure = moduleTransitiveClosures[modID]; // For main module or binary modules, no command-line to resolve. // For Clang modules, their dependencies are resolved by the clang Scanner // itself for us. - if (modID.second != ModuleDependencyKind::SwiftInterface) - continue; auto optionalDeps = cache.findDependency(modID.first, modID.second); assert(optionalDeps.has_value()); auto deps = optionalDeps.value(); - resolveExplicitModuleInputs(modID, *deps, dependencyClosure.second, cache); + if (auto E = resolveExplicitModuleInputs(modID, *deps, dependencyClosure, + cache, instance)) + instance.getDiags().diagnose(SourceLoc(), diag::error_cas, + toString(std::move(E))); } auto dependencyGraph = generateFullDependencyGraph( diff --git a/lib/Frontend/CachingUtils.cpp b/lib/Frontend/CachingUtils.cpp index 721158a735701..20f9bb262d550 100644 --- a/lib/Frontend/CachingUtils.cpp +++ b/lib/Frontend/CachingUtils.cpp @@ -342,4 +342,51 @@ Error storeCachedCompilerOutput(llvm::cas::ObjectStore &CAS, return Error::success(); } +namespace cas { + +CachingTool::CachingTool(StringRef Path) { + auto DB = llvm::cas::createOnDiskUnifiedCASDatabases(Path); + if (!DB) { + llvm::errs() << "Failed to create CAS at " << Path << ": " + << toString(DB.takeError()) << "\n"; + return; + } + + CAS = std::move(DB->first); + Cache = std::move(DB->second); +} + +std::string CachingTool::computeCacheKey(ArrayRef Args, + StringRef InputPath, + file_types::ID OutputKind) { + auto BaseKey = createCompileJobBaseCacheKey(*CAS, Args); + if (!BaseKey) { + llvm::errs() << "Failed to create cache key: " + << toString(BaseKey.takeError()) << "\n"; + return ""; + } + + auto Key = + createCompileJobCacheKeyForOutput(*CAS, *BaseKey, InputPath, OutputKind); + if (!Key) { + llvm::errs() << "Failed to create cache key: " << toString(Key.takeError()) + << "\n"; + return ""; + } + + return CAS->getID(*Key).toString(); +} + +std::string CachingTool::storeContent(StringRef Content) { + auto Result = CAS->storeFromString({}, Content); + if (!Result) { + llvm::errs() << "Failed to store to CAS: " << toString(Result.takeError()) + << "\n"; + return ""; + } + + return CAS->getID(*Result).toString(); +} + +} // namespace cas } // namespace swift diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp index 6849bc3f8e690..60ed04b9859fd 100644 --- a/lib/Frontend/CompilerInvocation.cpp +++ b/lib/Frontend/CompilerInvocation.cpp @@ -1461,6 +1461,8 @@ static bool ParseClangImporterArgs(ClangImporterOptions &Opts, if (auto *A = Args.getLastArg(OPT_import_objc_header)) Opts.BridgingHeader = A->getValue(); + Opts.BridgingHeaderPCHCacheKey = + Args.getLastArgValue(OPT_bridging_header_pch_key); Opts.DisableSwiftBridgeAttr |= Args.hasArg(OPT_disable_swift_bridge_attr); Opts.DisableOverlayModules |= Args.hasArg(OPT_emit_imported_modules); diff --git a/lib/Frontend/Frontend.cpp b/lib/Frontend/Frontend.cpp index b4ff90136cf24..ab2ed1d842b7e 100644 --- a/lib/Frontend/Frontend.cpp +++ b/lib/Frontend/Frontend.cpp @@ -580,6 +580,25 @@ bool CompilerInstance::setUpVirtualFileSystemOverlays() { SourceMgr.setFileSystem(std::move(*FS)); } + // If we have a bridging header cache key, try load it now and overlay it. + if (!Invocation.getClangImporterOptions().BridgingHeaderPCHCacheKey.empty() && + Invocation.getFrontendOptions().EnableCAS) { + auto loadedBridgingBuffer = loadCachedCompileResultFromCacheKey( + getObjectStore(), getActionCache(), Diagnostics, + Invocation.getClangImporterOptions().BridgingHeaderPCHCacheKey, + Invocation.getClangImporterOptions().BridgingHeader); + if (loadedBridgingBuffer) { + llvm::IntrusiveRefCntPtr PCHFS = + new llvm::vfs::InMemoryFileSystem(); + PCHFS->addFile(Invocation.getClangImporterOptions().BridgingHeader, 0, + std::move(loadedBridgingBuffer)); + llvm::IntrusiveRefCntPtr OverlayVFS = + new llvm::vfs::OverlayFileSystem(SourceMgr.getFileSystem()); + OverlayVFS->pushOverlay(PCHFS); + SourceMgr.setFileSystem(std::move(OverlayVFS)); + } + } + auto ExpectedOverlay = Invocation.getSearchPathOptions().makeOverlayFileSystem( SourceMgr.getFileSystem()); @@ -701,17 +720,24 @@ bool CompilerInstance::setUpModuleLoaders() { // If using `-explicit-swift-module-map-file`, create the explicit loader // before creating `ClangImporter` because the entries in the map influence // the Clang flags. The loader is added to the context below. - std::unique_ptr ESML = nullptr; + std::unique_ptr ESML = nullptr; bool ExplicitModuleBuild = Invocation.getFrontendOptions().DisableImplicitModules; if (ExplicitModuleBuild || !Invocation.getSearchPathOptions().ExplicitSwiftModuleMap.empty() || !Invocation.getSearchPathOptions().ExplicitSwiftModuleInputs.empty()) { - ESML = ExplicitSwiftModuleLoader::create( - *Context, getDependencyTracker(), MLM, - Invocation.getSearchPathOptions().ExplicitSwiftModuleMap, - Invocation.getSearchPathOptions().ExplicitSwiftModuleInputs, - IgnoreSourceInfoFile); + if (Invocation.getFrontendOptions().EnableCAS) + ESML = ExplicitCASModuleLoader::create( + *Context, getObjectStore(), getActionCache(), getDependencyTracker(), + MLM, Invocation.getSearchPathOptions().ExplicitSwiftModuleMap, + Invocation.getSearchPathOptions().ExplicitSwiftModuleInputs, + IgnoreSourceInfoFile); + else + ESML = ExplicitSwiftModuleLoader::create( + *Context, getDependencyTracker(), MLM, + Invocation.getSearchPathOptions().ExplicitSwiftModuleMap, + Invocation.getSearchPathOptions().ExplicitSwiftModuleInputs, + IgnoreSourceInfoFile); } // Wire up the Clang importer. If the user has specified an SDK, use it. diff --git a/lib/Frontend/ModuleInterfaceLoader.cpp b/lib/Frontend/ModuleInterfaceLoader.cpp index 437fa6090a0df..7a277a6cec1f5 100644 --- a/lib/Frontend/ModuleInterfaceLoader.cpp +++ b/lib/Frontend/ModuleInterfaceLoader.cpp @@ -20,6 +20,7 @@ #include "swift/AST/FileSystem.h" #include "swift/AST/Module.h" #include "swift/Basic/Platform.h" +#include "swift/Frontend/CachingUtils.h" #include "swift/Frontend/Frontend.h" #include "swift/Frontend/ModuleInterfaceSupport.h" #include "swift/Parse/ParseVersion.h" @@ -28,6 +29,7 @@ #include "swift/Serialization/Validation.h" #include "swift/Strings.h" #include "clang/Basic/Module.h" +#include "clang/Frontend/CompileJobCacheResult.h" #include "clang/Frontend/CompilerInstance.h" #include "clang/Lex/HeaderSearch.h" #include "clang/Lex/Preprocessor.h" @@ -36,9 +38,12 @@ #include "llvm/ADT/Hashing.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringExtras.h" +#include "llvm/CAS/ActionCache.h" +#include "llvm/CAS/ObjectStore.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/Errc.h" +#include "llvm/Support/Error.h" #include "llvm/Support/Path.h" #include "llvm/Support/VirtualOutputBackend.h" #include "llvm/Support/YAMLParser.h" @@ -1661,6 +1666,7 @@ InterfaceSubContextDelegateImpl::InterfaceSubContextDelegateImpl( // required by sourcekitd. subClangImporterOpts.DetailedPreprocessingRecord = clangImporterOpts.DetailedPreprocessingRecord; + subClangImporterOpts.CASPath = clangImporterOpts.CASPath; // If the compiler has been asked to be strict with ensuring downstream dependencies // get the parent invocation's context, or this is an Explicit build, inherit the @@ -1941,15 +1947,22 @@ struct ExplicitSwiftModuleLoader::Implementation { void parseSwiftExplicitModuleMap(StringRef fileName) { ExplicitModuleMapParser parser(Allocator); llvm::StringMap ExplicitClangModuleMap; - auto result = - parser.parseSwiftExplicitModuleMap(fileName, ExplicitModuleMap, - ExplicitClangModuleMap); - if (result == std::errc::invalid_argument) - Ctx.Diags.diagnose(SourceLoc(), diag::explicit_swift_module_map_corrupted, - fileName); - else if (result == std::errc::no_such_file_or_directory) + // Load the input file. + llvm::ErrorOr> fileBufOrErr = + llvm::MemoryBuffer::getFile(fileName); + if (!fileBufOrErr) { Ctx.Diags.diagnose(SourceLoc(), diag::explicit_swift_module_map_missing, fileName); + return; + } + + auto hasError = parser.parseSwiftExplicitModuleMap( + (*fileBufOrErr)->getMemBufferRef(), ExplicitModuleMap, + ExplicitClangModuleMap); + + if (hasError) + Ctx.Diags.diagnose(SourceLoc(), diag::explicit_swift_module_map_corrupted, + fileName); // A single module map can define multiple modules; keep track of the ones // we've seen so that we don't generate duplicate flags. @@ -2154,3 +2167,332 @@ ExplicitSwiftModuleLoader::create(ASTContext &ctx, return result; } + +struct ExplicitCASModuleLoader::Implementation { + ASTContext &Ctx; + llvm::BumpPtrAllocator Allocator; + llvm::cas::ObjectStore &CAS; + llvm::cas::ActionCache &Cache; + + llvm::StringMap ExplicitModuleMap; + + Implementation(ASTContext &Ctx, llvm::cas::ObjectStore &CAS, + llvm::cas::ActionCache &Cache) + : Ctx(Ctx), CAS(CAS), Cache(Cache) {} + + llvm::Expected> loadBuffer(StringRef ID) { + auto key = CAS.parseID(ID); + if (!key) + return key.takeError(); + + auto ref = CAS.getReference(*key); + if (!ref) + return nullptr; + + auto loaded = CAS.getProxy(*ref); + if (!loaded) + return loaded.takeError(); + + return loaded->getMemoryBuffer(); + } + + // Same as the regular explicit module map but must be loaded from + // CAS, instead of a file that is not tracked by the dependency. + void parseSwiftExplicitModuleMap(StringRef ID) { + ExplicitModuleMapParser parser(Allocator); + llvm::StringMap ExplicitClangModuleMap; + auto buf = loadBuffer(ID); + if (!buf) { + Ctx.Diags.diagnose(SourceLoc(), diag::error_cas, + toString(buf.takeError())); + return; + } + if (!*buf) { + Ctx.Diags.diagnose(SourceLoc(), diag::explicit_swift_module_map_missing, + ID); + return; + } + llvm::ErrorOr> fileBufOrErr = + llvm::MemoryBuffer::getFile(ID); + + auto hasError = parser.parseSwiftExplicitModuleMap( + (*buf)->getMemBufferRef(), ExplicitModuleMap, ExplicitClangModuleMap); + + if (hasError) + Ctx.Diags.diagnose(SourceLoc(), diag::explicit_swift_module_map_corrupted, + ID); + + std::set moduleMapsSeen; + std::vector &extraClangArgs = Ctx.ClangImporterOpts.ExtraArgs; + for (auto &entry : ExplicitClangModuleMap) { + const auto &moduleMapPath = entry.getValue().moduleMapPath; + if (!moduleMapPath.empty() && + moduleMapsSeen.find(moduleMapPath) == moduleMapsSeen.end()) { + moduleMapsSeen.insert(moduleMapPath); + extraClangArgs.push_back( + (Twine("-fmodule-map-file=") + moduleMapPath).str()); + } + + const auto &modulePath = entry.getValue().modulePath; + if (!modulePath.empty()) { + extraClangArgs.push_back( + (Twine("-fmodule-file=") + entry.getKey() + "=" + modulePath) + .str()); + } + auto cachePath = entry.getValue().moduleCacheKey; + if (cachePath) { + extraClangArgs.push_back("-Xclang"); + extraClangArgs.push_back("-fmodule-file-cache-key"); + extraClangArgs.push_back("-Xclang"); + extraClangArgs.push_back(modulePath); + extraClangArgs.push_back("-Xclang"); + extraClangArgs.push_back(*cachePath); + } + } + } + + void addCommandLineExplicitInputs( + const std::vector> + &commandLineExplicitInputs) { + for (const auto &moduleInput : commandLineExplicitInputs) { + ExplicitSwiftModuleInputInfo entry(moduleInput.second, {}, {}); + ExplicitModuleMap.try_emplace(moduleInput.first, std::move(entry)); + } + } + + llvm::Expected> + loadFileBuffer(StringRef ID, StringRef Name) { + auto key = CAS.parseID(ID); + if (!key) + return key.takeError(); + + auto moduleLookup = Cache.get(*key); + if (!moduleLookup) + return moduleLookup.takeError(); + if (!*moduleLookup) + return nullptr; + + auto moduleRef = CAS.getReference(**moduleLookup); + if (!moduleRef) + return nullptr; + + clang::cas::CompileJobResultSchema schema(CAS); + auto result = schema.load(*moduleRef); + if (!result) + return result.takeError(); + auto output = result->getOutput( + clang::cas::CompileJobCacheResult::OutputKind::MainOutput); + if (!output) + return nullptr; + + auto buf = CAS.getProxy(output->Object); + if (!buf) + return buf.takeError(); + + return buf->getMemoryBuffer(Name); + } + + llvm::Expected> + loadModuleFromPath(StringRef Path, DiagnosticEngine &Diags) { + for (auto &Deps : ExplicitModuleMap) { + if (Deps.second.modulePath == Path) { + if (!Deps.second.moduleCacheKey) + return nullptr; + return loadCachedCompileResultFromCacheKey( + CAS, Cache, Diags, *Deps.second.moduleCacheKey, Path); + } + } + return nullptr; + } +}; + +ExplicitCASModuleLoader::ExplicitCASModuleLoader(ASTContext &ctx, + llvm::cas::ObjectStore &CAS, + llvm::cas::ActionCache &cache, + DependencyTracker *tracker, + ModuleLoadingMode loadMode, + bool IgnoreSwiftSourceInfoFile) + : SerializedModuleLoaderBase(ctx, tracker, loadMode, + IgnoreSwiftSourceInfoFile), + Impl(*new Implementation(ctx, CAS, cache)) {} + +ExplicitCASModuleLoader::~ExplicitCASModuleLoader() { delete &Impl; } + +bool ExplicitCASModuleLoader::findModule( + ImportPath::Element ModuleID, SmallVectorImpl *ModuleInterfacePath, + SmallVectorImpl *ModuleInterfaceSourcePath, + std::unique_ptr *ModuleBuffer, + std::unique_ptr *ModuleDocBuffer, + std::unique_ptr *ModuleSourceInfoBuffer, + bool skipBuildingInterface, bool isTestableDependencyLookup, + bool &IsFramework, bool &IsSystemModule) { + // Find a module with an actual, physical name on disk, in case + // -module-alias is used (otherwise same). + // + // For example, if '-module-alias Foo=Bar' is passed in to the frontend, and + // an input file has 'import Foo', a module called Bar (real name) should be + // searched. + StringRef moduleName = Ctx.getRealModuleName(ModuleID.Item).str(); + + auto it = Impl.ExplicitModuleMap.find(moduleName); + // If no explicit module path is given matches the name, return with an + // error code. + if (it == Impl.ExplicitModuleMap.end()) { + return false; + } + auto &moduleInfo = it->getValue(); + + // Set IsFramework bit according to the moduleInfo + IsFramework = moduleInfo.isFramework; + IsSystemModule = moduleInfo.isSystem; + + // Fallback check for module cache key passed on command-line as module path. + std::string moduleCASID = moduleInfo.moduleCacheKey + ? *moduleInfo.moduleCacheKey + : moduleInfo.modulePath; + + // FIXME: the loaded module buffer doesn't set an identifier so it + // is not tracked in dependency tracker, which doesn't handle modules + // that are not located on disk. + auto moduleBuf = loadCachedCompileResultFromCacheKey(Impl.CAS, Impl.Cache, + Ctx.Diags, moduleCASID); + if (!moduleBuf) { + // We cannot read the module content, diagnose. + Ctx.Diags.diagnose(SourceLoc(), diag::error_opening_explicit_module_file, + moduleInfo.modulePath); + return false; + } + + const bool isForwardingModule = + !serialization::isSerializedAST(moduleBuf->getBuffer()); + // If the module is a forwarding module, read the actual content from the path + // encoded in the forwarding module as the actual module content. + if (isForwardingModule) { + auto forwardingModule = ForwardingModule::load(*moduleBuf.get()); + if (forwardingModule) { + // Look through ExplicitModuleMap for paths. + // TODO: need to have dependency scanner reports forwarded module as + // dependency for this compilation and ingested into CAS. + auto moduleOrErr = Impl.loadModuleFromPath( + forwardingModule->underlyingModulePath, Ctx.Diags); + if (!moduleOrErr) { + llvm::consumeError(moduleOrErr.takeError()); + Ctx.Diags.diagnose(SourceLoc(), + diag::error_opening_explicit_module_file, + moduleInfo.modulePath); + return false; + } + moduleBuf = std::move(*moduleOrErr); + if (!moduleBuf) { + // We cannot read the module content, diagnose. + Ctx.Diags.diagnose(SourceLoc(), + diag::error_opening_explicit_module_file, + moduleInfo.modulePath); + return false; + } + } else { + // We cannot read the module content, diagnose. + Ctx.Diags.diagnose(SourceLoc(), diag::error_opening_explicit_module_file, + moduleInfo.modulePath); + return false; + } + } + assert(moduleBuf); + // Move the opened module buffer to the caller. + *ModuleBuffer = std::move(moduleBuf); + + // TODO: support .swiftdoc file and .swiftsourceinfo file + return true; +} + +std::error_code ExplicitCASModuleLoader::findModuleFilesInDirectory( + ImportPath::Element ModuleID, const SerializedModuleBaseName &BaseName, + SmallVectorImpl *ModuleInterfacePath, + SmallVectorImpl *ModuleInterfaceSourcePath, + std::unique_ptr *ModuleBuffer, + std::unique_ptr *ModuleDocBuffer, + std::unique_ptr *ModuleSourceInfoBuffer, + bool skipBuildingInterface, bool IsFramework, + bool IsTestableDependencyLookup) { + llvm_unreachable("Not supported in the Explicit Swift Module Loader."); + return std::make_error_code(std::errc::not_supported); +} + +bool ExplicitCASModuleLoader::canImportModule( + ImportPath::Module path, ModuleVersionInfo *versionInfo, + bool isTestableDependencyLookup) { + // FIXME: Swift submodules? + if (path.hasSubmodule()) + return false; + ImportPath::Element mID = path.front(); + // Look up the module with the real name (physical name on disk); + // in case `-module-alias` is used, the name appearing in source files + // and the real module name are different. For example, '-module-alias + // Foo=Bar' maps Foo appearing in source files, e.g. 'import Foo', to the real + // module name Bar (on-disk name), which should be searched for loading. + StringRef moduleName = Ctx.getRealModuleName(mID.Item).str(); + auto it = Impl.ExplicitModuleMap.find(moduleName); + // If no provided explicit module matches the name, then it cannot be + // imported. + if (it == Impl.ExplicitModuleMap.end()) { + return false; + } + + // If the caller doesn't want version info we're done. + if (!versionInfo) + return true; + + // Open .swiftmodule file and read out the version + std::string moduleCASID = it->second.moduleCacheKey + ? *it->second.moduleCacheKey + : it->second.modulePath; + auto moduleBuf = Impl.loadFileBuffer(moduleCASID, it->second.modulePath); + if (!moduleBuf) { + Ctx.Diags.diagnose(SourceLoc(), diag::error_cas, + toString(moduleBuf.takeError())); + return false; + } + if (!*moduleBuf) { + Ctx.Diags.diagnose(SourceLoc(), diag::error_opening_explicit_module_file, + it->second.modulePath); + return false; + } + auto metaData = serialization::validateSerializedAST( + (*moduleBuf)->getBuffer(), Ctx.SILOpts.EnableOSSAModules, + Ctx.LangOpts.SDKName, !Ctx.LangOpts.DebuggerSupport); + versionInfo->setVersion(metaData.userModuleVersion, + ModuleVersionSourceKind::SwiftBinaryModule); + return true; +} + +void ExplicitCASModuleLoader::collectVisibleTopLevelModuleNames( + SmallVectorImpl &names) const { + for (auto &entry : Impl.ExplicitModuleMap) { + names.push_back(Ctx.getIdentifier(entry.getKey())); + } +} + +std::unique_ptr ExplicitCASModuleLoader::create( + ASTContext &ctx, llvm::cas::ObjectStore &CAS, llvm::cas::ActionCache &cache, + DependencyTracker *tracker, ModuleLoadingMode loadMode, + StringRef ExplicitSwiftModuleMap, + const std::vector> + &ExplicitSwiftModuleInputs, + bool IgnoreSwiftSourceInfoFile) { + auto result = + std::unique_ptr(new ExplicitCASModuleLoader( + ctx, CAS, cache, tracker, loadMode, IgnoreSwiftSourceInfoFile)); + auto &Impl = result->Impl; + // If the explicit module map is given, try parse it. + if (!ExplicitSwiftModuleMap.empty()) { + // Parse a JSON file to collect explicitly built modules. + Impl.parseSwiftExplicitModuleMap(ExplicitSwiftModuleMap); + } + // If some modules are provided with explicit + // '-swift-module-file' options, add those as well. + if (!ExplicitSwiftModuleInputs.empty()) { + Impl.addCommandLineExplicitInputs(ExplicitSwiftModuleInputs); + } + + return result; +} diff --git a/lib/Serialization/ModuleDependencyScanner.cpp b/lib/Serialization/ModuleDependencyScanner.cpp index 2d3de471e0247..1de7eace92cc5 100644 --- a/lib/Serialization/ModuleDependencyScanner.cpp +++ b/lib/Serialization/ModuleDependencyScanner.cpp @@ -22,6 +22,9 @@ #include "swift/Serialization/SerializedModuleLoader.h" #include "swift/Serialization/ModuleDependencyScanner.h" #include "swift/Subsystems.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" +#include "llvm/CAS/CachingOnDiskFileSystem.h" +#include "llvm/Support/VirtualFileSystem.h" #include "ModuleFileSharedCore.h" #include @@ -151,11 +154,6 @@ ErrorOr ModuleDependencyScanner::scanInterfaceFile( Args.push_back("-o"); Args.push_back(outputPathBase.str().str()); - std::vector ArgsRefs(Args.begin(), Args.end()); - Result = ModuleDependencyInfo::forSwiftInterfaceModule( - outputPathBase.str().str(), InPath, compiledCandidates, ArgsRefs, PCMArgs, - Hash, isFramework); - // Open the interface file. auto &fs = *Ctx.SourceMgr.getFileSystem(); auto interfaceBuf = fs.getBufferForFile(moduleInterfacePath); @@ -172,6 +170,23 @@ ErrorOr ModuleDependencyScanner::scanInterfaceFile( *moduleDecl, SourceFileKind::Interface, bufferID, parsingOpts); moduleDecl->addAuxiliaryFile(*sourceFile); + std::string RootID; + if (dependencyTracker) { + dependencyTracker->startTracking(); + dependencyTracker->trackFile(moduleInterfacePath); + RootID = cantFail(dependencyTracker->createTreeFromDependencies()) + .getID() + .toString(); + Args.push_back("-enable-cas"); + Args.push_back("-cas-path"); + Args.push_back(Ctx.ClangImporterOpts.CASPath); + } + + std::vector ArgsRefs(Args.begin(), Args.end()); + Result = ModuleDependencyInfo::forSwiftInterfaceModule( + outputPathBase.str().str(), InPath, compiledCandidates, ArgsRefs, + PCMArgs, Hash, isFramework, RootID, /*module-cache-key*/ ""); + // Walk the source file to find the import declarations. llvm::StringSet<> alreadyAddedModules; Result->addModuleImport(*sourceFile, alreadyAddedModules); @@ -255,9 +270,10 @@ Optional SerializedModuleLoaderBase::getModuleDepen // FIXME: submodules? scanners.push_back(std::make_unique( Ctx, LoadMode, moduleId, Ctx.SearchPathOpts.PlaceholderDependencyModuleMap, - delegate)); + delegate, cache.getScanService().createSwiftDependencyTracker())); scanners.push_back(std::make_unique( - Ctx, LoadMode, moduleId, delegate)); + Ctx, LoadMode, moduleId, delegate, ModuleDependencyScanner::MDS_plain, + cache.getScanService().createSwiftDependencyTracker())); // Check whether there is a module with this name that we can import. assert(isa(scanners[0].get()) && diff --git a/lib/Serialization/SerializedModuleLoader.cpp b/lib/Serialization/SerializedModuleLoader.cpp index 03984e311e507..54b3a6309a4ca 100644 --- a/lib/Serialization/SerializedModuleLoader.cpp +++ b/lib/Serialization/SerializedModuleLoader.cpp @@ -442,7 +442,8 @@ SerializedModuleLoaderBase::scanModuleFile(Twine modulePath, bool isFramework) { const std::string sourceInfoPath; // Map the set of dependencies over to the "module dependencies". auto dependencies = ModuleDependencyInfo::forSwiftBinaryModule( - modulePath.str(), moduleDocPath, sourceInfoPath, isFramework); + modulePath.str(), moduleDocPath, sourceInfoPath, isFramework, + /*module-cache-key*/ ""); // Some transitive dependencies of binary modules are not required to be // imported during normal builds. // TODO: This is worth revisiting for debugger purposes where diff --git a/test/CAS/Inputs/SwiftDepsExtractor.py b/test/CAS/Inputs/SwiftDepsExtractor.py new file mode 100755 index 0000000000000..a06df5b6e0add --- /dev/null +++ b/test/CAS/Inputs/SwiftDepsExtractor.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 + +import json +import sys + +## SwiftDepsExtractor.py file.json ModuleName Key +input_json = sys.argv[1] +module_name = sys.argv[2] +key = sys.argv[3] + +mode = "swift" + +if module_name.startswith("clang:"): + mode = "clang" + module_name = module_name[6:] +elif module_name.startswith("swiftPrebuiltExternal:"): + mode = "swiftPrebuiltExternal" + module_name = module_name[22:] + +with open(input_json, 'r') as file: + deps = json.load(file) + extract_next = False + for module in deps['modules']: + if extract_next: + if key in module.keys(): + print(module[key]) + break + print(module['details'][mode][key]) + break + + if module.get(mode, '') == module_name: + extract_next = True diff --git a/test/CAS/binary_module_deps.swift b/test/CAS/binary_module_deps.swift new file mode 100644 index 0000000000000..e59a88e17c09f --- /dev/null +++ b/test/CAS/binary_module_deps.swift @@ -0,0 +1,41 @@ +// REQUIRES: objc_interop + +// RUN: %empty-directory(%t) +// RUN: mkdir -p %t/clang-module-cache +// RUN: mkdir -p %t/cas + +// RUN: %target-swift-frontend -emit-module -module-cache-path %t/clang-module-cache %S/../ScanDependencies/Inputs/Swift/E.swiftinterface -o %t/E.swiftmodule -I %S/../ScanDependencies/Inputs/CHeaders -I %S/../ScanDependencies/Inputs/Swift -swift-version 4 +// RUN: %target-swift-frontend -emit-module -module-cache-path %t/clang-module-cache %S/../ScanDependencies/Inputs/Swift/A.swiftinterface -o %t/A.swiftmodule -I %S/../ScanDependencies/Inputs/CHeaders -I %S/../ScanDependencies/Inputs/Swift -swift-version 4 +// RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache %s -o %t/deps.json -I %t -swift-version 4 -enable-cas -cas-path %t/cas +// RUN: %FileCheck %s -DTEMP=%t < %t/deps.json + +/// Test binary module key: binary module key is the CASID of itself. +// RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps.json swiftPrebuiltExternal:A moduleCacheKey > %t/A.key.casid +// RUN: llvm-cas --cas %t/cas --cat-blob @%t/A.key.casid > %t/Loaded.swiftmodule +// RUN: diff %t/A.swiftmodule %t/Loaded.swiftmodule + +import A +import E + + +/// Main module +// CHECK-LABEL: "swift": "deps" +// CHECK: "directDependencies": [ +// CHECK: "swiftPrebuiltExternal": "A" +// CHECK: "swiftPrebuiltExternal": "E" +// CHECK: "details": { +// CHECK: "casFSRootID" + +/// E.swiftmodule +// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}E.swiftmodule", +// CHECK: "details": { +// CHECK: "swiftPrebuiltExternal": { +// CHECK: "compiledModulePath": +// CHECK: "moduleCacheKey": + +/// A.swiftmodule +// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}A.swiftmodule", +// CHECK: "details": { +// CHECK: "swiftPrebuiltExternal": { +// CHECK: "compiledModulePath": +// CHECK: "moduleCacheKey": diff --git a/test/CAS/cas-explicit-module-map.swift b/test/CAS/cas-explicit-module-map.swift new file mode 100644 index 0000000000000..f54aad1e0aae3 --- /dev/null +++ b/test/CAS/cas-explicit-module-map.swift @@ -0,0 +1,103 @@ +// RUN: %empty-directory(%t) +// RUN: mkdir -p %t/cas +// RUN: split-file %s %t +// RUN: %target-swift-frontend -emit-module -module-cache-path %t/clang-module-cache %t/A.swift -o %t/A.swiftmodule -swift-version 5 +// RUN: %target-swift-frontend -emit-module -module-cache-path %t/clang-module-cache %t/B.swift -o %t/B.swiftmodule -I %t -swift-version 5 +// RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache %t/Test.swift -o %t/deps.json -I %t -swift-version 5 -enable-cas -cas-path %t/cas + +// RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps.json swiftPrebuiltExternal:A moduleCacheKey | tr -d '\n' > %t/A.key +// RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps.json swiftPrebuiltExternal:B moduleCacheKey | tr -d '\n' > %t/B.key + +/// Prepare the cas objects that can be used to construct CompileJobResultSchema object. +// RUN: llvm-cas --cas %t/cas --get-cache-result @%t/A.key > %t/A.result +// RUN: llvm-cas --cas %t/cas --ls-node-refs @%t/A.result | tail -n 1 > %t/schema.casid +// RUN: llvm-cas --cas %t/cas --cat-blob @%t/A.result > %t/kind.blob + +/// Make keys for module loads. The result casid construction is tied with the actual structure of CompilerJobResultSchema. +// RUN: llvm-cas --cas %t/cas --make-blob --data %stdlib_module | tr -d '\n' > %t/Swift.key +// RUN: llvm-cas --cas %t/cas --make-node --data %t/kind.blob @%t/Swift.key @%t/schema.casid > %t/Swift.casid +// RUN: llvm-cas --cas %t/cas --put-cache-key @%t/Swift.key @%t/Swift.casid + +// RUN: llvm-cas --cas %t/cas --make-blob --data %ononesupport_module | tr -d '\n' > %t/ONone.key +// RUN: llvm-cas --cas %t/cas --make-node --data %t/kind.blob @%t/ONone.key @%t/schema.casid > %t/ONone.casid +// RUN: llvm-cas --cas %t/cas --put-cache-key @%t/ONone.key @%t/ONone.casid + +// RUN: llvm-cas --cas %t/cas --make-blob --data %concurrency_module | tr -d '\n' > %t/Concurrency.key +// RUN: llvm-cas --cas %t/cas --make-node --data %t/kind.blob @%t/Concurrency.key @%t/schema.casid > %t/Concurrency.casid +// RUN: llvm-cas --cas %t/cas --put-cache-key @%t/Concurrency.key @%t/Concurrency.casid + +// RUN: llvm-cas --cas %t/cas --make-blob --data %string_processing_module | tr -d '\n' > %t/String.key +// RUN: llvm-cas --cas %t/cas --make-node --data %t/kind.blob @%t/String.key @%t/schema.casid > %t/String.casid +// RUN: llvm-cas --cas %t/cas --put-cache-key @%t/String.key @%t/String.casid + +// RUN: echo "[{" > %/t/map.json +// RUN: echo "\"moduleName\": \"A\"," >> %/t/map.json +// RUN: echo "\"modulePath\": \"A.swiftmodule\"," >> %/t/map.json +// RUN: echo -n "\"moduleCacheKey\": \"" >> %/t/map.json +// RUN: cat %t/A.key >> %/t/map.json +// RUN: echo "\"," >> %/t/map.json +// RUN: echo "\"isFramework\": false" >> %/t/map.json +// RUN: echo "}," >> %/t/map.json +// RUN: echo "{" >> %/t/map.json +// RUN: echo "\"moduleName\": \"B\"," >> %/t/map.json +// RUN: echo "\"modulePath\": \"B.swiftmodule\"," >> %/t/map.json +// RUN: echo -n "\"moduleCacheKey\": \"" >> %/t/map.json +// RUN: cat %t/B.key >> %/t/map.json +// RUN: echo "\"," >> %/t/map.json +// RUN: echo "\"isFramework\": false" >> %/t/map.json +// RUN: echo "}," >> %/t/map.json +// RUN: echo "{" >> %/t/map.json +// RUN: echo "\"moduleName\": \"Swift\"," >> %/t/map.json +// RUN: echo "\"modulePath\": \"Swift.swiftmodule\"," >> %/t/map.json +// RUN: echo -n "\"moduleCacheKey\": \"" >> %/t/map.json +// RUN: cat %t/Swift.key >> %/t/map.json +// RUN: echo "\"," >> %/t/map.json +// RUN: echo "\"isFramework\": false" >> %/t/map.json +// RUN: echo "}," >> %/t/map.json +// RUN: echo "{" >> %/t/map.json +// RUN: echo "\"moduleName\": \"SwiftOnoneSupport\"," >> %/t/map.json +// RUN: echo "\"modulePath\": \"SwiftOnoneSupport.swiftmodule\"," >> %/t/map.json +// RUN: echo -n "\"moduleCacheKey\": \"" >> %/t/map.json +// RUN: cat %t/ONone.key >> %/t/map.json +// RUN: echo "\"," >> %/t/map.json +// RUN: echo "\"isFramework\": false" >> %/t/map.json +// RUN: echo "}," >> %/t/map.json +// RUN: echo "{" >> %/t/map.json +// RUN: echo "\"moduleName\": \"_Concurrency\"," >> %/t/map.json +// RUN: echo "\"modulePath\": \"_Concurrency.swiftmodule\"," >> %/t/map.json +// RUN: echo -n "\"moduleCacheKey\": \"" >> %/t/map.json +// RUN: cat %t/Concurrency.key >> %/t/map.json +// RUN: echo "\"," >> %/t/map.json +// RUN: echo "\"isFramework\": false" >> %/t/map.json +// RUN: echo "}," >> %/t/map.json +// RUN: echo "{" >> %/t/map.json +// RUN: echo "\"moduleName\": \"_StringProcessing\"," >> %/t/map.json +// RUN: echo "\"modulePath\": \"_StringProcessing.swiftmodule\"," >> %/t/map.json +// RUN: echo -n "\"moduleCacheKey\": \"" >> %/t/map.json +// RUN: cat %t/String.key >> %/t/map.json +// RUN: echo "\"," >> %/t/map.json +// RUN: echo "\"isFramework\": false" >> %/t/map.json +// RUN: echo "}]" >> %/t/map.json + +// RUN: llvm-cas --cas %t/cas --make-blob --data %t/map.json > %t/map.casid + +// RUN: %target-swift-frontend -emit-module -emit-module-path %t/Foo.swiftmodule -disable-implicit-swift-modules -module-cache-path %t.module-cache -explicit-swift-module-map-file @%t/map.casid -Rmodule-loading -Xcc -Rmodule-import %s -enable-cas -cas-path %t/cas -allow-unstable-cache-key-for-testing 2>&1 | %FileCheck %s + +// CHECK-DAG: loaded module 'A' +// CHECK-DAG: loaded module 'B' +// CHECK-DAG: loaded module 'Swift' +// CHECK-DAG: loaded module '_StringProcessing' +// CHECK-DAG: loaded module '_Concurrency' +// CHECK-DAG: loaded module 'SwiftOnoneSupport' + +//--- A.swift +func test() {} + +//--- B.swift +import A +func myTest() {} + +//--- Test.swift +import B + + diff --git a/test/CAS/cas_fs.swift b/test/CAS/cas_fs.swift index 97bb238ae6c3d..8020d2963abee 100644 --- a/test/CAS/cas_fs.swift +++ b/test/CAS/cas_fs.swift @@ -8,7 +8,6 @@ // RUN: llvm-cas --cas %t/cas --ingest --data %s > %t/source.casid // RUN: not %target-swift-frontend -typecheck -enable-cas -cas-fs @%t/source.casid -cas-path %t/cas %s 2>&1 | %FileCheck %s --check-prefix NO-RESOURCES -// NO-RESOURCES: error: unable to set working directory // NO-RESOURCES: error: unable to load standard library /// Ingest the resource directory to satisfy the file system requirement. Also switch CWD to resource dir. diff --git a/test/CAS/module_deps.swift b/test/CAS/module_deps.swift new file mode 100644 index 0000000000000..4d646ad049415 --- /dev/null +++ b/test/CAS/module_deps.swift @@ -0,0 +1,195 @@ +// REQUIRES: objc_interop + +// RUN: %empty-directory(%t) +// RUN: mkdir -p %t/clang-module-cache +// RUN: mkdir -p %t/cas + +// RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache %s -o %t/deps.json -I %S/../ScanDependencies/Inputs/CHeaders -I %S/../ScanDependencies/Inputs/Swift -emit-dependencies -emit-dependencies-path %t/deps.d -import-objc-header %S/../ScanDependencies/Inputs/CHeaders/Bridging.h -swift-version 4 -enable-cas -cas-path %t/cas +// Check the contents of the JSON output +// RUN: %FileCheck -check-prefix CHECK -check-prefix CHECK_NO_CLANG_TARGET %s < %t/deps.json + +// Check the contents of the JSON output +// RUN: %FileCheck %s -check-prefix CHECK -check-prefix CHECK-NO-SEARCH-PATHS < %t/deps.json + +// Check the make-style dependencies file +// RUN: %FileCheck %s -check-prefix CHECK-MAKE-DEPS < %t/deps.d + +// RUN: %target-swift-frontend -scan-dependencies -test-dependency-scan-cache-serialization -module-cache-path %t/clang-module-cache %s -o %t/deps.json -I %S/../ScanDependencies/Inputs/CHeaders -I %S/../ScanDependencies/Inputs/Swift -import-objc-header %S/../ScanDependencies/Inputs/CHeaders/Bridging.h -swift-version 4 -enable-cas -cas-path %t/cas +// RUN: %FileCheck -check-prefix CHECK -check-prefix CHECK_NO_CLANG_TARGET %s < %t/deps.json + +// Ensure that scanning with `-clang-target` makes sure that Swift modules' respective PCM-dependency-build-argument sets do not contain target triples. +// RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache %s -o %t/deps_clang_target.json -I %S/../ScanDependencies/Inputs/CHeaders -I %S/../ScanDependencies/Inputs/Swift -import-objc-header %S/../ScanDependencies/Inputs/CHeaders/Bridging.h -swift-version 4 -clang-target %target-cpu-apple-macosx10.14 -enable-cas -cas-path %t/cas +// Check the contents of the JSON output +// RUN: %FileCheck -check-prefix CHECK_CLANG_TARGET %s < %t/deps_clang_target.json + +/// check cas-fs content +// RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps.json E casFSRootID > %t/E_fs.casid +// RUN: llvm-cas --cas %t/cas --ls-tree-recursive @%t/E_fs.casid | %FileCheck %s -check-prefix FS_ROOT_E +// RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps.json clang:F casFSRootID > %t/F_fs.casid +// RUN: llvm-cas --cas %t/cas --ls-tree-recursive @%t/F_fs.casid | %FileCheck %s -check-prefix FS_ROOT_F + +// FS_ROOT_E-DAG: E.swiftinterface +// FS_ROOT_E-DAG: SDKSettings.json + +// FS_ROOT_F: CHeaders/A.h +// FS_ROOT_F: CHeaders/B.h +// FS_ROOT_F: CHeaders/C.h +// FS_ROOT_F: CHeaders/D.h +// FS_ROOT_F: CHeaders/F.h +// FS_ROOT_F: CHeaders/G.h +// FS_ROOT_F: CHeaders/H.h +// FS_ROOT_F: CHeaders/I.h +// FS_ROOT_F: CHeaders/X.h +// FS_ROOT_F: CHeaders/module.modulemap + +import C +import E +import G +import SubE + +// CHECK: "mainModuleName": "deps" + +/// --------Main module +// CHECK-LABEL: "modulePath": "deps.swiftmodule", +// CHECK-NEXT: sourceFiles +// CHECK-NEXT: module_deps.swift +// CHECK-NEXT: ], +// CHECK-NEXT: "directDependencies": [ +// CHECK-DAG: "swift": "A" +// CHECK-DAG: "clang": "C" +// CHECK-DAG: "swift": "E" +// CHECK-DAG: "swift": "F" +// CHECK-DAG: "swift": "G" +// CHECK-DAG: "swift": "SubE" +// CHECK-DAG: "swift": "Swift" +// CHECK-DAG: "swift": "SwiftOnoneSupport" +// CHECK-DAG: "swift": "_Concurrency" +// CHECK-DAG: "swift": "_cross_import_E" +// CHECK: ], + +// CHECK: "extraPcmArgs": [ +// CHECK-NEXT: "-Xcc", +// CHECK-NEXT: "-target", +// CHECK-NEXT: "-Xcc", +// CHECK: "-fapinotes-swift-version=4" +// CHECK-NOT: "error: cannot open Swift placeholder dependency module map from" +// CHECK: "bridgingHeader": +// CHECK-NEXT: "path": +// CHECK-SAME: Bridging.h + +// CHECK-NEXT: "sourceFiles": +// CHECK-NEXT: Bridging.h +// CHECK-NEXT: BridgingOther.h + +// CHECK: "moduleDependencies": [ +// CHECK-NEXT: "F" +// CHECK-NEXT: ] + +/// --------Swift module A +// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}A-{{.*}}.swiftmodule", + +// CHECK: directDependencies +// CHECK-NEXT: { +// CHECK-DAG: "clang": "A" +// CHECK-DAG: "swift": "Swift" +// CHECK-NEXT: }, +// CHECK: "details": +// CHECK: "moduleCacheKey": + +/// --------Swift module F +// CHECK: "modulePath": "{{.*}}{{/|\\}}F-{{.*}}.swiftmodule", +// CHECK-NEXT: "sourceFiles": [ +// CHECK-NEXT: ], +// CHECK-NEXT: "directDependencies": [ +// CHECK-NEXT: { +// CHECK-DAG: "clang": "F" +// CHECK-DAG: "swift": "Swift" +// CHECK-DAG: "swift": "SwiftOnoneSupport" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK: "details": +// CHECK: "moduleCacheKey": + +/// --------Swift module G +// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}G-{{.*}}.swiftmodule" +// CHECK: "directDependencies" +// CHECK-NEXT: { +// CHECK-DAG: "clang": "G" +// CHECK-DAG: "swift": "Swift" +// CHECK-DAG: "swift": "SwiftOnoneSupport" +// CHECK: ], +// CHECK-NEXT: "details": { + +// CHECK: "contextHash": "{{.*}}", +// CHECK: "commandLine": [ +// CHECK: "-compile-module-from-interface" +// CHECK: "-target" +// CHECK: "-module-name" +// CHECK: "G" +// CHECK: "-swift-version" +// CHECK: "5" +// CHECK: ], +// CHECK_NO_CLANG_TARGET: "extraPcmArgs": [ +// CHECK_NO_CLANG_TARGET-NEXT: "-Xcc", +// CHECK_NO_CLANG_TARGET-NEXT: "-target", +// CHECK_CLANG_TARGET: "extraPcmArgs": [ +// CHECK_CLANG_TARGET-NEXT: "-Xcc", +// CHECK_CLANG_TARGET-NEXT: "-fapinotes-swift-version={{.*}}" +// CHECK_CLANG_TARGET-NEXT: ] + +/// --------Swift module E +// CHECK: "swift": "E" +// CHECK-LABEL: modulePath": "{{.*}}{{/|\\}}E-{{.*}}.swiftmodule" +// CHECK: "directDependencies" +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "Swift" + +// CHECK: "moduleInterfacePath" +// CHECK-SAME: E.swiftinterface + +/// --------Swift module Swift +// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}Swift-{{.*}}.swiftmodule", + +// CHECK: directDependencies +// CHECK-NEXT: { +// CHECK-NEXT: "clang": "SwiftShims" + +/// --------Clang module SwiftShims +// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}SwiftShims-{{.*}}.pcm", +// CHECK: "contextHash": "[[SHIMS_CONTEXT:.*]]", +// CHECK: "-o" +// CHECK-NEXT: SwiftShims-{{.*}}[[SHIMS_CONTEXT]].pcm +// CHECK-NO-SEARCH-PATHS-NOT: "-prebuilt-module-cache-path" + +/// --------Clang module C +// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}C-{{.*}}.pcm", + +// CHECK: "sourceFiles": [ +// CHECK-DAG: module.modulemap +// CHECK-DAG: C.h + +// CHECK: directDependencies +// CHECK-NEXT: { +// CHECK-NEXT: "clang": "B" + +// CHECK: "moduleMapPath" +// CHECK-SAME: module.modulemap + +// CHECK: "contextHash" +// CHECK-SAME: "{{.*}}" + +/// --------Clang module B +// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}B-{{.*}}.pcm", +// CHECK: "contextHash": "[[B_CONTEXT:.*]]", +// CHECK: "-o" +// CHECK-NEXT: B-{{.*}}[[B_CONTEXT]].pcm + +// Check make-style dependencies +// CHECK-MAKE-DEPS: module_deps.swift +// CHECK-MAKE-DEPS-SAME: A.swiftinterface +// CHECK-MAKE-DEPS-SAME: G.swiftinterface +// CHECK-MAKE-DEPS-SAME: B.h +// CHECK-MAKE-DEPS-SAME: F.h +// CHECK-MAKE-DEPS-SAME: Bridging.h +// CHECK-MAKE-DEPS-SAME: BridgingOther.h +// CHECK-MAKE-DEPS-SAME: module.modulemap diff --git a/tools/libSwiftScan/libSwiftScan.cpp b/tools/libSwiftScan/libSwiftScan.cpp index 23c5b697ef26c..a3da81a3643fa 100644 --- a/tools/libSwiftScan/libSwiftScan.cpp +++ b/tools/libSwiftScan/libSwiftScan.cpp @@ -20,11 +20,15 @@ #include "swift/DependencyScan/DependencyScanImpl.h" #include "swift/DependencyScan/DependencyScanningTool.h" #include "swift/DependencyScan/StringUtils.h" +#include "swift/Frontend/CachingUtils.h" #include "swift/Option/Options.h" +#include "llvm/CAS/ObjectStore.h" using namespace swift::dependencies; +using namespace swift::cas; DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScanningTool, swiftscan_scanner_t) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(CachingTool, swiftscan_cas_t) //=== Private Cleanup Functions -------------------------------------------===// @@ -48,6 +52,10 @@ void swiftscan_dependency_info_details_dispose( swiftscan_string_set_dispose( details_impl->swift_textual_details.extra_pcm_args); swiftscan_string_dispose(details_impl->swift_textual_details.context_hash); + swiftscan_string_dispose( + details_impl->swift_textual_details.cas_fs_root_id); + swiftscan_string_dispose( + details_impl->swift_textual_details.module_cache_key); break; case SWIFTSCAN_DEPENDENCY_INFO_SWIFT_BINARY: swiftscan_string_dispose( @@ -56,6 +64,8 @@ void swiftscan_dependency_info_details_dispose( details_impl->swift_binary_details.module_doc_path); swiftscan_string_dispose( details_impl->swift_binary_details.module_source_info_path); + swiftscan_string_dispose( + details_impl->swift_binary_details.module_cache_key); break; case SWIFTSCAN_DEPENDENCY_INFO_SWIFT_PLACEHOLDER: swiftscan_string_dispose( @@ -70,6 +80,8 @@ void swiftscan_dependency_info_details_dispose( swiftscan_string_dispose(details_impl->clang_details.context_hash); swiftscan_string_set_dispose(details_impl->clang_details.command_line); swiftscan_string_set_dispose(details_impl->clang_details.captured_pcm_args); + swiftscan_string_dispose(details_impl->clang_details.cas_fs_root_id); + swiftscan_string_dispose(details_impl->clang_details.module_cache_key); break; } delete details_impl; @@ -296,6 +308,16 @@ swiftscan_string_set_t *swiftscan_swift_textual_detail_get_swift_overlay_depende return details->swift_textual_details.swift_overlay_module_dependencies; } +swiftscan_string_ref_t swiftscan_swift_textual_detail_get_cas_fs_root_id( + swiftscan_module_details_t details) { + return details->swift_textual_details.cas_fs_root_id; +} + +swiftscan_string_ref_t swiftscan_swift_textual_detail_get_module_cache_key( + swiftscan_module_details_t details) { + return details->swift_textual_details.module_cache_key; +} + //=== Swift Binary Module Details query APIs ------------------------------===// swiftscan_string_ref_t swiftscan_swift_binary_detail_get_compiled_module_path( @@ -319,6 +341,12 @@ bool swiftscan_swift_binary_detail_get_is_framework( return details->swift_binary_details.is_framework; } +swiftscan_string_ref_t swiftscan_swift_binary_detail_get_module_cache_key( + swiftscan_module_details_t details) { + return details->swift_binary_details.module_cache_key; +} + + //=== Swift Placeholder Module Details query APIs -------------------------===// swiftscan_string_ref_t @@ -360,6 +388,16 @@ swiftscan_clang_detail_get_captured_pcm_args(swiftscan_module_details_t details) return details->clang_details.captured_pcm_args; } +swiftscan_string_ref_t +swiftscan_clang_detail_get_cas_fs_root_id(swiftscan_module_details_t details) { + return details->clang_details.cas_fs_root_id; +} + +swiftscan_string_ref_t swiftscan_clang_detail_get_module_cache_key( + swiftscan_module_details_t details) { + return details->clang_details.module_cache_key; +} + //=== Batch Scan Input Functions ------------------------------------------===// swiftscan_batch_scan_input_t *swiftscan_batch_scan_input_create() { @@ -617,6 +655,60 @@ swiftscan_diagnostics_set_dispose(swiftscan_diagnostic_set_t* diagnostics){ delete diagnostics; } +//=== CAS Functions ----------------------------------------------------------// + +swiftscan_cas_t swiftscan_cas_create(const char *path) { + std::string CASPath(path); + if (CASPath.empty()) + CASPath = llvm::cas::getDefaultOnDiskCASPath(); + + CachingTool *tool = new CachingTool(CASPath); + if (!tool->isValid()) { + delete tool; + return nullptr; + } + return wrap(tool); +} + +void swiftscan_cas_dispose(swiftscan_cas_t cas) { delete unwrap(cas); } + +swiftscan_string_ref_t +swiftscan_cas_store(swiftscan_cas_t cas, uint8_t *data, unsigned size) { + llvm::StringRef StrContent((char*)data, size); + auto ID = unwrap(cas)->storeContent(StrContent); + return swift::c_string_utils::create_clone(ID.c_str()); +} + +static swift::file_types::ID +getFileTypeFromScanOutputKind(swiftscan_output_kind_t kind) { + switch (kind) { + case SWIFTSCAN_OUTPUT_TYPE_OBJECT: + return swift::file_types::ID::TY_Object; + case SWIFTSCAN_OUTPUT_TYPE_SWIFTMODULE: + return swift::file_types::ID::TY_SwiftModuleFile; + case SWIFTSCAN_OUTPUT_TYPE_SWIFTINTERFACE: + return swift::file_types::ID::TY_SwiftModuleInterfaceFile; + case SWIFTSCAN_OUTPUT_TYPE_SWIFTPRIAVEINTERFACE: + return swift::file_types::ID::TY_PrivateSwiftModuleInterfaceFile; + case SWIFTSCAN_OUTPUT_TYPE_CLANG_MODULE: + return swift::file_types::ID::TY_ClangModuleFile; + case SWIFTSCAN_OUTPUT_TYPE_CLANG_PCH: + return swift::file_types::ID::TY_PCH; + } +} + +swiftscan_string_ref_t +swiftscan_compute_cache_key(swiftscan_cas_t cas, int argc, const char **argv, + const char *input, swiftscan_output_kind_t kind) { + std::vector Compilation; + for (int i = 0; i < argc; ++i) + Compilation.push_back(argv[i]); + + auto ID = unwrap(cas)->computeCacheKey(Compilation, input, + getFileTypeFromScanOutputKind(kind)); + return swift::c_string_utils::create_clone(ID.c_str()); +} + //=== Experimental Compiler Invocation Functions ------------------------===// int invoke_swift_compiler(int argc, const char **argv) { diff --git a/tools/libSwiftScan/libSwiftScan.exports b/tools/libSwiftScan/libSwiftScan.exports index 4b940c96ea5f2..0486474c23ac6 100644 --- a/tools/libSwiftScan/libSwiftScan.exports +++ b/tools/libSwiftScan/libSwiftScan.exports @@ -16,10 +16,13 @@ swiftscan_swift_textual_detail_get_extra_pcm_args swiftscan_swift_textual_detail_get_context_hash swiftscan_swift_textual_detail_get_is_framework swiftscan_swift_textual_detail_get_swift_overlay_dependencies +swiftscan_swift_textual_detail_get_cas_fs_root_id +swiftscan_swift_textual_detail_get_module_cache_key swiftscan_swift_binary_detail_get_compiled_module_path swiftscan_swift_binary_detail_get_module_doc_path swiftscan_swift_binary_detail_get_module_source_info_path swiftscan_swift_binary_detail_get_is_framework +swiftscan_swift_binary_detail_get_module_cache_key swiftscan_swift_placeholder_detail_get_compiled_module_path swiftscan_swift_placeholder_detail_get_module_doc_path swiftscan_swift_placeholder_detail_get_module_source_info_path @@ -27,6 +30,8 @@ swiftscan_clang_detail_get_module_map_path swiftscan_clang_detail_get_context_hash swiftscan_clang_detail_get_command_line swiftscan_clang_detail_get_captured_pcm_args +swiftscan_clang_detail_get_cas_fs_root_id +swiftscan_clang_detail_get_module_cache_key swiftscan_batch_scan_input_set_modules swiftscan_batch_scan_entry_set_module_name swiftscan_batch_scan_entry_set_arguments @@ -68,4 +73,8 @@ swiftscan_scanner_diagnostics_reset swiftscan_diagnostic_get_message swiftscan_diagnostic_get_severity swiftscan_diagnostics_set_dispose +swiftscan_cas_create +swiftscan_cas_dispose +swiftscan_cas_store +swiftscan_compute_cache_key invoke_swift_compiler From 72f32d725517306dab8c45e4ddf0a249118111af Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Fri, 12 May 2023 13:32:05 -0700 Subject: [PATCH 2/4] [CAS] Support clang include tree Using clang include tree for dependency scanning and building for all clang modules and PCHs. --- .../swift-c/DependencyScan/DependencyScan.h | 4 + include/swift/AST/ModuleDependencies.h | 83 ++++++-- include/swift/Basic/LangOptions.h | 3 + include/swift/ClangImporter/ClangImporter.h | 5 + .../swift/DependencyScan/DependencyScanImpl.h | 9 + .../SerializedModuleDependencyCacheFormat.h | 6 +- include/swift/Frontend/CachingUtils.h | 5 + include/swift/Frontend/FrontendOptions.h | 5 +- include/swift/Option/Options.td | 8 + lib/AST/ModuleDependencies.cpp | 103 ++++++++-- lib/AST/ModuleLoader.cpp | 4 +- lib/ClangImporter/ClangImporter.cpp | 24 ++- .../ClangModuleDependencyScanner.cpp | 101 +++++++++- lib/DependencyScan/DependencyScanningTool.cpp | 3 - .../ModuleDependencyCacheSerialization.cpp | 63 +++++- lib/DependencyScan/ScanDependencies.cpp | 179 +++++++++++------ .../ArgsToFrontendOptionsConverter.cpp | 7 +- lib/Frontend/CachingUtils.cpp | 76 +++++++ lib/Frontend/CompilerInvocation.cpp | 16 +- lib/Frontend/Frontend.cpp | 39 ++-- lib/Frontend/ModuleInterfaceLoader.cpp | 15 ++ lib/Serialization/ModuleDependencyScanner.cpp | 11 +- test/CAS/module_deps_include_tree.swift | 190 ++++++++++++++++++ test/CMakeLists.txt | 1 + tools/libSwiftScan/libSwiftScan.cpp | 8 + tools/libSwiftScan/libSwiftScan.exports | 1 + 26 files changed, 804 insertions(+), 165 deletions(-) create mode 100644 test/CAS/module_deps_include_tree.swift diff --git a/include/swift-c/DependencyScan/DependencyScan.h b/include/swift-c/DependencyScan/DependencyScan.h index 5749b6cc9e9a2..ba6bc4a179194 100644 --- a/include/swift-c/DependencyScan/DependencyScan.h +++ b/include/swift-c/DependencyScan/DependencyScan.h @@ -139,6 +139,10 @@ SWIFTSCAN_PUBLIC swiftscan_string_set_t * swiftscan_swift_textual_detail_get_command_line( swiftscan_module_details_t details); +SWIFTSCAN_PUBLIC swiftscan_string_set_t * +swiftscan_swift_textual_detail_get_bridging_pch_command_line( + swiftscan_module_details_t details); + SWIFTSCAN_PUBLIC swiftscan_string_set_t * swiftscan_swift_textual_detail_get_extra_pcm_args( swiftscan_module_details_t details); diff --git a/include/swift/AST/ModuleDependencies.h b/include/swift/AST/ModuleDependencies.h index c872b7e8216e7..048bce85768cd 100644 --- a/include/swift/AST/ModuleDependencies.h +++ b/include/swift/AST/ModuleDependencies.h @@ -26,8 +26,10 @@ #include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/StringSet.h" +#include "llvm/CAS/CASProvidingFileSystem.h" #include "llvm/CAS/CASReference.h" #include "llvm/CAS/CachingOnDiskFileSystem.h" +#include "llvm/CAS/ObjectStore.h" #include "llvm/Support/Mutex.h" #include "llvm/Support/Error.h" #include "llvm/Support/VirtualFileSystem.h" @@ -128,9 +130,12 @@ class ModuleDependencyInfoStorageBase { struct CommonSwiftTextualModuleDependencyDetails { CommonSwiftTextualModuleDependencyDetails( ArrayRef extraPCMArgs, ArrayRef buildCommandLine, + ArrayRef bridgingHeaderBuildCommandLine, const std::string &CASFileSystemRootID) : extraPCMArgs(extraPCMArgs.begin(), extraPCMArgs.end()), buildCommandLine(buildCommandLine.begin(), buildCommandLine.end()), + bridgingHeaderBuildCommandLine(bridgingHeaderBuildCommandLine.begin(), + bridgingHeaderBuildCommandLine.end()), CASFileSystemRootID(CASFileSystemRootID) {} /// To build a PCM to be used by this Swift module, we need to append these @@ -155,8 +160,14 @@ struct CommonSwiftTextualModuleDependencyDetails { /// interface. std::vector buildCommandLine; + /// The Swift frontend invocation arguments to build bridging header. + std::vector bridgingHeaderBuildCommandLine; + /// CASID for the Root of CASFS. Empty if CAS is not used. std::string CASFileSystemRootID; + + /// CASID for the Root of bridgingHeaderClangIncludeTree. Empty if not used. + std::string bridgingHeaderIncludeTreeRoot; }; /// Describes the dependencies of a Swift module described by an Swift interface file. @@ -202,7 +213,7 @@ class SwiftInterfaceModuleDependenciesStorage : compiledModuleCandidates(compiledModuleCandidates.begin(), compiledModuleCandidates.end()), contextHash(contextHash), isFramework(isFramework), - textualModuleDetails(extraPCMArgs, buildCommandLine, RootID), + textualModuleDetails(extraPCMArgs, buildCommandLine, {}, RootID), moduleCacheKey(ModuleCacheKey) {} @@ -218,10 +229,6 @@ class SwiftInterfaceModuleDependenciesStorage : textualModuleDetails.buildCommandLine = newCommandLine; } - void updateCASFileSystemID(const std::string &ID) { - textualModuleDetails.CASFileSystemRootID = ID; - } - void updateModuleCacheKey(const std::string &Key) { moduleCacheKey = Key; } @@ -243,11 +250,13 @@ class SwiftSourceModuleDependenciesStorage : /// Collection of module imports that were detected to be `@Testable` llvm::StringSet<> testableImports; - SwiftSourceModuleDependenciesStorage(const std::string &RootID, - ArrayRef buildCommandLine, - ArrayRef extraPCMArgs) + SwiftSourceModuleDependenciesStorage( + const std::string &RootID, ArrayRef buildCommandLine, + ArrayRef bridgingHeaderBuildCommandLine, + ArrayRef extraPCMArgs) : ModuleDependencyInfoStorageBase(ModuleDependencyKind::SwiftSource), - textualModuleDetails(extraPCMArgs, buildCommandLine, RootID), + textualModuleDetails(extraPCMArgs, buildCommandLine, + bridgingHeaderBuildCommandLine, RootID), testableImports(llvm::StringSet<>()) {} ModuleDependencyInfoStorageBase *clone() const override { @@ -262,8 +271,9 @@ class SwiftSourceModuleDependenciesStorage : textualModuleDetails.buildCommandLine = newCommandLine; } - void updateCASFileSystemID(const std::string &ID) { - textualModuleDetails.CASFileSystemRootID = ID; + void updateBridgingHeaderCommandLine( + const std::vector &newCommandLine) { + textualModuleDetails.bridgingHeaderBuildCommandLine = newCommandLine; } void addTestableImport(ImportPath::Module module) { @@ -341,6 +351,9 @@ class ClangModuleDependencyStorage : public ModuleDependencyInfoStorageBase { /// CASID for the Root of CASFS. Empty if CAS is not used. std::string CASFileSystemRootID; + /// CASID for the Root of ClangIncludeTree. Empty if not used. + std::string clangIncludeTreeRoot; + /// The cache key for the produced module. std::string moduleCacheKey; @@ -352,6 +365,7 @@ class ClangModuleDependencyStorage : public ModuleDependencyInfoStorageBase { const std::vector &fileDependencies, const std::vector &capturedPCMArgs, const std::string &CASFileSystemRootID, + const std::string &clangIncludeTreeRoot, const std::string &ModuleCacheKey ) : ModuleDependencyInfoStorageBase(ModuleDependencyKind::Clang), pcmOutputPath(pcmOutputPath), @@ -361,6 +375,7 @@ class ClangModuleDependencyStorage : public ModuleDependencyInfoStorageBase { fileDependencies(fileDependencies), capturedPCMArgs(capturedPCMArgs), CASFileSystemRootID(CASFileSystemRootID), + clangIncludeTreeRoot(clangIncludeTreeRoot), moduleCacheKey(ModuleCacheKey) {} ModuleDependencyInfoStorageBase *clone() const override { @@ -473,10 +488,12 @@ class ModuleDependencyInfo { static ModuleDependencyInfo forSwiftSourceModule(const std::string &CASFileSystemRootID, ArrayRef buildCommands, + ArrayRef bridgingHeaderBuildCommands, ArrayRef extraPCMArgs) { return ModuleDependencyInfo( std::make_unique( - CASFileSystemRootID, buildCommands, extraPCMArgs)); + CASFileSystemRootID, buildCommands, bridgingHeaderBuildCommands, + extraPCMArgs)); } /// Describe the module dependencies for a Clang module that can be @@ -489,12 +506,13 @@ class ModuleDependencyInfo { const std::vector &fileDependencies, const std::vector &capturedPCMArgs, const std::string &CASFileSystemRootID, + const std::string &clangIncludeTreeRoot, const std::string &moduleCacheKey) { return ModuleDependencyInfo( std::make_unique( pcmOutputPath, moduleMapFile, contextHash, nonPathCommandLine, fileDependencies, capturedPCMArgs, - CASFileSystemRootID, moduleCacheKey)); + CASFileSystemRootID, clangIncludeTreeRoot, moduleCacheKey)); } /// Describe a placeholder dependency swift module. @@ -593,13 +611,17 @@ class ModuleDependencyInfo { llvm_unreachable("Unexpected type"); } - void updateCASFileSystemID(const std::string &ID) { - if (isSwiftInterfaceModule()) - return cast(storage.get()) - ->updateCASFileSystemID(ID); - else if (isSwiftSourceModule()) + std::vector getBridgingHeaderCommandline() const { + if (auto *detail = getAsSwiftSourceModule()) + return detail->textualModuleDetails.bridgingHeaderBuildCommandLine; + return {}; + } + + void updateBridgingHeaderCommandLine( + const std::vector &newCommandLine) { + if (isSwiftSourceModule()) return cast(storage.get()) - ->updateCASFileSystemID(ID); + ->updateBridgingHeaderCommandLine(newCommandLine); llvm_unreachable("Unexpected type"); } @@ -695,6 +717,12 @@ class ModuleDependencyInfo { /// Get CAS Filesystem RootID. Optional getCASFSRootID() const; + /// Get Clang Include Tree ID. + Optional getClangIncludeTree() const; + + /// Get bridging header Include Tree ID. + Optional getBridgingHeaderIncludeTree() const; + /// Get module output path. std::string getModuleOutputPath() const; @@ -711,6 +739,9 @@ class ModuleDependencyInfo { void addBridgingModuleDependency(StringRef module, llvm::StringSet<> &alreadyAddedModules); + /// Add bridging header include tree. + void addBridgingHeaderIncludeTree(StringRef ID); + /// Collect a map from a secondary module name to a list of cross-import /// overlays, when this current module serves as the primary module. llvm::StringMap> @@ -773,6 +804,12 @@ class SwiftDependencyScanningService { /// CachingOnDiskFileSystem for dependency tracking. llvm::IntrusiveRefCntPtr CacheFS; + /// If use clang include tree. + bool UseClangIncludeTree = false; + + /// CAS ObjectStore Instance. + std::shared_ptr CAS; + /// The common dependencies that is needed for every swift compiler instance. std::vector CommonDependencyFiles; @@ -826,7 +863,7 @@ class SwiftDependencyScanningService { return getCacheForScanningContextHash(scanningContextHash)->alreadySeenClangModules; } - bool usingCachingFS() const { return (bool)CacheFS; } + bool usingCachingFS() const { return !UseClangIncludeTree && (bool)CacheFS; } llvm::cas::CachingOnDiskFileSystem &getSharedCachingFS() const { assert(CacheFS && "Expect CachingOnDiskFileSystem"); @@ -841,9 +878,13 @@ class SwiftDependencyScanningService { } llvm::IntrusiveRefCntPtr getClangScanningFS() const { - if (CacheFS) + if (usingCachingFS()) return CacheFS->createProxyFS(); + if (UseClangIncludeTree) + return llvm::cas::createCASProvidingFileSystem( + CAS, llvm::vfs::createPhysicalFileSystem()); + return llvm::vfs::createPhysicalFileSystem(); } diff --git a/include/swift/Basic/LangOptions.h b/include/swift/Basic/LangOptions.h index 3e4766a69cd96..42bbf82bb8aab 100644 --- a/include/swift/Basic/LangOptions.h +++ b/include/swift/Basic/LangOptions.h @@ -890,6 +890,9 @@ namespace swift { /// built and provided to the compiler invocation. bool DisableImplicitClangModules = false; + /// Enable ClangIncludeTree for explicit module builds. + bool UseClangIncludeTree = false; + /// Return a hash code of any components from these options that should /// contribute to a Swift Bridging PCH hash. llvm::hash_code getPCHHashComponents() const { diff --git a/include/swift/ClangImporter/ClangImporter.h b/include/swift/ClangImporter/ClangImporter.h index 70fe747dd012f..7b6d817c107c3 100644 --- a/include/swift/ClangImporter/ClangImporter.h +++ b/include/swift/ClangImporter/ClangImporter.h @@ -54,6 +54,7 @@ namespace clang { namespace tooling { namespace dependencies { struct ModuleDeps; + struct TranslationUnitDeps; using ModuleDepsGraph = std::vector; } } @@ -431,6 +432,10 @@ class ClangImporter final : public ClangModuleLoader { ModuleDependenciesCache &cache, const clang::tooling::dependencies::ModuleDepsGraph &clangDependencies); + void recordBridgingHeaderOptions( + ModuleDependencyInfo &MDI, + const clang::tooling::dependencies::TranslationUnitDeps &deps); + Optional getModuleDependencies( StringRef moduleName, ModuleDependenciesCache &cache, InterfaceSubContextDelegate &delegate, diff --git a/include/swift/DependencyScan/DependencyScanImpl.h b/include/swift/DependencyScan/DependencyScanImpl.h index d335055648c17..92b3d45851a8a 100644 --- a/include/swift/DependencyScan/DependencyScanImpl.h +++ b/include/swift/DependencyScan/DependencyScanImpl.h @@ -86,6 +86,9 @@ typedef struct { /// Options to the compile command required to build this module interface swiftscan_string_set_t *command_line; + /// Options to the compile command required to build bridging header. + swiftscan_string_set_t *bridging_pch_command_line; + /// To build a PCM to be used by this Swift module, we need to append these /// arguments to the generic PCM build arguments reported from the dependency /// graph. @@ -100,6 +103,9 @@ typedef struct { /// The CASID for CASFileSystemRoot swiftscan_string_ref_t cas_fs_root_id; + /// The CASID for bridging header include tree + swiftscan_string_ref_t bridging_header_include_tree; + /// ModuleCacheKey swiftscan_string_ref_t module_cache_key; } swiftscan_swift_textual_details_t; @@ -152,6 +158,9 @@ typedef struct { /// The CASID for CASFileSystemRoot swiftscan_string_ref_t cas_fs_root_id; + /// The CASID for CASFileSystemRoot + swiftscan_string_ref_t clang_include_tree; + /// ModuleCacheKey swiftscan_string_ref_t module_cache_key; } swiftscan_clang_details_t; diff --git a/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h b/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h index e2d547fcfe870..3e9f4eeb4ca15 100644 --- a/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h +++ b/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h @@ -142,6 +142,7 @@ using SwiftInterfaceModuleDetailsLayout = FileIDArrayIDField, // bridgingModuleDependencies DependencyIDArrayIDField, // swiftOverlayDependencies IdentifierIDField, // CASFileSystemRootID + IdentifierIDField, // bridgingHeaderIncludeTree IdentifierIDField // moduleCacheKey >; @@ -154,7 +155,9 @@ using SwiftSourceModuleDetailsLayout = FileIDArrayIDField, // bridgingModuleDependencies DependencyIDArrayIDField, // swiftOverlayDependencies IdentifierIDField, // CASFileSystemRootID - FlagIDArrayIDField // buildCommandLine + IdentifierIDField, // bridgingHeaderIncludeTree + FlagIDArrayIDField, // buildCommandLine + FlagIDArrayIDField // bridgingHeaderBuildCommandLine >; using SwiftBinaryModuleDetailsLayout = @@ -182,6 +185,7 @@ using ClangModuleDetailsLayout = FileIDArrayIDField, // fileDependencies FlagIDArrayIDField, // capturedPCMArgs IdentifierIDField, // CASFileSystemRootID + IdentifierIDField, // clangIncludeTreeRoot IdentifierIDField // moduleCacheKey >; } // namespace graph_block diff --git a/include/swift/Frontend/CachingUtils.h b/include/swift/Frontend/CachingUtils.h index 97d92de02f8ef..501a65415d38e 100644 --- a/include/swift/Frontend/CachingUtils.h +++ b/include/swift/Frontend/CachingUtils.h @@ -19,6 +19,7 @@ #include "llvm/CAS/ActionCache.h" #include "llvm/CAS/ObjectStore.h" #include "llvm/CAS/CASReference.h" +#include "llvm/Support/VirtualFileSystem.h" #include "llvm/Support/VirtualOutputBackend.h" #include @@ -54,6 +55,10 @@ llvm::Error storeCachedCompilerOutput(llvm::cas::ObjectStore &CAS, StringRef CorrespondingInput, file_types::ID OutputKind); +llvm::Expected> +createCASFileSystem(llvm::cas::ObjectStore &CAS, ArrayRef FSRoots, + ArrayRef IncludeTreeRoots); + namespace cas { /// Helper class to manage CAS/Caching from libSwiftScan C APIs. class CachingTool { diff --git a/include/swift/Frontend/FrontendOptions.h b/include/swift/Frontend/FrontendOptions.h index 8809316ee9ca9..d19af926aafc8 100644 --- a/include/swift/Frontend/FrontendOptions.h +++ b/include/swift/Frontend/FrontendOptions.h @@ -129,7 +129,10 @@ class FrontendOptions { std::string CASPath; /// CASFS Root. - std::string CASFSRootID; + std::vector CASFSRootIDs; + + /// Clang Include Trees. + std::vector ClangIncludeTrees; /// Number of retry opening an input file if the previous opening returns /// bad file descriptor error. diff --git a/include/swift/Option/Options.td b/include/swift/Option/Options.td index c5854c65e8d54..1069d3ea2b10f 100644 --- a/include/swift/Option/Options.td +++ b/include/swift/Option/Options.td @@ -1833,6 +1833,10 @@ def bridging_header_pch_key : Separate<["-"], "bridging-header-pch-key">, Flags<[FrontendOption, HelpHidden, ArgumentIsPath]>, HelpText<"Cache Key for bridging header pch">; +def clang_include_tree: Flag<["-"], "clang-include-tree">, + Flags<[FrontendOption, NoDriverOption]>, + HelpText<"Use clang include tree">; + // END ONLY SUPPORTED IN NEW DRIVER @@ -1844,6 +1848,10 @@ def cas_fs: Separate<["-"], "cas-fs">, Flags<[FrontendOption, NoDriverOption]>, HelpText<"Root CASID for CAS FileSystem">, MetaVarName<"">; +def clang_include_tree_root: Separate<["-"], "clang-include-tree-root">, + Flags<[FrontendOption, NoDriverOption]>, + HelpText<"Clang Include Tree CASID">, MetaVarName<"">; + def load_plugin_library: Separate<["-"], "load-plugin-library">, Flags<[FrontendOption, DoesNotAffectIncrementalBuild, ArgumentIsPath]>, diff --git a/lib/AST/ModuleDependencies.cpp b/lib/AST/ModuleDependencies.cpp index a9660e6f96979..fb5c3c72aee9a 100644 --- a/lib/AST/ModuleDependencies.cpp +++ b/lib/AST/ModuleDependencies.cpp @@ -210,6 +210,50 @@ Optional ModuleDependencyInfo::getCASFSRootID() const { return Root; } +Optional ModuleDependencyInfo::getClangIncludeTree() const { + std::string Root; + switch (getKind()) { + case swift::ModuleDependencyKind::Clang: { + auto clangModuleStorage = cast(storage.get()); + Root = clangModuleStorage->clangIncludeTreeRoot; + break; + } + default: + return None; + } + if (Root.empty()) + return None; + + return Root; +} + +Optional +ModuleDependencyInfo::getBridgingHeaderIncludeTree() const { + std::string Root; + switch (getKind()) { + case swift::ModuleDependencyKind::SwiftInterface: { + auto swiftInterfaceStorage = + cast(storage.get()); + Root = swiftInterfaceStorage->textualModuleDetails + .bridgingHeaderIncludeTreeRoot; + break; + } + case swift::ModuleDependencyKind::SwiftSource: { + auto swiftSourceStorage = + cast(storage.get()); + Root = + swiftSourceStorage->textualModuleDetails.bridgingHeaderIncludeTreeRoot; + break; + } + default: + return None; + } + if (Root.empty()) + return None; + + return Root; +} + std::string ModuleDependencyInfo::getModuleOutputPath() const { switch (getKind()) { case swift::ModuleDependencyKind::SwiftInterface: { @@ -281,6 +325,27 @@ void ModuleDependencyInfo::addBridgingSourceFile(StringRef bridgingSourceFile) { } } +void ModuleDependencyInfo::addBridgingHeaderIncludeTree(StringRef ID) { + switch (getKind()) { + case swift::ModuleDependencyKind::SwiftInterface: { + auto swiftInterfaceStorage = + cast(storage.get()); + swiftInterfaceStorage->textualModuleDetails.bridgingHeaderIncludeTreeRoot = + ID.str(); + break; + } + case swift::ModuleDependencyKind::SwiftSource: { + auto swiftSourceStorage = + cast(storage.get()); + swiftSourceStorage->textualModuleDetails.bridgingHeaderIncludeTreeRoot = + ID.str(); + break; + } + default: + llvm_unreachable("Unexpected dependency kind"); + } +} + void ModuleDependencyInfo::addSourceFile(StringRef sourceFile) { switch (getKind()) { case swift::ModuleDependencyKind::SwiftSource: { @@ -320,7 +385,7 @@ void ModuleDependencyInfo::addBridgingModuleDependency( SwiftDependencyScanningService::SwiftDependencyScanningService() { ClangScanningService.emplace( clang::tooling::dependencies::ScanningMode::DependencyDirectivesScan, - clang::tooling::dependencies::ScanningOutputFormat::Full, + clang::tooling::dependencies::ScanningOutputFormat::FullTree, clang::CASOptions(), /* CAS (llvm::cas::ObjectStore) */ nullptr, /* Cache (llvm::cas::ActionCache) */ nullptr, @@ -341,14 +406,15 @@ SwiftDependencyTracker::createTreeFromDependencies() { return FS->createTreeFromNewAccesses(); } -void SwiftDependencyScanningService::overlaySharedFilesystemCacheForCompilation(CompilerInstance &Instance) { - auto existingFS = Instance.getSourceMgr().getFileSystem(); - llvm::IntrusiveRefCntPtr< - clang::tooling::dependencies::DependencyScanningWorkerFilesystem> - depFS = - new clang::tooling::dependencies::DependencyScanningWorkerFilesystem( - getSharedFilesystemCache(), existingFS); - Instance.getSourceMgr().setFileSystem(depFS); +void SwiftDependencyScanningService::overlaySharedFilesystemCacheForCompilation( + CompilerInstance &Instance) { + auto existingFS = Instance.getSourceMgr().getFileSystem(); + llvm::IntrusiveRefCntPtr< + clang::tooling::dependencies::DependencyScanningWorkerFilesystem> + depFS = + new clang::tooling::dependencies::DependencyScanningWorkerFilesystem( + getSharedFilesystemCache(), existingFS); + Instance.getSourceMgr().setFileSystem(depFS); } void SwiftDependencyScanningService::setupCachingDependencyScanningService( @@ -356,6 +422,9 @@ void SwiftDependencyScanningService::setupCachingDependencyScanningService( if (!Instance.getInvocation().getFrontendOptions().EnableCAS) return; + // Setup CAS. + CAS = Instance.getSharedCASInstance(); + // Add SDKSetting file. SmallString<256> SDKSettingPath; llvm::sys::path::append( @@ -376,7 +445,8 @@ void SwiftDependencyScanningService::setupCachingDependencyScanningService( CommonDependencyFiles.emplace_back(F->path().str()); } - auto CachingFS = llvm::cas::createCachingOnDiskFileSystem(Instance.getObjectStore()); + auto CachingFS = + llvm::cas::createCachingOnDiskFileSystem(Instance.getObjectStore()); if (!CachingFS) { Instance.getDiags().diagnose(SourceLoc(), diag::error_create_cas, "CachingOnDiskFS", @@ -389,11 +459,18 @@ void SwiftDependencyScanningService::setupCachingDependencyScanningService( CASOpts.CASPath = Instance.getInvocation().getFrontendOptions().CASPath; CASOpts.ensurePersistentCAS(); + UseClangIncludeTree = + Instance.getInvocation().getClangImporterOptions().UseClangIncludeTree; + const clang::tooling::dependencies::ScanningOutputFormat ClangScanningFormat = + UseClangIncludeTree + ? clang::tooling::dependencies::ScanningOutputFormat::FullIncludeTree + : clang::tooling::dependencies::ScanningOutputFormat::FullTree; + ClangScanningService.emplace( clang::tooling::dependencies::ScanningMode::DependencyDirectivesScan, - clang::tooling::dependencies::ScanningOutputFormat::FullTree, CASOpts, - Instance.getSharedCASInstance(), Instance.getSharedCacheInstance(), - CacheFS, + ClangScanningFormat, CASOpts, Instance.getSharedCASInstance(), + Instance.getSharedCacheInstance(), + UseClangIncludeTree ? nullptr : CacheFS, /* ReuseFileManager */ false, /* OptimizeArgs */ false); } diff --git a/lib/AST/ModuleLoader.cpp b/lib/AST/ModuleLoader.cpp index 643082931a7dc..5480c40ed6268 100644 --- a/lib/AST/ModuleLoader.cpp +++ b/lib/AST/ModuleLoader.cpp @@ -94,7 +94,9 @@ static bool findOverlayFilesInDirectory(ASTContext &ctx, StringRef path, callback(file); } - if (error && error != std::errc::no_such_file_or_directory) { + // A CAS file list returns operation not permitted on directory iterations. + if (error && error != std::errc::no_such_file_or_directory && + error != std::errc::operation_not_permitted) { ctx.Diags.diagnose(diagLoc, diag::cannot_list_swiftcrossimport_dir, moduleName, error.message(), path); } diff --git a/lib/ClangImporter/ClangImporter.cpp b/lib/ClangImporter/ClangImporter.cpp index 3688e347e8c7c..55b175d6836a2 100644 --- a/lib/ClangImporter/ClangImporter.cpp +++ b/lib/ClangImporter/ClangImporter.cpp @@ -1729,7 +1729,8 @@ ClangImporter::cloneCompilerInstanceForPrecompiling() { auto &FrontendOpts = invocation->getFrontendOpts(); FrontendOpts.DisableFree = false; - FrontendOpts.Inputs.clear(); + if (FrontendOpts.CASIncludeTreeID.empty()) + FrontendOpts.Inputs.clear(); auto clonedInstance = std::make_unique( Impl.Instance->getPCHContainerOperations(), @@ -1760,7 +1761,8 @@ bool ClangImporter::emitBridgingPCH( auto inputFile = clang::FrontendInputFile(headerPath, language); auto &FrontendOpts = invocation.getFrontendOpts(); - FrontendOpts.Inputs = {inputFile}; + if (invocation.getFrontendOpts().CASIncludeTreeID.empty()) + FrontendOpts.Inputs = {inputFile}; FrontendOpts.OutputFile = outputPCHPath.str(); FrontendOpts.ProgramAction = clang::frontend::GeneratePCH; @@ -1792,7 +1794,8 @@ bool ClangImporter::runPreprocessor( auto inputFile = clang::FrontendInputFile(inputPath, language); auto &FrontendOpts = invocation.getFrontendOpts(); - FrontendOpts.Inputs = {inputFile}; + if (invocation.getFrontendOpts().CASIncludeTreeID.empty()) + FrontendOpts.Inputs = {inputFile}; FrontendOpts.OutputFile = outputPath.str(); FrontendOpts.ProgramAction = clang::frontend::PrintPreprocessedInput; @@ -1815,11 +1818,13 @@ bool ClangImporter::emitPrecompiledModule( auto language = getLanguageFromOptions(LangOpts); auto &FrontendOpts = invocation.getFrontendOpts(); - auto inputFile = clang::FrontendInputFile( - moduleMapPath, clang::InputKind( - language, clang::InputKind::ModuleMap, false), - FrontendOpts.IsSystemModule); - FrontendOpts.Inputs = {inputFile}; + if (invocation.getFrontendOpts().CASIncludeTreeID.empty()) { + auto inputFile = clang::FrontendInputFile( + moduleMapPath, + clang::InputKind(language, clang::InputKind::ModuleMap, false), + FrontendOpts.IsSystemModule); + FrontendOpts.Inputs = {inputFile}; + } FrontendOpts.OriginalModuleMap = moduleMapPath.str(); FrontendOpts.OutputFile = outputPath.str(); FrontendOpts.ProgramAction = clang::frontend::GenerateModule; @@ -1847,7 +1852,8 @@ bool ClangImporter::dumpPrecompiledModule( clang::Language::Unknown, clang::InputKind::Precompiled, false)); auto &FrontendOpts = invocation.getFrontendOpts(); - FrontendOpts.Inputs = {inputFile}; + if (invocation.getFrontendOpts().CASIncludeTreeID.empty()) + FrontendOpts.Inputs = {inputFile}; FrontendOpts.OutputFile = outputPath.str(); auto action = std::make_unique(); diff --git a/lib/ClangImporter/ClangModuleDependencyScanner.cpp b/lib/ClangImporter/ClangModuleDependencyScanner.cpp index dc86bc5808692..bee9092c22c9d 100644 --- a/lib/ClangImporter/ClangModuleDependencyScanner.cpp +++ b/lib/ClangImporter/ClangModuleDependencyScanner.cpp @@ -20,6 +20,7 @@ #include "swift/ClangImporter/ClangImporter.h" #include "clang/Basic/Diagnostic.h" #include "clang/Frontend/CompilerInvocation.h" +#include "clang/Frontend/FrontendOptions.h" #include "clang/Tooling/DependencyScanning/DependencyScanningService.h" #include "clang/Tooling/DependencyScanning/DependencyScanningTool.h" #include "llvm/ADT/STLExtras.h" @@ -237,20 +238,32 @@ void ClangImporter::recordModuleDependencies( ? clangModuleDep.CASFileSystemRootID->toString() : ""; - if (!RootID.empty()) { + std::string IncludeTree = + clangModuleDep.IncludeTreeID ? *clangModuleDep.IncludeTreeID : ""; + + if (!RootID.empty() || !IncludeTree.empty()) { swiftArgs.push_back("-enable-cas"); swiftArgs.push_back("-cas-path"); swiftArgs.push_back(ctx.ClangImporterOpts.CASPath); + } + + if (!RootID.empty()) { swiftArgs.push_back("-cas-fs"); swiftArgs.push_back(RootID); } + if (!IncludeTree.empty()) { + swiftArgs.push_back("-clang-include-tree"); + swiftArgs.push_back("-clang-include-tree-root"); + swiftArgs.push_back(IncludeTree); + } + // Module-level dependencies. llvm::StringSet<> alreadyAddedModules; auto dependencies = ModuleDependencyInfo::forClangModule( pcmPath, clangModuleDep.ClangModuleMapFile, clangModuleDep.ID.ContextHash, swiftArgs, fileDeps, capturedPCMArgs, - RootID, /*module-cache-key*/ ""); + RootID, IncludeTree, /*module-cache-key*/ ""); for (const auto &moduleName : clangModuleDep.ClangModuleDeps) { dependencies.addModuleImport(moduleName.ModuleName, &alreadyAddedModules); // It is safe to assume that all dependencies of a Clang module are Clang modules. @@ -264,6 +277,85 @@ void ClangImporter::recordModuleDependencies( } } +void ClangImporter::recordBridgingHeaderOptions( + ModuleDependencyInfo &MDI, + const clang::tooling::dependencies::TranslationUnitDeps &deps) { + auto &ctx = Impl.SwiftContext; + + std::vector swiftArgs; + auto addClangArg = [&](Twine arg) { + swiftArgs.push_back("-Xcc"); + swiftArgs.push_back(arg.str()); + }; + + // We are using Swift frontend mode. + swiftArgs.push_back("-frontend"); + + // Swift frontend action: -emit-pcm + swiftArgs.push_back("-emit-pch"); + + // We pass the entire argument list via -Xcc, so the invocation should + // use extra clang options alone. + swiftArgs.push_back("-only-use-extra-clang-opts"); + + // Ensure that the resulting PCM build invocation uses Clang frontend + // directly + swiftArgs.push_back("-direct-clang-cc1-module-build"); + + // Add args reported by the scanner. + + // Round-trip clang args to canonicalize and clear the options that swift + // compiler doesn't need. + clang::CompilerInvocation depsInvocation; + clang::DiagnosticsEngine clangDiags(new clang::DiagnosticIDs(), + new clang::DiagnosticOptions(), + new clang::IgnoringDiagConsumer()); + + llvm::SmallVector clangArgs; + llvm::for_each(deps.Commands[0].Arguments, [&](const std::string &Arg) { + clangArgs.push_back(Arg.c_str()); + }); + + bool success = clang::CompilerInvocation::CreateFromArgs( + depsInvocation, clangArgs, clangDiags); + (void)success; + assert(success && "clang option from dep scanner round trip failed"); + + // Clear the cache key for module. The module key is computed from clang + // invocation, not swift invocation. + depsInvocation.getFrontendOpts().ProgramAction = + clang::frontend::ActionKind::GeneratePCH; + depsInvocation.getFrontendOpts().ModuleCacheKeys.clear(); + depsInvocation.getFrontendOpts().OutputFile = ""; + + llvm::BumpPtrAllocator allocator; + llvm::StringSaver saver(allocator); + clangArgs.clear(); + depsInvocation.generateCC1CommandLine( + clangArgs, + [&saver](const llvm::Twine &T) { return saver.save(T).data(); }); + + llvm::for_each(clangArgs, addClangArg); + + if (!ctx.ClangImporterOpts.CASPath.empty()) { + swiftArgs.push_back("-enable-cas"); + swiftArgs.push_back("-cas-path"); + swiftArgs.push_back(ctx.ClangImporterOpts.CASPath); + } + + if (auto Tree = deps.IncludeTreeID) { + swiftArgs.push_back("-clang-include-tree"); + swiftArgs.push_back("-clang-include-tree-root"); + swiftArgs.push_back(*Tree); + } + if (auto CASFS = deps.CASFileSystemRootID) { + swiftArgs.push_back("-cas-fs"); + swiftArgs.push_back(*CASFS); + } + + MDI.updateBridgingHeaderCommandLine(swiftArgs); +} + // The Swift compiler does not have a concept of a working directory. // It is instead handled by the Swift driver by resolving relative paths // according to the driver's notion of a working directory. On the other hand, @@ -401,6 +493,11 @@ bool ClangImporter::addBridgingHeaderDependencies( alreadyAddedModules); } + if (auto TreeID = clangModuleDependencies->IncludeTreeID) + targetModule.addBridgingHeaderIncludeTree(*TreeID); + + recordBridgingHeaderOptions(targetModule, *clangModuleDependencies); + // Update the cache with the new information for the module. cache.updateDependency({moduleName.str(), moduleKind}, targetModule); diff --git a/lib/DependencyScan/DependencyScanningTool.cpp b/lib/DependencyScan/DependencyScanningTool.cpp index 0c150f93ad6e1..6f67370390588 100644 --- a/lib/DependencyScan/DependencyScanningTool.cpp +++ b/lib/DependencyScan/DependencyScanningTool.cpp @@ -236,9 +236,6 @@ DependencyScanningTool::initCompilerInstanceForScan( auto Instance = std::make_unique(); Instance->addDiagnosticConsumer(&CDC); - // Wrap the filesystem with a caching `DependencyScanningWorkerFilesystem` - ScanningService->overlaySharedFilesystemCacheForCompilation(*Instance); - // Basic error checking on the arguments if (CommandArgs.empty()) { Instance->getDiags().diagnose(SourceLoc(), diag::error_no_frontend_args); diff --git a/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp b/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp index 99759a11653b2..63fff992acbe9 100644 --- a/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp +++ b/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp @@ -240,14 +240,14 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi extraPCMArgsArrayID, contextHashID, isFramework, bridgingHeaderFileID, sourceFilesArrayID, bridgingSourceFilesArrayID, bridgingModuleDependenciesArrayID, overlayDependencyIDArrayID, - CASFileSystemRootID, moduleCacheKeyID; + CASFileSystemRootID, bridgingHeaderIncludeTreeID, moduleCacheKeyID; SwiftInterfaceModuleDetailsLayout::readRecord( Scratch, outputPathFileID, interfaceFileID, compiledModuleCandidatesArrayID, buildCommandLineArrayID, extraPCMArgsArrayID, contextHashID, isFramework, bridgingHeaderFileID, sourceFilesArrayID, bridgingSourceFilesArrayID, bridgingModuleDependenciesArrayID, overlayDependencyIDArrayID, - CASFileSystemRootID, moduleCacheKeyID); + CASFileSystemRootID, bridgingHeaderIncludeTreeID, moduleCacheKeyID); auto outputModulePath = getIdentifier(outputPathFileID); if (!outputModulePath) @@ -337,6 +337,14 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi llvm::report_fatal_error("Bad overlay dependencies: no qualified dependencies"); moduleDep.setOverlayDependencies(overlayModuleDependencyIDs.value()); + // Add bridging header include tree + auto bridgingHeaderIncludeTree = + getIdentifier(bridgingHeaderIncludeTreeID); + if (!bridgingHeaderIncludeTree) + llvm::report_fatal_error("Bad bridging header include tree"); + if (!bridgingHeaderIncludeTree->empty()) + moduleDep.addBridgingHeaderIncludeTree(*bridgingHeaderIncludeTree); + cache.recordDependency(currentModuleName, std::move(moduleDep), getContextHash()); hasCurrentModule = false; @@ -355,12 +363,14 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi unsigned extraPCMArgsArrayID, bridgingHeaderFileID, sourceFilesArrayID, bridgingSourceFilesArrayID, bridgingModuleDependenciesArrayID, overlayDependencyIDArrayID, CASFileSystemRootID, - buildCommandLineArrayID; + bridgingHeaderIncludeTreeID, buildCommandLineArrayID, + bridgingHeaderBuildCommandLineArrayID; SwiftSourceModuleDetailsLayout::readRecord( Scratch, extraPCMArgsArrayID, bridgingHeaderFileID, sourceFilesArrayID, bridgingSourceFilesArrayID, bridgingModuleDependenciesArrayID, overlayDependencyIDArrayID, - CASFileSystemRootID, buildCommandLineArrayID); + CASFileSystemRootID, bridgingHeaderIncludeTreeID, + buildCommandLineArrayID, bridgingHeaderBuildCommandLineArrayID); auto extraPCMArgs = getStringArray(extraPCMArgsArrayID); if (!extraPCMArgs) @@ -378,10 +388,18 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi std::vector buildCommandRefs; for (auto &arg : *commandLine) buildCommandRefs.push_back(arg); + std::vector bridgingHeaderBuildCommandRefs; + auto bridgingHeaderCommandLine = + getStringArray(bridgingHeaderBuildCommandLineArrayID); + if (!bridgingHeaderCommandLine) + llvm::report_fatal_error("Bad bridging header command line"); + for (auto &arg : *bridgingHeaderCommandLine) + bridgingHeaderBuildCommandRefs.push_back(arg); // Form the dependencies storage object auto moduleDep = ModuleDependencyInfo::forSwiftSourceModule( - *rootFileSystemID, buildCommandRefs, extraPCMRefs); + *rootFileSystemID, buildCommandRefs, bridgingHeaderBuildCommandRefs, + extraPCMRefs); // Add dependencies of this module for (const auto &moduleName : *currentModuleImports) @@ -424,6 +442,14 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi llvm::report_fatal_error("Bad overlay dependencies: no qualified dependencies"); moduleDep.setOverlayDependencies(overlayModuleDependencyIDs.value()); + // Add bridging header include tree + auto bridgingHeaderIncludeTree = + getIdentifier(bridgingHeaderIncludeTreeID); + if (!bridgingHeaderIncludeTree) + llvm::report_fatal_error("Bad bridging header include tree"); + if (!bridgingHeaderIncludeTree->empty()) + moduleDep.addBridgingHeaderIncludeTree(*bridgingHeaderIncludeTree); + cache.recordDependency(currentModuleName, std::move(moduleDep), getContextHash()); hasCurrentModule = false; @@ -507,12 +533,13 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi cache.configureForContextHash(getContextHash()); unsigned pcmOutputPathID, moduleMapPathID, contextHashID, commandLineArrayID, fileDependenciesArrayID, capturedPCMArgsArrayID, CASFileSystemRootID, - moduleCacheKeyID; + clangIncludeTreeRootID, moduleCacheKeyID; ClangModuleDetailsLayout::readRecord(Scratch, pcmOutputPathID, moduleMapPathID, contextHashID, commandLineArrayID, fileDependenciesArrayID, capturedPCMArgsArrayID, CASFileSystemRootID, + clangIncludeTreeRootID, moduleCacheKeyID); auto pcmOutputPath = getIdentifier(pcmOutputPathID); if (!pcmOutputPath) @@ -535,14 +562,18 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi auto rootFileSystemID = getIdentifier(CASFileSystemRootID); if (!rootFileSystemID) llvm::report_fatal_error("Bad CASFileSystem RootID"); + auto clangIncludeTreeRoot = getIdentifier(clangIncludeTreeRootID); + if (!clangIncludeTreeRoot) + llvm::report_fatal_error("Bad clang include tree ID"); auto moduleCacheKey = getIdentifier(moduleCacheKeyID); if (!moduleCacheKeyID) llvm::report_fatal_error("Bad moduleCacheKey"); // Form the dependencies storage object - auto moduleDep = ModuleDependencyInfo::forClangModule(*pcmOutputPath, - *moduleMapPath, *contextHash, *commandLineArgs, *fileDependencies, - *capturedPCMArgs, *rootFileSystemID, *moduleCacheKey); + auto moduleDep = ModuleDependencyInfo::forClangModule( + *pcmOutputPath, *moduleMapPath, *contextHash, *commandLineArgs, + *fileDependencies, *capturedPCMArgs, *rootFileSystemID, + *clangIncludeTreeRoot, *moduleCacheKey); // Add dependencies of this module for (const auto &moduleName : *currentModuleImports) @@ -681,6 +712,7 @@ enum ModuleIdentifierArrayKind : uint8_t { BridgingSourceFiles, BridgingModuleDependencies, SwiftOverlayDependencyIDs, + BridgingHeaderBuildCommandLine, NonPathCommandLine, FileDependencies, CapturedPCMArgs, @@ -899,6 +931,7 @@ void ModuleDependenciesCacheSerializer::writeModuleInfo(ModuleDependencyID modul getArrayID(moduleID, ModuleIdentifierArrayKind::BridgingModuleDependencies), getArrayID(moduleID, ModuleIdentifierArrayKind::SwiftOverlayDependencyIDs), getIdentifier(swiftTextDeps->textualModuleDetails.CASFileSystemRootID), + getIdentifier(swiftTextDeps->textualModuleDetails.bridgingHeaderIncludeTreeRoot), getIdentifier(swiftTextDeps->moduleCacheKey)); break; } @@ -921,7 +954,9 @@ void ModuleDependenciesCacheSerializer::writeModuleInfo(ModuleDependencyID modul getArrayID(moduleID, ModuleIdentifierArrayKind::BridgingModuleDependencies), getArrayID(moduleID, ModuleIdentifierArrayKind::SwiftOverlayDependencyIDs), getIdentifier(swiftSourceDeps->textualModuleDetails.CASFileSystemRootID), - getArrayID(moduleID, ModuleIdentifierArrayKind::BuildCommandLine)); + getIdentifier(swiftSourceDeps->textualModuleDetails.bridgingHeaderIncludeTreeRoot), + getArrayID(moduleID, ModuleIdentifierArrayKind::BuildCommandLine), + getArrayID(moduleID, ModuleIdentifierArrayKind::BridgingHeaderBuildCommandLine)); break; } case swift::ModuleDependencyKind::SwiftBinary: { @@ -963,6 +998,7 @@ void ModuleDependenciesCacheSerializer::writeModuleInfo(ModuleDependencyID modul getArrayID(moduleID, ModuleIdentifierArrayKind::FileDependencies), getArrayID(moduleID, ModuleIdentifierArrayKind::CapturedPCMArgs), getIdentifier(clangDeps->CASFileSystemRootID), + getIdentifier(clangDeps->clangIncludeTreeRoot), getIdentifier(clangDeps->moduleCacheKey)); break; @@ -1097,6 +1133,8 @@ void ModuleDependenciesCacheSerializer::collectStringsAndArrays( moduleID, ModuleIdentifierArrayKind::SwiftOverlayDependencyIDs, swiftTextDeps->textualModuleDetails.swiftOverlayDependencies); addIdentifier(swiftTextDeps->textualModuleDetails.CASFileSystemRootID); + addIdentifier( + swiftTextDeps->textualModuleDetails.bridgingHeaderIncludeTreeRoot); addIdentifier(swiftTextDeps->moduleCacheKey); break; } @@ -1140,6 +1178,10 @@ void ModuleDependenciesCacheSerializer::collectStringsAndArrays( addStringArray( moduleID, ModuleIdentifierArrayKind::BuildCommandLine, swiftSourceDeps->textualModuleDetails.buildCommandLine); + addStringArray( + moduleID, ModuleIdentifierArrayKind::BridgingHeaderBuildCommandLine, + swiftSourceDeps->textualModuleDetails + .bridgingHeaderBuildCommandLine); addIdentifier( swiftSourceDeps->textualModuleDetails.CASFileSystemRootID); break; @@ -1157,6 +1199,7 @@ void ModuleDependenciesCacheSerializer::collectStringsAndArrays( addStringArray(moduleID, ModuleIdentifierArrayKind::CapturedPCMArgs, clangDeps->capturedPCMArgs); addIdentifier(clangDeps->CASFileSystemRootID); + addIdentifier(clangDeps->clangIncludeTreeRoot); addIdentifier(clangDeps->moduleCacheKey); break; } diff --git a/lib/DependencyScan/ScanDependencies.cpp b/lib/DependencyScan/ScanDependencies.cpp index 1dae3fef61877..d98120d30fa11 100644 --- a/lib/DependencyScan/ScanDependencies.cpp +++ b/lib/DependencyScan/ScanDependencies.cpp @@ -31,6 +31,7 @@ #include "swift/DependencyScan/ScanDependencies.h" #include "swift/DependencyScan/SerializedModuleDependencyCacheFormat.h" #include "swift/DependencyScan/StringUtils.h" +#include "swift/Frontend/CachingUtils.h" #include "swift/Frontend/CompileJobCacheKey.h" #include "swift/Frontend/Frontend.h" #include "swift/Frontend/FrontendOptions.h" @@ -46,7 +47,6 @@ #include "llvm/ADT/StringSet.h" #include "llvm/CAS/ActionCache.h" #include "llvm/CAS/CASReference.h" -#include "llvm/CAS/HierarchicalTreeBuilder.h" #include "llvm/CAS/ObjectStore.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Error.h" @@ -257,28 +257,6 @@ computeTransitiveClosureOfExplicitDependencies( return result; } -static llvm::Expected -mergeCASFileSystem(llvm::cas::ObjectStore &CAS, ArrayRef FSRoots) { - assert(!FSRoots.empty() && "no root ID provided"); - - llvm::cas::HierarchicalTreeBuilder Builder; - for (auto &Root : FSRoots) { - auto ID = CAS.parseID(Root); - if (!ID) - return ID.takeError(); - - auto Ref = CAS.getReference(*ID); - assert(Ref && "CASFSRootID is missing from the ObjectStore instance"); - Builder.pushTreeContent(*Ref, ""); - } - - auto NewRoot = Builder.create(CAS); - if (!NewRoot) - return NewRoot.takeError(); - - return NewRoot->getRef(); -} - static llvm::Expected updateModuleCacheKey(ModuleDependencyInfo &depInfo, llvm::cas::ObjectStore &CAS) { @@ -324,6 +302,10 @@ static llvm::Error resolveExplicitModuleInputs( if (auto ID = resolvingDepInfo.getCASFSRootID()) rootIDs.push_back(*ID); + std::vector includeTrees; + if (auto ID = resolvingDepInfo.getClangIncludeTree()) + includeTrees.push_back(*ID); + std::vector commandLine = resolvingDepInfo.getCommandline(); for (const auto &depModuleID : dependencies) { const auto optionalDepInfo = @@ -358,12 +340,18 @@ static llvm::Error resolveExplicitModuleInputs( case swift::ModuleDependencyKind::Clang: { auto clangDepDetails = depInfo->getAsClangModule(); assert(clangDepDetails && "Expected Clang Module dependency."); - commandLine.push_back("-Xcc"); - commandLine.push_back("-fmodule-file=" + depModuleID.first + "=" + - clangDepDetails->pcmOutputPath); - commandLine.push_back("-Xcc"); - commandLine.push_back("-fmodule-map-file=" + - clangDepDetails->moduleMapFile); + if (!resolvingDepInfo.isClangModule()) { + commandLine.push_back("-Xcc"); + commandLine.push_back("-fmodule-file=" + depModuleID.first + "=" + + clangDepDetails->pcmOutputPath); + if (!instance.getInvocation() + .getClangImporterOptions() + .UseClangIncludeTree) { + commandLine.push_back("-Xcc"); + commandLine.push_back("-fmodule-map-file=" + + clangDepDetails->moduleMapFile); + } + } if (!clangDepDetails->moduleCacheKey.empty()) { auto appendXclang = [&]() { if (!resolvingDepInfo.isClangModule()) { @@ -384,22 +372,31 @@ static llvm::Error resolveExplicitModuleInputs( // Only need to merge the CASFS from clang importer. if (auto ID = depInfo->getCASFSRootID()) rootIDs.push_back(*ID); + if (auto ID = depInfo->getClangIncludeTree()) + includeTrees.push_back(*ID); } break; case swift::ModuleDependencyKind::SwiftSource: { auto sourceDepDetails = depInfo->getAsSwiftSourceModule(); assert(sourceDepDetails && "Expected source dependency"); - if (!sourceDepDetails->textualModuleDetails.bridgingSourceFiles.empty()) { - if (auto tracker = - cache.getScanService().createSwiftDependencyTracker()) { - tracker->startTracking(); - for (auto &file : sourceDepDetails->textualModuleDetails.bridgingSourceFiles) - tracker->trackFile(file); - auto bridgeRoot = tracker->createTreeFromDependencies(); - if (!bridgeRoot) - return bridgeRoot.takeError(); - rootIDs.push_back(bridgeRoot->getID().toString()); + if (sourceDepDetails->textualModuleDetails.bridgingHeaderIncludeTreeRoot + .empty()) { + if (!sourceDepDetails->textualModuleDetails.bridgingSourceFiles + .empty()) { + if (auto tracker = + cache.getScanService().createSwiftDependencyTracker()) { + tracker->startTracking(); + for (auto &file : + sourceDepDetails->textualModuleDetails.bridgingSourceFiles) + tracker->trackFile(file); + auto bridgeRoot = tracker->createTreeFromDependencies(); + if (!bridgeRoot) + return bridgeRoot.takeError(); + rootIDs.push_back(bridgeRoot->getID().toString()); + } } - } + } else + includeTrees.push_back(sourceDepDetails->textualModuleDetails + .bridgingHeaderIncludeTreeRoot); break; } default: @@ -419,23 +416,46 @@ static llvm::Error resolveExplicitModuleInputs( auto &CASFS = cache.getScanService().getSharedCachingFS(); auto &CAS = CASFS.getCAS(); - // Update CASFS RootID. + // Update build command line. if (resolvingDepInfo.isSwiftInterfaceModule() || resolvingDepInfo.isSwiftSourceModule()) { - auto NewRoot = mergeCASFileSystem(CAS, rootIDs); - if (!NewRoot) - return NewRoot.takeError(); - auto NewID = CAS.getID(*NewRoot).toString(); - dependencyInfoCopy.updateCASFileSystemID(NewID); - // Update with casfs option. std::vector newCommandLine = dependencyInfoCopy.getCommandline(); - newCommandLine.push_back("-cas-fs"); - newCommandLine.push_back(NewID); + for (auto rootID : rootIDs) { + newCommandLine.push_back("-cas-fs"); + newCommandLine.push_back(rootID); + } + + for (auto tree : includeTrees) { + newCommandLine.push_back("-clang-include-tree-root"); + newCommandLine.push_back(tree); + } dependencyInfoCopy.updateCommandLine(newCommandLine); } + if (auto *sourceDep = resolvingDepInfo.getAsSwiftSourceModule()) { + std::vector newCommandLine = + dependencyInfoCopy.getBridgingHeaderCommandline(); + for (auto bridgingDep : + sourceDep->textualModuleDetails.bridgingModuleDependencies) { + auto dep = + cache.findDependency(bridgingDep, ModuleDependencyKind::Clang); + assert(dep && "unknown clang dependency"); + auto *clangDep = (*dep)->getAsClangModule(); + assert(clangDep && "wrong module dependency kind"); + if (!clangDep->moduleCacheKey.empty()) { + newCommandLine.push_back("-Xcc"); + newCommandLine.push_back("-fmodule-file-cache-key"); + newCommandLine.push_back("-Xcc"); + newCommandLine.push_back(clangDep->pcmOutputPath); + newCommandLine.push_back("-Xcc"); + newCommandLine.push_back(clangDep->moduleCacheKey); + } + dependencyInfoCopy.updateBridgingHeaderCommandLine(newCommandLine); + } + } + if (resolvingDepInfo.isClangModule() || resolvingDepInfo.isSwiftInterfaceModule()) { // Compute and update module cache key. @@ -528,11 +548,11 @@ resolveDirectDependencies(CompilerInstance &instance, ModuleDependencyID moduleI // A record of all of the Clang modules referenced from this Swift module. std::vector allClangModules; llvm::StringSet<> knownModules; + auto clangImporter = + static_cast(ctx.getClangModuleLoader()); // If the Swift module has a bridging header, add those dependencies. if (knownDependencies->getBridgingHeader()) { - auto clangImporter = - static_cast(ctx.getClangModuleLoader()); if (!clangImporter->addBridgingHeaderDependencies(moduleID.first, moduleID.second, cache)) { // Grab the updated module dependencies. @@ -643,7 +663,7 @@ static void discoverCrossImportOverlayDependencies( // overlays. StringRef dummyMainName = "DummyMainModuleForResolvingCrossImportOverlays"; auto dummyMainDependencies = - ModuleDependencyInfo::forSwiftSourceModule({}, {}, {}); + ModuleDependencyInfo::forSwiftSourceModule({}, {}, {}, {}); std::for_each(newOverlays.begin(), newOverlays.end(), [&](Identifier modName) { dummyMainDependencies.addModuleImport(modName.str()); @@ -1062,9 +1082,29 @@ static void writeJSON(llvm::raw_ostream &out, writeJSONSingleField(out, "sourceFiles", swiftTextualDeps->bridging_source_files, 6, /*trailingComma=*/true); + if (swiftTextualDeps->bridging_header_include_tree.length != 0) { + writeJSONSingleField(out, "includeTree", + swiftTextualDeps->bridging_header_include_tree, + 6, /*trailingComma=*/true); + } writeJSONSingleField(out, "moduleDependencies", swiftTextualDeps->bridging_module_dependencies, 6, - /*trailingComma=*/false); + /*trailingComma=*/true); + out.indent(6 * 2); + out << "\"commandLine\": [\n"; + for (int i = 0, + count = swiftTextualDeps->bridging_pch_command_line->count; + i < count; ++i) { + const auto &arg = get_C_string( + swiftTextualDeps->bridging_pch_command_line->strings[i]); + out.indent(7 * 2); + out << "\"" << quote(arg) << "\""; + if (i != count - 1) + out << ","; + out << "\n"; + } + out.indent(6 * 2); + out << "]\n"; out.indent(5 * 2); out << (hasOverlayDependencies ? "},\n" : "}\n"); } @@ -1143,6 +1183,10 @@ static void writeJSON(llvm::raw_ostream &out, if (clangDeps->cas_fs_root_id.length != 0) writeJSONSingleField(out, "casFSRootID", clangDeps->cas_fs_root_id, 5, /*trailingComma=*/true); + if (clangDeps->clang_include_tree.length != 0) + writeJSONSingleField(out, "clangIncludeTree", + clangDeps->clang_include_tree, 5, + /*trailingComma=*/true); if (clangDeps->module_cache_key.length != 0) writeJSONSingleField(out, "moduleCacheKey", clangDeps->module_cache_key, 5, @@ -1298,10 +1342,12 @@ generateFullDependencyGraph(CompilerInstance &instance, create_set(swiftTextualDeps->textualModuleDetails.bridgingModuleDependencies), create_set(bridgedOverlayDependencyNames), create_set(swiftTextualDeps->textualModuleDetails.buildCommandLine), + create_set(swiftTextualDeps->textualModuleDetails.bridgingHeaderBuildCommandLine), create_set(swiftTextualDeps->textualModuleDetails.extraPCMArgs), create_clone(swiftTextualDeps->contextHash.c_str()), swiftTextualDeps->isFramework, create_clone(swiftTextualDeps->textualModuleDetails.CASFileSystemRootID.c_str()), + create_clone(swiftTextualDeps->textualModuleDetails.bridgingHeaderIncludeTreeRoot.c_str()), create_clone(swiftTextualDeps->moduleCacheKey.c_str())}; } else if (swiftSourceDeps) { swiftscan_string_ref_t moduleInterfacePath = create_null(); @@ -1324,10 +1370,12 @@ generateFullDependencyGraph(CompilerInstance &instance, create_set(swiftSourceDeps->textualModuleDetails.bridgingModuleDependencies), create_set(bridgedOverlayDependencyNames), create_set(swiftSourceDeps->textualModuleDetails.buildCommandLine), + create_set(swiftSourceDeps->textualModuleDetails.bridgingHeaderBuildCommandLine), create_set(swiftSourceDeps->textualModuleDetails.extraPCMArgs), /*contextHash*/create_null(), /*isFramework*/false, /*CASFS*/create_clone(swiftSourceDeps->textualModuleDetails.CASFileSystemRootID.c_str()), + /*IncludeTree*/create_clone(swiftSourceDeps->textualModuleDetails.bridgingHeaderIncludeTreeRoot.c_str()), /*CacheKey*/create_clone("")}; } else if (swiftPlaceholderDeps) { details->kind = SWIFTSCAN_DEPENDENCY_INFO_SWIFT_PLACEHOLDER; @@ -1352,6 +1400,7 @@ generateFullDependencyGraph(CompilerInstance &instance, create_set(clangDeps->nonPathCommandLine), create_set(clangDeps->capturedPCMArgs), create_clone(clangDeps->CASFileSystemRootID.c_str()), + create_clone(clangDeps->clangIncludeTreeRoot.c_str()), create_clone(clangDeps->moduleCacheKey.c_str()) }; } @@ -1573,11 +1622,22 @@ static ModuleDependencyInfo identifyMainModuleDependencies( ExtraPCMArgs.insert(ExtraPCMArgs.begin(), {"-Xcc", "-target", "-Xcc", instance.getASTContext().LangOpts.Target.str()}); - auto mainDependencies = - ModuleDependencyInfo::forSwiftSourceModule({}, {}, ExtraPCMArgs); - if (tracker) + std::string rootID; + if (tracker) { tracker->startTracking(); + for (auto fileUnit : mainModule->getFiles()) { + auto sf = dyn_cast(fileUnit); + if (!sf) + continue; + tracker->trackFile(sf->getFilename()); + } + auto root = cantFail(tracker->createTreeFromDependencies()); + rootID = root.getID().toString(); + } + + auto mainDependencies = + ModuleDependencyInfo::forSwiftSourceModule(rootID, {}, {}, ExtraPCMArgs); // Compute Implicit dependencies of the main module { @@ -1588,8 +1648,6 @@ static ModuleDependencyInfo identifyMainModuleDependencies( continue; mainDependencies.addModuleImport(*sf, alreadyAddedModules); - if (tracker) - tracker->trackFile(sf->getFilename()); } const auto &importInfo = mainModule->getImplicitImportInfo(); @@ -1637,11 +1695,6 @@ static ModuleDependencyInfo identifyMainModuleDependencies( } } - if (tracker) { - auto rootID = cantFail(tracker->createTreeFromDependencies()); - mainDependencies.updateCASFileSystemID(rootID.getID().toString()); - } - return mainDependencies; } diff --git a/lib/Frontend/ArgsToFrontendOptionsConverter.cpp b/lib/Frontend/ArgsToFrontendOptionsConverter.cpp index 4074acd56a570..b7ff8bb6fb712 100644 --- a/lib/Frontend/ArgsToFrontendOptionsConverter.cpp +++ b/lib/Frontend/ArgsToFrontendOptionsConverter.cpp @@ -354,8 +354,11 @@ bool ArgsToFrontendOptionsConverter::convert( Opts.EnableCAS = Args.hasArg(OPT_enable_cas); Opts.CASPath = Args.getLastArgValue(OPT_cas_path, llvm::cas::getDefaultOnDiskCASPath()); - Opts.CASFSRootID = Args.getLastArgValue(OPT_cas_fs); - if (Opts.EnableCAS && Opts.CASFSRootID.empty() && + Opts.CASFSRootIDs = Args.getAllArgValues(OPT_cas_fs); + Opts.ClangIncludeTrees = Args.getAllArgValues(OPT_clang_include_tree_root); + + if (Opts.EnableCAS && Opts.CASFSRootIDs.empty() && + Opts.ClangIncludeTrees.empty() && FrontendOptions::supportCompilationCaching(Opts.RequestedAction)) { if (!Args.hasArg(OPT_allow_unstable_cache_key_for_testing)) { Diags.diagnose(SourceLoc(), diag::error_caching_no_cas_fs); diff --git a/lib/Frontend/CachingUtils.cpp b/lib/Frontend/CachingUtils.cpp index 20f9bb262d550..cb6435d887120 100644 --- a/lib/Frontend/CachingUtils.cpp +++ b/lib/Frontend/CachingUtils.cpp @@ -15,13 +15,19 @@ #include "swift/AST/DiagnosticsFrontend.h" #include "swift/Basic/FileTypes.h" #include "swift/Frontend/CompileJobCacheKey.h" +#include "clang/CAS/IncludeTree.h" #include "clang/Frontend/CompileJobCacheResult.h" #include "llvm/ADT/STLExtras.h" #include "llvm/CAS/BuiltinUnifiedCASDatabases.h" +#include "llvm/CAS/CASFileSystem.h" +#include "llvm/CAS/HierarchicalTreeBuilder.h" #include "llvm/CAS/ObjectStore.h" +#include "llvm/CAS/TreeEntry.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/Error.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/Path.h" +#include "llvm/Support/VirtualFileSystem.h" #include "llvm/Support/VirtualOutputBackends.h" #include "llvm/Support/VirtualOutputFile.h" #include @@ -342,6 +348,76 @@ Error storeCachedCompilerOutput(llvm::cas::ObjectStore &CAS, return Error::success(); } +static llvm::Error createCASObjectNotFoundError(const llvm::cas::CASID &ID) { + return createStringError(llvm::inconvertibleErrorCode(), + "CASID missing from Object Store " + ID.toString()); +} + +static Expected mergeCASFileSystem(ObjectStore &CAS, + ArrayRef FSRoots) { + llvm::cas::HierarchicalTreeBuilder Builder; + for (auto &Root : FSRoots) { + auto ID = CAS.parseID(Root); + if (!ID) + return ID.takeError(); + + auto Ref = CAS.getReference(*ID); + if (!Ref) + return createCASObjectNotFoundError(*ID); + Builder.pushTreeContent(*Ref, ""); + } + + auto NewRoot = Builder.create(CAS); + if (!NewRoot) + return NewRoot.takeError(); + + return NewRoot->getRef(); +} + +Expected> +createCASFileSystem(ObjectStore &CAS, ArrayRef FSRoots, + ArrayRef IncludeTrees) { + assert(!FSRoots.empty() || !IncludeTrees.empty() && "no root ID provided"); + if (FSRoots.size() == 1 && IncludeTrees.empty()) { + auto ID = CAS.parseID(FSRoots.front()); + if (!ID) + return ID.takeError(); + auto Ref = CAS.getReference(*ID); + if (!Ref) + return createCASObjectNotFoundError(*ID); + } + + auto NewRoot = mergeCASFileSystem(CAS, FSRoots); + if (!NewRoot) + return NewRoot.takeError(); + + auto FS = createCASFileSystem(CAS, CAS.getID(*NewRoot)); + if (!FS) + return FS.takeError(); + + auto CASFS = makeIntrusiveRefCnt(std::move(*FS)); + // Push all Include File System onto overlay. + for (auto &Tree : IncludeTrees) { + auto ID = CAS.parseID(Tree); + if (!ID) + return ID.takeError(); + + auto Ref = CAS.getReference(*ID); + if (!Ref) + return createCASObjectNotFoundError(*ID); + auto IT = clang::cas::IncludeTreeRoot::get(CAS, *Ref); + if (!IT) + return IT.takeError(); + + auto ITFS = clang::cas::createIncludeTreeFileSystem(*IT); + if (!ITFS) + return ITFS.takeError(); + CASFS->pushOverlay(std::move(*ITFS)); + } + + return CASFS; +} + namespace cas { CachingTool::CachingTool(StringRef Path) { diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp index 60ed04b9859fd..1cc523b744b6d 100644 --- a/lib/Frontend/CompilerInvocation.cpp +++ b/lib/Frontend/CompilerInvocation.cpp @@ -1436,12 +1436,15 @@ static bool ParseClangImporterArgs(ClangImporterOptions &Opts, Opts.ExtraArgs.push_back("-fdebug-prefix-map=" + Val); } - if (!workingDirectory.empty()) { - // Provide a working directory to Clang as well if there are any -Xcc - // options, in case some of them are search-related. But do it at the - // beginning, so that an explicit -Xcc -working-directory will win. - Opts.ExtraArgs.insert(Opts.ExtraArgs.begin(), - {"-working-directory", workingDirectory.str()}); + if (FrontendOpts.CASFSRootIDs.empty() && + FrontendOpts.ClangIncludeTrees.empty()) { + if (!workingDirectory.empty()) { + // Provide a working directory to Clang as well if there are any -Xcc + // options, in case some of them are search-related. But do it at the + // beginning, so that an explicit -Xcc -working-directory will win. + Opts.ExtraArgs.insert(Opts.ExtraArgs.begin(), + {"-working-directory", workingDirectory.str()}); + } } Opts.DumpClangDiagnostics |= Args.hasArg(OPT_dump_clang_diagnostics); @@ -1473,6 +1476,7 @@ static bool ParseClangImporterArgs(ClangImporterOptions &Opts, Opts.ExtraArgsOnly |= Args.hasArg(OPT_extra_clang_options_only); Opts.DirectClangCC1ModuleBuild |= Args.hasArg(OPT_direct_clang_cc1_module_build); + Opts.UseClangIncludeTree |= Args.hasArg(OPT_clang_include_tree); if (const Arg *A = Args.getLastArg(OPT_pch_output_dir)) { Opts.PrecompiledHeaderOutputDir = A->getValue(); diff --git a/lib/Frontend/Frontend.cpp b/lib/Frontend/Frontend.cpp index ab2ed1d842b7e..1168c2cba8239 100644 --- a/lib/Frontend/Frontend.cpp +++ b/lib/Frontend/Frontend.cpp @@ -439,22 +439,6 @@ bool CompilerInstance::setupCASIfNeeded(ArrayRef Args) { ResultCache = std::move(MaybeCache->second); // create baseline key. - llvm::Optional FSRef; - if (!Opts.CASFSRootID.empty()) { - auto CASFSID = CAS->parseID(Opts.CASFSRootID); - if (!CASFSID) { - Diagnostics.diagnose(SourceLoc(), diag::error_cas, - toString(CASFSID.takeError())); - return true; - } - FSRef = CAS->getReference(*CASFSID); - if (!FSRef) { - Diagnostics.diagnose(SourceLoc(), diag::error_cas, - "-cas-fs value does not exist in CAS"); - return true; - } - } - auto BaseKey = createCompileJobBaseCacheKey(*CAS, Args); if (!BaseKey) { Diagnostics.diagnose(SourceLoc(), diag::error_cas, @@ -561,19 +545,14 @@ bool CompilerInstance::setup(const CompilerInvocation &Invoke, bool CompilerInstance::setUpVirtualFileSystemOverlays() { if (Invocation.getFrontendOptions().EnableCAS && - !Invocation.getFrontendOptions().CASFSRootID.empty()) { + (!Invocation.getFrontendOptions().CASFSRootIDs.empty() || + !Invocation.getFrontendOptions().ClangIncludeTrees.empty())) { // Set up CASFS as BaseFS. - auto RootID = CAS->parseID(Invocation.getFrontendOptions().CASFSRootID); - if (!RootID) { - Diagnostics.diagnose(SourceLoc(), diag::error_invalid_cas_id, - Invocation.getFrontendOptions().CASFSRootID, - toString(RootID.takeError())); - return true; - } - auto FS = llvm::cas::createCASFileSystem(*CAS, *RootID); + const auto &Opts = getInvocation().getFrontendOptions(); + auto FS = + createCASFileSystem(*CAS, Opts.CASFSRootIDs, Opts.ClangIncludeTrees); if (!FS) { - Diagnostics.diagnose(SourceLoc(), diag::error_invalid_cas_id, - Invocation.getFrontendOptions().CASFSRootID, + Diagnostics.diagnose(SourceLoc(), diag::error_cas, toString(FS.takeError())); return true; } @@ -865,6 +844,12 @@ std::string CompilerInstance::getBridgingHeaderPath() const { } bool CompilerInstance::setUpInputs() { + // There is no input file when building PCM using ClangIncludeTree. + if (Invocation.getFrontendOptions().RequestedAction == + FrontendOptions::ActionType::EmitPCM && + Invocation.getClangImporterOptions().UseClangIncludeTree) + return false; + // Adds to InputSourceCodeBufferIDs, so may need to happen before the // per-input setup. const Optional ideInspectionTargetBufferID = diff --git a/lib/Frontend/ModuleInterfaceLoader.cpp b/lib/Frontend/ModuleInterfaceLoader.cpp index 7a277a6cec1f5..af9a42220d986 100644 --- a/lib/Frontend/ModuleInterfaceLoader.cpp +++ b/lib/Frontend/ModuleInterfaceLoader.cpp @@ -1577,6 +1577,20 @@ void InterfaceSubContextDelegateImpl::inheritOptionsForBuildingInterface( GenericArgs.push_back("-clang-build-session-file"); GenericArgs.push_back(clangImporterOpts.BuildSessionFilePath); } + + if (!clangImporterOpts.CASPath.empty()) { + genericSubInvocation.getClangImporterOptions().CASPath = + clangImporterOpts.CASPath; + GenericArgs.push_back("-enable-cas"); + GenericArgs.push_back("-cas-path"); + GenericArgs.push_back(clangImporterOpts.CASPath); + } + + if (clangImporterOpts.UseClangIncludeTree) { + genericSubInvocation.getClangImporterOptions().UseClangIncludeTree = + clangImporterOpts.UseClangIncludeTree; + GenericArgs.push_back("-clang-include-tree"); + } } bool InterfaceSubContextDelegateImpl::extractSwiftInterfaceVersionAndArgs( @@ -2227,6 +2241,7 @@ struct ExplicitCASModuleLoader::Implementation { for (auto &entry : ExplicitClangModuleMap) { const auto &moduleMapPath = entry.getValue().moduleMapPath; if (!moduleMapPath.empty() && + !Ctx.ClangImporterOpts.UseClangIncludeTree && moduleMapsSeen.find(moduleMapPath) == moduleMapsSeen.end()) { moduleMapsSeen.insert(moduleMapPath); extraClangArgs.push_back( diff --git a/lib/Serialization/ModuleDependencyScanner.cpp b/lib/Serialization/ModuleDependencyScanner.cpp index 1de7eace92cc5..ecac3a5160d83 100644 --- a/lib/Serialization/ModuleDependencyScanner.cpp +++ b/lib/Serialization/ModuleDependencyScanner.cpp @@ -24,6 +24,7 @@ #include "swift/Subsystems.h" #include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/CAS/CachingOnDiskFileSystem.h" +#include "llvm/Support/Error.h" #include "llvm/Support/VirtualFileSystem.h" #include "ModuleFileSharedCore.h" @@ -174,12 +175,10 @@ ErrorOr ModuleDependencyScanner::scanInterfaceFile( if (dependencyTracker) { dependencyTracker->startTracking(); dependencyTracker->trackFile(moduleInterfacePath); - RootID = cantFail(dependencyTracker->createTreeFromDependencies()) - .getID() - .toString(); - Args.push_back("-enable-cas"); - Args.push_back("-cas-path"); - Args.push_back(Ctx.ClangImporterOpts.CASPath); + auto RootOrError = dependencyTracker->createTreeFromDependencies(); + if (!RootOrError) + return llvm::errorToErrorCode(RootOrError.takeError()); + RootID = RootOrError->getID().toString(); } std::vector ArgsRefs(Args.begin(), Args.end()); diff --git a/test/CAS/module_deps_include_tree.swift b/test/CAS/module_deps_include_tree.swift new file mode 100644 index 0000000000000..85c113e361bd2 --- /dev/null +++ b/test/CAS/module_deps_include_tree.swift @@ -0,0 +1,190 @@ +// REQUIRES: objc_interop + +// RUN: %empty-directory(%t) +// RUN: mkdir -p %t/clang-module-cache +// RUN: mkdir -p %t/cas + +// RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache %s -o %t/deps.json -I %S/../ScanDependencies/Inputs/CHeaders -I %S/../ScanDependencies/Inputs/Swift -emit-dependencies -emit-dependencies-path %t/deps.d -import-objc-header %S/../ScanDependencies/Inputs/CHeaders/Bridging.h -swift-version 4 -enable-cas -cas-path %t/cas -clang-include-tree +// Check the contents of the JSON output +// RUN: %FileCheck -check-prefix CHECK -check-prefix CHECK_NO_CLANG_TARGET %s < %t/deps.json + +// Check the contents of the JSON output +// RUN: %FileCheck %s -check-prefix CHECK -check-prefix CHECK-NO-SEARCH-PATHS < %t/deps.json + +// Check the make-style dependencies file +// RUN: %FileCheck %s -check-prefix CHECK-MAKE-DEPS < %t/deps.d + +// RUN: %target-swift-frontend -scan-dependencies -test-dependency-scan-cache-serialization -module-cache-path %t/clang-module-cache %s -o %t/deps.json -I %S/../ScanDependencies/Inputs/CHeaders -I %S/../ScanDependencies/Inputs/Swift -import-objc-header %S/../ScanDependencies/Inputs/CHeaders/Bridging.h -swift-version 4 -enable-cas -cas-path %t/cas -clang-include-tree +// RUN: %FileCheck -check-prefix CHECK -check-prefix CHECK_NO_CLANG_TARGET %s < %t/deps.json + +// Ensure that scanning with `-clang-target` makes sure that Swift modules' respective PCM-dependency-build-argument sets do not contain target triples. +// RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache %s -o %t/deps_clang_target.json -I %S/../ScanDependencies/Inputs/CHeaders -I %S/../ScanDependencies/Inputs/Swift -import-objc-header %S/../ScanDependencies/Inputs/CHeaders/Bridging.h -swift-version 4 -clang-target %target-cpu-apple-macosx10.14 -enable-cas -cas-path %t/cas -clang-include-tree +// Check the contents of the JSON output +// RUN: %FileCheck -check-prefix CHECK_CLANG_TARGET %s < %t/deps_clang_target.json + +/// check cas-fs content +// RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps.json E casFSRootID > %t/E_fs.casid +// RUN: llvm-cas --cas %t/cas --ls-tree-recursive @%t/E_fs.casid | %FileCheck %s -check-prefix FS_ROOT_E + +// RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps.json clang:F clangIncludeTree > %t/F_tree.casid +// RUN: clang-cas-test --cas %t/cas --print-include-tree @%t/F_tree.casid | %FileCheck %s -check-prefix INCLUDE_TREE_F + +// FS_ROOT_E-DAG: E.swiftinterface +// FS_ROOT_E-DAG: SDKSettings.json + +// INCLUDE_TREE_F: +// INCLUDE_TREE_F: +// INCLUDE_TREE_F: Files: +// INCLUDE_TREE_F-NEXT: CHeaders/F.h + +import C +import E +import G +import SubE + +// CHECK: "mainModuleName": "deps" + +/// --------Main module +// CHECK-LABEL: "modulePath": "deps.swiftmodule", +// CHECK-NEXT: sourceFiles +// CHECK-NEXT: module_deps_include_tree.swift +// CHECK-NEXT: ], +// CHECK-NEXT: "directDependencies": [ +// CHECK-DAG: "swift": "A" +// CHECK-DAG: "clang": "C" +// CHECK-DAG: "swift": "E" +// CHECK-DAG: "swift": "F" +// CHECK-DAG: "swift": "G" +// CHECK-DAG: "swift": "SubE" +// CHECK-DAG: "swift": "Swift" +// CHECK-DAG: "swift": "SwiftOnoneSupport" +// CHECK-DAG: "swift": "_Concurrency" +// CHECK-DAG: "swift": "_cross_import_E" +// CHECK: ], + +// CHECK: "extraPcmArgs": [ +// CHECK-NEXT: "-Xcc", +// CHECK-NEXT: "-target", +// CHECK-NEXT: "-Xcc", +// CHECK: "-fapinotes-swift-version=4" +// CHECK-NOT: "error: cannot open Swift placeholder dependency module map from" +// CHECK: "bridgingHeader": +// CHECK-NEXT: "path": +// CHECK-SAME: Bridging.h + +// CHECK-NEXT: "sourceFiles": +// CHECK-NEXT: Bridging.h +// CHECK-NEXT: BridgingOther.h + +// CHECK: "moduleDependencies": [ +// CHECK-NEXT: "F" +// CHECK-NEXT: ] + +/// --------Swift module A +// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}A-{{.*}}.swiftmodule", + +// CHECK: directDependencies +// CHECK-NEXT: { +// CHECK-DAG: "clang": "A" +// CHECK-DAG: "swift": "Swift" +// CHECK-NEXT: }, +// CHECK: "details": +// CHECK: "moduleCacheKey": + +/// --------Swift module F +// CHECK: "modulePath": "{{.*}}{{/|\\}}F-{{.*}}.swiftmodule", +// CHECK-NEXT: "sourceFiles": [ +// CHECK-NEXT: ], +// CHECK-NEXT: "directDependencies": [ +// CHECK-NEXT: { +// CHECK-DAG: "clang": "F" +// CHECK-DAG: "swift": "Swift" +// CHECK-DAG: "swift": "SwiftOnoneSupport" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK: "details": +// CHECK: "moduleCacheKey": + +/// --------Swift module G +// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}G-{{.*}}.swiftmodule" +// CHECK: "directDependencies" +// CHECK-NEXT: { +// CHECK-DAG: "clang": "G" +// CHECK-DAG: "swift": "Swift" +// CHECK-DAG: "swift": "SwiftOnoneSupport" +// CHECK: ], +// CHECK-NEXT: "details": { + +// CHECK: "contextHash": "{{.*}}", +// CHECK: "commandLine": [ +// CHECK: "-compile-module-from-interface" +// CHECK: "-target" +// CHECK: "-module-name" +// CHECK: "G" +// CHECK: "-swift-version" +// CHECK: "5" +// CHECK: ], +// CHECK_NO_CLANG_TARGET: "extraPcmArgs": [ +// CHECK_NO_CLANG_TARGET-NEXT: "-Xcc", +// CHECK_NO_CLANG_TARGET-NEXT: "-target", +// CHECK_CLANG_TARGET: "extraPcmArgs": [ +// CHECK_CLANG_TARGET-NEXT: "-Xcc", +// CHECK_CLANG_TARGET-NEXT: "-fapinotes-swift-version={{.*}}" +// CHECK_CLANG_TARGET-NEXT: ] + +/// --------Swift module E +// CHECK: "swift": "E" +// CHECK-LABEL: modulePath": "{{.*}}{{/|\\}}E-{{.*}}.swiftmodule" +// CHECK: "directDependencies" +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "Swift" + +// CHECK: "moduleInterfacePath" +// CHECK-SAME: E.swiftinterface + +/// --------Swift module Swift +// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}Swift-{{.*}}.swiftmodule", + +// CHECK: directDependencies +// CHECK-NEXT: { +// CHECK-NEXT: "clang": "SwiftShims" + +/// --------Clang module SwiftShims +// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}SwiftShims-{{.*}}.pcm", +// CHECK: "contextHash": "[[SHIMS_CONTEXT:.*]]", +// CHECK: "-o" +// CHECK-NEXT: SwiftShims-{{.*}}[[SHIMS_CONTEXT]].pcm +// CHECK-NO-SEARCH-PATHS-NOT: "-prebuilt-module-cache-path" + +/// --------Clang module C +// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}C-{{.*}}.pcm", + +// CHECK: "sourceFiles": [ +// CHECK-DAG: module.modulemap +// CHECK-DAG: C.h + +// CHECK: directDependencies +// CHECK-NEXT: { +// CHECK-NEXT: "clang": "B" + +// CHECK: "moduleMapPath" +// CHECK-SAME: module.modulemap + +// CHECK: "contextHash" +// CHECK-SAME: "{{.*}}" + +/// --------Clang module B +// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}B-{{.*}}.pcm", +// CHECK: "contextHash": "[[B_CONTEXT:.*]]", +// CHECK: "-o" +// CHECK-NEXT: B-{{.*}}[[B_CONTEXT]].pcm + +// Check make-style dependencies +// CHECK-MAKE-DEPS: module_deps_include_tree.swift +// CHECK-MAKE-DEPS-SAME: A.swiftinterface +// CHECK-MAKE-DEPS-SAME: G.swiftinterface +// CHECK-MAKE-DEPS-SAME: B.h +// CHECK-MAKE-DEPS-SAME: F.h +// CHECK-MAKE-DEPS-SAME: Bridging.h +// CHECK-MAKE-DEPS-SAME: BridgingOther.h +// CHECK-MAKE-DEPS-SAME: module.modulemap diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 64dbbcaa0df45..cbbfdeffcbe74 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -71,6 +71,7 @@ function(get_test_dependencies SDK result_var_name) c-arcmt-test c-index-test clang + clang-cas-test count dsymutil FileCheck diff --git a/tools/libSwiftScan/libSwiftScan.cpp b/tools/libSwiftScan/libSwiftScan.cpp index a3da81a3643fa..8e2e69e6c6820 100644 --- a/tools/libSwiftScan/libSwiftScan.cpp +++ b/tools/libSwiftScan/libSwiftScan.cpp @@ -54,6 +54,8 @@ void swiftscan_dependency_info_details_dispose( swiftscan_string_dispose(details_impl->swift_textual_details.context_hash); swiftscan_string_dispose( details_impl->swift_textual_details.cas_fs_root_id); + swiftscan_string_dispose( + details_impl->swift_textual_details.bridging_header_include_tree); swiftscan_string_dispose( details_impl->swift_textual_details.module_cache_key); break; @@ -288,6 +290,12 @@ swiftscan_string_set_t *swiftscan_swift_textual_detail_get_command_line( return details->swift_textual_details.command_line; } +swiftscan_string_set_t * +swiftscan_swift_textual_detail_get_bridging_pch_command_line( + swiftscan_module_details_t details) { + return details->swift_textual_details.bridging_pch_command_line; +} + swiftscan_string_set_t *swiftscan_swift_textual_detail_get_extra_pcm_args( swiftscan_module_details_t details) { return details->swift_textual_details.extra_pcm_args; diff --git a/tools/libSwiftScan/libSwiftScan.exports b/tools/libSwiftScan/libSwiftScan.exports index 0486474c23ac6..ff1d5337276fe 100644 --- a/tools/libSwiftScan/libSwiftScan.exports +++ b/tools/libSwiftScan/libSwiftScan.exports @@ -12,6 +12,7 @@ swiftscan_swift_textual_detail_get_bridging_header_path swiftscan_swift_textual_detail_get_bridging_source_files swiftscan_swift_textual_detail_get_bridging_module_dependencies swiftscan_swift_textual_detail_get_command_line +swiftscan_swift_textual_detail_get_bridging_pch_command_line swiftscan_swift_textual_detail_get_extra_pcm_args swiftscan_swift_textual_detail_get_context_hash swiftscan_swift_textual_detail_get_is_framework From bfa356b3ecc546631af5dcbe30e2684a52abdf4d Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Wed, 7 Jun 2023 10:49:49 -0700 Subject: [PATCH 3/4] Address review feedback --- .../swift-c/DependencyScan/DependencyScan.h | 2 +- include/swift/AST/ModuleDependencies.h | 153 +++++++----------- lib/AST/ModuleDependencies.cpp | 33 ++-- .../ClangModuleDependencyScanner.cpp | 21 ++- .../ModuleDependencyCacheSerialization.cpp | 46 +++--- lib/DependencyScan/ScanDependencies.cpp | 49 +++--- test/CAS/Inputs/SwiftDepsExtractor.py | 32 ++-- test/CAS/cas-explicit-module-map.swift | 26 +++ 8 files changed, 180 insertions(+), 182 deletions(-) diff --git a/include/swift-c/DependencyScan/DependencyScan.h b/include/swift-c/DependencyScan/DependencyScan.h index ba6bc4a179194..998c8309bf352 100644 --- a/include/swift-c/DependencyScan/DependencyScan.h +++ b/include/swift-c/DependencyScan/DependencyScan.h @@ -25,7 +25,7 @@ /// SWIFTSCAN_VERSION_MINOR should increase when there are API additions. /// SWIFTSCAN_VERSION_MAJOR is intended for "major" source/ABI breaking changes. #define SWIFTSCAN_VERSION_MAJOR 0 -#define SWIFTSCAN_VERSION_MINOR 3 +#define SWIFTSCAN_VERSION_MINOR 4 SWIFTSCAN_BEGIN_DECLS diff --git a/include/swift/AST/ModuleDependencies.h b/include/swift/AST/ModuleDependencies.h index 048bce85768cd..98dbc1de8fda6 100644 --- a/include/swift/AST/ModuleDependencies.h +++ b/include/swift/AST/ModuleDependencies.h @@ -106,8 +106,10 @@ class ModuleDependencyInfoStorageBase { const ModuleDependencyKind dependencyKind; ModuleDependencyInfoStorageBase(ModuleDependencyKind dependencyKind, + StringRef moduleCacheKey = "", bool resolved = false) - : dependencyKind(dependencyKind), resolved(resolved) { } + : dependencyKind(dependencyKind), moduleCacheKey(moduleCacheKey.str()), + resolved(resolved) {} virtual ModuleDependencyInfoStorageBase *clone() const = 0; @@ -124,18 +126,19 @@ class ModuleDependencyInfoStorageBase { /// The set of modules on which this module depends, resolved /// to Module IDs, qualified by module kind: Swift, Clang, etc. std::vector resolvedModuleDependencies; + + /// The cache key for the produced module. + std::string moduleCacheKey; + bool resolved; }; struct CommonSwiftTextualModuleDependencyDetails { CommonSwiftTextualModuleDependencyDetails( ArrayRef extraPCMArgs, ArrayRef buildCommandLine, - ArrayRef bridgingHeaderBuildCommandLine, const std::string &CASFileSystemRootID) : extraPCMArgs(extraPCMArgs.begin(), extraPCMArgs.end()), buildCommandLine(buildCommandLine.begin(), buildCommandLine.end()), - bridgingHeaderBuildCommandLine(bridgingHeaderBuildCommandLine.begin(), - bridgingHeaderBuildCommandLine.end()), CASFileSystemRootID(CASFileSystemRootID) {} /// To build a PCM to be used by this Swift module, we need to append these @@ -160,14 +163,11 @@ struct CommonSwiftTextualModuleDependencyDetails { /// interface. std::vector buildCommandLine; - /// The Swift frontend invocation arguments to build bridging header. - std::vector bridgingHeaderBuildCommandLine; - /// CASID for the Root of CASFS. Empty if CAS is not used. std::string CASFileSystemRootID; /// CASID for the Root of bridgingHeaderClangIncludeTree. Empty if not used. - std::string bridgingHeaderIncludeTreeRoot; + std::string CASBridgingHeaderIncludeTreeRootID; }; /// Describes the dependencies of a Swift module described by an Swift interface file. @@ -194,28 +194,21 @@ class SwiftInterfaceModuleDependenciesStorage : /// Details common to Swift textual (interface or source) modules CommonSwiftTextualModuleDependencyDetails textualModuleDetails; - /// The cache key for the produced module. - std::string moduleCacheKey; - SwiftInterfaceModuleDependenciesStorage( const std::string &moduleOutputPath, const std::string &swiftInterfaceFile, ArrayRef compiledModuleCandidates, - ArrayRef buildCommandLine, - ArrayRef extraPCMArgs, - StringRef contextHash, - bool isFramework, - const std::string &RootID, - const std::string &ModuleCacheKey - ) : ModuleDependencyInfoStorageBase(ModuleDependencyKind::SwiftInterface), - moduleOutputPath(moduleOutputPath), - swiftInterfaceFile(swiftInterfaceFile), - compiledModuleCandidates(compiledModuleCandidates.begin(), - compiledModuleCandidates.end()), - contextHash(contextHash), isFramework(isFramework), - textualModuleDetails(extraPCMArgs, buildCommandLine, {}, RootID), - moduleCacheKey(ModuleCacheKey) - {} + ArrayRef buildCommandLine, ArrayRef extraPCMArgs, + StringRef contextHash, bool isFramework, const std::string &RootID, + const std::string &moduleCacheKey) + : ModuleDependencyInfoStorageBase(ModuleDependencyKind::SwiftInterface, + moduleCacheKey), + moduleOutputPath(moduleOutputPath), + swiftInterfaceFile(swiftInterfaceFile), + compiledModuleCandidates(compiledModuleCandidates.begin(), + compiledModuleCandidates.end()), + contextHash(contextHash), isFramework(isFramework), + textualModuleDetails(extraPCMArgs, buildCommandLine, RootID) {} ModuleDependencyInfoStorageBase *clone() const override { return new SwiftInterfaceModuleDependenciesStorage(*this); @@ -228,10 +221,6 @@ class SwiftInterfaceModuleDependenciesStorage : void updateCommandLine(const std::vector &newCommandLine) { textualModuleDetails.buildCommandLine = newCommandLine; } - - void updateModuleCacheKey(const std::string &Key) { - moduleCacheKey = Key; - } }; /// Describes the dependencies of a Swift module @@ -250,14 +239,18 @@ class SwiftSourceModuleDependenciesStorage : /// Collection of module imports that were detected to be `@Testable` llvm::StringSet<> testableImports; + /// The Swift frontend invocation arguments to build bridging header. + std::vector bridgingHeaderBuildCommandLine; + SwiftSourceModuleDependenciesStorage( const std::string &RootID, ArrayRef buildCommandLine, ArrayRef bridgingHeaderBuildCommandLine, ArrayRef extraPCMArgs) : ModuleDependencyInfoStorageBase(ModuleDependencyKind::SwiftSource), - textualModuleDetails(extraPCMArgs, buildCommandLine, - bridgingHeaderBuildCommandLine, RootID), - testableImports(llvm::StringSet<>()) {} + textualModuleDetails(extraPCMArgs, buildCommandLine, RootID), + testableImports(llvm::StringSet<>()), + bridgingHeaderBuildCommandLine(bridgingHeaderBuildCommandLine.begin(), + bridgingHeaderBuildCommandLine.end()) {} ModuleDependencyInfoStorageBase *clone() const override { return new SwiftSourceModuleDependenciesStorage(*this); @@ -273,7 +266,7 @@ class SwiftSourceModuleDependenciesStorage : void updateBridgingHeaderCommandLine( const std::vector &newCommandLine) { - textualModuleDetails.bridgingHeaderBuildCommandLine = newCommandLine; + bridgingHeaderBuildCommandLine = newCommandLine; } void addTestableImport(ImportPath::Module module) { @@ -290,11 +283,11 @@ class SwiftBinaryModuleDependencyStorage : public ModuleDependencyInfoStorageBas const std::string &moduleDocPath, const std::string &sourceInfoPath, const bool isFramework, - const std::string ModuleCacheKey) - : ModuleDependencyInfoStorageBase(ModuleDependencyKind::SwiftBinary), + const std::string &moduleCacheKey) + : ModuleDependencyInfoStorageBase(ModuleDependencyKind::SwiftBinary, + moduleCacheKey), compiledModulePath(compiledModulePath), moduleDocPath(moduleDocPath), - sourceInfoPath(sourceInfoPath), isFramework(isFramework), - moduleCacheKey(ModuleCacheKey) {} + sourceInfoPath(sourceInfoPath), isFramework(isFramework) {} ModuleDependencyInfoStorageBase *clone() const override { return new SwiftBinaryModuleDependencyStorage(*this); @@ -312,16 +305,9 @@ class SwiftBinaryModuleDependencyStorage : public ModuleDependencyInfoStorageBas /// A flag that indicates this dependency is a framework const bool isFramework; - /// The cache key for the produced module. - std::string moduleCacheKey; - static bool classof(const ModuleDependencyInfoStorageBase *base) { return base->dependencyKind == ModuleDependencyKind::SwiftBinary; } - - void updateModuleCacheKey(const std::string &Key) { - moduleCacheKey = Key; - } }; /// Describes the dependencies of a Clang module. @@ -339,7 +325,7 @@ class ClangModuleDependencyStorage : public ModuleDependencyInfoStorageBase { const std::string contextHash; /// Partial (Clang) command line that can be used to build this module. - std::vector nonPathCommandLine; + std::vector buildCommandLine; /// The file dependencies const std::vector fileDependencies; @@ -352,31 +338,24 @@ class ClangModuleDependencyStorage : public ModuleDependencyInfoStorageBase { std::string CASFileSystemRootID; /// CASID for the Root of ClangIncludeTree. Empty if not used. - std::string clangIncludeTreeRoot; - - /// The cache key for the produced module. - std::string moduleCacheKey; - - ClangModuleDependencyStorage( - const std::string &pcmOutputPath, - const std::string &moduleMapFile, - const std::string &contextHash, - const std::vector &nonPathCommandLine, - const std::vector &fileDependencies, - const std::vector &capturedPCMArgs, - const std::string &CASFileSystemRootID, - const std::string &clangIncludeTreeRoot, - const std::string &ModuleCacheKey - ) : ModuleDependencyInfoStorageBase(ModuleDependencyKind::Clang), - pcmOutputPath(pcmOutputPath), - moduleMapFile(moduleMapFile), - contextHash(contextHash), - nonPathCommandLine(nonPathCommandLine), - fileDependencies(fileDependencies), - capturedPCMArgs(capturedPCMArgs), - CASFileSystemRootID(CASFileSystemRootID), - clangIncludeTreeRoot(clangIncludeTreeRoot), - moduleCacheKey(ModuleCacheKey) {} + std::string CASClangIncludeTreeRootID; + + ClangModuleDependencyStorage(const std::string &pcmOutputPath, + const std::string &moduleMapFile, + const std::string &contextHash, + const std::vector &buildCommandLine, + const std::vector &fileDependencies, + const std::vector &capturedPCMArgs, + const std::string &CASFileSystemRootID, + const std::string &clangIncludeTreeRoot, + const std::string &moduleCacheKey) + : ModuleDependencyInfoStorageBase(ModuleDependencyKind::Clang, + moduleCacheKey), + pcmOutputPath(pcmOutputPath), moduleMapFile(moduleMapFile), + contextHash(contextHash), buildCommandLine(buildCommandLine), + fileDependencies(fileDependencies), capturedPCMArgs(capturedPCMArgs), + CASFileSystemRootID(CASFileSystemRootID), + CASClangIncludeTreeRootID(clangIncludeTreeRoot) {} ModuleDependencyInfoStorageBase *clone() const override { return new ClangModuleDependencyStorage(*this); @@ -387,12 +366,7 @@ class ClangModuleDependencyStorage : public ModuleDependencyInfoStorageBase { } void updateCommandLine(const std::vector &newCommandLine) { - nonPathCommandLine = newCommandLine; - } - - void updateModuleCacheKey(const std::string &Key) { - assert(moduleCacheKey.empty()); - moduleCacheKey = Key; + buildCommandLine = newCommandLine; } }; @@ -543,28 +517,11 @@ class ModuleDependencyInfo { std::string getModuleCacheKey() const { assert(storage->resolved); - if (auto *dep = getAsSwiftInterfaceModule()) - return dep->moduleCacheKey; - else if (auto *dep = getAsSwiftBinaryModule()) - return dep->moduleCacheKey; - else if (auto *dep = getAsClangModule()) - return dep->moduleCacheKey; - - llvm_unreachable("Unexpected type"); + return storage->moduleCacheKey; } void updateModuleCacheKey(const std::string &key) { - if (isSwiftInterfaceModule()) - return cast(storage.get()) - ->updateModuleCacheKey(key); - else if (isSwiftBinaryModule()) - return cast(storage.get()) - ->updateModuleCacheKey(key); - else if (isClangModule()) - return cast(storage.get()) - ->updateModuleCacheKey(key); - - llvm_unreachable("Unexpected type"); + storage->moduleCacheKey = key; } /// Resolve a dependency's set of `imports` with qualified Module IDs @@ -590,7 +547,7 @@ class ModuleDependencyInfo { std::vector getCommandline() const { if (auto *detail = getAsClangModule()) - return detail->nonPathCommandLine; + return detail->buildCommandLine; else if (auto *detail = getAsSwiftInterfaceModule()) return detail->textualModuleDetails.buildCommandLine; else if (auto *detail = getAsSwiftSourceModule()) @@ -613,7 +570,7 @@ class ModuleDependencyInfo { std::vector getBridgingHeaderCommandline() const { if (auto *detail = getAsSwiftSourceModule()) - return detail->textualModuleDetails.bridgingHeaderBuildCommandLine; + return detail->bridgingHeaderBuildCommandLine; return {}; } diff --git a/lib/AST/ModuleDependencies.cpp b/lib/AST/ModuleDependencies.cpp index fb5c3c72aee9a..c3d3a59f590c2 100644 --- a/lib/AST/ModuleDependencies.cpp +++ b/lib/AST/ModuleDependencies.cpp @@ -215,7 +215,7 @@ Optional ModuleDependencyInfo::getClangIncludeTree() const { switch (getKind()) { case swift::ModuleDependencyKind::Clang: { auto clangModuleStorage = cast(storage.get()); - Root = clangModuleStorage->clangIncludeTreeRoot; + Root = clangModuleStorage->CASClangIncludeTreeRootID; break; } default: @@ -235,14 +235,14 @@ ModuleDependencyInfo::getBridgingHeaderIncludeTree() const { auto swiftInterfaceStorage = cast(storage.get()); Root = swiftInterfaceStorage->textualModuleDetails - .bridgingHeaderIncludeTreeRoot; + .CASBridgingHeaderIncludeTreeRootID; break; } case swift::ModuleDependencyKind::SwiftSource: { auto swiftSourceStorage = cast(storage.get()); - Root = - swiftSourceStorage->textualModuleDetails.bridgingHeaderIncludeTreeRoot; + Root = swiftSourceStorage->textualModuleDetails + .CASBridgingHeaderIncludeTreeRootID; break; } default: @@ -330,15 +330,15 @@ void ModuleDependencyInfo::addBridgingHeaderIncludeTree(StringRef ID) { case swift::ModuleDependencyKind::SwiftInterface: { auto swiftInterfaceStorage = cast(storage.get()); - swiftInterfaceStorage->textualModuleDetails.bridgingHeaderIncludeTreeRoot = - ID.str(); + swiftInterfaceStorage->textualModuleDetails + .CASBridgingHeaderIncludeTreeRootID = ID.str(); break; } case swift::ModuleDependencyKind::SwiftSource: { auto swiftSourceStorage = cast(storage.get()); - swiftSourceStorage->textualModuleDetails.bridgingHeaderIncludeTreeRoot = - ID.str(); + swiftSourceStorage->textualModuleDetails + .CASBridgingHeaderIncludeTreeRootID = ID.str(); break; } default: @@ -435,14 +435,15 @@ void SwiftDependencyScanningService::setupCachingDependencyScanningService( SDKSettingPath.size()); // Add Legacy layout file (maybe just hard code instead of searching). - StringRef RuntimeLibPath = - Instance.getInvocation().getSearchPathOptions().RuntimeLibraryPaths[0]; - auto &FS = Instance.getFileSystem(); - std::error_code EC; - for (auto F = FS.dir_begin(RuntimeLibPath, EC); - !EC && F != llvm::vfs::directory_iterator(); F.increment(EC)) { - if (F->path().endswith(".yaml")) - CommonDependencyFiles.emplace_back(F->path().str()); + for (auto RuntimeLibPath : + Instance.getInvocation().getSearchPathOptions().RuntimeLibraryPaths) { + auto &FS = Instance.getFileSystem(); + std::error_code EC; + for (auto F = FS.dir_begin(RuntimeLibPath, EC); + !EC && F != llvm::vfs::directory_iterator(); F.increment(EC)) { + if (F->path().endswith(".yaml")) + CommonDependencyFiles.emplace_back(F->path().str()); + } } auto CachingFS = diff --git a/lib/ClangImporter/ClangModuleDependencyScanner.cpp b/lib/ClangImporter/ClangModuleDependencyScanner.cpp index bee9092c22c9d..0855acffb882a 100644 --- a/lib/ClangImporter/ClangModuleDependencyScanner.cpp +++ b/lib/ClangImporter/ClangModuleDependencyScanner.cpp @@ -187,18 +187,6 @@ void ClangImporter::recordModuleDependencies( swiftArgs.push_back("-vfsoverlay"); swiftArgs.push_back(overlay); } - } else { - // HACK: find the -ivfsoverlay option from clang scanner and pass to - // swift. - bool addOption = false; - for (auto &arg : clangModuleDep.BuildArguments) { - if (addOption) { - swiftArgs.push_back("-vfsoverlay"); - swiftArgs.push_back(arg); - addOption = false; - } else if (arg == "-ivfsoverlay") - addOption = true; - } } // Add args reported by the scanner. @@ -224,6 +212,15 @@ void ClangImporter::recordModuleDependencies( // invocation, not swift invocation. depsInvocation.getFrontendOpts().ModuleCacheKeys.clear(); + // FIXME: workaround for rdar://105684525: find the -ivfsoverlay option + // from clang scanner and pass to swift. + for (auto overlay : depsInvocation.getHeaderSearchOpts().VFSOverlayFiles) { + if (llvm::is_contained(ctx.SearchPathOpts.VFSOverlayFiles, overlay)) + continue; + swiftArgs.push_back("-vfsoverlay"); + swiftArgs.push_back(overlay); + } + llvm::BumpPtrAllocator allocator; llvm::StringSaver saver(allocator); clangArgs.clear(); diff --git a/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp b/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp index 63fff992acbe9..184f356ef78c4 100644 --- a/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp +++ b/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp @@ -918,20 +918,22 @@ void ModuleDependenciesCacheSerializer::writeModuleInfo(ModuleDependencyID modul : 0; SwiftInterfaceModuleDetailsLayout::emitRecord( Out, ScratchRecord, AbbrCodes[SwiftInterfaceModuleDetailsLayout::Code], - outputModulePathFileId, - swiftInterfaceFileId, - getArrayID(moduleID, ModuleIdentifierArrayKind::CompiledModuleCandidates), + outputModulePathFileId, swiftInterfaceFileId, + getArrayID(moduleID, + ModuleIdentifierArrayKind::CompiledModuleCandidates), getArrayID(moduleID, ModuleIdentifierArrayKind::BuildCommandLine), getArrayID(moduleID, ModuleIdentifierArrayKind::ExtraPCMArgs), - getIdentifier(swiftTextDeps->contextHash), - swiftTextDeps->isFramework, + getIdentifier(swiftTextDeps->contextHash), swiftTextDeps->isFramework, bridgingHeaderFileId, getArrayID(moduleID, ModuleIdentifierArrayKind::SourceFiles), getArrayID(moduleID, ModuleIdentifierArrayKind::BridgingSourceFiles), - getArrayID(moduleID, ModuleIdentifierArrayKind::BridgingModuleDependencies), - getArrayID(moduleID, ModuleIdentifierArrayKind::SwiftOverlayDependencyIDs), + getArrayID(moduleID, + ModuleIdentifierArrayKind::BridgingModuleDependencies), + getArrayID(moduleID, + ModuleIdentifierArrayKind::SwiftOverlayDependencyIDs), getIdentifier(swiftTextDeps->textualModuleDetails.CASFileSystemRootID), - getIdentifier(swiftTextDeps->textualModuleDetails.bridgingHeaderIncludeTreeRoot), + getIdentifier(swiftTextDeps->textualModuleDetails + .CASBridgingHeaderIncludeTreeRootID), getIdentifier(swiftTextDeps->moduleCacheKey)); break; } @@ -951,12 +953,17 @@ void ModuleDependenciesCacheSerializer::writeModuleInfo(ModuleDependencyID modul bridgingHeaderFileId, getArrayID(moduleID, ModuleIdentifierArrayKind::SourceFiles), getArrayID(moduleID, ModuleIdentifierArrayKind::BridgingSourceFiles), - getArrayID(moduleID, ModuleIdentifierArrayKind::BridgingModuleDependencies), - getArrayID(moduleID, ModuleIdentifierArrayKind::SwiftOverlayDependencyIDs), - getIdentifier(swiftSourceDeps->textualModuleDetails.CASFileSystemRootID), - getIdentifier(swiftSourceDeps->textualModuleDetails.bridgingHeaderIncludeTreeRoot), + getArrayID(moduleID, + ModuleIdentifierArrayKind::BridgingModuleDependencies), + getArrayID(moduleID, + ModuleIdentifierArrayKind::SwiftOverlayDependencyIDs), + getIdentifier( + swiftSourceDeps->textualModuleDetails.CASFileSystemRootID), + getIdentifier(swiftSourceDeps->textualModuleDetails + .CASBridgingHeaderIncludeTreeRootID), getArrayID(moduleID, ModuleIdentifierArrayKind::BuildCommandLine), - getArrayID(moduleID, ModuleIdentifierArrayKind::BridgingHeaderBuildCommandLine)); + getArrayID(moduleID, + ModuleIdentifierArrayKind::BridgingHeaderBuildCommandLine)); break; } case swift::ModuleDependencyKind::SwiftBinary: { @@ -998,7 +1005,7 @@ void ModuleDependenciesCacheSerializer::writeModuleInfo(ModuleDependencyID modul getArrayID(moduleID, ModuleIdentifierArrayKind::FileDependencies), getArrayID(moduleID, ModuleIdentifierArrayKind::CapturedPCMArgs), getIdentifier(clangDeps->CASFileSystemRootID), - getIdentifier(clangDeps->clangIncludeTreeRoot), + getIdentifier(clangDeps->CASClangIncludeTreeRootID), getIdentifier(clangDeps->moduleCacheKey)); break; @@ -1133,8 +1140,8 @@ void ModuleDependenciesCacheSerializer::collectStringsAndArrays( moduleID, ModuleIdentifierArrayKind::SwiftOverlayDependencyIDs, swiftTextDeps->textualModuleDetails.swiftOverlayDependencies); addIdentifier(swiftTextDeps->textualModuleDetails.CASFileSystemRootID); - addIdentifier( - swiftTextDeps->textualModuleDetails.bridgingHeaderIncludeTreeRoot); + addIdentifier(swiftTextDeps->textualModuleDetails + .CASBridgingHeaderIncludeTreeRootID); addIdentifier(swiftTextDeps->moduleCacheKey); break; } @@ -1180,8 +1187,7 @@ void ModuleDependenciesCacheSerializer::collectStringsAndArrays( swiftSourceDeps->textualModuleDetails.buildCommandLine); addStringArray( moduleID, ModuleIdentifierArrayKind::BridgingHeaderBuildCommandLine, - swiftSourceDeps->textualModuleDetails - .bridgingHeaderBuildCommandLine); + swiftSourceDeps->bridgingHeaderBuildCommandLine); addIdentifier( swiftSourceDeps->textualModuleDetails.CASFileSystemRootID); break; @@ -1193,13 +1199,13 @@ void ModuleDependenciesCacheSerializer::collectStringsAndArrays( addIdentifier(clangDeps->moduleMapFile); addIdentifier(clangDeps->contextHash); addStringArray(moduleID, ModuleIdentifierArrayKind::NonPathCommandLine, - clangDeps->nonPathCommandLine); + clangDeps->buildCommandLine); addStringArray(moduleID, ModuleIdentifierArrayKind::FileDependencies, clangDeps->fileDependencies); addStringArray(moduleID, ModuleIdentifierArrayKind::CapturedPCMArgs, clangDeps->capturedPCMArgs); addIdentifier(clangDeps->CASFileSystemRootID); - addIdentifier(clangDeps->clangIncludeTreeRoot); + addIdentifier(clangDeps->CASClangIncludeTreeRootID); addIdentifier(clangDeps->moduleCacheKey); break; } diff --git a/lib/DependencyScan/ScanDependencies.cpp b/lib/DependencyScan/ScanDependencies.cpp index d98120d30fa11..2feaba20cbce0 100644 --- a/lib/DependencyScan/ScanDependencies.cpp +++ b/lib/DependencyScan/ScanDependencies.cpp @@ -378,8 +378,8 @@ static llvm::Error resolveExplicitModuleInputs( case swift::ModuleDependencyKind::SwiftSource: { auto sourceDepDetails = depInfo->getAsSwiftSourceModule(); assert(sourceDepDetails && "Expected source dependency"); - if (sourceDepDetails->textualModuleDetails.bridgingHeaderIncludeTreeRoot - .empty()) { + if (sourceDepDetails->textualModuleDetails + .CASBridgingHeaderIncludeTreeRootID.empty()) { if (!sourceDepDetails->textualModuleDetails.bridgingSourceFiles .empty()) { if (auto tracker = @@ -396,7 +396,7 @@ static llvm::Error resolveExplicitModuleInputs( } } else includeTrees.push_back(sourceDepDetails->textualModuleDetails - .bridgingHeaderIncludeTreeRoot); + .CASBridgingHeaderIncludeTreeRootID); break; } default: @@ -1338,16 +1338,20 @@ generateFullDependencyGraph(CompilerInstance &instance, moduleInterfacePath, create_set(swiftTextualDeps->compiledModuleCandidates), bridgingHeaderPath, - create_set(swiftTextualDeps->textualModuleDetails.bridgingSourceFiles), - create_set(swiftTextualDeps->textualModuleDetails.bridgingModuleDependencies), + create_set( + swiftTextualDeps->textualModuleDetails.bridgingSourceFiles), + create_set(swiftTextualDeps->textualModuleDetails + .bridgingModuleDependencies), create_set(bridgedOverlayDependencyNames), create_set(swiftTextualDeps->textualModuleDetails.buildCommandLine), - create_set(swiftTextualDeps->textualModuleDetails.bridgingHeaderBuildCommandLine), + /*bridgingHeaderBuildCommand*/ create_set({}), create_set(swiftTextualDeps->textualModuleDetails.extraPCMArgs), create_clone(swiftTextualDeps->contextHash.c_str()), swiftTextualDeps->isFramework, - create_clone(swiftTextualDeps->textualModuleDetails.CASFileSystemRootID.c_str()), - create_clone(swiftTextualDeps->textualModuleDetails.bridgingHeaderIncludeTreeRoot.c_str()), + create_clone(swiftTextualDeps->textualModuleDetails + .CASFileSystemRootID.c_str()), + create_clone(swiftTextualDeps->textualModuleDetails + .CASBridgingHeaderIncludeTreeRootID.c_str()), create_clone(swiftTextualDeps->moduleCacheKey.c_str())}; } else if (swiftSourceDeps) { swiftscan_string_ref_t moduleInterfacePath = create_null(); @@ -1366,17 +1370,23 @@ generateFullDependencyGraph(CompilerInstance &instance, moduleInterfacePath, create_empty_set(), bridgingHeaderPath, - create_set(swiftSourceDeps->textualModuleDetails.bridgingSourceFiles), - create_set(swiftSourceDeps->textualModuleDetails.bridgingModuleDependencies), + create_set( + swiftSourceDeps->textualModuleDetails.bridgingSourceFiles), + create_set(swiftSourceDeps->textualModuleDetails + .bridgingModuleDependencies), create_set(bridgedOverlayDependencyNames), create_set(swiftSourceDeps->textualModuleDetails.buildCommandLine), - create_set(swiftSourceDeps->textualModuleDetails.bridgingHeaderBuildCommandLine), + create_set(swiftSourceDeps->bridgingHeaderBuildCommandLine), create_set(swiftSourceDeps->textualModuleDetails.extraPCMArgs), - /*contextHash*/create_null(), - /*isFramework*/false, - /*CASFS*/create_clone(swiftSourceDeps->textualModuleDetails.CASFileSystemRootID.c_str()), - /*IncludeTree*/create_clone(swiftSourceDeps->textualModuleDetails.bridgingHeaderIncludeTreeRoot.c_str()), - /*CacheKey*/create_clone("")}; + /*contextHash*/ create_null(), + /*isFramework*/ false, + /*CASFS*/ + create_clone(swiftSourceDeps->textualModuleDetails + .CASFileSystemRootID.c_str()), + /*IncludeTree*/ + create_clone(swiftSourceDeps->textualModuleDetails + .CASBridgingHeaderIncludeTreeRootID.c_str()), + /*CacheKey*/ create_clone("")}; } else if (swiftPlaceholderDeps) { details->kind = SWIFTSCAN_DEPENDENCY_INFO_SWIFT_PLACEHOLDER; details->swift_placeholder_details = { @@ -1397,12 +1407,11 @@ generateFullDependencyGraph(CompilerInstance &instance, details->clang_details = { create_clone(clangDeps->moduleMapFile.c_str()), create_clone(clangDeps->contextHash.c_str()), - create_set(clangDeps->nonPathCommandLine), + create_set(clangDeps->buildCommandLine), create_set(clangDeps->capturedPCMArgs), create_clone(clangDeps->CASFileSystemRootID.c_str()), - create_clone(clangDeps->clangIncludeTreeRoot.c_str()), - create_clone(clangDeps->moduleCacheKey.c_str()) - }; + create_clone(clangDeps->CASClangIncludeTreeRootID.c_str()), + create_clone(clangDeps->moduleCacheKey.c_str())}; } return details; }; diff --git a/test/CAS/Inputs/SwiftDepsExtractor.py b/test/CAS/Inputs/SwiftDepsExtractor.py index a06df5b6e0add..9666127c2ed86 100755 --- a/test/CAS/Inputs/SwiftDepsExtractor.py +++ b/test/CAS/Inputs/SwiftDepsExtractor.py @@ -1,32 +1,34 @@ #!/usr/bin/env python3 +# +# Usage: SwiftDepsExtractor.py file.json ModuleName Key import json import sys -## SwiftDepsExtractor.py file.json ModuleName Key input_json = sys.argv[1] module_name = sys.argv[2] key = sys.argv[3] -mode = "swift" +mode = 'swift' -if module_name.startswith("clang:"): - mode = "clang" +if module_name.startswith('clang:'): + mode = 'clang' module_name = module_name[6:] -elif module_name.startswith("swiftPrebuiltExternal:"): - mode = "swiftPrebuiltExternal" +elif module_name.startswith('swiftPrebuiltExternal:'): + mode = 'swiftPrebuiltExternal' module_name = module_name[22:] with open(input_json, 'r') as file: deps = json.load(file) - extract_next = False - for module in deps['modules']: - if extract_next: - if key in module.keys(): - print(module[key]) - break - print(module['details'][mode][key]) + module_names = deps['modules'][::2] + module_details = deps['modules'][1::2] + for name, detail in zip(module_names, module_details): + if name.get(mode, '') != module_name: + continue + + if key in detail.keys(): + print(detail[key]) break - if module.get(mode, '') == module_name: - extract_next = True + print(detail['details'][mode][key]) + break diff --git a/test/CAS/cas-explicit-module-map.swift b/test/CAS/cas-explicit-module-map.swift index f54aad1e0aae3..fec0dd188d2ec 100644 --- a/test/CAS/cas-explicit-module-map.swift +++ b/test/CAS/cas-explicit-module-map.swift @@ -1,6 +1,8 @@ // RUN: %empty-directory(%t) // RUN: mkdir -p %t/cas // RUN: split-file %s %t +// RUN: %target-swift-emit-pcm -module-cache-path %t/clang-module-cache -module-name SwiftShims %swift-lib-dir/swift/shims/module.modulemap -o %t/SwiftShims.pcm +// RUN: %target-swift-emit-pcm -module-cache-path %t/clang-module-cache -module-name _SwiftConcurrencyShims %swift-lib-dir/swift/shims/module.modulemap -o %t/_SwiftConcurrencyShims.pcm // RUN: %target-swift-frontend -emit-module -module-cache-path %t/clang-module-cache %t/A.swift -o %t/A.swiftmodule -swift-version 5 // RUN: %target-swift-frontend -emit-module -module-cache-path %t/clang-module-cache %t/B.swift -o %t/B.swiftmodule -I %t -swift-version 5 // RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache %t/Test.swift -o %t/deps.json -I %t -swift-version 5 -enable-cas -cas-path %t/cas @@ -30,6 +32,14 @@ // RUN: llvm-cas --cas %t/cas --make-node --data %t/kind.blob @%t/String.key @%t/schema.casid > %t/String.casid // RUN: llvm-cas --cas %t/cas --put-cache-key @%t/String.key @%t/String.casid +// RUN: llvm-cas --cas %t/cas --make-blob --data %t/SwiftShims.pcm | tr -d '\n' > %t/Shims.key +// RUN: llvm-cas --cas %t/cas --make-node --data %t/kind.blob @%t/Shims.key @%t/schema.casid > %t/Shims.casid +// RUN: llvm-cas --cas %t/cas --put-cache-key @%t/Shims.key @%t/Shims.casid + +// RUN: llvm-cas --cas %t/cas --make-blob --data %t/_SwiftConcurrencyShims.pcm | tr -d '\n' > %t/ConcurrencyShims.key +// RUN: llvm-cas --cas %t/cas --make-node --data %t/kind.blob @%t/ConcurrencyShims.key @%t/schema.casid > %t/ConcurrencyShims.casid +// RUN: llvm-cas --cas %t/cas --put-cache-key @%t/ConcurrencyShims.key @%t/ConcurrencyShims.casid + // RUN: echo "[{" > %/t/map.json // RUN: echo "\"moduleName\": \"A\"," >> %/t/map.json // RUN: echo "\"modulePath\": \"A.swiftmodule\"," >> %/t/map.json @@ -77,6 +87,22 @@ // RUN: cat %t/String.key >> %/t/map.json // RUN: echo "\"," >> %/t/map.json // RUN: echo "\"isFramework\": false" >> %/t/map.json +// RUN: echo "}," >> %/t/map.json +// RUN: echo "{" >> %/t/map.json +// RUN: echo "\"moduleName\": \"SwiftShims\"," >> %/t/map.json +// RUN: echo "\"clangModulePath\": \"SwiftShims.pcm\"," >> %/t/map.json +// RUN: echo -n "\"clangModuleCacheKey\": \"" >> %/t/map.json +// RUN: cat %t/Shims.key >> %/t/map.json +// RUN: echo "\"," >> %/t/map.json +// RUN: echo "\"isFramework\": false" >> %/t/map.json +// RUN: echo "}," >> %/t/map.json +// RUN: echo "{" >> %/t/map.json +// RUN: echo "\"moduleName\": \"_SwiftConcurrencyShims\"," >> %/t/map.json +// RUN: echo "\"clangModulePath\": \"_SwiftConcurrency.pcm\"," >> %/t/map.json +// RUN: echo -n "\"clangModuleCacheKey\": \"" >> %/t/map.json +// RUN: cat %t/ConcurrencyShims.key >> %/t/map.json +// RUN: echo "\"," >> %/t/map.json +// RUN: echo "\"isFramework\": false" >> %/t/map.json // RUN: echo "}]" >> %/t/map.json // RUN: llvm-cas --cas %t/cas --make-blob --data %t/map.json > %t/map.casid From 56617acdaaddf51983c4fb9d02b4761926f91308 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Fri, 9 Jun 2023 09:44:01 -0700 Subject: [PATCH 4/4] [ScanDependency] Fix dependency output json format Remove a trailing common that makes the JSON output from `-scan-dependencies` to be invalide JSON format. --- lib/DependencyScan/ScanDependencies.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/DependencyScan/ScanDependencies.cpp b/lib/DependencyScan/ScanDependencies.cpp index 2feaba20cbce0..a3a09efb75b30 100644 --- a/lib/DependencyScan/ScanDependencies.cpp +++ b/lib/DependencyScan/ScanDependencies.cpp @@ -1111,7 +1111,7 @@ static void writeJSON(llvm::raw_ostream &out, if (hasOverlayDependencies) { writeDependencies(out, swiftTextualDeps->swift_overlay_module_dependencies, "swiftOverlayDependencies", 5, - /*trailingComma=*/true); + /*trailingComma=*/false); } } else if (swiftPlaceholderDeps) { out << "\"swiftPlaceholder\": {\n";