Add proof of concept for ASTGen.

Author: zoecarver

CMakeLists.txt

@@ -0,0 +1,11 @@
+cmake_minimum_required(VERSION 3.18)
+
+project(SwiftSyntax LANGUAGES Swift)
+
+set(min_supported_swift_version 5.7)
+if(CMAKE_Swift_COMPILER_VERSION VERSION_LESS "${min_supported_swift_version}")
+    message(FATAL_ERROR "Outdated Swift compiler: "
+            "Swift ${min_supported_swift_version} or newer is required.")
+endif()
+
+add_subdirectory(Sources)

Changelog.md

@@ -7,7 +7,7 @@ Note: This is in reverse chronological order, so newer entries are added to the
 * To clarify that the edits passed to `IncrementalParseTransition` are applied concurrently, introduce a new `ConcurrentEdit` type that provides the guarantee and allows translation of sequentially applied edits to the expected concurrent form.
 * The `SwiftSyntaxParser` type and a few related types now live in their own module (also named `SwiftSyntaxParser`). This allows using `SwiftSyntax` for code generation purposes without having a compatible `_InternalSwiftSyntaxParser.dylib` around. 
 
-  `import SwiftSyntaxParser` where necessary.
+  `import SSwiftSyntaxParser` where necessary.
 * `DiagnosticEngine` has been removed. Instead, `SyntaxParser` takes a closure through which it emits parser diagnostics. Depending on your needs, use the closure to handle the diagnostics or write + hook up your own diagnostics engine.
 
 ## Swift 5.3

Examples/AddOneToIntegerLiterals.swift

@@ -1,5 +1,5 @@
-import SwiftSyntax
-import SwiftSyntaxParser
+import SSwiftSyntax
+import SSwiftSyntaxParser
 import Foundation
 
 /// AddOneToIntegerLiterals will visit each token in the Syntax tree, and

Package.swift

@@ -46,7 +46,7 @@ let package = Package(
   ],
   products: [
     .library(name: "SwiftParser", type: .static, targets: ["SwiftParser"]),
-    .library(name: "SwiftSyntax", type: .static, targets: ["SwiftSyntax"]),
+    .library(name: "SSwiftSyntax", type: .static, targets: ["SSwiftSyntax"]),
     .library(name: "SwiftSyntaxParser", type: .static, targets: ["SwiftSyntaxParser"]),
     .library(name: "SwiftSyntaxBuilder", type: .static, targets: ["SwiftSyntaxBuilder"]),
     .executable(name: "generate-swift-syntax-builder", targets: ["generate-swift-syntax-builder"])
@@ -60,10 +60,10 @@ let package = Package(
     ),
     .target(
       name: "SwiftDiagnostics",
-      dependencies: ["SwiftSyntax"]
+      dependencies: ["SSwiftSyntax"]
     ),
     .target(
-      name: "SwiftSyntax",
+      name: "SSwiftSyntax",
       dependencies: ["_CSwiftSyntax"],
       exclude: [
         "Misc.swift.gyb",
@@ -88,15 +88,15 @@ let package = Package(
     ),
     .target(
       name: "SwiftSyntaxBuilder",
-      dependencies: ["SwiftSyntax"],
+      dependencies: ["SSwiftSyntax"],
       exclude: [
         "gyb_helpers",
         "ResultBuilders.swift.gyb",
       ]
     ),
     .target(
       name: "SwiftSyntaxParser",
-      dependencies: ["SwiftSyntax"],
+      dependencies: ["SSwiftSyntax"],
       exclude: [
         "NodeDeclarationHash.swift.gyb",
         "Serialization.swift.gyb",
@@ -105,22 +105,22 @@ let package = Package(
     ),
     .target(
       name: "_SwiftSyntaxTestSupport",
-      dependencies: ["SwiftSyntax"]
+      dependencies: ["SSwiftSyntax"]
     ),
     .target(
       name: "SwiftParser",
-      dependencies: ["SwiftDiagnostics", "SwiftSyntax"],
+      dependencies: ["SwiftDiagnostics", "SSwiftSyntax"],
       exclude: [
         "README.md"
       ]
     ),
     .executableTarget(
       name: "lit-test-helper",
-      dependencies: ["SwiftSyntax", "SwiftSyntaxParser"]
+      dependencies: ["SSwiftSyntax", "SwiftSyntaxParser"]
     ),
     .executableTarget(
       name: "swift-parser-test",
-      dependencies: ["SwiftDiagnostics", "SwiftSyntax", "SwiftParser", .product(name: "ArgumentParser", package: "swift-argument-parser")]
+      dependencies: ["SwiftDiagnostics", "SSwiftSyntax", "SwiftParser", .product(name: "ArgumentParser", package: "swift-argument-parser")]
     ),
     .executableTarget(
         name: "generate-swift-syntax-builder",
@@ -148,7 +148,7 @@ let package = Package(
     ),
     .testTarget(
       name: "SwiftSyntaxTest",
-      dependencies: ["SwiftSyntax", "_SwiftSyntaxTestSupport"]
+      dependencies: ["SSwiftSyntax", "_SwiftSyntaxTestSupport"]
     ),
     .testTarget(
       name: "SwiftSyntaxBuilderTest",
@@ -161,7 +161,7 @@ let package = Package(
     ),
     .testTarget(
       name: "PerformanceTest",
-      dependencies: ["SwiftSyntax", "SwiftSyntaxParser", "SwiftParser"],
+      dependencies: ["SSwiftSyntax", "SwiftSyntaxParser", "SwiftParser"],
       exclude: ["Inputs"]
     ),
     .testTarget(

Sources/ASTGen/ASTGen.swift

@@ -0,0 +1,73 @@
+import SwiftParser
+import SSwiftSyntax
+
+import CASTBridging
+
+extension Array {
+  public func withBridgedArrayRef<T>(_ c: (BridgedArrayRef) -> T) -> T {
+    withUnsafeBytes { buf in
+      c(BridgedArrayRef(data: buf.baseAddress!, numElements: count))
+    }
+  }
+}
+
+extension UnsafePointer {
+  public var raw: UnsafeMutableRawPointer {
+    UnsafeMutableRawPointer(mutating: self)
+  }
+}
+
+struct ASTGenVisitor: SyntaxTransformVisitor {
+  let ctx: UnsafeMutableRawPointer
+  let base: UnsafePointer<CChar>
+  
+  public func visit(_ node: FunctionCallExprSyntax) -> [UnsafeMutableRawPointer] {
+    let args = visit(node.argumentList).first!
+    // TODO: hack
+    let callee = visit(node.calledExpression.as(IdentifierExprSyntax.self)!).first!
+    let call = SwiftFunctionCallExpr_create(self.ctx, callee, args)
+    
+    return [call]
+  }
+  
+  public func visit(_ node: IdentifierExprSyntax) -> [UnsafeMutableRawPointer] {
+    let loc = self.base.advanced(by: node.position.utf8Offset).raw
+    
+    var text = node.identifier.text
+    let id = text.withUTF8 { buf in
+      return SwiftASTContext_getIdentifier(ctx, buf.baseAddress, buf.count)
+    }
+
+    return [SwiftIdentifierExpr_create(ctx, id, loc)]
+  }
+  
+  public func visit(_ node: TupleExprElementListSyntax) -> [UnsafeMutableRawPointer] {
+    let elements = node.map { visit($0).first! }
+    
+    // TODO: find correct paren locs.
+    let lParenLoc = self.base.advanced(by: node.position.utf8Offset).raw
+    let rParenLoc = self.base.advanced(by: node.position.utf8Offset).raw
+    
+    return [elements.withBridgedArrayRef { elementsRef in
+      SwiftTupleExpr_create(self.ctx, lParenLoc, elementsRef, rParenLoc)
+    }]
+  }
+    
+  public func visit(_ node: StringLiteralExprSyntax) -> [UnsafeMutableRawPointer] {
+    let loc = self.base.advanced(by: node.position.utf8Offset).raw
+    var segment = node.segments.first!.as(StringSegmentSyntax.self)!.content.text
+    return segment.withUTF8 { buf in
+      let id = SwiftASTContext_getIdentifier(ctx, buf.baseAddress, buf.count)
+      return [SwiftStringLiteralExpr_create(ctx, id, buf.count, loc)]
+    }
+  }
+}
+
+@_cdecl("parseTopLevelSwift")
+public func parseTopLevelSwift(
+    buffer: UnsafePointer<CChar>, ctx: UnsafeMutableRawPointer
+) -> UnsafeMutableRawPointer {
+    let syntax = try! Parser.parse(source: String(cString: buffer))
+    let count = ASTGenVisitor(ctx: ctx, base: buffer).visit(syntax)
+    return count.first!
+}

Sources/ASTGen/CMakeLists.txt

@@ -0,0 +1,27 @@
+add_library(ASTGen
+        ASTGen.swift)
+
+# TOOD: error when SWIFT_SOURCE_DIR is not defined.
+
+target_compile_options(ASTGen PUBLIC
+        "-enable-library-evolution"
+        "-emit-module-interface"
+        "-g"
+        # Includes we shouldn't need
+        "-I${CMAKE_CURRENT_BINARY_DIR}/../SwiftSyntax"
+        "-I${CMAKE_CURRENT_BINARY_DIR}/../SwiftParser"
+        "-I${CMAKE_CURRENT_BINARY_DIR}/../SwiftDiagnostics")
+
+target_include_directories(SwiftDiagnostics PUBLIC
+        "${SWIFT_SOURCE_DIR}/include/"
+        "${CMAKE_CURRENT_BINARY_DIR}/../SwiftSyntax"
+        "${CMAKE_CURRENT_BINARY_DIR}/../SwiftParser"
+        "${CMAKE_CURRENT_BINARY_DIR}/../SwiftDiagnostics")
+
+target_link_directories(SwiftDiagnostics PUBLIC
+        "${SWIFT_SOURCE_DIR}/include/"
+        "${CMAKE_CURRENT_BINARY_DIR}/../SwiftSyntax"
+        "${CMAKE_CURRENT_BINARY_DIR}/../SwiftParser"
+        "${CMAKE_CURRENT_BINARY_DIR}/../SwiftDiagnostics")
+
+add_dependencies(ASTGen SwiftParser)

Sources/CMakeLists.txt

@@ -0,0 +1,4 @@
+add_subdirectory(SwiftSyntax)
+add_subdirectory(SwiftDiagnostics)
+add_subdirectory(SwiftParser)
+add_subdirectory(ASTGen)

Sources/SwiftDiagnostics/CMakeLists.txt

@@ -0,0 +1,20 @@
+file(GLOB_RECURSE _LIBSWIFT_SWIFT_SYNTAX_PARSER_SOURCES
+        "${CMAKE_CURRENT_SOURCE_DIR}/*.swift")
+set(LIBSWIFT_SWIFT_SYNTAX_PARSER_SOURCES)
+foreach(source ${_LIBSWIFT_SWIFT_SYNTAX_PARSER_SOURCES})
+    file(TO_CMAKE_PATH "${source}" source)
+    list(APPEND LIBSWIFT_SWIFT_SYNTAX_PARSER_SOURCES ${source})
+endforeach()
+message(STATUS "Using Swift Parser library for libswift SwiftParser (${SWIFT_SYNTAX_SOURCE_DIR}).")
+
+add_library(SwiftDiagnostics
+        "${LIBSWIFT_SWIFT_SYNTAX_PARSER_SOURCES}")
+add_dependencies(SwiftDiagnostics SSwiftSyntax)
+
+target_include_directories(SwiftDiagnostics PUBLIC
+        "${CMAKE_CURRENT_BINARY_DIR}/../SwiftSyntax"
+        "${CMAKE_CURRENT_BINARY_DIR}/../SwiftParser")
+
+target_link_directories(SwiftDiagnostics PUBLIC
+        "${CMAKE_CURRENT_BINARY_DIR}/../SwiftSyntax"
+        "${CMAKE_CURRENT_BINARY_DIR}/../SwiftParser")

Sources/SwiftDiagnostics/Diagnostic.swift

@@ -10,7 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-import SwiftSyntax
+import SSwiftSyntax
 
 public struct Diagnostic: CustomDebugStringConvertible {
   /// The message that should be displayed to the user

Sources/SwiftDiagnostics/FixIt.swift

@@ -10,7 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-import SwiftSyntax
+import SSwiftSyntax
 
 /// Types conforming to this protocol represent Fix-It messages that can be
 /// shown to the client.

Sources/SwiftParser/Attributes.swift

@@ -10,7 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-@_spi(RawSyntax) import SwiftSyntax
+@_spi(RawSyntax) import SSwiftSyntax
 
 extension Parser {
   mutating func parseAttributeList() -> RawAttributeListSyntax? {

Sources/SwiftParser/Availability.swift

@@ -10,7 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-@_spi(RawSyntax) import SwiftSyntax
+@_spi(RawSyntax) import SSwiftSyntax
 
 extension Parser {
   enum AvailabilitySpecSource {

Sources/SwiftParser/CMakeLists.txt

@@ -0,0 +1,26 @@
+file(GLOB_RECURSE _LIBSWIFT_SWIFT_SYNTAX_PARSER_SOURCES
+        "${CMAKE_CURRENT_SOURCE_DIR}/*.swift")
+set(LIBSWIFT_SWIFT_SYNTAX_PARSER_SOURCES)
+foreach(source ${_LIBSWIFT_SWIFT_SYNTAX_PARSER_SOURCES})
+    file(TO_CMAKE_PATH "${source}" source)
+    list(APPEND LIBSWIFT_SWIFT_SYNTAX_PARSER_SOURCES ${source})
+endforeach()
+message(STATUS "Using Swift Parser library for libswift SwiftParser (${SWIFT_SYNTAX_SOURCE_DIR}).")
+
+add_library(SwiftParser
+        "${LIBSWIFT_SWIFT_SYNTAX_PARSER_SOURCES}")
+add_dependencies(SwiftParser SSwiftSyntax)
+add_dependencies(SwiftParser SwiftDiagnostics)
+
+target_include_directories(SwiftDiagnostics PUBLIC
+        "${CMAKE_CURRENT_BINARY_DIR}/../SwiftSyntax"
+        "${CMAKE_CURRENT_BINARY_DIR}/../SwiftDiagnostics")
+
+target_link_directories(SwiftDiagnostics PUBLIC
+        "${CMAKE_CURRENT_BINARY_DIR}/../SwiftSyntax"
+        "${CMAKE_CURRENT_BINARY_DIR}/../SwiftDiagnostics")
+        
+# TODO: why do we need to do this manually?
+target_compile_options(SwiftParser PUBLIC
+        "-I${CMAKE_CURRENT_BINARY_DIR}/../SwiftSyntax"
+        "-I${CMAKE_CURRENT_BINARY_DIR}/../SwiftDiagnostics")

Sources/SwiftParser/CharacterInfo.swift

@@ -93,7 +93,6 @@ extension Unicode.Scalar {
   }
 }
 
-#if swift(<5.7)
 extension UnsafeRawBufferPointer {
   /// Returns a typed buffer to the memory referenced by this buffer,
   /// assuming that the memory is already bound to the specified type.
@@ -122,7 +121,6 @@ extension UnsafeRawBufferPointer {
     return .init(start: s.assumingMemoryBound(to: T.self), count: n)
   }
 }
-#endif
 
 private var InfoTable: CharacterInfoTable = (
   // 0 NUL        1 SOH         2 STX         3 ETX

Sources/SwiftParser/Declarations.swift

@@ -10,7 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-@_spi(RawSyntax) import SwiftSyntax
+@_spi(RawSyntax) import SSwiftSyntax
 
 extension Parser {
   @_spi(RawSyntax)

Sources/SwiftParser/Diagnostics/ParseDiagnosticsGenerator.swift

@@ -11,7 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 import SwiftDiagnostics
-import SwiftSyntax
+import SSwiftSyntax
 
 extension UnexpectedNodesSyntax {
   func tokens(satisfying isIncluded: (TokenSyntax) -> Bool) -> [TokenSyntax] {

Sources/SwiftParser/Diagnostics/ParserDiagnosticMessages.swift

@@ -11,7 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 import SwiftDiagnostics
-import SwiftSyntax
+import SSwiftSyntax
 
 let diagnosticDomain: String = "SwiftParser"
 

Sources/SwiftParser/Directives.swift

@@ -10,7 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-@_spi(RawSyntax) import SwiftSyntax
+@_spi(RawSyntax) import SSwiftSyntax
 
 extension Parser {
   /// Parse a conditional compilation block.

Sources/SwiftParser/Expressions.swift

@@ -10,7 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-@_spi(RawSyntax) import SwiftSyntax
+@_spi(RawSyntax) import SSwiftSyntax
 
 extension Parser {
   public enum ExprFlavor {

Sources/SwiftParser/Lexer.swift

@@ -10,7 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-@_spi(RawSyntax) import SwiftSyntax
+@_spi(RawSyntax) import SSwiftSyntax
 
 /// A lexical analyzer for the Swift programming language.
 ///

Sources/SwiftParser/Lookahead.swift

@@ -10,7 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-@_spi(RawSyntax) import SwiftSyntax
+@_spi(RawSyntax) import SSwiftSyntax
 
 extension Parser {
   /// Token lookahead for the parser.

Sources/SwiftParser/LoopProgressCondition.swift

@@ -10,7 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-@_spi(RawSyntax) import SwiftSyntax
+@_spi(RawSyntax) import SSwiftSyntax
 
 /// A type that can be used in place of a `while true` loop.
 /// See `evaluate` for more detail.