From 15ca3d3064e20486ce16f93d8382cdea6bbbb11b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leutha=CC=88user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 13 Aug 2025 07:50:03 +0200 Subject: [PATCH 1/3] Remove parallel invocation of AST parsing Crashes when built with x86_64-swift-linux-musl. We remove it for now until further investigation. --- .../SwiftAstGenLib/SwiftAstGenerator.swift | 50 ++++++++----------- 1 file changed, 20 insertions(+), 30 deletions(-) diff --git a/Sources/SwiftAstGenLib/SwiftAstGenerator.swift b/Sources/SwiftAstGenLib/SwiftAstGenerator.swift index 34d0cd7..e2c4ac2 100644 --- a/Sources/SwiftAstGenLib/SwiftAstGenerator.swift +++ b/Sources/SwiftAstGenLib/SwiftAstGenerator.swift @@ -5,10 +5,6 @@ public class SwiftAstGenerator { private var srcDir: URL private var outputDir: URL private var prettyPrint: Bool - private let availableProcessors = ProcessInfo.processInfo.activeProcessorCount - - // Private serial queue for directory creation operations - private let fileQueue = DispatchQueue(label: "io.joern.swiftastgen") public init(srcDir: URL, outputDir: URL, prettyPrint: Bool) throws { self.srcDir = srcDir @@ -33,6 +29,16 @@ public class SwiftAstGenerator { || nameLowercased.contains("/spec/") } + private func writeStringToFileAsync(string: String, toFilePath filePath: String) { + DispatchQueue.global(qos: .background).async { + do { + let data = Data(string.utf8) + let fileURL = URL(fileURLWithPath: filePath) + try data.write(to: fileURL, options: .atomic) + } catch {} + } + } + private func parseFile(fileUrl: URL) { do { let relativeFilePath = fileUrl.relativePath(from: srcDir)! @@ -47,34 +53,22 @@ public class SwiftAstGenerator { .appendingPathExtension("json") let outfileDirUrl = outFileUrl.deletingLastPathComponent() - do { - try fileQueue.sync { - try FileManager.default.createDirectory( - atPath: outfileDirUrl.path, - withIntermediateDirectories: true, - attributes: nil - ) - } - } catch {} + if !FileManager.default.fileExists(atPath: outfileDirUrl.path) { + try FileManager.default.createDirectory( + atPath: outfileDirUrl.path, + withIntermediateDirectories: true, + attributes: nil + ) + } - try astJsonString.write( - to: outFileUrl, - atomically: true, - encoding: String.Encoding.utf8 - ) - - print("Generated AST for file: `\(fileUrl.path)`") + writeStringToFileAsync(string: astJsonString, toFilePath: outFileUrl.path) + print("Generated AST for file: `\(fileUrl.path)`") } catch { print("Parsing failed for file: `\(fileUrl.path)` (\(error))") } } private func iterateSwiftFiles(at url: URL) { - let queue = OperationQueue() - queue.name = "SwiftAstGen" - queue.qualityOfService = .userInitiated - queue.maxConcurrentOperationCount = availableProcessors - if let enumerator = FileManager.default.enumerator( at: url, includingPropertiesForKeys: [.isRegularFileKey], @@ -84,15 +78,11 @@ public class SwiftAstGenerator { if fileAttributes.isRegularFile! && fileURL.pathExtension == "swift" { let relativeFilePath = fileURL.relativePath(from: srcDir)! if !ignoreDirectory(name: "/\(relativeFilePath)") { - queue.addOperation { - self.parseFile(fileUrl: fileURL) - } + self.parseFile(fileUrl: fileURL) } } } } - - queue.waitUntilAllOperationsAreFinished() } public func generate() throws { From 59087c84fe1f31b4b169f75192cf4e0ef770fcf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leutha=CC=88user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 13 Aug 2025 08:05:25 +0200 Subject: [PATCH 2/3] Fix for background writing --- .../SwiftAstGenLib/SwiftAstGenerator.swift | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/Sources/SwiftAstGenLib/SwiftAstGenerator.swift b/Sources/SwiftAstGenLib/SwiftAstGenerator.swift index e2c4ac2..6b24b9c 100644 --- a/Sources/SwiftAstGenLib/SwiftAstGenerator.swift +++ b/Sources/SwiftAstGenLib/SwiftAstGenerator.swift @@ -29,13 +29,15 @@ public class SwiftAstGenerator { || nameLowercased.contains("/spec/") } - private func writeStringToFileAsync(string: String, toFilePath filePath: String) { - DispatchQueue.global(qos: .background).async { - do { - let data = Data(string.utf8) - let fileURL = URL(fileURLWithPath: filePath) - try data.write(to: fileURL, options: .atomic) - } catch {} + private func writeStringToFileAsync(string: String, fileUrl: URL) { + DispatchQueue.global(qos: .background).sync { + do { + try string.write( + to: fileUrl, + atomically: true, + encoding: String.Encoding.utf8 + ) + } catch {} } } @@ -61,7 +63,11 @@ public class SwiftAstGenerator { ) } - writeStringToFileAsync(string: astJsonString, toFilePath: outFileUrl.path) + try astJsonString.write( + to: outFileUrl, + atomically: true, + encoding: String.Encoding.utf8 + ) print("Generated AST for file: `\(fileUrl.path)`") } catch { print("Parsing failed for file: `\(fileUrl.path)` (\(error))") From 59a1dce036637504ddf04c7bfc7950663dbe815f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leutha=CC=88user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 13 Aug 2025 08:06:39 +0200 Subject: [PATCH 3/3] remove --- Sources/SwiftAstGenLib/SwiftAstGenerator.swift | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/Sources/SwiftAstGenLib/SwiftAstGenerator.swift b/Sources/SwiftAstGenLib/SwiftAstGenerator.swift index 6b24b9c..4d5faf7 100644 --- a/Sources/SwiftAstGenLib/SwiftAstGenerator.swift +++ b/Sources/SwiftAstGenLib/SwiftAstGenerator.swift @@ -29,18 +29,6 @@ public class SwiftAstGenerator { || nameLowercased.contains("/spec/") } - private func writeStringToFileAsync(string: String, fileUrl: URL) { - DispatchQueue.global(qos: .background).sync { - do { - try string.write( - to: fileUrl, - atomically: true, - encoding: String.Encoding.utf8 - ) - } catch {} - } - } - private func parseFile(fileUrl: URL) { do { let relativeFilePath = fileUrl.relativePath(from: srcDir)!