diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..1812e74 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,27 @@ +{ + "name": "Putty", + "dockerComposeFile": "docker-compose.yaml", + "features": { + "ghcr.io/devcontainers/features/common-utils:2": { + "installZsh": "true", + "username": "user", + "upgradePackages": "false" + }, + "ghcr.io/devcontainers/features/git:1": { + "version": "os-provided", + "ppa": "false" + } + }, + "runArgs": [ + "--cap-add=SYS_PTRACE", + "--security-opt", + "seccomp=unconfined" + ], + "service": "workspace", + "workspaceFolder": "/workspace", + "forwardPorts": [ + 8080 + ], + "postCreateCommand": "sh ./.devcontainer/postCreateCommand.sh", + "remoteUser": "user" +} diff --git a/.devcontainer/docker-compose.yaml b/.devcontainer/docker-compose.yaml new file mode 100644 index 0000000..c5a489d --- /dev/null +++ b/.devcontainer/docker-compose.yaml @@ -0,0 +1,14 @@ +--- +services: + workspace: + image: swift:6.1 + command: sleep infinity + depends_on: [browser] + volumes: [..:/workspace:cached, build-tmp:/workspace/.build/] + env_file: ../.env + browser: + image: selenium/standalone-chromium:latest + shm_size: 2gb + ports: [4444:4444, 7900:7900] +volumes: + build-tmp: {} diff --git a/.devcontainer/postCreateCommand.sh b/.devcontainer/postCreateCommand.sh new file mode 100644 index 0000000..ba2f534 --- /dev/null +++ b/.devcontainer/postCreateCommand.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env sh + +# postCreateCommand.sh +sudo chown user:user -R /workspace/.build +# TODO: Install swiftformat diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..24e5b0a --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +.build diff --git a/.swift-version b/.swift-version index 0cda48a..a435f5a 100644 --- a/.swift-version +++ b/.swift-version @@ -1 +1 @@ -6.2 +6.1 diff --git a/CLI.d b/CLI.d new file mode 100644 index 0000000..72f400e --- /dev/null +++ b/CLI.d @@ -0,0 +1 @@ +CLI.o : /Users/bkreeger/src/kreeger/putty/Sources/CLI/CLI.swift /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX26.0.sdk/usr/lib/swift/_DarwinFoundation1.swiftmodule/arm64e-apple-macos.swiftinterface /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX26.0.sdk/usr/lib/swift/_DarwinFoundation2.swiftmodule/arm64e-apple-macos.swiftinterface /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX26.0.sdk/usr/lib/swift/_DarwinFoundation3.swiftmodule/arm64e-apple-macos.swiftinterface /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX26.0.sdk/usr/lib/swift/XPC.swiftmodule/arm64e-apple-macos.swiftinterface /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX26.0.sdk/usr/lib/swift/ObjectiveC.swiftmodule/arm64e-apple-macos.swiftinterface /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX26.0.sdk/System/Library/Frameworks/Combine.framework/Modules/Combine.swiftmodule/arm64e-apple-macos.swiftinterface /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX26.0.sdk/usr/lib/swift/_StringProcessing.swiftmodule/arm64e-apple-macos.swiftinterface /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX26.0.sdk/usr/lib/swift/Dispatch.swiftmodule/arm64e-apple-macos.swiftinterface /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX26.0.sdk/usr/lib/swift/System.swiftmodule/arm64e-apple-macos.swiftinterface /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX26.0.sdk/usr/lib/swift/Darwin.swiftmodule/arm64e-apple-macos.swiftinterface /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX26.0.sdk/System/Library/Frameworks/Foundation.framework/Modules/Foundation.swiftmodule/arm64e-apple-macos.swiftinterface /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX26.0.sdk/usr/lib/swift/CoreFoundation.swiftmodule/arm64e-apple-macos.swiftinterface /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX26.0.sdk/usr/lib/swift/Observation.swiftmodule/arm64e-apple-macos.swiftinterface /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX26.0.sdk/usr/lib/swift/_Builtin_float.swiftmodule/arm64e-apple-macos.swiftinterface /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX26.0.sdk/usr/lib/swift/Swift.swiftmodule/arm64e-apple-macos.swiftinterface /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX26.0.sdk/usr/lib/swift/IOKit.swiftmodule/arm64e-apple-macos.swiftinterface /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX26.0.sdk/usr/lib/swift/SwiftOnoneSupport.swiftmodule/arm64e-apple-macos.swiftinterface /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX26.0.sdk/usr/lib/swift/_Concurrency.swiftmodule/arm64e-apple-macos.swiftinterface /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/macosx/prebuilt-modules/26.0/_DarwinFoundation1.swiftmodule/arm64e-apple-macos.swiftmodule /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/macosx/prebuilt-modules/26.0/_DarwinFoundation2.swiftmodule/arm64e-apple-macos.swiftmodule /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/macosx/prebuilt-modules/26.0/_DarwinFoundation3.swiftmodule/arm64e-apple-macos.swiftmodule /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/macosx/prebuilt-modules/26.0/XPC.swiftmodule/arm64e-apple-macos.swiftmodule /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/macosx/prebuilt-modules/26.0/ObjectiveC.swiftmodule/arm64e-apple-macos.swiftmodule /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/macosx/prebuilt-modules/26.0/Combine.swiftmodule/arm64e-apple-macos.swiftmodule /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/macosx/prebuilt-modules/26.0/_StringProcessing.swiftmodule/arm64e-apple-macos.swiftmodule /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/macosx/prebuilt-modules/26.0/Dispatch.swiftmodule/arm64e-apple-macos.swiftmodule /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/macosx/prebuilt-modules/26.0/System.swiftmodule/arm64e-apple-macos.swiftmodule /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/macosx/prebuilt-modules/26.0/Darwin.swiftmodule/arm64e-apple-macos.swiftmodule /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/macosx/prebuilt-modules/26.0/Foundation.swiftmodule/arm64e-apple-macos.swiftmodule /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/macosx/prebuilt-modules/26.0/CoreFoundation.swiftmodule/arm64e-apple-macos.swiftmodule /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/macosx/prebuilt-modules/26.0/Observation.swiftmodule/arm64e-apple-macos.swiftmodule /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/macosx/prebuilt-modules/26.0/_Builtin_float.swiftmodule/arm64e-apple-macos.swiftmodule /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/macosx/prebuilt-modules/26.0/Swift.swiftmodule/arm64e-apple-macos.swiftmodule /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/macosx/prebuilt-modules/26.0/IOKit.swiftmodule/arm64e-apple-macos.swiftmodule /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/macosx/prebuilt-modules/26.0/SwiftOnoneSupport.swiftmodule/arm64e-apple-macos.swiftmodule /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/macosx/prebuilt-modules/26.0/_Concurrency.swiftmodule/arm64e-apple-macos.swiftmodule /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX26.0.sdk/usr/include/_DarwinFoundation2.apinotes /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX26.0.sdk/usr/include/XPC.apinotes /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX26.0.sdk/usr/include/ObjectiveC.apinotes /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX26.0.sdk/usr/include/Dispatch.apinotes /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX26.0.sdk/System/Library/Frameworks/Foundation.framework/Headers/Foundation.apinotes /Applications/Xcode-26.0.0-beta.6.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX26.0.sdk/System/Library/Frameworks/Security.framework/Headers/Security.apinotes /Users/bkreeger/src/kreeger/putty/.build/arm64-apple-macosx/debug/Modules/ArgumentParserToolInfo.swiftmodule /Users/bkreeger/src/kreeger/putty/.build/arm64-apple-macosx/debug/Modules/ArgumentParser.swiftmodule diff --git a/CLI.dia b/CLI.dia new file mode 100644 index 0000000..48d7a77 Binary files /dev/null and b/CLI.dia differ diff --git a/Package.swift b/Package.swift index b61906a..e2952d6 100644 --- a/Package.swift +++ b/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version: 6.2 +// swift-tools-version: 6.1 // The swift-tools-version declares the minimum version of Swift required to build this package. import PackageDescription @@ -6,14 +6,20 @@ import PackageDescription let package = Package( name: "Putty", platforms: [.macOS(.v15)], + products: [ + .library(name: "PuttyKit", targets: ["PuttyKit"]), + ], dependencies: [ .package(url: "https://github.com/apple/swift-argument-parser", from: "1.6.1"), ], targets: [ // Targets are the basic building blocks of a package, defining a module or a test suite. // Targets can depend on other targets in this package and products from dependencies. - .executableTarget(name: "putty", dependencies: [ - .product(name: "ArgumentParser", package: "swift-argument-parser"), - ]), - ], -) + .target(name: "PuttyKit"), + .executableTarget(name: "putty", + dependencies: [ + .product(name: "ArgumentParser", package: "swift-argument-parser"), + .target(name: "PuttyKit"), + ], + path: "Sources/CLI"), + ]) diff --git a/Sources/Putty/Putty.swift b/Sources/CLI/CLI.swift similarity index 68% rename from Sources/Putty/Putty.swift rename to Sources/CLI/CLI.swift index 893f5b5..c2002c9 100644 --- a/Sources/Putty/Putty.swift +++ b/Sources/CLI/CLI.swift @@ -1,12 +1,12 @@ import ArgumentParser +import PuttyKit @main struct CLI: AsyncParsableCommand { static let configuration: CommandConfiguration = .init( commandName: "putty", abstract: "A utility for getting comic data from GoComics.com.", - subcommands: [Scrape.self], - ) + subcommands: [Scrape.self]) } struct Scrape: AsyncParsableCommand { @@ -14,5 +14,8 @@ struct Scrape: AsyncParsableCommand { mutating func run() async throws { print("scrape") + let fetcher = PageFetcher(webDriverURL: "http://browser:4444") + _ = try await fetcher.fetchAToZ() + print("Fetch complete!") } } diff --git a/Sources/PuttyKit/Fetchers/PageFetcher.swift b/Sources/PuttyKit/Fetchers/PageFetcher.swift new file mode 100644 index 0000000..dc870b4 --- /dev/null +++ b/Sources/PuttyKit/Fetchers/PageFetcher.swift @@ -0,0 +1,125 @@ +import Foundation +#if canImport(FoundationNetworking) + import FoundationNetworking +#endif + +/// A class that connects to a Selenium WebDriver running Chromium to fetch HTML page content +public final class PageFetcher { + private let webDriverURL: URL + private let session = URLSession.shared + + // MARK: - Initialization + + /// Initialize PageFetcher with WebDriver URL + /// - Parameter webDriverURL: The URL where Selenium WebDriver is running (default: http://localhost:4444) + public init(webDriverURL: String) { + self.webDriverURL = URL(string: webDriverURL)! + } + + // MARK: - Public Methods + + /// Fetch the raw HTML content from the specified URL. + /// + /// - Parameter url: The URL to fetch content from + /// - Returns: The raw HTML content as a string + /// - Throws: PageFetcherError for various failure scenarios + public func fetchHTML(from url: String) async throws -> String { + let sessionId = try await startSession() + try await navigateToURL(url, sessionId: sessionId) + let source = try await getPageSource(sessionId: sessionId) + try await endSession(sessionId: sessionId) + return source + } + + // MARK: - Private Methods + + /// Start a new WebDriver session with Chrome capabilities. + private func startSession() async throws -> String { + let capabilities = [ + "capabilities": [ + "alwaysMatch": [ + "browserName": "chrome", + "goog:chromeOptions": [ + "args": [ + "--headless", + "--no-sandbox", + "--disable-dev-shm-usage", + "--disable-gpu", + "--window-size=1920,1080", + ], + ], + ], + ], + ] + + let (data, _) = try await makeRequest(verb: "POST", path: "wd/hub/session", body: capabilities) + guard let json = try JSONSerialization.jsonObject(with: data) as? [String: Any], + let value = json["value"] as? [String: Any], + let sessionId = value["sessionId"] as? String + else { + throw PageFetcherError.invalidSessionResponse + } + return sessionId + } + + private func navigateToURL(_ url: String, sessionId: String) async throws { + _ = try await makeRequest(verb: "POST", path: "url", sessionId: sessionId, body: ["url": url]) + } + + private func getPageSource(sessionId: String) async throws -> String { + let (data, _) = try await makeRequest(verb: "GET", path: "source", sessionId: sessionId) + guard let json = try JSONSerialization.jsonObject(with: data) as? [String: Any], + let value = json["value"] as? String + else { + throw PageFetcherError.invalidPageSourceResponse + } + + return value + } + + private func endSession(sessionId: String) async throws { + _ = try await makeRequest(verb: "DELETE", path: "", sessionId: sessionId) + } + + private func makeRequest(verb: String, path: String, + body: [String: Any]? = nil) async throws -> (Data, HTTPURLResponse) + { + var request = URLRequest(url: webDriverURL.appendingPathComponent(path)) + print("\(verb) \(request.url?.absoluteString ?? "")") + request.httpMethod = verb + if verb == "POST" { + request.setValue("application/json", forHTTPHeaderField: "Content-Type") + request.httpBody = try JSONSerialization.data(withJSONObject: body as Any) + } + let (data, response) = try await session.data(for: request) + guard let httpResponse = response as? HTTPURLResponse else { + throw PageFetcherError.invalidSessionResponse + } + guard httpResponse.statusCode == 200 else { + let maybeJSON = try? JSONSerialization.jsonObject(with: data) as? + [String: Any] + let value = maybeJSON?["value"] as? [String: Any] + let message = value?["message"] as? String ?? "" + throw PageFetcherError.seleniumError(httpResponse.statusCode, message) + } + return (data, httpResponse) + } + + private func makeRequest(verb: String, path: String, sessionId: String, + body: [String: Any]? = nil) async throws -> (Data, HTTPURLResponse) + { + let addendum = [sessionId, path].filter { !$0.isEmpty }.joined(separator: "/") + return try await makeRequest(verb: verb, path: "wd/hub/session/\(addendum)", body: body) + } +} + +// MARK: - Errors + +// MARK: - Usage Example + +public extension PageFetcher { + /// Convenience method to fetch GoComics A-Z page + func fetchAToZ() async throws -> String { + try await fetchHTML(from: "https://www.gocomics.com/comics/a-to-z") + } +} diff --git a/Sources/PuttyKit/Fetchers/PageFetcherError.swift b/Sources/PuttyKit/Fetchers/PageFetcherError.swift new file mode 100644 index 0000000..0a5e923 --- /dev/null +++ b/Sources/PuttyKit/Fetchers/PageFetcherError.swift @@ -0,0 +1,28 @@ +import Foundation + +/// Describes any errors that can happen with the PageFetcher. +public enum PageFetcherError: Error, LocalizedError { + case sessionCreationFailed + case invalidSessionResponse + case noActiveSession + case seleniumError(Int, String) + case requestFailed + case invalidPageSourceResponse + + public var errorDescription: String? { + switch self { + case .sessionCreationFailed: + "Failed to create WebDriver session" + case .invalidSessionResponse: + "Invalid response when creating WebDriver session" + case .noActiveSession: + "No active WebDriver session" + case let .seleniumError(code, message): + "Selenium error (HTTP \(code)): \(message)" + case .requestFailed: + "Failed to make request" + case .invalidPageSourceResponse: + "Invalid response when getting page source" + } + } +}