From 772ba73fad95cbd5e0aa380a2143ddf1e955616f Mon Sep 17 00:00:00 2001 From: Tim Morgan Date: Mon, 14 Sep 2026 13:03:47 -0700 Subject: [PATCH 1/7] Modernize for Swift 6.4 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adopt typed throws across the parsing surface: `DOF.init(data:)`, `DOF.init(url:)`, the `DOF.from(…)` factories, `DOFByteParser`, and the DOF file line reader's `AsyncIteratorProtocol.Failure` now carry `DOFError`, while `AsyncBytesLineReader` propagates its source sequence's own `Failure` type. Extract the chunked file reader into `FileLineReader` so both `AsyncDOFLineReader` and the new synchronous streaming initializer share one implementation, and route `DOF.from(filePath:)` through it instead of reading the whole file into memory. Match the "CURRENCY DATE = " header marker against an `InlineArray<16, UInt8>`, removing a heap allocation and the crash on input shorter than the marker. Replace the force-unwraps in `Cycle.previous`, `Cycle.next`, and the cycle datum date with a shared failable helper and a precondition. Add Swift 6.4 CI legs alongside the newest leg for each OS already in the matrix. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01EbZbx5D2gGeXT8UxuiKEdq --- .github/workflows/ci.yml | 8 +- CHANGELOG.md | 12 ++ Sources/SwiftDOF/Cycle.swift | 58 +++---- Sources/SwiftDOF/DOF.swift | 64 ++++++-- Sources/SwiftDOF/Parser/ByteParsing.swift | 2 +- Sources/SwiftDOF/Parser/DOFByteParser.swift | 56 ++++--- Sources/SwiftDOF/Parser/DOFLineReader.swift | 166 ++++++++++++-------- Tests/SwiftDOFTests/DOFTests.swift | 58 +++++++ 8 files changed, 295 insertions(+), 129 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b219a92..199231a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,11 +5,9 @@ # Category C (tools-version 6.3): macos-15 + 6.3, macos-26 + 6.3 # Linux: ubuntu + Swift 6.3 # -# This package is Category A, plus a Linux leg on Swift 6.3. Test names are raw +# This package is Category A, plus a Linux leg. Test names are raw # identifiers (SE-0451), so a leg must be on Swift 6.2 or newer to run `swift test`; # a leg on an older compiler is limited to `swift build -v`. -# -# When Swift 6.4 ships: add 6.4 legs alongside 6.2 name: CI @@ -32,8 +30,12 @@ jobs: include: - os: macos-26 swift: "6.2" + - os: macos-26 + swift: "6.4" - os: ubuntu-latest swift: "6.3" + - os: ubuntu-latest + swift: "6.4" runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v6 diff --git a/CHANGELOG.md b/CHANGELOG.md index bd53185..0dcc62c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Change Log +## [Unreleased] + +### Changed + +- Adopt typed throws across the parsing surface: `DOF.init(data:)`, `DOF.init(url:)`, the `DOF.from(…)` factories, and `DOFByteParser` now declare `throws(DOFError)`, and the DOF file line reader's `AsyncIteratorProtocol.Failure` is `DOFError`. `AsyncBytesLineReader` propagates its source sequence's own `Failure` type. +- `DOF.from(filePath:)` streams the file in chunks instead of reading it into memory in its entirety. A file that cannot be opened now throws `DOFError.fileNotFound` rather than a Foundation file-read error. +- Match the header's "CURRENCY DATE = " marker against an `InlineArray<16, UInt8>`, removing a heap allocation from currency date parsing. + +### Fixed + +- `Cycle.previous`, `Cycle.next`, and the cycle datum date no longer force-unwrap optionals. + ## [1.3.0] - 2026-09-14 ### Changed diff --git a/Sources/SwiftDOF/Cycle.swift b/Sources/SwiftDOF/Cycle.swift index e087ea9..6600b8b 100644 --- a/Sources/SwiftDOF/Cycle.swift +++ b/Sources/SwiftDOF/Cycle.swift @@ -26,7 +26,10 @@ public struct Cycle: Sendable, Codable, Equatable, Hashable { month: datum.month, day: datum.day ) - return calendar.date(from: components)! + guard let date = calendar.date(from: components) else { + preconditionFailure("The DOF datum is not a valid Gregorian date") + } + return date } /// The currently effective cycle based on today's date. @@ -42,34 +45,10 @@ public struct Cycle: Sendable, Codable, Equatable, Hashable { public let day: UInt8 /// The cycle preceding this one (56 days earlier). - public var previous: Self? { - guard let firstDate, - let previousDate = Self.calendar.date(byAdding: .day, value: -Self.period, to: firstDate) - else { - return nil - } - let components = Self.calendar.dateComponents([.year, .month, .day], from: previousDate) - return Self( - year: UInt(components.year!), - month: UInt8(components.month!), - day: UInt8(components.day!) - ) - } + public var previous: Self? { cycle(offsetByDays: -Self.period) } /// The cycle following this one (56 days later). - public var next: Self? { - guard let firstDate, - let nextDate = Self.calendar.date(byAdding: .day, value: Self.period, to: firstDate) - else { - return nil - } - let components = Self.calendar.dateComponents([.year, .month, .day], from: nextDate) - return Self( - year: UInt(components.year!), - month: UInt8(components.month!), - day: UInt8(components.day!) - ) - } + public var next: Self? { cycle(offsetByDays: Self.period) } /// Whether this cycle falls on a valid cycle boundary. /// @@ -213,6 +192,31 @@ public struct Cycle: Sendable, Codable, Equatable, Hashable { self.init(year: UInt(year), month: UInt8(month), day: UInt8(day)) } + + /// Creates a cycle from date components that already fall on a cycle boundary. + /// + /// - Parameter components: Components carrying a year, month, and day. + private init?(dateComponents components: DateComponents) { + guard let year = components.year, + let month = components.month, + let day = components.day + else { + return nil + } + self.init(year: UInt(year), month: UInt8(month), day: UInt8(day)) + } + + /// The cycle whose start date is `days` away from this one's. + private func cycle(offsetByDays days: Int) -> Self? { + guard let firstDate, + let shiftedDate = Self.calendar.date(byAdding: .day, value: days, to: firstDate) + else { + return nil + } + return Self( + dateComponents: Self.calendar.dateComponents([.year, .month, .day], from: shiftedDate) + ) + } } extension Cycle: LosslessStringConvertible { diff --git a/Sources/SwiftDOF/DOF.swift b/Sources/SwiftDOF/DOF.swift index 2bc231b..fcc0e06 100644 --- a/Sources/SwiftDOF/DOF.swift +++ b/Sources/SwiftDOF/DOF.swift @@ -37,7 +37,7 @@ public struct DOF: Sendable, Codable { data: Data, progressHandler: @Sendable (Progress) -> Void = { _ in }, errorCallback: ((any Error, Int) -> Void)? = nil - ) throws { + ) throws(DOFError) { var obstacles: [String: Obstacle] = [:] obstacles.reserveCapacity(Self.estimatedObstacleCount) @@ -84,7 +84,7 @@ public struct DOF: Sendable, Codable { url: URL, progressHandler: @Sendable (Progress) -> Void = { _ in }, errorCallback: ((any Error, Int) -> Void)? = nil - ) async throws { + ) async throws(DOFError) { var obstacles: [String: Obstacle] = [:] obstacles.reserveCapacity(Self.estimatedObstacleCount) @@ -180,6 +180,47 @@ public struct DOF: Sendable, Codable { self.obstaclesByID = obstacles } + /// Creates a DOF container by streaming a file from disk without holding it all in memory. + private init( + streamingFrom url: URL, + progressHandler: @Sendable (Progress) -> Void, + errorCallback: ((any Error, Int) -> Void)? + ) throws(DOFError) { + var obstacles: [String: Obstacle] = [:] + obstacles.reserveCapacity(Self.estimatedObstacleCount) + + var lineNumber = 0 + var cycle: Cycle? + + var reader = FileLineReader(url: url) + + // Setup progress tracking based on file size + let progress = Progress(totalUnitCount: reader.fileSize ?? -1) + progressHandler(progress) + + while let line = try reader.next() { + lineNumber += 1 + try Self.processLine( + line[...], + lineNumber: lineNumber, + cycle: &cycle, + obstacles: &obstacles, + errorCallback: errorCallback + ) + progress.completedUnitCount = reader.bytesRead + } + if progress.totalUnitCount > 0 { + progress.completedUnitCount = progress.totalUnitCount + } + + guard let cycle else { + throw DOFError.invalidFormat(.missingCurrencyDate) + } + + self.cycle = cycle + self.obstaclesByID = obstacles + } + /// Process a single line from the DOF file. private static func processLine( _ line: ArraySlice, @@ -187,7 +228,7 @@ public struct DOF: Sendable, Codable { cycle: inout Cycle?, obstacles: inout [String: Obstacle], errorCallback: ((any Error, Int) -> Void)? - ) throws { + ) throws(DOFError) { // Line 1: Parse currency date if lineNumber == 1 { cycle = try DOFByteParser.parseCurrencyDate(line) @@ -213,7 +254,7 @@ public struct DOF: Sendable, Codable { // MARK: - Static Factory Methods - /// Load DOF data from a file path (synchronous). + /// Load DOF data from a file path (synchronous), streaming the file from disk. /// /// - Parameters: /// - filePath: The URL of the DOF file. @@ -221,14 +262,17 @@ public struct DOF: Sendable, Codable { /// object that you can use to track parsing progress. /// - errorCallback: Optional callback for parse errors. /// - Returns: The parsed DOF. - /// - Throws: Error if loading or parsing fails. + /// - Throws: ``DOFError`` if loading or parsing fails. public static func from( filePath: URL, progressHandler: @Sendable (Progress) -> Void = { _ in }, errorCallback: ((any Error, Int) -> Void)? = nil - ) throws -> Self { - let data = try Data(contentsOf: filePath) - return try Self(data: data, progressHandler: progressHandler, errorCallback: errorCallback) + ) throws(DOFError) -> Self { + try Self( + streamingFrom: filePath, + progressHandler: progressHandler, + errorCallback: errorCallback + ) } /// Load DOF data from raw data. @@ -244,7 +288,7 @@ public struct DOF: Sendable, Codable { data: Data, progressHandler: @Sendable (Progress) -> Void = { _ in }, errorCallback: ((any Error, Int) -> Void)? = nil - ) throws -> Self { + ) throws(DOFError) -> Self { try Self(data: data, progressHandler: progressHandler, errorCallback: errorCallback) } @@ -261,7 +305,7 @@ public struct DOF: Sendable, Codable { url: URL, progressHandler: @Sendable (Progress) -> Void = { _ in }, errorCallback: ((any Error, Int) -> Void)? = nil - ) async throws -> Self { + ) async throws(DOFError) -> Self { try await Self(url: url, progressHandler: progressHandler, errorCallback: errorCallback) } diff --git a/Sources/SwiftDOF/Parser/ByteParsing.swift b/Sources/SwiftDOF/Parser/ByteParsing.swift index 5fd37ce..b652705 100644 --- a/Sources/SwiftDOF/Parser/ByteParsing.swift +++ b/Sources/SwiftDOF/Parser/ByteParsing.swift @@ -102,7 +102,7 @@ extension RandomAccessCollection where Element == UInt8, Index == Int { /// Convert to trimmed String (only when actually needed). /// - Throws: DOFError.invalidEncoding if bytes cannot be decoded as Latin-1. @inlinable - func toString() throws -> String { + func toString() throws(DOFError) -> String { guard let string = String(bytes: Array(self), encoding: .isoLatin1) else { throw DOFError.invalidEncoding } diff --git a/Sources/SwiftDOF/Parser/DOFByteParser.swift b/Sources/SwiftDOF/Parser/DOFByteParser.swift index 765eac1..86bfeb7 100644 --- a/Sources/SwiftDOF/Parser/DOFByteParser.swift +++ b/Sources/SwiftDOF/Parser/DOFByteParser.swift @@ -34,8 +34,13 @@ struct DOFByteParser: Sendable { /// Minimum line length required for parsing. static let minimumLineLength = 127 - /// Pattern to match in currency date header. - private static let currencyDatePattern: [UInt8] = Array("CURRENCY DATE = ".utf8) + /// The bytes of "CURRENCY DATE = ", the pattern preceding the date in the DOF header. + private static let currencyDatePattern: InlineArray<16, UInt8> = [ + UInt8(ascii: "C"), UInt8(ascii: "U"), UInt8(ascii: "R"), UInt8(ascii: "R"), + UInt8(ascii: "E"), UInt8(ascii: "N"), UInt8(ascii: "C"), UInt8(ascii: "Y"), + UInt8(ascii: " "), UInt8(ascii: "D"), UInt8(ascii: "A"), UInt8(ascii: "T"), + UInt8(ascii: "E"), UInt8(ascii: " "), UInt8(ascii: "="), UInt8(ascii: " ") + ] // MARK: Public API @@ -43,7 +48,7 @@ struct DOFByteParser: Sendable { static func parseLine( _ bytes: T, lineNumber: Int = 0 - ) throws -> Obstacle where T.Element == UInt8, T.Index == Int { + ) throws(DOFError) -> Obstacle where T.Element == UInt8, T.Index == Int { guard bytes.count >= minimumLineLength else { throw DOFError.lineTooShort( expected: minimumLineLength, @@ -170,26 +175,12 @@ struct DOFByteParser: Sendable { /// Expects format: "CURRENCY DATE = MM/DD/YY" static func parseCurrencyDate( _ bytes: T - ) throws -> Cycle where T.Element == UInt8, T.Index == Int { - // Find pattern - var matchStart: Int? - for i in bytes.startIndex..<(bytes.endIndex - currencyDatePattern.count) { - let slice = bytes[i..<(i + currencyDatePattern.count)] - guard zip(slice, currencyDatePattern).allSatisfy({ $0 == $1 }) else { continue } - matchStart = i + currencyDatePattern.count - break - } - - guard let start = matchStart else { + ) throws(DOFError) -> Cycle where T.Element == UInt8, T.Index == Int { + guard let start = currencyDateStart(in: bytes) else { throw DOFError.invalidFormat(.currencyDateHeaderNotFound) } - // Find slash positions in date portion - let dateBytes = bytes[start...] - var slashPositions: [Int] = [] - for (i, byte) in dateBytes.enumerated() where byte == ASCII.slash { - slashPositions.append(start + i) - } + let slashPositions = (start..= 2 else { throw DOFError.invalidFormat(.invalidCurrencyDateFormat) @@ -212,6 +203,25 @@ struct DOFByteParser: Sendable { // MARK: Private Helpers + /// The index just past the currency date pattern, or `nil` when the pattern is absent. + private static func currencyDateStart( + in bytes: T + ) -> Int? where T.Element == UInt8, T.Index == Int { + let patternLength = currencyDatePattern.count + guard bytes.count >= patternLength else { return nil } + + return (bytes.startIndex..<(bytes.endIndex - patternLength)) + .first { matchesCurrencyDatePattern(bytes, at: $0) } + .map { $0 + patternLength } + } + + private static func matchesCurrencyDatePattern( + _ bytes: T, + at start: Int + ) -> Bool where T.Element == UInt8, T.Index == Int { + currencyDatePattern.indices.allSatisfy { bytes[start + $0] == currencyDatePattern[$0] } + } + private static func slice( _ bytes: T, _ base: Int, @@ -224,7 +234,7 @@ struct DOFByteParser: Sendable { _ bytes: T, base: Int, lineNumber: Int - ) throws -> Double where T.Element == UInt8, T.Index == Int { + ) throws(DOFError) -> Double where T.Element == UInt8, T.Index == Int { let degSlice = slice(bytes, base, fields.latDegrees) let minSlice = slice(bytes, base, fields.latMinutes) let secSlice = slice(bytes, base, fields.latSeconds) @@ -268,7 +278,7 @@ struct DOFByteParser: Sendable { _ bytes: T, base: Int, lineNumber: Int - ) throws -> Double where T.Element == UInt8, T.Index == Int { + ) throws(DOFError) -> Double where T.Element == UInt8, T.Index == Int { let degSlice = slice(bytes, base, fields.lonDegrees) let minSlice = slice(bytes, base, fields.lonMinutes) let secSlice = slice(bytes, base, fields.lonSeconds) @@ -314,7 +324,7 @@ struct DOFByteParser: Sendable { private static func parseJulianDate( _ bytes: T, lineNumber: Int - ) throws -> DateComponents where T.Element == UInt8, T.Index == Int { + ) throws(DOFError) -> DateComponents where T.Element == UInt8, T.Index == Int { // Format: YYYYDDD (e.g., 2014138 = year 2014, day 138) let yearSlice = bytes.prefix(4) let daySlice = bytes.dropFirst(4) diff --git a/Sources/SwiftDOF/Parser/DOFLineReader.swift b/Sources/SwiftDOF/Parser/DOFLineReader.swift index 511902b..e78eaa9 100644 --- a/Sources/SwiftDOF/Parser/DOFLineReader.swift +++ b/Sources/SwiftDOF/Parser/DOFLineReader.swift @@ -45,13 +45,11 @@ struct DOFLineReader: Sequence, IteratorProtocol, Sendable { } } -// MARK: - AsyncDOFLineReader - -/// Async line reader for streaming DOF data from a file URL. -/// Reads in chunks to minimize memory usage for large files. -struct AsyncDOFLineReader: AsyncSequence, Sendable { - typealias Element = [UInt8] +// MARK: - FileLineReader +/// Line reader that streams DOF data from a file on disk. +/// Reads in chunks so a large file is never held in memory in its entirety. +struct FileLineReader: Sendable { /// Default read buffer size (64KB). static let defaultBufferSize = 65536 @@ -60,83 +58,120 @@ struct AsyncDOFLineReader: AsyncSequence, Sendable { private let url: URL private let bufferSize: Int + private var handle: FileHandle? + private var buffer: [UInt8] = [] + private var bufferPosition = 0 + private var lineBuffer: [UInt8] = [] + private var isAtEnd = false /// The total size of the file in bytes, if known. let fileSize: Int64? + /// Total bytes read from the file so far. + private(set) var bytesRead: Int64 = 0 + + private var bufferIsExhausted: Bool { bufferPosition >= buffer.count } + init(url: URL, bufferSize: Int = defaultBufferSize) { self.url = url self.bufferSize = bufferSize - // Try to get file size for progress tracking - if let attrs = try? FileManager.default.attributesOfItem(atPath: url.path), - let size = attrs[.size] as? Int64 - { - self.fileSize = size - } else { - self.fileSize = nil - } + self.fileSize = Self.sizeOfFile(at: url) + lineBuffer.reserveCapacity(Self.lineBufferCapacity) } - func makeAsyncIterator() -> AsyncIterator { - AsyncIterator(url: url, bufferSize: bufferSize) + /// The size in bytes of the file at `url`, if it can be determined. + static func sizeOfFile(at url: URL) -> Int64? { + guard let attributes = try? FileManager.default.attributesOfItem(atPath: url.path) else { + return nil + } + return attributes[.size] as? Int64 } - struct AsyncIterator: AsyncIteratorProtocol { - private let url: URL - private let bufferSize: Int - private var handle: FileHandle? - private var buffer: [UInt8] = [] - private var bufferPos: Int = 0 - private var lineBuffer: [UInt8] = [] - private var isEOF = false + /// Returns the next line, or `nil` once the file is exhausted. + mutating func next() throws(DOFError) -> [UInt8]? { + guard !isAtEnd else { return nil } - /// Total bytes read from the file so far. - private(set) var bytesRead: Int64 = 0 + let handle = try openedHandle() + lineBuffer.removeAll(keepingCapacity: true) - init(url: URL, bufferSize: Int) { - self.url = url - self.bufferSize = bufferSize - self.lineBuffer.reserveCapacity(lineBufferCapacity) - } + while true { + if bufferIsExhausted { + guard let chunk = try readChunk(from: handle), !chunk.isEmpty else { + isAtEnd = true + return lineBuffer.isEmpty ? nil : lineBuffer + } + bytesRead += Int64(chunk.count) + buffer = Array(chunk) + bufferPosition = 0 + } - mutating func next() throws -> [UInt8]? { - guard !isEOF else { return nil } + let byte = buffer[bufferPosition] + bufferPosition += 1 - // Lazily open file handle on first call - if handle == nil { - handle = try FileHandle(forReadingFrom: url) + if byte == ASCII.LF { + // Strip trailing CR if present (handles CRLF) + if lineBuffer.last == ASCII.CR { + lineBuffer.removeLast() + } + return lineBuffer } - guard let handle else { preconditionFailure("handle was nil") } + lineBuffer.append(byte) + } + } - lineBuffer.removeAll(keepingCapacity: true) + private mutating func openedHandle() throws(DOFError) -> FileHandle { + if let handle { return handle } + guard let opened = try? FileHandle(forReadingFrom: url) else { + throw DOFError.fileNotFound(url) + } + handle = opened + return opened + } - while true { - // Refill buffer if exhausted - if bufferPos >= buffer.count { - guard let chunk = try handle.read(upToCount: bufferSize), - !chunk.isEmpty - else { - isEOF = true - // Return any remaining content as final line - return lineBuffer.isEmpty ? nil : lineBuffer - } - bytesRead += Int64(chunk.count) - buffer = Array(chunk) - bufferPos = 0 - } + private func readChunk(from handle: FileHandle) throws(DOFError) -> Data? { + do { + return try handle.read(upToCount: bufferSize) + } catch { + throw DOFError.streamError(error) + } + } +} - let byte = buffer[bufferPos] - bufferPos += 1 +// MARK: - AsyncDOFLineReader - if byte == ASCII.LF { - // Strip trailing CR if present (handles CRLF) - if lineBuffer.last == ASCII.CR { - lineBuffer.removeLast() - } - return lineBuffer - } - lineBuffer.append(byte) - } +/// Async façade over ``FileLineReader`` for `for await` iteration of a DOF file. +struct AsyncDOFLineReader: AsyncSequence, Sendable { + typealias Element = [UInt8] + typealias Failure = DOFError + + private let url: URL + private let bufferSize: Int + + /// The total size of the file in bytes, if known. + let fileSize: Int64? + + init(url: URL, bufferSize: Int = FileLineReader.defaultBufferSize) { + self.url = url + self.bufferSize = bufferSize + self.fileSize = FileLineReader.sizeOfFile(at: url) + } + + func makeAsyncIterator() -> AsyncIterator { + AsyncIterator(reader: FileLineReader(url: url, bufferSize: bufferSize)) + } + + struct AsyncIterator: AsyncIteratorProtocol { + private var reader: FileLineReader + + /// Total bytes read from the file so far. + var bytesRead: Int64 { reader.bytesRead } + + init(reader: FileLineReader) { + self.reader = reader + } + + mutating func next() throws(DOFError) -> [UInt8]? { + try reader.next() } } } @@ -148,6 +183,7 @@ struct AsyncDOFLineReader: AsyncSequence, Sendable { struct AsyncBytesLineReader: AsyncSequence, Sendable where Source.Element == UInt8, Source: Sendable { typealias Element = [UInt8] + typealias Failure = Source.Failure /// Pre-allocated capacity for line buffer (DOF lines are ~128 bytes). private static var lineBufferCapacity: Int { 256 } @@ -172,10 +208,10 @@ where Source.Element == UInt8, Source: Sendable { } @concurrent - mutating func next() async throws -> [UInt8]? { + mutating func next() async throws(Source.Failure) -> [UInt8]? { lineBuffer.removeAll(keepingCapacity: true) - while let byte = try await iterator.next() { + while let byte = try await iterator.next(isolation: nil) { if byte == ASCII.LF { // Strip trailing CR if present (handles CRLF) if lineBuffer.last == ASCII.CR { diff --git a/Tests/SwiftDOFTests/DOFTests.swift b/Tests/SwiftDOFTests/DOFTests.swift index 1c1ad44..e49ffb1 100644 --- a/Tests/SwiftDOFTests/DOFTests.swift +++ b/Tests/SwiftDOFTests/DOFTests.swift @@ -23,6 +23,11 @@ struct DOFTests { sampleDOFContent.data(using: .utf8)! } + /// A URL in the temporary directory that no file occupies. + private static func temporaryFileURL() -> URL { + FileManager.default.temporaryDirectory.appendingPathComponent("\(UUID().uuidString).dat") + } + @Test func `parses obstacles and the currency date from DOF data`() throws { let dof = try DOF(data: sampleDOFData) @@ -172,6 +177,41 @@ struct DOFTests { #expect(dof.count == 3) } + @Test + func `parses obstacles streamed from a file on disk`() throws { + let dof = try withTemporaryFile(containing: sampleDOFContent) { + try DOF.from(filePath: $0) + } + + #expect(dof.count == 3) + #expect(dof.cycle.year == 2025) + } + + @Test(arguments: [1, 7, 64, 4096]) + func `reads the same lines from a file whatever the buffer size`(_ bufferSize: Int) throws { + let lines = try withTemporaryFile(containing: sampleDOFContent) { url in + var reader = FileLineReader(url: url, bufferSize: bufferSize) + var lines: [[UInt8]] = [] + while let line = try reader.next() { + lines.append(line) + } + return lines + } + + #expect(lines == sampleDOFContent.split(separator: "\n").map { Array($0.utf8) }) + } + + @Test + func `throws a file-not-found error for a missing file`() { + let missingFile = Self.temporaryFileURL() + + let error = #expect(throws: DOFError.self) { + try DOF.from(filePath: missingFile) + } + + #expect(error?.isFileNotFound == true) + } + @Test func `leaves the error callback uncalled for valid data`() throws { var errorCalled = false @@ -186,4 +226,22 @@ struct DOFTests { #expect(dof.count == 3) #expect(!errorCalled) // No errors in valid content } + + /// Writes `content` to a temporary file, hands its URL to `body`, and removes the file after. + private func withTemporaryFile( + containing content: String, + _ body: (URL) throws -> T + ) throws -> T { + let url = Self.temporaryFileURL() + try content.write(to: url, atomically: true, encoding: .utf8) + defer { try? FileManager.default.removeItem(at: url) } + return try body(url) + } +} + +extension DOFError { + fileprivate var isFileNotFound: Bool { + if case .fileNotFound = self { return true } + return false + } } From fb43673e34cfd2281d41c91a2419e89600b6cc11 Mon Sep 17 00:00:00 2001 From: Tim Morgan Date: Mon, 14 Sep 2026 14:53:07 -0700 Subject: [PATCH 2/7] Drop the unused bytesRead forward on the line iterator Extracting the chunked reader out of the async iterator left `AsyncIterator.bytesRead` forwarding to a reader nothing asks it about; progress reporting reads `FileLineReader.bytesRead` directly. Periphery flags it, failing the strict scan. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01EbZbx5D2gGeXT8UxuiKEdq --- Sources/SwiftDOF/Parser/DOFLineReader.swift | 3 --- 1 file changed, 3 deletions(-) diff --git a/Sources/SwiftDOF/Parser/DOFLineReader.swift b/Sources/SwiftDOF/Parser/DOFLineReader.swift index e78eaa9..c195d93 100644 --- a/Sources/SwiftDOF/Parser/DOFLineReader.swift +++ b/Sources/SwiftDOF/Parser/DOFLineReader.swift @@ -163,9 +163,6 @@ struct AsyncDOFLineReader: AsyncSequence, Sendable { struct AsyncIterator: AsyncIteratorProtocol { private var reader: FileLineReader - /// Total bytes read from the file so far. - var bytesRead: Int64 { reader.bytesRead } - init(reader: FileLineReader) { self.reader = reader } From 94424d921ef96bb284457da7b677d34e4be245ec Mon Sep 17 00:00:00 2001 From: Tim Morgan Date: Mon, 14 Sep 2026 16:06:34 -0700 Subject: [PATCH 3/7] Keep the macOS 26 floor for the InlineArray work `main` lowered this package's floor to what its code there requires. The byte-parsing work on this branch uses `InlineArray`, which is macOS 26, so the branch declares the floor its own code needs. Merging this therefore raises the floor. That is the trade the branch asks for and it should be decided on the merge, not worked around in the source. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01EbZbx5D2gGeXT8UxuiKEdq --- Package.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Package.swift b/Package.swift index 71bf0e7..2701820 100644 --- a/Package.swift +++ b/Package.swift @@ -15,7 +15,7 @@ let upcomingFeatures: [SwiftSetting] = [ let package = Package( name: "SwiftDOF", defaultLocalization: "en", - platforms: [.macOS(.v15), .iOS(.v18), .watchOS(.v11), .tvOS(.v18), .visionOS(.v2)], + platforms: [.macOS(.v26), .iOS(.v26), .watchOS(.v26), .tvOS(.v26), .visionOS(.v26)], products: [ // Products define the executables and libraries a package produces, making them visible to other packages. .library( From de059d68a4512f522e96e31e8c5815bd97fa8c43 Mon Sep 17 00:00:00 2001 From: Tim Morgan Date: Mon, 14 Sep 2026 18:01:52 -0700 Subject: [PATCH 4/7] Adopt strict memory safety Enable `.strictMemorySafety()` (SE-0458) alongside the existing upcoming feature flags and audit every unsafe construct it surfaces, marking each with the `unsafe` expression marker. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01EbZbx5D2gGeXT8UxuiKEdq --- Package.swift | 3 ++- Sources/SwiftDOF/Cycle.swift | 2 +- Sources/SwiftDOF/Parser/DOFLineReader.swift | 6 +++--- Sources/SwiftDOF_E2E/OutputFormatter.swift | 14 ++++++++------ Sources/SwiftDOF_E2E/SwiftDOF_E2E.swift | 2 +- 5 files changed, 15 insertions(+), 12 deletions(-) diff --git a/Package.swift b/Package.swift index 2701820..fbab8a3 100644 --- a/Package.swift +++ b/Package.swift @@ -9,7 +9,8 @@ let upcomingFeatures: [SwiftSetting] = [ .enableUpcomingFeature("ImmutableWeakCaptures"), .enableUpcomingFeature("MemberImportVisibility"), .enableUpcomingFeature("ExistentialAny"), - .enableUpcomingFeature("InternalImportsByDefault") + .enableUpcomingFeature("InternalImportsByDefault"), + .strictMemorySafety() ] let package = Package( diff --git a/Sources/SwiftDOF/Cycle.swift b/Sources/SwiftDOF/Cycle.swift index 6600b8b..c5013e6 100644 --- a/Sources/SwiftDOF/Cycle.swift +++ b/Sources/SwiftDOF/Cycle.swift @@ -241,7 +241,7 @@ extension Cycle: Comparable { extension Cycle: Identifiable { /// The unique identifier for this cycle in YYYYMMDD format. public var id: String { - String(format: "%04d%02d%02d", year, month, day) + unsafe String(format: "%04d%02d%02d", year, month, day) } } diff --git a/Sources/SwiftDOF/Parser/DOFLineReader.swift b/Sources/SwiftDOF/Parser/DOFLineReader.swift index c195d93..34af64e 100644 --- a/Sources/SwiftDOF/Parser/DOFLineReader.swift +++ b/Sources/SwiftDOF/Parser/DOFLineReader.swift @@ -26,10 +26,10 @@ struct DOFLineReader: Sequence, IteratorProtocol, Sendable { lineBuffer.removeAll(keepingCapacity: true) // Scan until LF or end of data - data.withUnsafeBytes { buffer in - let bytes = buffer.bindMemory(to: UInt8.self) + unsafe data.withUnsafeBytes { buffer in + let bytes = unsafe buffer.bindMemory(to: UInt8.self) while position < bytes.count { - let byte = bytes[position] + let byte = unsafe bytes[position] position += 1 if byte == ASCII.LF { return } lineBuffer.append(byte) diff --git a/Sources/SwiftDOF_E2E/OutputFormatter.swift b/Sources/SwiftDOF_E2E/OutputFormatter.swift index 0c6f5c4..1f79e19 100644 --- a/Sources/SwiftDOF_E2E/OutputFormatter.swift +++ b/Sources/SwiftDOF_E2E/OutputFormatter.swift @@ -17,9 +17,10 @@ protocol OutputFormatter { extension OutputStream { func write(_ string: String) { guard let data = string.data(using: .utf8) else { return } - data.withUnsafeBytes { buffer in - guard let pointer = buffer.baseAddress?.assumingMemoryBound(to: UInt8.self) else { return } - write(pointer, maxLength: buffer.count) + unsafe data.withUnsafeBytes { buffer in + guard let pointer = unsafe buffer.baseAddress?.assumingMemoryBound(to: UInt8.self) + else { return } + unsafe write(pointer, maxLength: buffer.count) } } @@ -48,9 +49,10 @@ struct JSONOutputFormatter: OutputFormatter { let encoder = JSONEncoder() encoder.outputFormatting = [.prettyPrinted, .sortedKeys] let jsonData = try encoder.encode(dof.all) - jsonData.withUnsafeBytes { buffer in - guard let pointer = buffer.baseAddress?.assumingMemoryBound(to: UInt8.self) else { return } - stream.write(pointer, maxLength: buffer.count) + unsafe jsonData.withUnsafeBytes { buffer in + guard let pointer = unsafe buffer.baseAddress?.assumingMemoryBound(to: UInt8.self) + else { return } + unsafe stream.write(pointer, maxLength: buffer.count) } } } diff --git a/Sources/SwiftDOF_E2E/SwiftDOF_E2E.swift b/Sources/SwiftDOF_E2E/SwiftDOF_E2E.swift index 9d87e91..9aeb685 100644 --- a/Sources/SwiftDOF_E2E/SwiftDOF_E2E.swift +++ b/Sources/SwiftDOF_E2E/SwiftDOF_E2E.swift @@ -42,7 +42,7 @@ struct SwiftDOF_E2E: AsyncParsableCommand { else { fatalError("Current cycle could not be determined") } - let filename = String(format: "DOF_%02d%02d%02d.zip", year % 100, month, day) + let filename = unsafe String(format: "DOF_%02d%02d%02d.zip", year % 100, month, day) guard let url = URL(string: "https://aeronav.faa.gov/Obst_Data/\(filename)") else { fatalError("Current DOF URL could not be determined") } From a7e86e31383deb2c44b983ce77fb23dcfff796af Mon Sep 17 00:00:00 2001 From: Tim Morgan Date: Mon, 14 Sep 2026 20:37:57 -0700 Subject: [PATCH 5/7] Match the README to this branch's raised floor `main` lowered the floor to macOS 15 and the README followed. This branch raises it to 26 for `InlineArray`, so the README has to say so too. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01EbZbx5D2gGeXT8UxuiKEdq --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2f4b108..7e795fc 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ The DOF format is documented at Date: Thu, 17 Sep 2026 00:38:49 -0700 Subject: [PATCH 6/7] Read the marking and accuracy columns the DOF actually defines Obstacle.marking was read from column 100, which holds the vertical accuracy code, and the real mark indicator in column 102 was declared as a field and never read. The old MarkingType raw values were A-I, exactly the vertical accuracy code set, so every record in every cycle parsed without error and reported a marking derived from the obstacle's height tolerance: a +/-50 foot obstacle (D) read as .paintAndFlags. Split AccuracyCategory, which mixed both code sets, into HorizontalAccuracy (1-9) and VerticalAccuracy (A-I), and expose the vertical column that had no representation at all. Cases are named for the code the FAA publishes, which it has used since 1979 and still writes as "Accuracy Code 1A". Verified against cycle 20260802: all 652,785 records now agree with an independent read of the raw columns across every parsed field. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 25 +++++++++ .../Documentation.docc/Documentation.md | 3 +- .../SwiftDOF/Documentation.docc/Obstacle.md | 4 ++ Sources/SwiftDOF/Obstacle.swift | 11 ++-- Sources/SwiftDOF/Parser/ASCII.swift | 2 +- Sources/SwiftDOF/Parser/DOFByteParser.swift | 40 +++++++++----- Sources/SwiftDOF/Types/AccuracyCategory.swift | 53 ------------------- .../SwiftDOF/Types/HorizontalAccuracy.swift | 51 ++++++++++++++++++ Sources/SwiftDOF/Types/MarkingType.swift | 36 ++++++------- Sources/SwiftDOF/Types/VerticalAccuracy.swift | 51 ++++++++++++++++++ Tests/SwiftDOFTests/ObstacleTests.swift | 50 +++++++++++++---- 11 files changed, 223 insertions(+), 103 deletions(-) delete mode 100644 Sources/SwiftDOF/Types/AccuracyCategory.swift create mode 100644 Sources/SwiftDOF/Types/HorizontalAccuracy.swift create mode 100644 Sources/SwiftDOF/Types/VerticalAccuracy.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index 0dcc62c..8d44a57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,14 +2,39 @@ ## [Unreleased] +### Added + +- `Obstacle.verticalAccuracy`, the accuracy category of an obstacle's reported + height, read from column 100 of the DOF record. + ### Changed +- **Breaking.** `AccuracyCategory` is replaced by two enums that each carry only + the codes the FAA defines for their column: `HorizontalAccuracy` (`1`-`9`) and + `VerticalAccuracy` (`A`-`I`). `Obstacle.horizontalAccuracy` is now a + `HorizontalAccuracy`. The `.survey` case (±3 feet) becomes + `VerticalAccuracy.codeA`, the vertical column it belongs to. Cases are named + for the code the FAA publishes — `.code1`, `.codeA` — rather than a "category" + the criteria never use, and the tolerance each one guarantees is now spelled + `tolerance` rather than `accuracy`. +- **Breaking.** `MarkingType`'s cases are the mark indicator codes the DOF + actually uses — `orangeOrOrangeWhitePaint` (`P`), `whitePaintOnly` (`W`), + `marked` (`M`), `flagMarker` (`F`), `sphericalMarker` (`S`), `none` (`N`), and + `unknown` (`U`) — in place of the `A`-`I` cases, which described a code set + that appears nowhere in the DOF. A blank column parses as `.unknown`. + - Adopt typed throws across the parsing surface: `DOF.init(data:)`, `DOF.init(url:)`, the `DOF.from(…)` factories, and `DOFByteParser` now declare `throws(DOFError)`, and the DOF file line reader's `AsyncIteratorProtocol.Failure` is `DOFError`. `AsyncBytesLineReader` propagates its source sequence's own `Failure` type. - `DOF.from(filePath:)` streams the file in chunks instead of reading it into memory in its entirety. A file that cannot be opened now throws `DOFError.fileNotFound` rather than a Foundation file-read error. - Match the header's "CURRENCY DATE = " marker against an `InlineArray<16, UInt8>`, removing a heap allocation from currency date parsing. ### Fixed +- `Obstacle.marking` reads the mark indicator from column 102 rather than column + 100, which holds the vertical accuracy code. Because the old `MarkingType` + cases `A`-`I` coincided exactly with the vertical accuracy code set, every + record in every cycle parsed without error and reported the wrong marking — + an obstacle with a ±50 foot height tolerance (`D`) read as `.paintAndFlags`. + The vertical accuracy column is now exposed as `Obstacle.verticalAccuracy`. - `Cycle.previous`, `Cycle.next`, and the cycle datum date no longer force-unwrap optionals. ## [1.3.0] - 2026-09-14 diff --git a/Sources/SwiftDOF/Documentation.docc/Documentation.md b/Sources/SwiftDOF/Documentation.docc/Documentation.md index e329739..6803a48 100644 --- a/Sources/SwiftDOF/Documentation.docc/Documentation.md +++ b/Sources/SwiftDOF/Documentation.docc/Documentation.md @@ -43,7 +43,8 @@ the ``DOF/init(data:progressHandler:errorCallback:)`` callback parameter. - ``MarkingType`` - ``ActionCode`` - ``VerificationStatus`` -- ``AccuracyCategory`` +- ``HorizontalAccuracy`` +- ``VerticalAccuracy`` ### Errors diff --git a/Sources/SwiftDOF/Documentation.docc/Obstacle.md b/Sources/SwiftDOF/Documentation.docc/Obstacle.md index 5b44d89..7826577 100644 --- a/Sources/SwiftDOF/Documentation.docc/Obstacle.md +++ b/Sources/SwiftDOF/Documentation.docc/Obstacle.md @@ -32,7 +32,11 @@ - ``lighting`` - ``marking`` + +### Accuracy + - ``horizontalAccuracy`` +- ``verticalAccuracy`` ### Administrative diff --git a/Sources/SwiftDOF/Obstacle.swift b/Sources/SwiftDOF/Obstacle.swift index f657e9d..4b5758e 100644 --- a/Sources/SwiftDOF/Obstacle.swift +++ b/Sources/SwiftDOF/Obstacle.swift @@ -65,8 +65,11 @@ public struct Obstacle: Sendable, Codable { /// The type of lighting installed on the obstacle. public let lighting: LightingType - /// The horizontal accuracy category. - public let horizontalAccuracy: AccuracyCategory + /// The accuracy category of the obstacle's reported position. + public let horizontalAccuracy: HorizontalAccuracy + + /// The accuracy category of the obstacle's reported height. + public let verticalAccuracy: VerticalAccuracy /// The marking type (paint, flags, etc.). public let marking: MarkingType @@ -103,7 +106,8 @@ public struct Obstacle: Sendable, Codable { heightFtAGL: Int, heightFtMSL: Int, lighting: LightingType, - horizontalAccuracy: AccuracyCategory, + horizontalAccuracy: HorizontalAccuracy, + verticalAccuracy: VerticalAccuracy, marking: MarkingType, studyNumber: String, action: ActionCode, @@ -122,6 +126,7 @@ public struct Obstacle: Sendable, Codable { self.heightFtMSL = heightFtMSL self.lighting = lighting self.horizontalAccuracy = horizontalAccuracy + self.verticalAccuracy = verticalAccuracy self.marking = marking self.studyNumber = studyNumber self.action = action diff --git a/Sources/SwiftDOF/Parser/ASCII.swift b/Sources/SwiftDOF/Parser/ASCII.swift index 2d9011a..6cf4b23 100644 --- a/Sources/SwiftDOF/Parser/ASCII.swift +++ b/Sources/SwiftDOF/Parser/ASCII.swift @@ -11,8 +11,8 @@ enum ASCII { @usableFromInline static let slash: UInt8 = 0x2F // '/' @usableFromInline static let zero: UInt8 = 0x30 // '0' @usableFromInline static let nine: UInt8 = 0x39 // '9' - @usableFromInline static let A: UInt8 = 0x41 @usableFromInline static let E: UInt8 = 0x45 + @usableFromInline static let I: UInt8 = 0x49 @usableFromInline static let N: UInt8 = 0x4E @usableFromInline static let O: UInt8 = 0x4F @usableFromInline static let S: UInt8 = 0x53 diff --git a/Sources/SwiftDOF/Parser/DOFByteParser.swift b/Sources/SwiftDOF/Parser/DOFByteParser.swift index 86bfeb7..a41daf0 100644 --- a/Sources/SwiftDOF/Parser/DOFByteParser.swift +++ b/Sources/SwiftDOF/Parser/DOFByteParser.swift @@ -23,9 +23,9 @@ struct DOFByteParser: Sendable { aglHeight: 83..<88, mslHeight: 89..<94, lighting: 95..<96, - accuracyH: 97..<98, - marking: 99..<100, - faaIndicator: 101..<102, + horizontalAccuracy: 97..<98, + verticalAccuracy: 99..<100, + marking: 101..<102, studyNumber: 103..<117, action: 118..<119, lastUpdated: 120..<127 @@ -94,14 +94,29 @@ struct DOFByteParser: Sendable { ) } - // AccuracyCategory: space means unknown (category9) - let accuracyByte = bytes[base + fields.accuracyH.lowerBound] + // Both accuracy columns are blank when the FAA has not categorized the obstacle. + let horizontalAccuracyByte = bytes[base + fields.horizontalAccuracy.lowerBound] guard - let accuracy = AccuracyCategory(byte: accuracyByte == ASCII.space ? ASCII.nine : accuracyByte) + let horizontalAccuracy = HorizontalAccuracy( + byte: horizontalAccuracyByte == ASCII.space ? ASCII.nine : horizontalAccuracyByte + ) + else { + throw DOFError.parseError( + field: "horizontalAccuracy", + value: String(UnicodeScalar(horizontalAccuracyByte)), + line: lineNumber + ) + } + + let verticalAccuracyByte = bytes[base + fields.verticalAccuracy.lowerBound] + guard + let verticalAccuracy = VerticalAccuracy( + byte: verticalAccuracyByte == ASCII.space ? ASCII.I : verticalAccuracyByte + ) else { throw DOFError.parseError( - field: "accuracy", - value: String(UnicodeScalar(accuracyByte)), + field: "verticalAccuracy", + value: String(UnicodeScalar(verticalAccuracyByte)), line: lineNumber ) } @@ -115,11 +130,9 @@ struct DOFByteParser: Sendable { ) } - // MarkingType: 'N' and space are aliases for 'A' (none) + // MarkingType: a blank column means the marking is unknown. let markingByte = bytes[base + fields.marking.lowerBound] - let normalizedMarkingByte = - (markingByte == ASCII.N || markingByte == ASCII.space) ? ASCII.A : markingByte - guard let marking = MarkingType(byte: normalizedMarkingByte) else { + guard let marking = MarkingType(byte: markingByte == ASCII.space ? ASCII.U : markingByte) else { throw DOFError.parseError( field: "marking", value: String(UnicodeScalar(markingByte)), @@ -163,7 +176,8 @@ struct DOFByteParser: Sendable { heightFtAGL: agl, heightFtMSL: msl, lighting: lighting, - horizontalAccuracy: accuracy, + horizontalAccuracy: horizontalAccuracy, + verticalAccuracy: verticalAccuracy, marking: marking, studyNumber: studyNumber, action: action, diff --git a/Sources/SwiftDOF/Types/AccuracyCategory.swift b/Sources/SwiftDOF/Types/AccuracyCategory.swift deleted file mode 100644 index 164ca93..0000000 --- a/Sources/SwiftDOF/Types/AccuracyCategory.swift +++ /dev/null @@ -1,53 +0,0 @@ -public import Foundation - -/// FAA horizontal accuracy category for obstacle position data. -/// -/// Categories 1-9 and A indicate the accuracy of the obstacle's reported position. -/// Lower numbers indicate higher accuracy. -public enum AccuracyCategory: Character, Sendable, Codable, CaseIterable, ByteInitializable { - /// Survey data accuracy (approximately ±3 feet). - case survey = "A" - - /// ±20 feet horizontal accuracy. - case category1 = "1" - - /// ±50 feet horizontal accuracy. - case category2 = "2" - - /// ±100 feet horizontal accuracy. - case category3 = "3" - - /// ±250 feet horizontal accuracy. - case category4 = "4" - - /// ±500 feet horizontal accuracy. - case category5 = "5" - - /// ±1,000 feet horizontal accuracy. - case category6 = "6" - - /// ±0.5 nautical mile horizontal accuracy. - case category7 = "7" - - /// ±1 nautical mile horizontal accuracy. - case category8 = "8" - - /// Unknown accuracy. - case category9 = "9" - - /// Approximate accuracy, if known. - public var accuracy: Measurement? { - switch self { - case .survey: .init(value: 3, unit: .feet) - case .category1: .init(value: 20, unit: .feet) - case .category2: .init(value: 50, unit: .feet) - case .category3: .init(value: 100, unit: .feet) - case .category4: .init(value: 250, unit: .feet) - case .category5: .init(value: 500, unit: .feet) - case .category6: .init(value: 1000, unit: .feet) - case .category7: .init(value: 0.5, unit: .nauticalMiles) - case .category8: .init(value: 1, unit: .nauticalMiles) - case .category9: nil - } - } -} diff --git a/Sources/SwiftDOF/Types/HorizontalAccuracy.swift b/Sources/SwiftDOF/Types/HorizontalAccuracy.swift new file mode 100644 index 0000000..a8ba5e6 --- /dev/null +++ b/Sources/SwiftDOF/Types/HorizontalAccuracy.swift @@ -0,0 +1,51 @@ +public import Foundation + +/// FAA horizontal accuracy code for an obstacle's reported position. +/// +/// The FAA has used these codes since 1979 to declare how tightly an obstacle's +/// location is known. Lower code numbers indicate a more accurate position. A code +/// is typically paired with its ``VerticalAccuracy`` counterpart and spoken as a +/// single accuracy code, such as "1A" or "4D". +public enum HorizontalAccuracy: Character, Sendable, Codable, CaseIterable, ByteInitializable { + /// ±20 feet horizontal accuracy. + case code1 = "1" + + /// ±50 feet horizontal accuracy. + case code2 = "2" + + /// ±100 feet horizontal accuracy. + case code3 = "3" + + /// ±250 feet horizontal accuracy. + case code4 = "4" + + /// ±500 feet horizontal accuracy. + case code5 = "5" + + /// ±1,000 feet horizontal accuracy. + case code6 = "6" + + /// ±0.5 nautical mile horizontal accuracy. + case code7 = "7" + + /// ±1 nautical mile horizontal accuracy. + case code8 = "8" + + /// Unknown accuracy. + case code9 = "9" + + /// The tolerance this code guarantees, or `nil` when the accuracy is unknown. + public var tolerance: Measurement? { + switch self { + case .code1: .init(value: 20, unit: .feet) + case .code2: .init(value: 50, unit: .feet) + case .code3: .init(value: 100, unit: .feet) + case .code4: .init(value: 250, unit: .feet) + case .code5: .init(value: 500, unit: .feet) + case .code6: .init(value: 1000, unit: .feet) + case .code7: .init(value: 0.5, unit: .nauticalMiles) + case .code8: .init(value: 1, unit: .nauticalMiles) + case .code9: nil + } + } +} diff --git a/Sources/SwiftDOF/Types/MarkingType.swift b/Sources/SwiftDOF/Types/MarkingType.swift index c721901..c1748f4 100644 --- a/Sources/SwiftDOF/Types/MarkingType.swift +++ b/Sources/SwiftDOF/Types/MarkingType.swift @@ -1,33 +1,27 @@ import Foundation -/// Obstacle marking indicator from the DOF. +/// Obstacle mark indicator from the DOF. /// /// Indicates how an obstacle is marked for visibility to aircraft. public enum MarkingType: Character, Sendable, Codable, CaseIterable, ByteInitializable { - /// No marking or unknown. - case none = "A" + /// Marked with orange, or orange and white, paint. + case orangeOrOrangeWhitePaint = "P" - /// Marked with high visibility orange and white paint. - case orangeWhitePaint = "B" + /// Marked with white paint only. + case whitePaintOnly = "W" - /// Marked with flag markers. - case flagMarkers = "C" - - /// Marked with orange and white paint and has flag markers. - case paintAndFlags = "D" - - /// Marked with high visibility lighting only. - case lightingOnly = "E" + /// Marked, by a means the DOF does not identify. + case marked = "M" - /// Marked with orange and white paint and high visibility lighting. - case paintAndLighting = "F" + /// Marked with flag markers. + case flagMarker = "F" - /// Marked with flag markers and high visibility lighting. - case flagsAndLighting = "G" + /// Marked with spherical markers (typically on power lines). + case sphericalMarker = "S" - /// Marked with orange and white paint, flag markers, and high visibility lighting. - case paintFlagsAndLighting = "H" + /// Not marked. + case none = "N" - /// Marked with spherical markers (typically on power lines). - case sphericalMarkers = "I" + /// Marking unknown. + case unknown = "U" } diff --git a/Sources/SwiftDOF/Types/VerticalAccuracy.swift b/Sources/SwiftDOF/Types/VerticalAccuracy.swift new file mode 100644 index 0000000..1282a9b --- /dev/null +++ b/Sources/SwiftDOF/Types/VerticalAccuracy.swift @@ -0,0 +1,51 @@ +public import Foundation + +/// FAA vertical accuracy code for an obstacle's reported height. +/// +/// The FAA has used these codes since 1979 to declare how tightly an obstacle's +/// height is known. Earlier code letters indicate a more accurate height. A code +/// is typically paired with its ``HorizontalAccuracy`` counterpart and spoken as a +/// single accuracy code, such as "1A" or "4D". +public enum VerticalAccuracy: Character, Sendable, Codable, CaseIterable, ByteInitializable { + /// ±3 feet vertical accuracy. + case codeA = "A" + + /// ±10 feet vertical accuracy. + case codeB = "B" + + /// ±20 feet vertical accuracy. + case codeC = "C" + + /// ±50 feet vertical accuracy. + case codeD = "D" + + /// ±125 feet vertical accuracy. + case codeE = "E" + + /// ±250 feet vertical accuracy. + case codeF = "F" + + /// ±500 feet vertical accuracy. + case codeG = "G" + + /// ±1,000 feet vertical accuracy. + case codeH = "H" + + /// Unknown accuracy. + case codeI = "I" + + /// The tolerance this code guarantees, or `nil` when the accuracy is unknown. + public var tolerance: Measurement? { + switch self { + case .codeA: .init(value: 3, unit: .feet) + case .codeB: .init(value: 10, unit: .feet) + case .codeC: .init(value: 20, unit: .feet) + case .codeD: .init(value: 50, unit: .feet) + case .codeE: .init(value: 125, unit: .feet) + case .codeF: .init(value: 250, unit: .feet) + case .codeG: .init(value: 500, unit: .feet) + case .codeH: .init(value: 1000, unit: .feet) + case .codeI: nil + } + } +} diff --git a/Tests/SwiftDOFTests/ObstacleTests.swift b/Tests/SwiftDOFTests/ObstacleTests.swift index 74ec3f1..346be39 100644 --- a/Tests/SwiftDOFTests/ObstacleTests.swift +++ b/Tests/SwiftDOFTests/ObstacleTests.swift @@ -9,11 +9,17 @@ import Foundation struct ObstacleTests { - // Sample DOF record line as bytes - let sampleLineBytes: [UInt8] = Array( + // Sample DOF record line + let sampleLine = "01-001307 O US AL DAUPHIN ISLAND 30 10 45.00N 088 04 39.00W RIG 1 00236 00236 R 5 D M 1990ASO01578OE C 2014138 " - .utf8 - ) + + var sampleLineBytes: [UInt8] { Array(sampleLine.utf8) } + + /// The range of a single one-indexed DOF column, as the FAA README numbers them. + private func column(_ oneIndexed: Int) -> Range { + let start = sampleLine.index(sampleLine.startIndex, offsetBy: oneIndexed - 1) + return start.. Date: Thu, 17 Sep 2026 00:38:59 -0700 Subject: [PATCH 7/7] Watch each DOF cycle for parsing errors The FAA releases each 56-day DOF the day after it takes effect, so there is no window in which the incoming cycle can be parsed early. A daily probe checks whether the cycle in effect is downloadable, and the first time it is, parses it end to end. The report artifact, named for the cycle, is both the record that the cycle was checked and the baseline the next cycle's counts are compared against. SwiftDOF_E2E gains the report the workflow reads: --report writes counts, per-region totals, and the field each failed line failed on, --baseline records counts that moved more than their tolerance, and the tool now exits non-zero when any line fails to parse. Periphery retains Codable properties, since jq is the only reader the report has. Tolerances are calibrated against real data: cycle 20260607 to 20260802 moved the total by 1.8%, well inside the 5% allowed. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/dof-cycle-watch.yml | 480 ++++++++++++++++++++++++ .periphery.yml | 3 + CHANGELOG.md | 8 + README.md | 18 + Sources/SwiftDOF_E2E/Report.swift | 200 ++++++++++ Sources/SwiftDOF_E2E/SwiftDOF_E2E.swift | 56 +++ 6 files changed, 765 insertions(+) create mode 100644 .github/workflows/dof-cycle-watch.yml create mode 100644 Sources/SwiftDOF_E2E/Report.swift diff --git a/.github/workflows/dof-cycle-watch.yml b/.github/workflows/dof-cycle-watch.yml new file mode 100644 index 0000000..a0fbe80 --- /dev/null +++ b/.github/workflows/dof-cycle-watch.yml @@ -0,0 +1,480 @@ +name: DOF Cycle Watch + +# The FAA releases each 56-day DOF the day after it takes effect — the published schedule runs +# "reflects changes to" August 02, released August 04, for the cycle effective August 03 — so +# unlike NASR there is no window in which the incoming cycle can be parsed early. The watch +# therefore tracks the cycle already in effect. A cheap probe gates the expensive parse; the +# report artifact, named for the cycle, doubles as the record of what has already been checked. + +on: + schedule: + - cron: "0 16 * * *" + workflow_dispatch: + inputs: + cycle: + description: "Cycle to check, as the YYYY-MM-DD currency date in its filename. Blank uses the cycle in effect." + required: false + type: string + +permissions: {} + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +jobs: + probe: + name: Look for a new cycle + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + actions: read + issues: write + outputs: + cycle: ${{ steps.target.outputs.cycle }} + cycle-id: ${{ steps.target.outputs.cycle-id }} + previous-cycle: ${{ steps.target.outputs.previous-cycle }} + url: ${{ steps.availability.outputs.url }} + parse: ${{ steps.decide.outputs.parse }} + steps: + - name: Resolve the target cycle + id: target + env: + REQUESTED: ${{ inputs.cycle }} + run: | + set -euo pipefail + + # Cycles take effect every 56 days from 2025-09-01. The FAA names the file for the last + # day of coverage, the day before, which is also the currency date in its header — so + # that is the identity used throughout, and it is what the report has to agree with. + period=$(( 56 * 86400 )) + datum=$(date -u -d 2025-09-01 +%s) + index=$(( ($(date -u +%s) - datum) / period )) + currency=$(( datum + index * period - 86400 )) + + if [ -n "$REQUESTED" ]; then + if ! requested=$(date -u -d "$REQUESTED" +%s 2>/dev/null); then + echo "::error::'$REQUESTED' is not a date. Use YYYY-MM-DD." + exit 1 + fi + if [ $(( (requested - (datum - 86400)) % period )) -ne 0 ]; then + echo "::error::$REQUESTED is not a DOF currency date; they fall every 56 days from 2025-08-31." + exit 1 + fi + currency=$requested + fi + + { + echo "cycle=$(date -u -d "@$currency" +%F)" + echo "cycle-id=$(date -u -d "@$currency" +%Y%m%d)" + echo "previous-cycle=$(date -u -d "@$(( currency - period ))" +%F)" + echo "effective=$(date -u -d "@$(( currency + 86400 ))" +%F)" + } >> "$GITHUB_OUTPUT" + + - name: Check whether the FAA has published it + id: availability + env: + CYCLE: ${{ steps.target.outputs.cycle }} + EFFECTIVE: ${{ steps.target.outputs.effective }} + run: | + set -euo pipefail + + # aeronav.faa.gov answers HEAD honestly, so no range-GET workaround is needed here. + url="https://aeronav.faa.gov/Obst_Data/DOF_$(date -u -d "$CYCLE" +%y%m%d).zip" + code=$(curl --silent --show-error --location --head \ + --retry 3 --retry-delay 10 --retry-all-errors \ + --dump-header headers.txt --output /dev/null \ + --write-out '%{http_code}' "$url" || echo 000) + echo "HEAD $url -> $code" + + if [ "$code" = "200" ]; then + available=true + else + available=false + fi + + # Last-Modified is the mirror's re-sync time, not the release date: the file released on + # 2026-08-04 reported a Last-Modified of 2026-09-14. It is recorded, never relied on. + modified=$(grep -i '^last-modified:' headers.txt | tail -n 1 | cut -d' ' -f2- | tr -d '\r') + size=$(grep -i '^content-length:' headers.txt | tail -n 1 | cut -d' ' -f2- | tr -d '\r') + days_since=$(( ( $(date -u +%s) - $(date -u -d "$EFFECTIVE" +%s) ) / 86400 )) + + # Release lands one day after the cycle takes effect. A week of silence means the FAA + # slipped or the URL shape changed, either of which looks like an ordinary quiet day. + if [ "$available" != "true" ] && [ "$days_since" -ge 7 ]; then + overdue=true + else + overdue=false + fi + + { + echo "available=$available" + echo "overdue=$overdue" + echo "days-since=$days_since" + echo "last-modified=$modified" + echo "size=$size" + echo "code=$code" + echo "url=$url" + } >> "$GITHUB_OUTPUT" + + - name: Decide whether to parse + id: decide + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + CYCLE: ${{ steps.target.outputs.cycle }} + AVAILABLE: ${{ steps.availability.outputs.available }} + DAYS_SINCE: ${{ steps.availability.outputs.days-since }} + SIZE: ${{ steps.availability.outputs.size }} + REQUESTED: ${{ inputs.cycle }} + ARTIFACT: dof-report-${{ steps.target.outputs.cycle }} + run: | + set -euo pipefail + + note() { echo "$1"; echo "$1" >> "$GITHUB_STEP_SUMMARY"; } + + if [ "$AVAILABLE" != "true" ]; then + note "Cycle $CYCLE is not downloadable yet; it took effect $DAYS_SINCE day(s) ago." + echo "parse=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + if [ -n "$REQUESTED" ]; then + note "Re-checking cycle $CYCLE on request." + echo "parse=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # An existing report artifact is the record that this cycle was already checked. + found=$(gh api "repos/$GH_REPO/actions/artifacts?name=$ARTIFACT&per_page=100" \ + --jq '[.artifacts[] | select(.name == env.ARTIFACT and .expired == false)] | length') + + if [ "$found" -gt 0 ]; then + note "Cycle $CYCLE has already been checked." + echo "parse=false" >> "$GITHUB_OUTPUT" + else + note "Cycle $CYCLE is new ($SIZE bytes, in effect $DAYS_SINCE day(s))." + echo "parse=true" >> "$GITHUB_OUTPUT" + fi + + - name: Report a cycle the FAA never published + if: steps.availability.outputs.overdue == 'true' + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + CYCLE: ${{ steps.target.outputs.cycle }} + EFFECTIVE: ${{ steps.target.outputs.effective }} + DAYS_SINCE: ${{ steps.availability.outputs.days-since }} + URL: ${{ steps.availability.outputs.url }} + CODE: ${{ steps.availability.outputs.code }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + + { + echo "Cycle **$CYCLE** took effect on $EFFECTIVE, $DAYS_SINCE day(s) ago, and is still" + echo "not downloadable. The FAA normally releases the day after a cycle takes effect," + echo "so either this cycle slipped or the download URL has changed shape." + echo + echo "| URL | HEAD |" + echo "|---|---|" + echo "| \`$URL\` | $CODE |" + echo + echo "[View the run]($RUN_URL)" + } > issue-body.md + + title="DOF cycle $CYCLE was never published" + number=$(TITLE="$title" gh issue list --state all --search "DOF cycle $CYCLE in:title" \ + --json number,title --jq 'map(select(.title == env.TITLE)) | .[0].number // empty') + + if [ -n "$number" ]; then + gh issue comment "$number" --body-file issue-body.md + else + gh issue create --title "$title" --body-file issue-body.md + fi + + parse: + name: Parse the cycle + needs: probe + if: needs.probe.outputs.parse == 'true' + runs-on: ubuntu-latest + timeout-minutes: 45 + permissions: + contents: read + actions: read + issues: write + env: + CYCLE: ${{ needs.probe.outputs.cycle }} + CYCLE_ID: ${{ needs.probe.outputs.cycle-id }} + PREVIOUS_CYCLE: ${{ needs.probe.outputs.previous-cycle }} + URL: ${{ needs.probe.outputs.url }} + steps: + - uses: actions/checkout@v7 + + - name: Name the report and the log + run: | + set -euo pipefail + { + echo "REPORT=$RUNNER_TEMP/report.json" + echo "PARSE_LOG=$RUNNER_TEMP/parse.log" + } >> "$GITHUB_ENV" + + - uses: SwiftyLab/setup-swift@latest + with: + swift-version: "6.4" + + - name: Cache SwiftPM build products + # Usually misses: 56 days between runs is well past the seven-day idle eviction. It earns + # its place when a cycle is re-checked by hand after a fix. + uses: actions/cache@v6 + with: + path: | + .build + ~/.cache/org.swift.swiftpm + key: swiftpm-cyclewatch-ubuntu-latest-swift6.4-${{ hashFiles('Package.resolved') }} + + - name: Build + # Release: a debug build cannot get through 650,000 fixed-width records in any useful time. + run: swift build -c release --product SwiftDOF_E2E + + - name: Fetch the previous cycle's report + id: baseline + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + ARTIFACT: dof-report-${{ needs.probe.outputs.previous-cycle }} + run: | + set -euo pipefail + + # download-artifact cannot reach another run's artifact without that run's id, which is + # itself state to keep somewhere; the name already carries the cycle. + id=$(gh api "repos/$GH_REPO/actions/artifacts?name=$ARTIFACT&per_page=100" \ + --jq '[.artifacts[] | select(.name == env.ARTIFACT and .expired == false)] + | max_by(.id) | .id // empty') + + if [ -z "$id" ]; then + echo "No report for $PREVIOUS_CYCLE; this cycle becomes the baseline for the next one." + echo "found=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + gh api "repos/$GH_REPO/actions/artifacts/$id/zip" > previous.zip + unzip -o -q previous.zip -d previous + + # A run that crashed before writing its report still uploaded its log, so the artifact + # existing does not mean there is anything to compare against. + if [ -f previous/report.json ]; then + echo "found=true" >> "$GITHUB_OUTPUT" + else + echo "The report for $PREVIOUS_CYCLE holds no counts; the comparison is skipped." + echo "found=false" >> "$GITHUB_OUTPUT" + fi + + - name: Parse the cycle + id: parse + env: + BASELINE_FOUND: ${{ steps.baseline.outputs.found }} + run: | + set -euo pipefail + + args=(--input "$URL" --report "$REPORT") + if [ "$BASELINE_FOUND" = "true" ]; then + args+=(--baseline previous/report.json) + fi + + # The exit code is captured, not propagated: the report still has to be uploaded, + # summarized, and turned into an issue. + status=0 + .build/release/SwiftDOF_E2E "${args[@]}" 2>&1 | tee "$PARSE_LOG" || status=$? + echo "status=$status" >> "$GITHUB_OUTPUT" + + - name: Upload the report + # Uploaded before anything reads it, and on failure too: this artifact is also the record + # that the cycle was checked, so a broken cycle is reported once rather than every day. + if: always() + uses: actions/upload-artifact@v7 + with: + name: dof-report-${{ needs.probe.outputs.cycle }} + path: | + ${{ runner.temp }}/report.json + ${{ runner.temp }}/parse.log + retention-days: 90 + if-no-files-found: warn + + - name: Read the verdict out of the report + id: verdict + if: always() + run: | + set -euo pipefail + + failed=unknown + drift=0 + declared= + if [ -f "$REPORT" ]; then + failed=$(jq -r '.failed' "$REPORT") + drift=$(jq '.drift | length' "$REPORT") + declared=$(jq -r '.cycle' "$REPORT") + fi + + # The header's currency date is computed independently of this workflow's arithmetic, so + # a disagreement means one of the two is wrong about which cycle this file even is. + if [ -n "$declared" ] && [ "$declared" != "$CYCLE_ID" ]; then + mismatch=true + echo "::error::The file at $URL declares cycle $declared, not $CYCLE_ID." + else + mismatch=false + fi + + { + echo "failed=$failed" + echo "drift=$drift" + echo "declared=$declared" + echo "mismatch=$mismatch" + } >> "$GITHUB_OUTPUT" + + - name: Write the job summary + if: always() + env: + DECLARED: ${{ steps.verdict.outputs.declared }} + run: | + set -euo pipefail + + if [ ! -f "$REPORT" ]; then + echo "The run produced no report for cycle \`$CYCLE\`; see the log." >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + { + echo "## DOF cycle $CYCLE" + echo + echo "- Source: \`$URL\`" + echo "- Cycle the file declares: \`$DECLARED\`" + echo "- Obstacles parsed: $(jq -r '.obstacleCount' "$REPORT")" + echo "- Parse errors: $(jq -r '.parseErrorCount' "$REPORT")" + echo "- Outcome: $(jq -r 'if .failed then "**failed**" else "clean" end' "$REPORT")" + jq -r '.failureReasons[]? | " - \(.)"' "$REPORT" + echo + + if [ "$(jq '.drift | length' "$REPORT")" -gt 0 ]; then + echo "### Count drift against $PREVIOUS_CYCLE" + echo + echo "| Region | $PREVIOUS_CYCLE | $CYCLE | Reason |" + echo "|---|---:|---:|---|" + jq -r '.drift[] | "| \(.scope) | \(.was) | \(.now // "missing") | \(.reason) |"' "$REPORT" + echo + fi + + if [ "$(jq '.errorSamples | length' "$REPORT")" -gt 0 ]; then + echo "### Error samples" + echo + echo '```' + jq -r '.errorSamples[] | "line \(.line): \(.reason // .message)"' "$REPORT" + echo '```' + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Open or update the cycle's issue + # Drift files an issue as readily as a parse failure: a region that quietly empties produces + # no errors at all, so the count diff is the only thing that would ever mention it. + if: >- + always() && steps.parse.outcome != 'skipped' + && (steps.parse.outputs.status != '0' || steps.verdict.outputs.drift != '0' + || steps.verdict.outputs.mismatch == 'true') + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + STATUS: ${{ steps.parse.outputs.status }} + DECLARED: ${{ steps.verdict.outputs.declared }} + MISMATCH: ${{ steps.verdict.outputs.mismatch }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + + # Everything from the report reaches the body through a file, so nothing the FAA ships is + # ever read as shell. + { + echo "The end-to-end check for cycle **$CYCLE** found problems." + echo + echo "- Source: \`$URL\`" + echo "[View the run]($RUN_URL) — the \`dof-report-$CYCLE\` artifact holds the full" + echo "report and the tool's log." + echo + + if [ "$MISMATCH" = "true" ]; then + echo "### Wrong cycle" + echo + echo "The file declares currency date \`$DECLARED\`, but this URL should hold" + echo "\`$CYCLE_ID\`. Either the FAA published the wrong file or the cycle arithmetic" + echo "in \`Cycle\` and in this workflow has drifted apart." + echo + fi + + if [ -f "$REPORT" ]; then + if [ "${STATUS:-1}" != "0" ]; then + echo "### Failures" + echo + jq -r '.failureReasons[]? | "- \(.)"' "$REPORT" + echo + fi + + if [ "$(jq '.drift | length' "$REPORT")" -gt 0 ]; then + echo "### Count drift against $PREVIOUS_CYCLE" + echo + echo "| Region | $PREVIOUS_CYCLE | $CYCLE | Reason |" + echo "|---|---:|---:|---|" + jq -r '.drift[] | "| \(.scope) | \(.was) | \(.now // "missing") | \(.reason) |"' "$REPORT" + echo + fi + + echo "### Counts" + echo + echo "- Obstacles parsed: $(jq -r '.obstacleCount' "$REPORT")" + echo "- Parse errors: $(jq -r '.parseErrorCount' "$REPORT")" + echo + + if [ "$(jq '.errorSamples | length' "$REPORT")" -gt 0 ]; then + echo "### Error samples" + echo + echo '```' + jq -r '.errorSamples[] | "line \(.line): \(.reason // .message)"' "$REPORT" + echo '```' + fi + else + echo "The run produced no report. The tail of its log:" + echo + echo '```' + tail -c 8000 "$PARSE_LOG" 2>/dev/null || echo "No log was produced either." + echo '```' + fi + } > issue-body.md + + # The title is a function of the cycle alone, which is what keeps a re-check idempotent. + # The search omits the punctuation so GitHub's query parser cannot read it as a qualifier. + title="DOF cycle $CYCLE: end-to-end check failed" + number=$(TITLE="$title" gh issue list --state all --search "DOF cycle $CYCLE in:title" \ + --json number,title --jq 'map(select(.title == env.TITLE)) | .[0].number // empty') + + if [ -n "$number" ]; then + gh issue reopen "$number" >/dev/null 2>&1 || true + gh issue comment "$number" --body-file issue-body.md + else + gh issue create --title "$title" --body-file issue-body.md + fi + + - name: Annotate the outcome + if: always() && steps.parse.outcome != 'skipped' + env: + STATUS: ${{ steps.parse.outputs.status }} + DRIFT: ${{ steps.verdict.outputs.drift }} + run: | + set -euo pipefail + + drift=${DRIFT:-0} + + if [ "${STATUS:-1}" != "0" ]; then + echo "::error::Cycle $CYCLE failed its end-to-end check; an issue has been filed." + elif [ "$drift" != "0" ]; then + echo "::warning::Cycle $CYCLE parsed cleanly but $drift count(s) drifted." + else + echo "Cycle $CYCLE parsed cleanly with no count drift." + fi diff --git a/.periphery.yml b/.periphery.yml index 85b884a..b5d9e03 100644 --- a/.periphery.yml +++ b/.periphery.yml @@ -1 +1,4 @@ retain_public: true +# The report the cycle-watch workflow reads is written by JSONEncoder and read by jq, so its +# properties have no Swift reader for Periphery to find. +retain_codable_properties: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d44a57..efb2b59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ - `Obstacle.verticalAccuracy`, the accuracy category of an obstacle's reported height, read from column 100 of the DOF record. +- A `DOF Cycle Watch` workflow that parses each 56-day cycle as the FAA releases + it and opens an issue when a cycle fails to parse, when counts drift from the + previous cycle, or when the file declares a currency date other than the one + its URL should hold. +- `SwiftDOF_E2E --report` writes a JSON report of a parse — counts, per-region + totals, and the field each failed line failed on — and `--baseline` compares + those counts against an earlier report. The tool now exits non-zero when any + line fails to parse. ### Changed diff --git a/README.md b/README.md index 7e795fc..7d940ca 100644 --- a/README.md +++ b/README.md @@ -195,3 +195,21 @@ Options: - `-i, --input `: Path or URL to DOF file (.dat or .zip). Defaults to current FAA cycle. - `-f, --format `: Output format. Defaults to summary. +- `--report `: Write a JSON report of the parse — obstacle count, parse + errors and the field each one failed on, and per-region counts. +- `--baseline `: Compare counts against an earlier report and record any + that moved more than their tolerance. + +The tool exits non-zero when any line fails to parse. + +### Cycle Watch + +The `DOF Cycle Watch` workflow parses each new 56-day cycle as the FAA releases +it. A daily probe checks whether the cycle in effect is downloadable yet, and +the first time it is, the cycle is parsed end to end and its report kept as an +artifact — both the record that the cycle was checked and the baseline the next +cycle's counts are compared against. Parse errors, counts that drift, or a file +declaring the wrong currency date each open an issue naming the cycle. + +To re-check a cycle by hand, run the workflow and give it the currency date from +the filename, as `YYYY-MM-DD`. diff --git a/Sources/SwiftDOF_E2E/Report.swift b/Sources/SwiftDOF_E2E/Report.swift new file mode 100644 index 0000000..a9ab5d1 --- /dev/null +++ b/Sources/SwiftDOF_E2E/Report.swift @@ -0,0 +1,200 @@ +import Foundation +import SwiftDOF + +/// A machine-readable record of one end-to-end parse, written with `--report`. +/// +/// The report is what the cycle-watch workflow reads to decide whether a cycle parsed cleanly, +/// and it doubles as the baseline the next cycle is compared against. +struct Report: Codable { + + /// The share of the total obstacle count that may change between cycles before it counts as + /// drift. Obstacle data turns over slowly; a swing this large is a parsing problem, not news. + private static let totalDriftTolerance = 0.05 + + /// The share of a single region's obstacle count that may change between cycles. Regions are + /// small enough to be noisier than the file as a whole. + private static let regionDriftTolerance = 0.10 + + /// The greatest number of parse errors quoted in the report. + private static let maximumErrorSamples = 50 + + /// The cycle the parsed file declares in its currency date header. + let cycle: String + + /// Where the data was read from. + let source: String + + /// The number of obstacles parsed. + let obstacleCount: Int + + /// The number of lines that failed to parse. + let parseErrorCount: Int + + /// How long loading and parsing took, in seconds. + let elapsed: TimeInterval + + /// Obstacle counts keyed by region, as `COUNTRY-STATE` or bare `COUNTRY` where the DOF carries + /// no state. The country is always part of the key because state and country codes collide — + /// `CA` is both California and Canada. + let countsByRegion: [String: Int] + + /// Up to ``maximumErrorSamples`` of the parse errors. + let errorSamples: [ErrorSample] + + /// Whether this cycle failed its check. + let failed: Bool + + /// Why the cycle failed, empty when it did not. + let failureReasons: [String] + + /// Counts that moved more than their tolerance against the baseline. + let drift: [Drift] + + /// Builds a report from a completed parse, optionally compared against an earlier cycle's report. + init( + dof: DOF, + source: String, + parseErrorCount: Int, + errorSamples: [ErrorSample], + elapsed: TimeInterval, + baseline: Self? + ) { + let countsByRegion = Self.countsByRegion(of: dof) + + self.cycle = dof.cycle.id + self.source = source + self.obstacleCount = dof.count + self.parseErrorCount = parseErrorCount + self.elapsed = elapsed + self.countsByRegion = countsByRegion + self.errorSamples = Array(errorSamples.prefix(Self.maximumErrorSamples)) + self.failureReasons = Self.failureReasons( + obstacleCount: dof.count, + parseErrorCount: parseErrorCount + ) + self.failed = !failureReasons.isEmpty + self.drift = + baseline.map { + Self.drift(from: $0, totalNow: dof.count, regionsNow: countsByRegion) + } ?? [] + } + + /// Reads a report written by an earlier run. + static func read(from url: URL) throws -> Self { + try JSONDecoder().decode(Self.self, from: Data(contentsOf: url)) + } + + /// Obstacle counts keyed by country and state, so that California and Canada stay apart. + private static func countsByRegion(of dof: DOF) -> [String: Int] { + dof.reduce(into: [:]) { counts, obstacle in + counts[regionKey(of: obstacle), default: 0] += 1 + } + } + + private static func regionKey(of obstacle: Obstacle) -> String { + guard let state = obstacle.state else { return obstacle.country } + return "\(obstacle.country)-\(state)" + } + + private static func failureReasons(obstacleCount: Int, parseErrorCount: Int) -> [String] { + var reasons: [String] = [] + if parseErrorCount > 0 { + reasons.append("\(parseErrorCount) line(s) failed to parse.") + } + if obstacleCount == 0 { + reasons.append("The file parsed to zero obstacles.") + } + return reasons + } + + private static func drift( + from baseline: Self, + totalNow: Int, + regionsNow: [String: Int] + ) -> [Drift] { + var drift: [Drift] = [] + + if let total = Drift( + scope: "total", + was: baseline.obstacleCount, + now: totalNow, + tolerance: totalDriftTolerance + ) { + drift.append(total) + } + + for (region, was) in baseline.countsByRegion.sorted(by: { $0.key < $1.key }) { + if let regionDrift = Drift( + scope: region, + was: was, + now: regionsNow[region], + tolerance: regionDriftTolerance + ) { + drift.append(regionDrift) + } + } + + return drift + } + + /// One parse error, quoted for the report. + struct ErrorSample: Codable { + /// The line the error was reported on. + let line: Int + + /// The error's general description. + let message: String + + /// Which field failed and why, when the error carries that detail. + let reason: String? + + init(line: Int, error: any Error) { + self.line = line + self.message = error.localizedDescription + self.reason = (error as? (any LocalizedError))?.failureReason + } + } + + /// A count that moved more than its tolerance between two cycles. + struct Drift: Codable { + /// What the count covers: `total`, or a region key from ``Report/countsByRegion``. + let scope: String + + /// The count in the baseline cycle. + let was: Int + + /// The count in this cycle, or `nil` when the scope vanished entirely. + let now: Int? + + /// Why the movement was recorded. + let reason: String + + /// Records drift, or returns `nil` when the count held within `tolerance`. + init?(scope: String, was: Int, now: Int?, tolerance: Double) { + guard let now else { + self.init(scope: scope, was: was, now: nil, reason: "The region is no longer present.") + return + } + + guard was > 0 else { return nil } + + let change = Double(now - was) / Double(was) + guard abs(change) > tolerance else { return nil } + + let percent = (abs(change) * 100).formatted(.number.precision(.fractionLength(1))) + self.init( + scope: scope, + was: was, + now: now, + reason: "The count \(change < 0 ? "fell" : "rose") by \(percent)%." + ) + } + + private init(scope: String, was: Int, now: Int?, reason: String) { + self.scope = scope + self.was = was + self.now = now + self.reason = reason + } + } +} diff --git a/Sources/SwiftDOF_E2E/SwiftDOF_E2E.swift b/Sources/SwiftDOF_E2E/SwiftDOF_E2E.swift index 9aeb685..4f324b6 100644 --- a/Sources/SwiftDOF_E2E/SwiftDOF_E2E.swift +++ b/Sources/SwiftDOF_E2E/SwiftDOF_E2E.swift @@ -32,6 +32,22 @@ struct SwiftDOF_E2E: AsyncParsableCommand { @Option(name: .shortAndLong, help: "Output format: summary or json") var format: OutputFormat = .summary + @Option( + name: .long, + help: "Path to write a JSON report of the parse to", + completion: .file(extensions: ["json"]), + transform: { URL(filePath: $0) } + ) + var report: URL? + + @Option( + name: .long, + help: "Path to an earlier report to compare counts against", + completion: .file(extensions: ["json"]), + transform: { URL(filePath: $0) } + ) + var baseline: URL? + private var currentCycleURL: URL { get throws { let cycle = Cycle.effective @@ -56,6 +72,7 @@ struct SwiftDOF_E2E: AsyncParsableCommand { let formatter = makeFormatter(for: format) var errorCount = 0 + var errorSamples: [Report.ErrorSample] = [] let startTime = Date() let showProgress = format != .json @@ -70,6 +87,7 @@ struct SwiftDOF_E2E: AsyncParsableCommand { }, errorCallback: { error, line in errorCount += 1 + errorSamples.append(.init(line: line, error: error)) var message = "Error at line \(line): \(error.localizedDescription)" if let reason = (error as? (any LocalizedError))?.failureReason { message += "\n - \(reason)" @@ -93,6 +111,44 @@ struct SwiftDOF_E2E: AsyncParsableCommand { defer { stdout.close() } try formatter.format(dof: dof, errorCount: errorCount, elapsed: elapsed, to: stdout) + + if let report { + try writeReport( + to: report, + dof: dof, + source: inputURL, + errorCount: errorCount, + errorSamples: errorSamples, + elapsed: elapsed + ) + } + + if errorCount > 0 { + throw ExitCode.failure + } + } + + /// Writes the JSON report, comparing counts against `baseline` when one was given. + private func writeReport( + to url: URL, + dof: DOF, + source: URL, + errorCount: Int, + errorSamples: [Report.ErrorSample], + elapsed: TimeInterval + ) throws { + let report = Report( + dof: dof, + source: source.absoluteString, + parseErrorCount: errorCount, + errorSamples: errorSamples, + elapsed: elapsed, + baseline: try baseline.map { try Report.read(from: $0) } + ) + + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + try encoder.encode(report).write(to: url) } private func makeLoader(for url: URL) -> any DOFDataLoader {