From 929786105030d05af9ac4badc2badf71aacb462e Mon Sep 17 00:00:00 2001 From: yash Date: Sat, 24 Oct 2026 11:00:00 +0530 Subject: [PATCH 01/19] swift: Add zero-fill and little-endian store helpers to ByteBuffer. A standard row is written in two moves: reserve a zeroed fixed region for the null bitmap and the 8-byte slots, then come back and patch each slot once its value, or the offset and size of its variable-width data, is known. ByteBuffer owns the storage, so it gains the two primitives that make those moves possible. appendZeros reserves the region and keeps null slots and padding deterministic. storeLittleEndian patches an integer at any absolute offset, aligned or not, because a row may begin anywhere in the buffer. --- swift/Sources/Fory/ByteBuffer.swift | 24 ++++++++++++++++++ swift/Tests/ForyTests/ByteBufferTests.swift | 27 +++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/swift/Sources/Fory/ByteBuffer.swift b/swift/Sources/Fory/ByteBuffer.swift index 0c82279830..f2646a5735 100644 --- a/swift/Sources/Fory/ByteBuffer.swift +++ b/swift/Sources/Fory/ByteBuffer.swift @@ -875,3 +875,27 @@ public final class ByteBuffer { } } } + +extension ByteBuffer { + + // For row support, a writer first reserves the fixed slots and patches them in place later. + // This appends `count` zero bytes and returns the absolute offset of the first one. + @inlinable + @inline(__always) + internal func appendZeros(_ count: Int) -> Int { + let start = storage.count + storage.append(contentsOf: repeatElement(0, count: count)) + return start + } + + // Writes `value` at `offset` with the smallest byte first, replacing the bytes already there. + // Row writers use this to fill a slot they reserved earlier. The offset doesn't need to be + // aligned, but those bytes must already exist. + @inlinable + @inline(__always) + internal func storeLittleEndian(_ value: T, at offset: Int) { + storage.withUnsafeMutableBytes { raw in + raw.storeBytes(of: value.littleEndian, toByteOffset: offset, as: T.self) + } + } +} diff --git a/swift/Tests/ForyTests/ByteBufferTests.swift b/swift/Tests/ForyTests/ByteBufferTests.swift index 71ff055cd7..da87ed6704 100644 --- a/swift/Tests/ForyTests/ByteBufferTests.swift +++ b/swift/Tests/ForyTests/ByteBufferTests.swift @@ -283,3 +283,30 @@ func byteBufferRejectsInvalidVarintsAndUtf8() throws { #expect("\(error)".contains("invalid UTF-8")) } } + +@Test +func byteBufferAppendZeros() { + let buffer = ByteBuffer(bytes: [0x01, 0x02, 0x03]) + #expect(buffer.appendZeros(5) == 3) + #expect(buffer.storage == [0x01, 0x02, 0x03, 0, 0, 0, 0, 0]) + + buffer.writeBytes([0xFF, 0xFF]) + buffer.clear() + #expect(buffer.appendZeros(2) == 0) + #expect(buffer.storage == [0, 0]) +} + +@Test +func byteBufferStoreLittleEndian() { + let buffer = ByteBuffer(bytes: [UInt8](repeating: 0, count: 16)) + buffer.storeLittleEndian(UInt32(0x0102_0304), at: 3) + buffer.storeLittleEndian(UInt64(0x1122_3344_5566_7788), at: 7) + #expect( + buffer.storage == [ + 0x00, 0x00, 0x00, + 0x04, 0x03, 0x02, 0x01, + 0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11, + 0x00 + ] + ) +} From d14da6ef687a115a8ee80e6768ea686c1b0103db Mon Sep 17 00:00:00 2001 From: yash Date: Sat, 24 Oct 2026 12:00:00 +0530 Subject: [PATCH 02/19] swift: Add null bitmap helpers for the row format. Every row and array opens with a null bitmap: one bit per field or element, packed eight to a byte and padded to whole 8-byte words. Bit i sits in byte i / 8 of the bitmap, counted from the least significant bit, and the bitmap itself starts wherever its row or array starts. --- swift/Sources/Fory/Row/RowBitmap.swift | 44 +++++++++++++++ swift/Tests/ForyTests/RowBitmapTests.swift | 62 ++++++++++++++++++++++ 2 files changed, 106 insertions(+) create mode 100644 swift/Sources/Fory/Row/RowBitmap.swift create mode 100644 swift/Tests/ForyTests/RowBitmapTests.swift diff --git a/swift/Sources/Fory/Row/RowBitmap.swift b/swift/Sources/Fory/Row/RowBitmap.swift new file mode 100644 index 0000000000..fe15dbc616 --- /dev/null +++ b/swift/Sources/Fory/Row/RowBitmap.swift @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// The null bitmap keeps one bit per field or array element, and a set bit means the value is null. +// Bit `index` lives in byte `bitmapOffset + index / 8`, counting bits from the least significant end. + +// Returns the bitmap size in bytes for `count` fields or elements, rounded up to whole 8-byte +// words. `count` must not be negative. +@inline(__always) +func rowBitmapWidth(_ count: Int) -> Int { + ((count + 63) / 64) * 8 +} + +// Rounds `size` up to the next multiple of 8. +@inline(__always) +func rowRoundToWord(_ size: Int) -> Int { + (size + 7) & ~7 +} + +// Reports whether bit `index` is set in the bitmap that starts at `bitmapOffset`. +@inline(__always) +func rowIsBitSet(_ bytes: UnsafeRawBufferPointer, bitmapOffset: Int, index: Int) -> Bool { + bytes[bitmapOffset + (index >> 3)] & (UInt8(1) << UInt8(index & 7)) != 0 +} + +// Sets bit `index` in the bitmap that starts at `bitmapOffset`. +@inline(__always) +func rowSetBit(_ bytes: UnsafeMutableRawBufferPointer, bitmapOffset: Int, index: Int) { + bytes[bitmapOffset + (index >> 3)] |= UInt8(1) << UInt8(index & 7) +} diff --git a/swift/Tests/ForyTests/RowBitmapTests.swift b/swift/Tests/ForyTests/RowBitmapTests.swift new file mode 100644 index 0000000000..537f18e1ec --- /dev/null +++ b/swift/Tests/ForyTests/RowBitmapTests.swift @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation +import Testing +@testable import Fory + +@Test +func rowBitmapWidths() { + let cases: [(count: Int, width: Int)] = [ + (0, 0), (1, 8), (63, 8), (64, 8), (65, 16), (128, 16), (129, 24) + ] + for (count, width) in cases { + #expect(rowBitmapWidth(count) == width) + } +} + +@Test +func rowBitmapWordRounding() { + let cases: [(size: Int, rounded: Int)] = [(0, 0), (1, 8), (7, 8), (8, 8), (9, 16)] + for (size, rounded) in cases { + #expect(rowRoundToWord(size) == rounded) + } +} + +@Test +func rowBitmapBits() { + let nullIndexes: Set = [0, 7, 8, 63, 64] + var bytes = [UInt8](repeating: 0, count: 24) + bytes.withUnsafeMutableBytes { raw in + for index in nullIndexes { + rowSetBit(raw, bitmapOffset: 8, index: index) + } + } + + var expected = [UInt8](repeating: 0, count: 24) + expected[8] = 0x81 + expected[9] = 0x01 + expected[15] = 0x80 + expected[16] = 0x01 + #expect(bytes == expected) + + bytes.withUnsafeBytes { raw in + for index in 0..<72 { + #expect(rowIsBitSet(raw, bitmapOffset: 8, index: index) == nullIndexes.contains(index)) + } + } +} From 87a81d77fe6d70ee9dfc32655a1d5a2d2a9b1acf Mon Sep 17 00:00:00 2001 From: yash Date: Sat, 24 Oct 2026 13:00:00 +0530 Subject: [PATCH 03/19] swift: Add offset and size slot encoding for the row format. A variable-width value lives after the fixed slots, and its slot holds a single little-endian UInt64 ticket: the offset from the start of its row or array in the upper 32 bits and the byte size in the lower 32 bits. Decoding keeps both halves as UInt32 and checks offset plus size in UInt64, so a corrupt ticket throws instead of trapping, even where Int is 32 bits. --- swift/Sources/Fory/Row/RowSlot.swift | 58 ++++++++++++++++++++++ swift/Tests/ForyTests/RowSlotTests.swift | 62 ++++++++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 swift/Sources/Fory/Row/RowSlot.swift create mode 100644 swift/Tests/ForyTests/RowSlotTests.swift diff --git a/swift/Sources/Fory/Row/RowSlot.swift b/swift/Sources/Fory/Row/RowSlot.swift new file mode 100644 index 0000000000..46736b2c2b --- /dev/null +++ b/swift/Sources/Fory/Row/RowSlot.swift @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// A variable-width value such as a string, array, map, or nested struct is stored after the fixed +// slots. Its slot holds one UInt64 ticket: the offset from the start of the enclosing row or array +// in the upper 32 bits and the byte size in the lower 32 bits, so the size comes first in memory. + +// Packs an offset and a size into one slot value. Both must fit in 32 bits. +func rowEncodeOffsetAndSize(offset: Int, size: Int) throws -> UInt64 { + guard let offset32 = UInt32(exactly: offset), let size32 = UInt32(exactly: size) else { + throw rowValueTooLarge(offset: offset, size: size) + } + return UInt64(offset32) << 32 | UInt64(size32) +} + +// Splits a slot value back into its offset and size. They stay UInt32 so a large offset cannot +// trap on platforms where Int is 32 bits. +@inline(__always) +func rowDecodeOffsetAndSize(_ slot: UInt64) -> (offset: UInt32, size: UInt32) { + (UInt32(truncatingIfNeeded: slot >> 32), UInt32(truncatingIfNeeded: slot)) +} + +// Checks that `size` bytes at `offset` fit inside a region of `regionCount` bytes and returns their +// absolute range in the buffer. The sum is taken in UInt64, so it cannot overflow. +@inline(__always) +func rowCheckedRange(offset: UInt32, size: UInt32, regionBase: Int, regionCount: Int) throws -> Range { + guard UInt64(offset) + UInt64(size) <= UInt64(regionCount) else { + throw rowOutOfBounds(offset: offset, size: size, regionCount: regionCount) + } + let start = regionBase + Int(offset) + return start..<(start + Int(size)) +} + +@inline(never) +func rowValueTooLarge(offset: Int, size: Int) -> ForyError { + .encodingError( + "row value at offset \(offset) with \(size) bytes does not fit the row format's 32-bit offset and size" + ) +} + +@inline(never) +func rowOutOfBounds(offset: UInt32, size: UInt32, regionCount: Int) -> ForyError { + .outOfBounds(cursor: Int(clamping: offset), need: Int(clamping: size), length: regionCount) +} diff --git a/swift/Tests/ForyTests/RowSlotTests.swift b/swift/Tests/ForyTests/RowSlotTests.swift new file mode 100644 index 0000000000..188dee6e37 --- /dev/null +++ b/swift/Tests/ForyTests/RowSlotTests.swift @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation +import Testing +@testable import Fory + +@Test +func rowSlotEncode() throws { + let slot = try rowEncodeOffsetAndSize(offset: 0x20, size: 2) + #expect(slot == 0x0000_0020_0000_0002) + #expect(withUnsafeBytes(of: slot.littleEndian, Array.init) == [0x02, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00]) +} + +@Test +func rowSlotDecode() { + let ticket = rowDecodeOffsetAndSize(0x0000_0020_0000_0002) + #expect(ticket.offset == 0x20) + #expect(ticket.size == 2) + + let large = rowDecodeOffsetAndSize(0xFFFF_FFFF_0000_0001) + #expect(large.offset == UInt32.max) + #expect(large.size == 1) +} + +@Test +func rowSlotEncodeLimits() throws { + #expect(throws: ForyError.self) { + try rowEncodeOffsetAndSize(offset: Int(UInt32.max) + 1, size: 0) + } + #expect(throws: ForyError.self) { + try rowEncodeOffsetAndSize(offset: 0, size: -1) + } + let slot = try rowEncodeOffsetAndSize(offset: Int(UInt32.max), size: Int(UInt32.max)) + #expect(slot == UInt64.max) +} + +@Test +func rowSlotCheckedRange() throws { + #expect(try rowCheckedRange(offset: 8, size: 8, regionBase: 100, regionCount: 16) == 108..<116) + #expect(try rowCheckedRange(offset: 0, size: 0, regionBase: 0, regionCount: 0).isEmpty) + #expect(throws: ForyError.self) { + try rowCheckedRange(offset: 8, size: 9, regionBase: 0, regionCount: 16) + } + #expect(throws: ForyError.self) { + try rowCheckedRange(offset: UInt32.max, size: UInt32.max, regionBase: 0, regionCount: 16) + } +} From b44887dac5091307f2c0f1f35eac8c4dfa9e0763 Mon Sep 17 00:00:00 2001 From: yash Date: Mon, 26 Oct 2026 11:00:00 +0530 Subject: [PATCH 04/19] swift: Add the row schema model. A row stores only values in fixed seats, so the writer and the reader must agree on a schema that says how many fields there are, what type each one has, and whether it can be null. RowDataType, RowField, and RowSchema describe that, using the cross-language ids from TypeId and the list and map child names Java uses, so a Swift schema matches Java's exactly once it is written as bytes and hashed. --- swift/Sources/Fory/Row/RowSchema.swift | 159 +++++++++++++++++++++ swift/Tests/ForyTests/RowSchemaTests.swift | 107 ++++++++++++++ 2 files changed, 266 insertions(+) create mode 100644 swift/Sources/Fory/Row/RowSchema.swift create mode 100644 swift/Tests/ForyTests/RowSchemaTests.swift diff --git a/swift/Sources/Fory/Row/RowSchema.swift b/swift/Sources/Fory/Row/RowSchema.swift new file mode 100644 index 0000000000..df9de2081b --- /dev/null +++ b/swift/Sources/Fory/Row/RowSchema.swift @@ -0,0 +1,159 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/// The type of a row field, an array element, or a map key or value. +public indirect enum RowDataType: Equatable, Sendable, CustomStringConvertible { + case bool + case int8 + case int16 + case int32 + case int64 + /// Only parsed from schemas; its values are not read or written yet. + case float16 + case float32 + case float64 + case string + case binary + case date32 + case timestamp + case duration + /// Only parsed from schemas; its values are not read or written yet. + case decimal(precision: Int, scale: Int) + case list(RowField) + case map(key: RowField, value: RowField) + case `struct`([RowField]) + + /// The cross-language type id written in schema bytes. + public var typeId: TypeId { + switch self { + case .bool: return .bool + case .int8: return .int8 + case .int16: return .int16 + case .int32: return .int32 + case .int64: return .int64 + case .float16: return .float16 + case .float32: return .float32 + case .float64: return .float64 + case .string: return .string + case .binary: return .binary + case .date32: return .date + case .timestamp: return .timestamp + case .duration: return .duration + case .decimal: return .decimal + case .list: return .list + case .map: return .map + case .struct: return .structType + } + } + + /// The size of one value in bytes, or nil when the value lives after the slots and is + /// reached through an offset and size. + public var byteWidth: Int? { + switch self { + case .bool, .int8: return 1 + case .int16, .float16: return 2 + case .int32, .float32, .date32: return 4 + case .int64, .float64, .timestamp, .duration: return 8 + case .string, .binary, .decimal, .list, .map, .struct: return nil + } + } + + /// A list whose element field is named `item` and can be null. + public static func listOf(_ element: RowDataType) -> RowDataType { + .list(RowField(name: "item", dataType: element, nullable: true)) + } + + /// A map whose key field is named `key` and is never null, and whose value field is named + /// `value` and can be null. + public static func mapOf(_ key: RowDataType, _ value: RowDataType) -> RowDataType { + .map( + key: RowField(name: "key", dataType: key, nullable: false), + value: RowField(name: "value", dataType: value, nullable: true) + ) + } + + public var description: String { + switch self { + case .bool: return "bool" + case .int8: return "int8" + case .int16: return "int16" + case .int32: return "int32" + case .int64: return "int64" + case .float16: return "float16" + case .float32: return "float32" + case .float64: return "float64" + case .string: return "string" + case .binary: return "binary" + case .date32: return "date32" + case .timestamp: return "timestamp" + case .duration: return "duration" + case .decimal(let precision, let scale): return "decimal(\(precision), \(scale))" + case .list(let element): return "list<\(element.dataType)>" + case .map(let key, let value): return "map<\(key.dataType), \(value.dataType)>" + case .struct(let fields): return "struct<\(rowFieldListDescription(fields))>" + } + } +} + +/// A named slot in a schema, or the element of a list, or the key or value of a map. +public struct RowField: Equatable, Sendable { + public var name: String + public var dataType: RowDataType + public var nullable: Bool + + public init(name: String, dataType: RowDataType, nullable: Bool) { + self.name = name + self.dataType = dataType + self.nullable = nullable + } +} + +/// The fields of a row in slot order. A row can only be read with the schema it was written with. +public struct RowSchema: Equatable, Sendable, CustomStringConvertible { + public let fields: [RowField] + // Built once from `fields`, so it never disagrees with them. + private let indexByName: [String: Int] + + public init(_ fields: [RowField]) { + self.fields = fields + var indexByName: [String: Int] = [:] + indexByName.reserveCapacity(fields.count) + for (index, field) in fields.enumerated() { + indexByName[field.name] = index + } + self.indexByName = indexByName + } + + public var fieldCount: Int { + fields.count + } + + /// The slot index of the field named `name`, or nil if there is none. When a name repeats, the + /// last field with that name wins. + public func fieldIndex(of name: String) -> Int? { + indexByName[name] + } + + public var description: String { + "schema<\(rowFieldListDescription(fields))>" + } +} + +// Shared by the struct and schema descriptions: "a: int32, b: string". +private func rowFieldListDescription(_ fields: [RowField]) -> String { + fields.map { "\($0.name): \($0.dataType)" }.joined(separator: ", ") +} diff --git a/swift/Tests/ForyTests/RowSchemaTests.swift b/swift/Tests/ForyTests/RowSchemaTests.swift new file mode 100644 index 0000000000..7c590766cf --- /dev/null +++ b/swift/Tests/ForyTests/RowSchemaTests.swift @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation +import Testing +@testable import Fory + +private let rowSchemaPrimitives: [RowDataType] = [ + .bool, .int8, .int16, .int32, .int64, .float16, .float32, .float64, + .string, .binary, .date32, .timestamp, .duration +] + +@Test +func rowSchemaTypeIds() { + let cases: [(type: RowDataType, id: UInt32)] = [ + (.bool, 1), (.int8, 2), (.int16, 3), (.int32, 4), (.int64, 6), + (.float16, 17), (.float32, 19), (.float64, 20), (.string, 21), (.binary, 41), + (.date32, 39), (.timestamp, 38), (.duration, 37), (.decimal(precision: 10, scale: 2), 40), + (.listOf(.int32), 22), (.mapOf(.string, .int32), 24), (.struct([]), 27) + ] + for (type, id) in cases { + #expect(type.typeId.rawValue == id, "\(type)") + } +} + +@Test +func rowSchemaByteWidths() { + let widths: [Int?] = [1, 1, 2, 4, 8, 2, 4, 8, nil, nil, 4, 8, 8] + for (type, width) in zip(rowSchemaPrimitives, widths) { + #expect(type.byteWidth == width, "\(type)") + } + #expect(RowDataType.decimal(precision: 10, scale: 2).byteWidth == nil) + #expect(RowDataType.listOf(.int32).byteWidth == nil) + #expect(RowDataType.mapOf(.string, .int32).byteWidth == nil) + #expect(RowDataType.struct([]).byteWidth == nil) +} + +@Test +func rowSchemaFactories() { + #expect(RowDataType.listOf(.string) == .list(RowField(name: "item", dataType: .string, nullable: true))) + #expect( + RowDataType.mapOf(.string, .int32) + == .map( + key: RowField(name: "key", dataType: .string, nullable: false), + value: RowField(name: "value", dataType: .int32, nullable: true) + ) + ) +} + +@Test +func rowSchemaFieldIndex() { + let schema = RowSchema([ + RowField(name: "a", dataType: .int32, nullable: false), + RowField(name: "b", dataType: .string, nullable: true), + RowField(name: "a", dataType: .int64, nullable: false) + ]) + #expect(schema.fieldCount == 3) + #expect(schema.fieldIndex(of: "a") == 2) + #expect(schema.fieldIndex(of: "b") == 1) + #expect(schema.fieldIndex(of: "missing") == nil) +} + +@Test +func rowSchemaEquality() { + let field = RowField(name: "a", dataType: .int32, nullable: true) + #expect(RowSchema([field]) == RowSchema([field])) + #expect(RowSchema([field]) != RowSchema([RowField(name: "b", dataType: .int32, nullable: true)])) + #expect(RowSchema([field]) != RowSchema([RowField(name: "a", dataType: .int32, nullable: false)])) + #expect(RowSchema([field]) != RowSchema([RowField(name: "a", dataType: .int64, nullable: true)])) + #expect(RowDataType.decimal(precision: 10, scale: 2) != .decimal(precision: 10, scale: 3)) + #expect(RowDataType.decimal(precision: 10, scale: 2) != .decimal(precision: 9, scale: 2)) + #expect(RowDataType.listOf(.int32) != .listOf(.int64)) +} + +@Test +func rowSchemaDescription() { + let names = [ + "bool", "int8", "int16", "int32", "int64", "float16", "float32", "float64", + "string", "binary", "date32", "timestamp", "duration" + ] + for (type, name) in zip(rowSchemaPrimitives, names) { + #expect(type.description == name) + } + #expect(RowDataType.decimal(precision: 10, scale: 2).description == "decimal(10, 2)") + #expect(RowDataType.listOf(.string).description == "list") + #expect(RowDataType.mapOf(.string, .int32).description == "map") + let structType = RowDataType.struct([ + RowField(name: "a", dataType: .int32, nullable: false), + RowField(name: "b", dataType: .string, nullable: true) + ]) + #expect(structType.description == "struct") + #expect(RowSchema([RowField(name: "a", dataType: .int32, nullable: false)]).description == "schema") +} From c38ea5a3009358ff632d39b7446cea5a17cde92a Mon Sep 17 00:00:00 2001 From: yash Date: Mon, 26 Oct 2026 14:00:00 +0530 Subject: [PATCH 05/19] swift: Add the Row reader. A Row is a view over the bytes of one row. It checks once, when it is created, that the bytes are long enough to hold the bitmap and every slot, so the fixed-width getters can read straight from their slots. A string, binary, or nested struct getter first checks the null bit, then checks its offset and size against the row before reading. Strings go through the same UTF-8 check ByteBuffer already used, now shared. --- swift/Sources/Fory/ByteBuffer.swift | 74 ++++---- swift/Sources/Fory/Row/Row.swift | 160 ++++++++++++++++++ swift/Sources/Fory/Row/RowSlot.swift | 14 ++ swift/Tests/ForyTests/RowReaderTests.swift | 188 +++++++++++++++++++++ 4 files changed, 403 insertions(+), 33 deletions(-) create mode 100644 swift/Sources/Fory/Row/Row.swift create mode 100644 swift/Tests/ForyTests/RowReaderTests.swift diff --git a/swift/Sources/Fory/ByteBuffer.swift b/swift/Sources/Fory/ByteBuffer.swift index f2646a5735..d49130e71b 100644 --- a/swift/Sources/Fory/ByteBuffer.swift +++ b/swift/Sources/Fory/ByteBuffer.swift @@ -819,39 +819,7 @@ public final class ByteBuffer { guard let base = buffer.baseAddress else { return nil } - let utf8Base = base.advanced(by: start) - let utf8Bytes = UnsafeBufferPointer(start: utf8Base, count: count) - var index = 0 - var isASCII = true - // An ASCII byte has a clear high bit, so this mask checks eight bytes at once - // regardless of native byte order. The bounded loop makes the unaligned load safe. - while index <= count - MemoryLayout.size { - let word = UnsafeRawPointer(utf8Base.advanced(by: index)) - .loadUnaligned(as: UInt64.self) - if word & 0x8080_8080_8080_8080 != 0 { - isASCII = false - break - } - index += MemoryLayout.size - } - if isASCII { - while index < count { - if utf8Bytes[index] >= 0x80 { - isASCII = false - break - } - index += 1 - } - } - if isASCII { - // ASCII bytes are always valid UTF-8. - // swiftlint:disable:next optional_data_string_conversion - return String(decoding: utf8Bytes, as: UTF8.self) - } - if #available(macOS 15.0, iOS 18.0, tvOS 18.0, watchOS 11.0, visionOS 2.0, *) { - return String(validating: utf8Bytes, as: UTF8.self) - } - return String(bytes: utf8Bytes, encoding: .utf8) + return decodeValidatedUTF8(UnsafeBufferPointer(start: base.advanced(by: start), count: count)) } guard let decoded else { throw ForyError.invalidData("invalid UTF-8 sequence") @@ -899,3 +867,43 @@ extension ByteBuffer { } } } + +@inlinable +@inline(__always) +internal func decodeValidatedUTF8(_ utf8Bytes: UnsafeBufferPointer) -> String? { + guard let utf8Base = utf8Bytes.baseAddress else { + return "" + } + let count = utf8Bytes.count + var index = 0 + var isASCII = true + // An ASCII byte has a clear high bit, so this mask checks eight bytes at once + // regardless of native byte order. The bounded loop makes the unaligned load safe. + while index <= count - MemoryLayout.size { + let word = UnsafeRawPointer(utf8Base.advanced(by: index)) + .loadUnaligned(as: UInt64.self) + if word & 0x8080_8080_8080_8080 != 0 { + isASCII = false + break + } + index += MemoryLayout.size + } + if isASCII { + while index < count { + if utf8Bytes[index] >= 0x80 { + isASCII = false + break + } + index += 1 + } + } + if isASCII { + // ASCII bytes are always valid UTF-8. + // swiftlint:disable:next optional_data_string_conversion + return String(decoding: utf8Bytes, as: UTF8.self) + } + if #available(macOS 15.0, iOS 18.0, tvOS 18.0, watchOS 11.0, visionOS 2.0, *) { + return String(validating: utf8Bytes, as: UTF8.self) + } + return String(bytes: utf8Bytes, encoding: .utf8) +} diff --git a/swift/Sources/Fory/Row/Row.swift b/swift/Sources/Fory/Row/Row.swift new file mode 100644 index 0000000000..eab33b02c4 --- /dev/null +++ b/swift/Sources/Fory/Row/Row.swift @@ -0,0 +1,160 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +/// A read-only view of one row's bytes. Each getter reads its field in place and returns nil when the +/// field is null. An index outside `0..= 0 && count >= 0 && count <= bytes.count - base, "row range is outside its buffer") + let bitmapWidth = rowBitmapWidth(fields.count) + let fixedSize = bitmapWidth + fields.count * 8 + guard count >= fixedSize else { + throw rowTooShort(count: count, fixedSize: fixedSize) + } + self.fields = fields + self.sizeInBytes = count + self.bytes = bytes + self.base = base + self.bitmapWidth = bitmapWidth + } + + public var fieldCount: Int { + fields.count + } + + public func isNull(at index: Int) -> Bool { + precondition(index >= 0 && index < fields.count, "field index \(index) is outside 0..<\(fields.count)") + return bytes.withUnsafeBytes { rowIsBitSet($0, bitmapOffset: base, index: index) } + } + + public func bool(at index: Int) -> Bool? { + isNull(at: index) ? nil : loadSlot(UInt8.self, at: index) != 0 + } + + public func int8(at index: Int) -> Int8? { + isNull(at: index) ? nil : Int8(bitPattern: loadSlot(UInt8.self, at: index)) + } + + public func int16(at index: Int) -> Int16? { + isNull(at: index) ? nil : Int16(bitPattern: loadSlot(UInt16.self, at: index)) + } + + public func int32(at index: Int) -> Int32? { + isNull(at: index) ? nil : Int32(bitPattern: loadSlot(UInt32.self, at: index)) + } + + public func int64(at index: Int) -> Int64? { + isNull(at: index) ? nil : Int64(bitPattern: loadSlot(UInt64.self, at: index)) + } + + public func float32(at index: Int) -> Float? { + isNull(at: index) ? nil : Float(bitPattern: loadSlot(UInt32.self, at: index)) + } + + public func float64(at index: Int) -> Double? { + isNull(at: index) ? nil : Double(bitPattern: loadSlot(UInt64.self, at: index)) + } + + public func localDate(at index: Int) -> LocalDate? { + int32(at: index).map { LocalDate(epochDay: Int64($0)) } + } + + public func timestamp(at index: Int) -> Date? { + int64(at: index).map(rowDate(micros:)) + } + + public func duration(at index: Int) -> Duration? { + int64(at: index).map { .microseconds($0) } + } + + /// Throws if the stored offset and size point outside this row, or the bytes are not valid UTF-8. + public func string(at index: Int) throws -> String? { + guard let range = try valueRange(at: index) else { + return nil + } + let decoded = bytes.withUnsafeBufferPointer { decodeValidatedUTF8(UnsafeBufferPointer(rebasing: $0[range])) } + guard let decoded else { + throw rowInvalidUTF8() + } + return decoded + } + + /// Throws if the stored offset and size point outside this row. + public func binary(at index: Int) throws -> [UInt8]? { + guard let range = try valueRange(at: index) else { + return nil + } + return Array(bytes[range]) + } + + /// The field must have a struct type. + public func row(at index: Int) throws -> Row? { + guard case .struct(let childFields) = fields[index].dataType else { + preconditionFailure("field \(index) has type \(fields[index].dataType), not a struct") + } + guard let range = try valueRange(at: index) else { + return nil + } + return try Row(fields: childFields, bytes: bytes, base: range.lowerBound, count: range.count) + } + + /// A copy of only this row's bytes, even when it sits inside a larger buffer. + public func toBytes() -> [UInt8] { + Array(bytes[base..<(base + sizeInBytes)]) + } + + @inline(__always) + private func loadSlot(_: T.Type, at index: Int) -> T { + bytes.withUnsafeBytes { + T(littleEndian: $0.loadUnaligned(fromByteOffset: base + bitmapWidth + index * 8, as: T.self)) + } + } + + // The null bit is checked before the slot is read, because a null field's slot can hold anything. + private func valueRange(at index: Int) throws -> Range? { + if isNull(at: index) { + return nil + } + let ticket = rowDecodeOffsetAndSize(loadSlot(UInt64.self, at: index)) + return try rowCheckedRange(offset: ticket.offset, size: ticket.size, regionBase: base, regionCount: sizeInBytes) + } +} + +@inline(never) +func rowTooShort(count: Int, fixedSize: Int) -> ForyError { + .invalidData("row of \(count) bytes is shorter than its \(fixedSize)-byte fixed region") +} + +@inline(never) +func rowInvalidUTF8() -> ForyError { + .invalidData("invalid UTF-8 sequence") +} diff --git a/swift/Sources/Fory/Row/RowSlot.swift b/swift/Sources/Fory/Row/RowSlot.swift index 46736b2c2b..d82570d8af 100644 --- a/swift/Sources/Fory/Row/RowSlot.swift +++ b/swift/Sources/Fory/Row/RowSlot.swift @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +import Foundation + // A variable-width value such as a string, array, map, or nested struct is stored after the fixed // slots. Its slot holds one UInt64 ticket: the offset from the start of the enclosing row or array // in the upper 32 bits and the byte size in the lower 32 bits, so the size comes first in memory. @@ -45,6 +47,18 @@ func rowCheckedRange(offset: UInt32, size: UInt32, regionBase: Int, regionCount: return start..<(start + Int(size)) } +// `micros` counts from 1970 but a Date counts from 2001, so the whole seconds are shifted as integers +// and the fraction is added once; going through timeIntervalSince1970 would round twice. Floor +// division keeps the fraction positive. +func rowDate(micros: Int64) -> Date { + var (seconds, remainder) = micros.quotientAndRemainder(dividingBy: 1_000_000) + if remainder < 0 { + seconds -= 1 + remainder += 1_000_000 + } + return Date(timeIntervalSinceReferenceDate: Double(seconds - 978_307_200) + Double(remainder) / 1e6) +} + @inline(never) func rowValueTooLarge(offset: Int, size: Int) -> ForyError { .encodingError( diff --git a/swift/Tests/ForyTests/RowReaderTests.swift b/swift/Tests/ForyTests/RowReaderTests.swift new file mode 100644 index 0000000000..2f1e40d0fd --- /dev/null +++ b/swift/Tests/ForyTests/RowReaderTests.swift @@ -0,0 +1,188 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation +import Testing +@testable import Fory + +private let rowStringSchema = RowSchema([RowField(name: "s", dataType: .string, nullable: true)]) + +// A one-string row: 8-byte bitmap, then the given slot, then the given data. +private func rowWithString(slot: [UInt8], data: [UInt8] = [0x61, 0, 0, 0, 0, 0, 0, 0]) throws -> Row { + try Row(schema: rowStringSchema, bytes: [UInt8](repeating: 0, count: 8) + slot + data) +} + +@Test +func rowReaderLayout() throws { + let schema = RowSchema([ + RowField(name: "a", dataType: .int32, nullable: true), + RowField(name: "b", dataType: .int32, nullable: false), + RowField(name: "c", dataType: .string, nullable: true) + ]) + let bytes: [UInt8] = [ + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // bitmap: a is null + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // slot a + 0x2C, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // slot b = 300 + 0x02, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, // slot c: size 2, offset 0x20 + 0x68, 0x69, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // "hi" + ] + let row = try Row(schema: schema, bytes: bytes) + #expect(row.fieldCount == 3) + #expect(row.sizeInBytes == 40) + #expect(row.isNull(at: 0)) + #expect(row.int32(at: 0) == nil) + #expect(row.int32(at: 1) == 300) + #expect(try row.string(at: 2) == "hi") + #expect(row.toBytes() == bytes) +} + +@Test +func rowReaderGoLayout() throws { + let schema = RowSchema([ + RowField(name: "f1", dataType: .int64, nullable: false), + RowField(name: "f2", dataType: .string, nullable: true) + ]) + let bytes: [UInt8] = [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x08, 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, + 0x02, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, + 0x61, 0x62, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + ] + let row = try Row(schema: schema, bytes: bytes) + #expect(row.int64(at: 0) == 0x0102_0304_0506_0708) + #expect(try row.string(at: 1) == "ab") +} + +@Test +func rowReaderNested() throws { + let schema = RowSchema([ + RowField(name: "id", dataType: .int32, nullable: false), + RowField(name: "child", dataType: .struct([RowField(name: "x", dataType: .int32, nullable: false)]), nullable: true) + ]) + let bytes: [UInt8] = [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // parent bitmap + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // slot id = 1 + 0x10, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, // slot child: size 16, offset 0x18 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // child bitmap + 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // child slot x = 5 + ] + let row = try Row(schema: schema, bytes: bytes) + #expect(row.int32(at: 0) == 1) + let child = try #require(try row.row(at: 1)) + #expect(child.sizeInBytes == 16) + #expect(child.int32(at: 0) == 5) + #expect(child.toBytes() == Array(bytes[0x18..<0x28])) +} + +@Test +func rowReaderNullSkipsData() throws { + let schema = RowSchema([ + RowField(name: "s", dataType: .string, nullable: true), + RowField(name: "b", dataType: .binary, nullable: true), + RowField(name: "e", dataType: .string, nullable: true) + ]) + let bytes: [UInt8] = [ + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // s and b are null + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // slot s: garbage + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // slot b: garbage + 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00 // slot e: empty, not null + ] + let row = try Row(schema: schema, bytes: bytes) + #expect(try row.string(at: 0) == nil) + #expect(try row.binary(at: 1) == nil) + #expect(!row.isNull(at: 2)) + #expect(try row.string(at: 2) == "") +} + +@Test +func rowReaderFieldCounts() throws { + for (count, size) in [(0, 0), (63, 512), (64, 520), (65, 536)] { + let schema = RowSchema((0.. 0 { + bytes[(count - 1) / 8] |= 1 << ((count - 1) % 8) + // The first slot starts right after the bitmap, which grows to 16 bytes at 65 fields. + bytes[rowBitmapWidth(count)] = 7 + } + let row = try Row(schema: schema, bytes: bytes) + #expect(row.sizeInBytes == size) + if count > 0 { + #expect(row.isNull(at: count - 1)) + #expect(!row.isNull(at: count - 2)) + #expect(row.int64(at: 0) == 7) + } + } +} + +@Test +func rowReaderMultiByteUTF8() throws { + let row = try rowWithString( + slot: [0x06, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00], + data: [0x68, 0xC3, 0xA9, 0x6C, 0x6C, 0x6F, 0x00, 0x00] + ) + #expect(try row.string(at: 0) == "héllo") +} + +@Test +func rowReaderTemporal() throws { + let schema = RowSchema([ + RowField(name: "ts", dataType: .timestamp, nullable: false), + RowField(name: "d", dataType: .date32, nullable: false), + RowField(name: "dur", dataType: .duration, nullable: false) + ]) + let bytes: [UInt8] = [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // ts = -1 microsecond + 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, // d = -1 day + 0x24, 0xFA, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF // dur = -1500 microseconds + ] + let row = try Row(schema: schema, bytes: bytes) + // Counted back from 2001 in whole microseconds, so the check itself adds no rounding. + func micros(_ date: Date) -> Int64 { + Int64((date.timeIntervalSinceReferenceDate * 1_000_000).rounded()) + 978_307_200_000_000 + } + let timestamp = try #require(row.timestamp(at: 0)) + #expect(micros(timestamp) == -1) + #expect(timestamp < Date(timeIntervalSince1970: 0)) + #expect(micros(rowDate(micros: 1_234_567_890_123_456)) == 1_234_567_890_123_456) + #expect(row.localDate(at: 1) == LocalDate(epochDay: -1)) + #expect(row.duration(at: 2) == .microseconds(-1500)) +} + +@Test +func rowReaderCorrupt() throws { + #expect(throws: ForyError.invalidData("row of 15 bytes is shorter than its 16-byte fixed region")) { + try Row(schema: rowStringSchema, bytes: [UInt8](repeating: 0, count: 15)) + } + // Offset 0x18 is the end of the 24-byte row, so even one byte there is past it. + #expect(throws: ForyError.outOfBounds(cursor: 0x18, need: 1, length: 24)) { + try rowWithString(slot: [0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00]).string(at: 0) + } + #expect(throws: ForyError.outOfBounds(cursor: 0x10, need: 9, length: 24)) { + try rowWithString(slot: [0x09, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00]).string(at: 0) + } + let max = Int(clamping: UInt32.max) + #expect(throws: ForyError.outOfBounds(cursor: max, need: max, length: 24)) { + try rowWithString(slot: [UInt8](repeating: 0xFF, count: 8)).binary(at: 0) + } + #expect(throws: ForyError.invalidData("invalid UTF-8 sequence")) { + try rowWithString( + slot: [0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00], + data: [0xFF, 0xFE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ).string(at: 0) + } +} From 46c9edb2e8fd80a68eadbdc07c67ba60cbc9932d Mon Sep 17 00:00:00 2001 From: yash Date: Mon, 26 Oct 2026 16:35:21 +0530 Subject: [PATCH 06/19] swift: Add the RowArray and RowMap readers. An array starts with its element count, and a forged count could make the size arithmetic overflow, so the count is first compared with the bytes that are actually there and only then multiplied by the element width. Like Row, RowArray and RowMap check everything once when they are created, including that a map has as many values as keys, so no getter can read past the end. --- swift/Sources/Fory/Row/Row.swift | 253 +++++++++++++++++- .../Tests/ForyTests/RowArrayReaderTests.swift | 159 +++++++++++ 2 files changed, 401 insertions(+), 11 deletions(-) create mode 100644 swift/Tests/ForyTests/RowArrayReaderTests.swift diff --git a/swift/Sources/Fory/Row/Row.swift b/swift/Sources/Fory/Row/Row.swift index eab33b02c4..0ebb4762ac 100644 --- a/swift/Sources/Fory/Row/Row.swift +++ b/swift/Sources/Fory/Row/Row.swift @@ -98,14 +98,7 @@ public struct Row { /// Throws if the stored offset and size point outside this row, or the bytes are not valid UTF-8. public func string(at index: Int) throws -> String? { - guard let range = try valueRange(at: index) else { - return nil - } - let decoded = bytes.withUnsafeBufferPointer { decodeValidatedUTF8(UnsafeBufferPointer(rebasing: $0[range])) } - guard let decoded else { - throw rowInvalidUTF8() - } - return decoded + try valueRange(at: index).map { try rowString(bytes, in: $0) } } /// Throws if the stored offset and size point outside this row. @@ -127,6 +120,28 @@ public struct Row { return try Row(fields: childFields, bytes: bytes, base: range.lowerBound, count: range.count) } + /// The field must have a list type. + public func array(at index: Int) throws -> RowArray? { + guard case .list(let element) = fields[index].dataType else { + preconditionFailure("field \(index) has type \(fields[index].dataType), not a list") + } + guard let range = try valueRange(at: index) else { + return nil + } + return try RowArray(element: element, bytes: bytes, base: range.lowerBound, count: range.count) + } + + /// The field must have a map type. + public func map(at index: Int) throws -> RowMap? { + guard case .map(let key, let value) = fields[index].dataType else { + preconditionFailure("field \(index) has type \(fields[index].dataType), not a map") + } + guard let range = try valueRange(at: index) else { + return nil + } + return try RowMap(key: key, value: value, bytes: bytes, base: range.lowerBound, count: range.count) + } + /// A copy of only this row's bytes, even when it sits inside a larger buffer. public func toBytes() -> [UInt8] { Array(bytes[base..<(base + sizeInBytes)]) @@ -134,9 +149,7 @@ public struct Row { @inline(__always) private func loadSlot(_: T.Type, at index: Int) -> T { - bytes.withUnsafeBytes { - T(littleEndian: $0.loadUnaligned(fromByteOffset: base + bitmapWidth + index * 8, as: T.self)) - } + rowLoad(T.self, bytes, at: base + bitmapWidth + index * 8) } // The null bit is checked before the slot is read, because a null field's slot can hold anything. @@ -149,6 +162,199 @@ public struct Row { } } +/// A read-only view of one array's bytes, with the same getters as ``Row`` taking element indexes. +public struct RowArray { + public let element: RowField + public let elementCount: Int + public let sizeInBytes: Int + private let bytes: [UInt8] + private let base: Int + private let headerSize: Int + private let elementWidth: Int + + // An array is an 8-byte element count, a null bitmap, then one slot per element. A fixed-width + // element takes only its own width, so three int8 values take three bytes, not three slots of 8. + init(element: RowField, bytes: [UInt8], base: Int, count: Int) throws { + precondition(base >= 0 && count >= 0 && count <= bytes.count - base, "array range is outside its buffer") + guard count >= 8 else { + throw rowArrayTooShort(count: count) + } + // Every element takes at least one byte, so a count above the byte count is rejected before + // any arithmetic that a forged count could overflow. + let declaredCount = rowLoad(UInt64.self, bytes, at: base) + guard declaredCount <= UInt64(count) else { + throw rowArrayCountTooLarge(elementCount: declaredCount, count: count) + } + let elementCount = Int(declaredCount) + let headerSize = 8 + rowBitmapWidth(elementCount) + let elementWidth = element.dataType.byteWidth ?? 8 + let (elementsSize, multiplyOverflow) = elementCount.multipliedReportingOverflow(by: elementWidth) + let (fixedSize, addOverflow) = headerSize.addingReportingOverflow(elementsSize) + guard !multiplyOverflow, !addOverflow, fixedSize <= count else { + throw rowArrayCountTooLarge(elementCount: declaredCount, count: count) + } + self.element = element + self.elementCount = elementCount + self.sizeInBytes = count + self.bytes = bytes + self.base = base + self.headerSize = headerSize + self.elementWidth = elementWidth + } + + public func isNull(at index: Int) -> Bool { + precondition(index >= 0 && index < elementCount, "element index \(index) is outside 0..<\(elementCount)") + return bytes.withUnsafeBytes { rowIsBitSet($0, bitmapOffset: base + 8, index: index) } + } + + public func bool(at index: Int) -> Bool? { + isNull(at: index) ? nil : loadSlot(UInt8.self, at: index) != 0 + } + + public func int8(at index: Int) -> Int8? { + isNull(at: index) ? nil : Int8(bitPattern: loadSlot(UInt8.self, at: index)) + } + + public func int16(at index: Int) -> Int16? { + isNull(at: index) ? nil : Int16(bitPattern: loadSlot(UInt16.self, at: index)) + } + + public func int32(at index: Int) -> Int32? { + isNull(at: index) ? nil : Int32(bitPattern: loadSlot(UInt32.self, at: index)) + } + + public func int64(at index: Int) -> Int64? { + isNull(at: index) ? nil : Int64(bitPattern: loadSlot(UInt64.self, at: index)) + } + + public func float32(at index: Int) -> Float? { + isNull(at: index) ? nil : Float(bitPattern: loadSlot(UInt32.self, at: index)) + } + + public func float64(at index: Int) -> Double? { + isNull(at: index) ? nil : Double(bitPattern: loadSlot(UInt64.self, at: index)) + } + + public func localDate(at index: Int) -> LocalDate? { + int32(at: index).map { LocalDate(epochDay: Int64($0)) } + } + + public func timestamp(at index: Int) -> Date? { + int64(at: index).map(rowDate(micros:)) + } + + public func duration(at index: Int) -> Duration? { + int64(at: index).map { .microseconds($0) } + } + + public func string(at index: Int) throws -> String? { + try valueRange(at: index).map { try rowString(bytes, in: $0) } + } + + public func binary(at index: Int) throws -> [UInt8]? { + guard let range = try valueRange(at: index) else { + return nil + } + return Array(bytes[range]) + } + + public func row(at index: Int) throws -> Row? { + guard case .struct(let childFields) = element.dataType else { + preconditionFailure("array element type is \(element.dataType), not a struct") + } + guard let range = try valueRange(at: index) else { + return nil + } + return try Row(fields: childFields, bytes: bytes, base: range.lowerBound, count: range.count) + } + + public func array(at index: Int) throws -> RowArray? { + guard case .list(let childElement) = element.dataType else { + preconditionFailure("array element type is \(element.dataType), not a list") + } + guard let range = try valueRange(at: index) else { + return nil + } + return try RowArray(element: childElement, bytes: bytes, base: range.lowerBound, count: range.count) + } + + public func map(at index: Int) throws -> RowMap? { + guard case .map(let key, let value) = element.dataType else { + preconditionFailure("array element type is \(element.dataType), not a map") + } + guard let range = try valueRange(at: index) else { + return nil + } + return try RowMap(key: key, value: value, bytes: bytes, base: range.lowerBound, count: range.count) + } + + public func toBytes() -> [UInt8] { + Array(bytes[base..<(base + sizeInBytes)]) + } + + // Slots here can be narrower than 8 bytes, so a getter that does not match the element type + // could read past the last slot and out of the buffer. That is a bug in the caller, so it stops. + @inline(__always) + private func loadSlot(_: T.Type, at index: Int) -> T { + precondition(MemoryLayout.size <= elementWidth, "array element type is \(element.dataType)") + return rowLoad(T.self, bytes, at: base + headerSize + index * elementWidth) + } + + private func valueRange(at index: Int) throws -> Range? { + if isNull(at: index) { + return nil + } + let ticket = rowDecodeOffsetAndSize(loadSlot(UInt64.self, at: index)) + return try rowCheckedRange(offset: ticket.offset, size: ticket.size, regionBase: base, regionCount: sizeInBytes) + } +} + +/// A read-only view of one map: an array of keys and an array of values, paired by index. +public struct RowMap { + public let keys: RowArray + public let values: RowArray + + // A map is the byte size of its keys array, then the keys array, then the values array. Each + // array counts its offsets from its own start, not from the map's. + init(key: RowField, value: RowField, bytes: [UInt8], base: Int, count: Int) throws { + precondition(base >= 0 && count >= 0 && count <= bytes.count - base, "map range is outside its buffer") + guard count >= 8 else { + throw rowMapTooShort(count: count) + } + let declaredKeysSize = rowLoad(UInt64.self, bytes, at: base) + guard declaredKeysSize <= UInt64(count - 8) else { + throw rowMapKeysTooLarge(keysSize: declaredKeysSize, count: count) + } + let keysSize = Int(declaredKeysSize) + let keys = try RowArray(element: key, bytes: bytes, base: base + 8, count: keysSize) + let values = try RowArray(element: value, bytes: bytes, base: base + 8 + keysSize, count: count - 8 - keysSize) + // Readers walk both arrays with one index, so unequal counts would run past the shorter one. + guard keys.elementCount == values.elementCount else { + throw rowMapCountMismatch(keyCount: keys.elementCount, valueCount: values.elementCount) + } + self.keys = keys + self.values = values + } + + public var count: Int { + keys.elementCount + } +} + +// Reads the little-endian integer at an absolute position that the caller has already checked. +@inline(__always) +private func rowLoad(_: T.Type, _ bytes: [UInt8], at offset: Int) -> T { + bytes.withUnsafeBytes { T(littleEndian: $0.loadUnaligned(fromByteOffset: offset, as: T.self)) } +} + +private func rowString(_ bytes: [UInt8], in range: Range) throws -> String { + let decoded = bytes.withUnsafeBufferPointer { decodeValidatedUTF8(UnsafeBufferPointer(rebasing: $0[range])) } + guard let decoded else { + throw rowInvalidUTF8() + } + return decoded +} + @inline(never) func rowTooShort(count: Int, fixedSize: Int) -> ForyError { .invalidData("row of \(count) bytes is shorter than its \(fixedSize)-byte fixed region") @@ -158,3 +364,28 @@ func rowTooShort(count: Int, fixedSize: Int) -> ForyError { func rowInvalidUTF8() -> ForyError { .invalidData("invalid UTF-8 sequence") } + +@inline(never) +func rowArrayTooShort(count: Int) -> ForyError { + .invalidData("array of \(count) bytes is shorter than its 8-byte element count") +} + +@inline(never) +func rowArrayCountTooLarge(elementCount: UInt64, count: Int) -> ForyError { + .invalidData("array declares \(elementCount) elements but holds only \(count) bytes") +} + +@inline(never) +func rowMapTooShort(count: Int) -> ForyError { + .invalidData("map of \(count) bytes is shorter than its 8-byte keys size") +} + +@inline(never) +func rowMapKeysTooLarge(keysSize: UInt64, count: Int) -> ForyError { + .invalidData("map keys array of \(keysSize) bytes does not fit in a \(count)-byte map") +} + +@inline(never) +func rowMapCountMismatch(keyCount: Int, valueCount: Int) -> ForyError { + .invalidData("map has \(keyCount) keys but \(valueCount) values") +} diff --git a/swift/Tests/ForyTests/RowArrayReaderTests.swift b/swift/Tests/ForyTests/RowArrayReaderTests.swift new file mode 100644 index 0000000000..8c5174ecac --- /dev/null +++ b/swift/Tests/ForyTests/RowArrayReaderTests.swift @@ -0,0 +1,159 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation +import Testing +@testable import Fory + +// A one-field row whose only slot points at `value`, so the value starts at byte 16 rather than 0. +private func rowHolding(_ value: [UInt8], as dataType: RowDataType) throws -> Row { + let slot = withUnsafeBytes(of: (UInt64(16) << 32 | UInt64(value.count)).littleEndian, Array.init) + let schema = RowSchema([RowField(name: "v", dataType: dataType, nullable: true)]) + return try Row(schema: schema, bytes: [UInt8](repeating: 0, count: 8) + slot + value) +} + +private func rowArray(_ bytes: [UInt8], of dataType: RowDataType) throws -> RowArray { + try RowArray(element: RowField(name: "item", dataType: dataType, nullable: true), bytes: bytes, base: 0, count: bytes.count) +} + +@Test +func rowArrayStrings() throws { + let bytes: [UInt8] = [ + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 3 elements + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // bitmap: element 1 is null + 0x02, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, // element 0: size 2, offset 0x28 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // element 1: null + 0x03, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, // element 2: size 3, offset 0x30 + 0x61, 0x62, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // "ab" + 0x78, 0x79, 0x7A, 0x00, 0x00, 0x00, 0x00, 0x00 // "xyz" + ] + let array = try #require(try rowHolding(bytes, as: .listOf(.string)).array(at: 0)) + #expect(array.elementCount == 3) + #expect(array.sizeInBytes == 56) + #expect(try array.string(at: 0) == "ab") + #expect(array.isNull(at: 1)) + #expect(try array.string(at: 1) == nil) + #expect(try array.string(at: 2) == "xyz") + #expect(array.toBytes() == bytes) +} + +@Test +func rowArrayEmpty() throws { + let array = try rowArray([UInt8](repeating: 0, count: 8), of: .int32) + #expect(array.elementCount == 0) + #expect(array.sizeInBytes == 8) +} + +@Test +func rowArrayCounts() throws { + for (count, headerSize, size) in [(64, 16, 80), (65, 24, 96)] { + var bytes = [UInt8](repeating: 0, count: size) + bytes[0] = UInt8(count) + bytes[8 + (count - 1) / 8] |= 1 << ((count - 1) % 8) + bytes[headerSize] = 7 + let array = try rowArray(bytes, of: .int8) + #expect(array.elementCount == count) + #expect(array.isNull(at: count - 1)) + #expect(!array.isNull(at: count - 2)) + #expect(array.int8(at: 0) == 7) + } +} + +@Test +func rowArrayWidths() throws { + let header: [UInt8] = [0x03, 0, 0, 0, 0, 0, 0, 0] + [UInt8](repeating: 0, count: 8) + + let int8s = try rowArray(header + [0x01, 0xFE, 0x7F, 0, 0, 0, 0, 0], of: .int8) + #expect(int8s.sizeInBytes == 24) + #expect((0..<3).map { int8s.int8(at: $0) } == [1, -2, 127]) + + let int16s = try rowArray(header + [0x02, 0x01, 0xFE, 0xFF, 0x03, 0x00, 0, 0], of: .int16) + #expect(int16s.sizeInBytes == 24) + #expect((0..<3).map { int16s.int16(at: $0) } == [0x0102, -2, 3]) + + let int32s = try rowArray( + header + [0x04, 0x03, 0x02, 0x01, 0xFE, 0xFF, 0xFF, 0xFF, 0x03, 0x00, 0x00, 0x00, 0, 0, 0, 0], + of: .int32 + ) + #expect(int32s.sizeInBytes == 32) + #expect((0..<3).map { int32s.int32(at: $0) } == [0x0102_0304, -2, 3]) + + let int64s = try rowArray( + header + [0x08, 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01] + + [0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF] + + [0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00], + of: .int64 + ) + #expect(int64s.sizeInBytes == 40) + #expect((0..<3).map { int64s.int64(at: $0) } == [0x0102_0304_0506_0708, -2, 3]) +} + +@Test +func rowMapLayout() throws { + let bytes: [UInt8] = [ + 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // keys array is 0x30 bytes + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // keys: 2 elements + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // keys bitmap + 0x02, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, // key 0: size 2, offset 0x20 from the keys array + 0x02, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, // key 1: size 2, offset 0x28 from the keys array + 0x6B, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // "k1" + 0x6B, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // "k2" + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // values: 2 elements + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // values bitmap + 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00 // int32 1 and 2 + ] + let map = try #require(try rowHolding(bytes, as: .mapOf(.string, .int32)).map(at: 0)) + #expect(map.count == 2) + #expect(try map.keys.string(at: 0) == "k1") + #expect(try map.keys.string(at: 1) == "k2") + #expect(map.values.int32(at: 0) == 1) + #expect(map.values.int32(at: 1) == 2) +} + +@Test +func rowArrayCorrupt() throws { + #expect(throws: ForyError.invalidData("array of 4 bytes is shorter than its 8-byte element count")) { + try rowArray([0, 0, 0, 0], of: .int8) + } + #expect(throws: ForyError.invalidData("array declares \(UInt64.max) elements but holds only 16 bytes")) { + try rowArray([UInt8](repeating: 0xFF, count: 8) + [UInt8](repeating: 0, count: 8), of: .int8) + } + // Four int64 elements need 16 + 32 bytes, but only 32 are there. + #expect(throws: ForyError.invalidData("array declares 4 elements but holds only 32 bytes")) { + try rowArray([0x04, 0, 0, 0, 0, 0, 0, 0] + [UInt8](repeating: 0, count: 24), of: .int64) + } + let pastEnd = try rowArray( + [0x01, 0, 0, 0, 0, 0, 0, 0] + [UInt8](repeating: 0, count: 8) + [0x01, 0, 0, 0, 0x18, 0, 0, 0], + of: .string + ) + #expect(throws: ForyError.outOfBounds(cursor: 0x18, need: 1, length: 24)) { + try pastEnd.string(at: 0) + } +} + +@Test +func rowMapCorrupt() throws { + let mapType = RowDataType.mapOf(.int32, .int32) + #expect(throws: ForyError.invalidData("map keys array of 16 bytes does not fit in a 16-byte map")) { + try rowHolding([0x10, 0, 0, 0, 0, 0, 0, 0] + [UInt8](repeating: 0, count: 8), as: mapType).map(at: 0) + } + let oneKey: [UInt8] = [0x01, 0, 0, 0, 0, 0, 0, 0] + [UInt8](repeating: 0, count: 16) + let noValues = [UInt8](repeating: 0, count: 8) + #expect(throws: ForyError.invalidData("map has 1 keys but 0 values")) { + try rowHolding([0x18, 0, 0, 0, 0, 0, 0, 0] + oneKey + noValues, as: mapType).map(at: 0) + } +} From c5eb1ad8b004683d464461fe88bf68f971d97afe Mon Sep 17 00:00:00 2001 From: yash Date: Thu, 29 Oct 2026 16:35:21 +0530 Subject: [PATCH 07/19] swift: Add RowWriter for fixed and variable-width fields. RowWriter reserves a zeroed bitmap and slot region, then fills each slot in place. Strings and bytes go after the slots, padded to 8 bytes, and their slot records the offset from the row's own start, so the same code can later write a nested row inside its parent. A Date is rounded to the nearest microsecond, so a timestamp read from a row and written back gives the same bytes. --- swift/Sources/Fory/Row/RowSlot.swift | 42 ++++ swift/Sources/Fory/Row/RowWriter.swift | 149 ++++++++++++ swift/Tests/ForyTests/RowWriterTests.swift | 254 +++++++++++++++++++++ 3 files changed, 445 insertions(+) create mode 100644 swift/Sources/Fory/Row/RowWriter.swift create mode 100644 swift/Tests/ForyTests/RowWriterTests.swift diff --git a/swift/Sources/Fory/Row/RowSlot.swift b/swift/Sources/Fory/Row/RowSlot.swift index d82570d8af..8a186a42d0 100644 --- a/swift/Sources/Fory/Row/RowSlot.swift +++ b/swift/Sources/Fory/Row/RowSlot.swift @@ -59,6 +59,48 @@ func rowDate(micros: Int64) -> Date { return Date(timeIntervalSinceReferenceDate: Double(seconds - 978_307_200) + Double(remainder) / 1e6) } +// The reverse of rowDate. The fraction is rounded to the nearest microsecond rather than floored, +// because the Double inside a Date made from `m` microseconds can land just below `m`. A fraction +// that rounds up to a whole second needs no carry: adding 1_000_000 to the scaled seconds is the same. +func rowTimestampMicros(_ date: Date) throws -> Int64 { + let interval = date.timeIntervalSinceReferenceDate + let wholeSeconds = interval.rounded(.down) + // Int64(exactly:) is nil for infinity and NaN as well as for values too large. + guard let seconds = Int64(exactly: wholeSeconds) else { + throw rowTimestampOutOfRange(interval) + } + let fraction = Int64(((interval - wholeSeconds) * 1_000_000).rounded(.toNearestOrAwayFromZero)) + let (unixSeconds, shiftOverflow) = seconds.addingReportingOverflow(978_307_200) + let (scaled, scaleOverflow) = unixSeconds.multipliedReportingOverflow(by: 1_000_000) + let (micros, addOverflow) = scaled.addingReportingOverflow(fraction) + guard !shiftOverflow, !scaleOverflow, !addOverflow else { + throw rowTimestampOutOfRange(interval) + } + return micros +} + +// Both parts of `components` carry the same sign, so dividing each toward zero truncates the +// whole duration toward zero. +func rowDurationMicros(_ duration: Duration) throws -> Int64 { + let (seconds, attoseconds) = duration.components + let (scaled, scaleOverflow) = seconds.multipliedReportingOverflow(by: 1_000_000) + let (micros, addOverflow) = scaled.addingReportingOverflow(attoseconds / 1_000_000_000_000) + guard !scaleOverflow, !addOverflow else { + throw rowDurationOutOfRange(duration) + } + return micros +} + +@inline(never) +func rowTimestampOutOfRange(_ interval: Double) -> ForyError { + .encodingError("timestamp \(interval) seconds from 2001 does not fit in 64-bit microseconds") +} + +@inline(never) +func rowDurationOutOfRange(_ duration: Duration) -> ForyError { + .encodingError("duration \(duration) does not fit in 64-bit microseconds") +} + @inline(never) func rowValueTooLarge(offset: Int, size: Int) -> ForyError { .encodingError( diff --git a/swift/Sources/Fory/Row/RowWriter.swift b/swift/Sources/Fory/Row/RowWriter.swift new file mode 100644 index 0000000000..a90e171ecf --- /dev/null +++ b/swift/Sources/Fory/Row/RowWriter.swift @@ -0,0 +1,149 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +/// Writes one row into a buffer: the null bitmap, one slot per field, then strings, bytes, and +/// nested values after the slots. Call `reset()` before each row. An index outside +/// `0.. [UInt8] { + buffer.copyBytes(start: base, end: buffer.count) + } + + private func slotOffset(_ index: Int) -> Int { + precondition(index >= 0 && index < fields.count, "field index \(index) is outside 0..<\(fields.count)") + return base + bitmapWidth + index * 8 + } + + // Pads the value that was just appended from `start` to a whole word and points the slot at it. + // An empty value adds no padding, so its slot points at the current end with size 0. + private func finishValue(at index: Int, start: Int) throws { + let size = buffer.count - start + _ = buffer.appendZeros(rowRoundToWord(size) - size) + try setOffsetAndSize(at: index, absoluteStart: start, size: size) + } +} + +@inline(never) +func rowDateOutOfRange(_ epochDay: Int64) -> ForyError { + .encodingError("date \(epochDay) days from 1970 does not fit in 32 bits") +} diff --git a/swift/Tests/ForyTests/RowWriterTests.swift b/swift/Tests/ForyTests/RowWriterTests.swift new file mode 100644 index 0000000000..38c184c17d --- /dev/null +++ b/swift/Tests/ForyTests/RowWriterTests.swift @@ -0,0 +1,254 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation +import Testing +@testable import Fory + +private let rowSection31Schema = RowSchema([ + RowField(name: "a", dataType: .int32, nullable: true), + RowField(name: "b", dataType: .int32, nullable: false), + RowField(name: "c", dataType: .string, nullable: true) +]) + +private let rowSection31Bytes: [UInt8] = [ + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // bitmap: a is null + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // slot a + 0x2C, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // slot b = 300 + 0x02, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, // slot c: size 2, offset 0x20 + 0x68, 0x69, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // "hi" +] + +private func writeSection31Row(into buffer: ByteBuffer) throws -> RowWriter { + var writer = RowWriter(schema: rowSection31Schema, buffer: buffer) + writer.reset() + writer.setNullAt(0) + writer.writeInt32(300, at: 1) + try writer.writeString("hi", at: 2) + return writer +} + +@Test +func rowWriterLayout() throws { + #expect(try writeSection31Row(into: ByteBuffer()).toBytes() == rowSection31Bytes) + + // A row that starts after other bytes still counts its offsets from its own start. + let buffer = ByteBuffer() + buffer.writeBytes([UInt8](repeating: 0xAA, count: 8)) + let writer = try writeSection31Row(into: buffer) + #expect(writer.base == 8) + #expect(writer.size == 40) + #expect(writer.toBytes() == rowSection31Bytes) +} + +@Test +func rowWriterGoLayout() throws { + let schema = RowSchema([ + RowField(name: "f1", dataType: .int64, nullable: false), + RowField(name: "f2", dataType: .string, nullable: true) + ]) + var writer = RowWriter(schema: schema) + writer.reset() + writer.writeInt64(0x0102_0304_0506_0708, at: 0) + try writer.writeString("ab", at: 1) + #expect(writer.toBytes() == [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x08, 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, + 0x02, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, + 0x61, 0x62, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + ]) +} + +@Test +func rowWriterPrimitives() throws { + let schema = RowSchema([ + RowField(name: "b", dataType: .bool, nullable: false), + RowField(name: "i8", dataType: .int8, nullable: false), + RowField(name: "i16", dataType: .int16, nullable: false), + RowField(name: "i32", dataType: .int32, nullable: false), + RowField(name: "i64", dataType: .int64, nullable: false) + ]) + for useMin in [true, false] { + var writer = RowWriter(schema: schema) + writer.reset() + writer.writeBool(useMin, at: 0) + writer.writeInt8(useMin ? .min : .max, at: 1) + writer.writeInt16(useMin ? .min : .max, at: 2) + writer.writeInt32(useMin ? .min : .max, at: 3) + writer.writeInt64(useMin ? .min : .max, at: 4) + let row = try Row(schema: schema, bytes: writer.toBytes()) + #expect(row.bool(at: 0) == useMin) + #expect(row.int8(at: 1) == (useMin ? .min : .max)) + #expect(row.int16(at: 2) == (useMin ? .min : .max)) + #expect(row.int32(at: 3) == (useMin ? .min : .max)) + #expect(row.int64(at: 4) == (useMin ? .min : .max)) + } +} + +@Test +func rowWriterFloatBits() throws { + let floats: [Float] = [Float(bitPattern: 0x7FC0_0001), -0.0, .infinity, -.infinity] + let doubles: [Double] = [Double(bitPattern: 0x7FF8_0000_0000_0001), -0.0, .infinity, -.infinity] + let schema = RowSchema( + floats.indices.map { RowField(name: "f\($0)", dataType: .float32, nullable: false) } + + doubles.indices.map { RowField(name: "d\($0)", dataType: .float64, nullable: false) } + ) + var writer = RowWriter(schema: schema) + writer.reset() + for (index, value) in floats.enumerated() { + writer.writeFloat32(value, at: index) + } + for (index, value) in doubles.enumerated() { + writer.writeFloat64(value, at: floats.count + index) + } + let row = try Row(schema: schema, bytes: writer.toBytes()) + // NaN never equals itself, so the bits are compared instead of the values. + for (index, value) in floats.enumerated() { + #expect(row.float32(at: index)?.bitPattern == value.bitPattern) + } + for (index, value) in doubles.enumerated() { + #expect(row.float64(at: floats.count + index)?.bitPattern == value.bitPattern) + } +} + +@Test +func rowWriterNullEveryType() throws { + let types: [RowDataType] = [ + .bool, .int8, .int16, .int32, .int64, .float32, .float64, .date32, .timestamp, .duration, .string, .binary + ] + let schema = RowSchema(types.enumerated().map { RowField(name: "f\($0.offset)", dataType: $0.element, nullable: true) }) + var writer = RowWriter(schema: schema) + writer.reset() + // Fill the fixed-width slots first, so setNullAt has something to zero. + for index in 0..<10 { + writer.writeInt8(-1, at: index) + } + for index in types.indices { + writer.setNullAt(index) + } + let bytes = writer.toBytes() + #expect(writer.size == 8 + types.count * 8) + #expect(Array(bytes[0..<8]) == [0xFF, 0x0F, 0, 0, 0, 0, 0, 0]) + #expect(bytes[8...].allSatisfy { $0 == 0 }) +} + +@Test +func rowWriterEmptyString() throws { + var writer = RowWriter(schema: RowSchema([RowField(name: "s", dataType: .string, nullable: true)])) + writer.reset() + try writer.writeString("", at: 0) + #expect(writer.toBytes() == [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00 // size 0, offset 0x10 (the end of the slots) + ]) +} + +@Test +func rowWriterMultiByteUTF8() throws { + var writer = RowWriter(schema: RowSchema([RowField(name: "s", dataType: .string, nullable: true)])) + writer.reset() + try writer.writeString("héllo", at: 0) + #expect(Array(writer.toBytes()[8...]) == [ + 0x06, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x68, 0xC3, 0xA9, 0x6C, 0x6C, 0x6F, 0x00, 0x00 + ]) +} + +@Test +func rowWriterFieldCounts() throws { + for (count, size) in [(0, 0), (63, 512), (64, 520), (65, 536)] { + let schema = RowSchema((0.. 0 { + writer.setNullAt(count - 1) + let bytes = writer.toBytes() + #expect(bytes[(count - 1) / 8] == 1 << ((count - 1) % 8)) + let row = try Row(schema: schema, bytes: bytes) + #expect(row.isNull(at: count - 1)) + #expect(!row.isNull(at: count - 2)) + } + } +} + +@Test +func rowWriterDeterministic() throws { + let first = try writeSection31Row(into: ByteBuffer()).toBytes() + let second = try writeSection31Row(into: ByteBuffer()).toBytes() + let reused = ByteBuffer() + reused.writeBytes([UInt8](repeating: 0xFF, count: 128)) + reused.clear() + let third = try writeSection31Row(into: reused).toBytes() + #expect(first == second) + #expect(first == third) +} + +@Test +func rowWriterTimestamp() throws { + let window = Int64(8_589_934_591) + let samples: [Int64] = [ + 0, -1, 1, 1_700_000_000_123_457, -1_000_000_000_000_000, + (978_307_200 + window) * 1_000_000 + 999_999, + (978_307_200 - window) * 1_000_000 + 999_999 + ] + for micros in samples { + #expect(try rowTimestampMicros(rowDate(micros: micros)) == micros) + } + + let schema = RowSchema([RowField(name: "t", dataType: .timestamp, nullable: false)]) + var writer = RowWriter(schema: schema) + writer.reset() + try writer.writeTimestamp(rowDate(micros: -1), at: 0) + #expect(try Row(schema: schema, bytes: writer.toBytes()).int64(at: 0) == -1) + + #expect(throws: ForyError.encodingError("timestamp inf seconds from 2001 does not fit in 64-bit microseconds")) { + try writer.writeTimestamp(Date(timeIntervalSinceReferenceDate: .infinity), at: 0) + } +} + +@Test +func rowWriterDuration() throws { + #expect(try rowDurationMicros(.nanoseconds(-1)) == 0) + #expect(try rowDurationMicros(.nanoseconds(1_999)) == 1) + #expect(try rowDurationMicros(.nanoseconds(-1_999)) == -1) + #expect(try rowDurationMicros(.microseconds(Int64.min)) == Int64.min) + + let schema = RowSchema([RowField(name: "d", dataType: .duration, nullable: false)]) + var writer = RowWriter(schema: schema) + writer.reset() + try writer.writeDuration(.microseconds(-1_500), at: 0) + #expect(try Row(schema: schema, bytes: writer.toBytes()).duration(at: 0) == .microseconds(-1_500)) + + #expect(throws: ForyError.self) { + try writer.writeDuration(.seconds(Int64.max), at: 0) + } +} + +@Test +func rowWriterLocalDate() throws { + let schema = RowSchema([RowField(name: "d", dataType: .date32, nullable: false)]) + var writer = RowWriter(schema: schema) + writer.reset() + for epochDay in [-1, Int64(Int32.max)] { + try writer.writeLocalDate(LocalDate(epochDay: epochDay), at: 0) + #expect(try Row(schema: schema, bytes: writer.toBytes()).localDate(at: 0) == LocalDate(epochDay: epochDay)) + } + #expect(throws: ForyError.encodingError("date 2147483648 days from 1970 does not fit in 32 bits")) { + try writer.writeLocalDate(LocalDate(epochDay: Int64(Int32.max) + 1), at: 0) + } +} From 774e48f71cc77efd304cbfb562361467cd02fa5d Mon Sep 17 00:00:00 2001 From: yash Date: Fri, 30 Oct 2026 16:35:21 +0530 Subject: [PATCH 08/19] swift: Add array, map, and nested struct writers. RowArrayWriter writes the element count and bitmap, then packs each element at its natural width, with strings and nested values after the elements. RowMapWriter only reserves the keys-size word: the keys and values are two arrays written one after the other, and finishKeys fills in the size between them. A nested struct, list, or map is a child writer on the parent's buffer whose start and size go in the parent's slot, so no extra API is needed for nesting. --- swift/Sources/Fory/Row/RowSlot.swift | 12 + swift/Sources/Fory/Row/RowWriter.swift | 172 +++++++++++-- .../Tests/ForyTests/RowArrayWriterTests.swift | 232 ++++++++++++++++++ 3 files changed, 402 insertions(+), 14 deletions(-) create mode 100644 swift/Tests/ForyTests/RowArrayWriterTests.swift diff --git a/swift/Sources/Fory/Row/RowSlot.swift b/swift/Sources/Fory/Row/RowSlot.swift index 8a186a42d0..d6f8c746e6 100644 --- a/swift/Sources/Fory/Row/RowSlot.swift +++ b/swift/Sources/Fory/Row/RowSlot.swift @@ -79,6 +79,13 @@ func rowTimestampMicros(_ date: Date) throws -> Int64 { return micros } +func rowEpochDay(_ date: LocalDate) throws -> Int32 { + guard let days = Int32(exactly: date.epochDay) else { + throw rowDateOutOfRange(date.epochDay) + } + return days +} + // Both parts of `components` carry the same sign, so dividing each toward zero truncates the // whole duration toward zero. func rowDurationMicros(_ duration: Duration) throws -> Int64 { @@ -91,6 +98,11 @@ func rowDurationMicros(_ duration: Duration) throws -> Int64 { return micros } +@inline(never) +func rowDateOutOfRange(_ epochDay: Int64) -> ForyError { + .encodingError("date \(epochDay) days from 1970 does not fit in 32 bits") +} + @inline(never) func rowTimestampOutOfRange(_ interval: Double) -> ForyError { .encodingError("timestamp \(interval) seconds from 2001 does not fit in 64-bit microseconds") diff --git a/swift/Sources/Fory/Row/RowWriter.swift b/swift/Sources/Fory/Row/RowWriter.swift index a90e171ecf..5bf36d1887 100644 --- a/swift/Sources/Fory/Row/RowWriter.swift +++ b/swift/Sources/Fory/Row/RowWriter.swift @@ -86,10 +86,7 @@ public struct RowWriter { /// Throws if the date is too far from 1970 to fit in 32-bit days. public func writeLocalDate(_ value: LocalDate, at index: Int) throws { - guard let days = Int32(exactly: value.epochDay) else { - throw rowDateOutOfRange(value.epochDay) - } - writeInt32(days, at: index) + writeInt32(try rowEpochDay(value), at: index) } /// Stores the date rounded to the nearest microsecond. Throws if it does not fit in 64-bit @@ -108,14 +105,14 @@ public struct RowWriter { public func writeString(_ value: String, at index: Int) throws { let start = buffer.count buffer.writeBytes(value.utf8) - try finishValue(at: index, start: start) + try setOffsetAndSize(at: index, absoluteStart: start, size: rowPadValue(buffer, from: start)) } /// Throws if the row grows past the 32-bit offsets and sizes a slot can hold. public func writeBytes(_ value: some ContiguousBytes, at index: Int) throws { let start = buffer.count value.withUnsafeBytes { buffer.writeBytes($0) } - try finishValue(at: index, start: start) + try setOffsetAndSize(at: index, absoluteStart: start, size: rowPadValue(buffer, from: start)) } /// Points slot `index` at `size` bytes that already sit in the buffer at `absoluteStart`. @@ -133,17 +130,164 @@ public struct RowWriter { precondition(index >= 0 && index < fields.count, "field index \(index) is outside 0..<\(fields.count)") return base + bitmapWidth + index * 8 } +} + +/// Writes one array into a buffer: the element count, the null bitmap, each element at its natural +/// width, then strings, bytes, and nested values. It has the same write methods as ``RowWriter``, +/// taking element indexes. +public struct RowArrayWriter { + public let elementWidth: Int + public let buffer: ByteBuffer + public private(set) var base: Int + public private(set) var elementCount: Int + private var headerSize: Int + + public init(element: RowField, buffer: ByteBuffer) { + self.init(elementWidth: element.dataType.byteWidth ?? 8, buffer: buffer) + } + + init(elementWidth: Int, buffer: ByteBuffer) { + self.elementWidth = elementWidth + self.buffer = buffer + self.base = buffer.count + self.elementCount = 0 + self.headerSize = 8 + } + + /// Throws if `count` is negative or the elements would not fit in 2 GiB. + public mutating func reset(count: Int) throws { + // Dividing instead of multiplying keeps a huge count from overflowing before it is rejected. + guard count >= 0, count <= (Int(Int32.max) - 15) / elementWidth else { + throw rowArrayCountOutOfRange(count, elementWidth: elementWidth) + } + headerSize = 8 + rowBitmapWidth(count) + base = buffer.appendZeros(headerSize + rowRoundToWord(count * elementWidth)) + elementCount = count + buffer.storeLittleEndian(UInt64(count), at: base) + } + + public var size: Int { + buffer.count - base + } + + public func setNullAt(_ index: Int) { + let offset = elementOffset(index) + buffer.storage.withUnsafeMutableBytes { raw in + rowSetBit(raw, bitmapOffset: base + 8, index: index) + for byte in offset.. [UInt8] { + buffer.copyBytes(start: base, end: buffer.count) } + + private func elementOffset(_ index: Int) -> Int { + precondition(index >= 0 && index < elementCount, "element index \(index) is outside 0..<\(elementCount)") + return base + headerSize + index * elementWidth + } + + // A value wider than the element would overwrite its neighbors or run past the buffer. + private func store(_ value: T, at index: Int) { + precondition( + MemoryLayout.size <= elementWidth, + "\(MemoryLayout.size)-byte value written to a \(elementWidth)-byte element" + ) + buffer.storeLittleEndian(value, at: elementOffset(index)) + } +} + +/// Writes one map into a buffer: the size of the keys array, the keys array, then the values array. +/// Call `reset()`, write the keys with a ``RowArrayWriter``, call `finishKeys()`, then write the +/// values with another ``RowArrayWriter`` on the same buffer. +public struct RowMapWriter { + public let buffer: ByteBuffer + public private(set) var base: Int + + public init(buffer: ByteBuffer) { + self.buffer = buffer + self.base = buffer.count + } + + public mutating func reset() { + base = buffer.appendZeros(8) + } + + public func finishKeys() { + buffer.storeLittleEndian(UInt64(buffer.count - base - 8), at: base) + } + + public var size: Int { + buffer.count - base + } +} + +// Pads the value appended since `start` to a whole word and returns its unpadded size. +private func rowPadValue(_ buffer: ByteBuffer, from start: Int) -> Int { + let size = buffer.count - start + _ = buffer.appendZeros(rowRoundToWord(size) - size) + return size } @inline(never) -func rowDateOutOfRange(_ epochDay: Int64) -> ForyError { - .encodingError("date \(epochDay) days from 1970 does not fit in 32 bits") +func rowArrayCountOutOfRange(_ count: Int, elementWidth: Int) -> ForyError { + .encodingError("array of \(count) elements of \(elementWidth) bytes each cannot be written") } diff --git a/swift/Tests/ForyTests/RowArrayWriterTests.swift b/swift/Tests/ForyTests/RowArrayWriterTests.swift new file mode 100644 index 0000000000..60e1467773 --- /dev/null +++ b/swift/Tests/ForyTests/RowArrayWriterTests.swift @@ -0,0 +1,232 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation +import Testing +@testable import Fory + +private let pointFields = [RowField(name: "x", dataType: .int32, nullable: false)] + +private func field(_ dataType: RowDataType) -> RowField { + RowField(name: "item", dataType: dataType, nullable: true) +} + +private func writePoint(_ x: Int32, into buffer: ByteBuffer) -> (start: Int, size: Int) { + var point = RowWriter(fields: pointFields, buffer: buffer) + point.reset() + point.writeInt32(x, at: 0) + return (point.base, point.size) +} + +@Test +func rowArrayWriterStrings() throws { + var writer = RowArrayWriter(element: field(.string), buffer: ByteBuffer()) + try writer.reset(count: 3) + try writer.writeString("ab", at: 0) + writer.setNullAt(1) + try writer.writeString("xyz", at: 2) + #expect(writer.toBytes() == [ + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 3 elements + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // bitmap: element 1 is null + 0x02, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, // element 0: size 2, offset 0x28 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // element 1: null + 0x03, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, // element 2: size 3, offset 0x30 + 0x61, 0x62, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // "ab" + 0x78, 0x79, 0x7A, 0x00, 0x00, 0x00, 0x00, 0x00 // "xyz" + ]) +} + +@Test +func rowMapWriterLayout() throws { + let buffer = ByteBuffer() + var map = RowMapWriter(buffer: buffer) + map.reset() + var keys = RowArrayWriter(element: field(.string), buffer: buffer) + try keys.reset(count: 2) + try keys.writeString("k1", at: 0) + try keys.writeString("k2", at: 1) + map.finishKeys() + var values = RowArrayWriter(element: field(.int32), buffer: buffer) + try values.reset(count: 2) + values.writeInt32(1, at: 0) + values.writeInt32(2, at: 1) + #expect(map.size == 80) + #expect(buffer.copyBytes(start: 0, end: buffer.count) == [ + 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // keys array is 48 bytes + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 2 keys + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // key bitmap + 0x02, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, // key 0: offset counted from the keys array + 0x02, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, // key 1 + 0x6B, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // "k1" + 0x6B, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // "k2" + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 2 values + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // value bitmap + 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00 // 1 and 2 + ]) +} + +@Test +func rowWriterNested() throws { + let schema = RowSchema([ + RowField(name: "id", dataType: .int32, nullable: false), + RowField(name: "child", dataType: .struct(pointFields), nullable: true) + ]) + var parent = RowWriter(schema: schema) + parent.reset() + parent.writeInt32(1, at: 0) + let child = writePoint(5, into: parent.buffer) + try parent.setOffsetAndSize(at: 1, absoluteStart: child.start, size: child.size) + #expect(parent.toBytes() == [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // parent bitmap + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // id = 1 + 0x10, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, // child: size 16, offset 0x18 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // child bitmap + 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // x = 5 + ]) +} + +@Test +func rowArrayWriterWidths() throws { + for (dataType, size) in [(RowDataType.int8, 24), (.int16, 24), (.int32, 32), (.int64, 40)] { + var writer = RowArrayWriter(element: field(dataType), buffer: ByteBuffer()) + try writer.reset(count: 3) + for index in 0..<3 { + switch dataType { + case .int8: writer.writeInt8(-1, at: index) + case .int16: writer.writeInt16(-1, at: index) + case .int32: writer.writeInt32(-1, at: index) + default: writer.writeInt64(-1, at: index) + } + } + let filled = 3 * writer.elementWidth + var expected: [UInt8] = [0x03, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + expected += [UInt8](repeating: 0xFF, count: filled) + expected += [UInt8](repeating: 0, count: size - 16 - filled) + #expect(writer.toBytes() == expected) + } +} + +@Test +func rowArrayWriterCounts() throws { + for (count, headerSize) in [(0, 8), (64, 16), (65, 24)] { + var writer = RowArrayWriter(element: field(.int64), buffer: ByteBuffer()) + try writer.reset(count: count) + #expect(writer.size == headerSize + count * 8) + if count == 0 { + #expect(writer.toBytes() == [UInt8](repeating: 0, count: 8)) + continue + } + writer.writeInt64(7, at: 0) + writer.setNullAt(count - 1) + let array = try RowArray(element: field(.int64), bytes: writer.toBytes(), base: 0, count: writer.size) + #expect(array.elementCount == count) + #expect(array.int64(at: 0) == 7) + #expect(array.isNull(at: count - 1)) + #expect(!array.isNull(at: count - 2)) + } +} + +@Test +func rowMapWriterEmpty() throws { + let buffer = ByteBuffer() + var map = RowMapWriter(buffer: buffer) + map.reset() + var keys = RowArrayWriter(element: field(.string), buffer: buffer) + try keys.reset(count: 0) + map.finishKeys() + var values = RowArrayWriter(element: field(.int32), buffer: buffer) + try values.reset(count: 0) + #expect(buffer.copyBytes(start: 0, end: buffer.count) + == [0x08, 0, 0, 0, 0, 0, 0, 0] + [UInt8](repeating: 0, count: 16)) +} + +@Test +func rowWriterNullElements() throws { + let schema = RowSchema([ + RowField(name: "names", dataType: .listOf(.string), nullable: true), + RowField(name: "points", dataType: .listOf(.struct(pointFields)), nullable: true) + ]) + var writer = RowWriter(schema: schema) + writer.reset() + + var names = RowArrayWriter(element: field(.string), buffer: writer.buffer) + try names.reset(count: 2) + try names.writeString("a", at: 0) + names.setNullAt(1) + try writer.setOffsetAndSize(at: 0, absoluteStart: names.base, size: names.size) + + var points = RowArrayWriter(element: field(.struct(pointFields)), buffer: writer.buffer) + try points.reset(count: 2) + let point = writePoint(1, into: writer.buffer) + try points.setOffsetAndSize(at: 0, absoluteStart: point.start, size: point.size) + points.setNullAt(1) + try writer.setOffsetAndSize(at: 1, absoluteStart: points.base, size: points.size) + + let row = try Row(schema: schema, bytes: writer.toBytes()) + let readNames = try #require(try row.array(at: 0)) + #expect(try readNames.string(at: 0) == "a") + #expect(try readNames.string(at: 1) == nil) + let readPoints = try #require(try row.array(at: 1)) + #expect(try readPoints.row(at: 0)?.int32(at: 0) == 1) + #expect(try readPoints.row(at: 1) == nil) +} + +@Test +func rowWriterNestedInListInMap() throws { + let listType = RowDataType.listOf(.struct(pointFields)) + let schema = RowSchema([RowField(name: "m", dataType: .mapOf(.string, listType), nullable: true)]) + var writer = RowWriter(schema: schema) + writer.reset() + let buffer = writer.buffer + + var map = RowMapWriter(buffer: buffer) + map.reset() + var keys = RowArrayWriter(element: field(.string), buffer: buffer) + try keys.reset(count: 1) + try keys.writeString("k", at: 0) + map.finishKeys() + var values = RowArrayWriter(element: field(listType), buffer: buffer) + try values.reset(count: 1) + var list = RowArrayWriter(element: field(.struct(pointFields)), buffer: buffer) + try list.reset(count: 2) + for (index, x) in [Int32(1), 2].enumerated() { + let point = writePoint(x, into: buffer) + try list.setOffsetAndSize(at: index, absoluteStart: point.start, size: point.size) + } + try values.setOffsetAndSize(at: 0, absoluteStart: list.base, size: list.size) + try writer.setOffsetAndSize(at: 0, absoluteStart: map.base, size: map.size) + + let readMap = try #require(try Row(schema: schema, bytes: writer.toBytes()).map(at: 0)) + #expect(try readMap.keys.string(at: 0) == "k") + let readList = try #require(try readMap.values.array(at: 0)) + #expect(readList.elementCount == 2) + #expect(try readList.row(at: 0)?.int32(at: 0) == 1) + #expect(try readList.row(at: 1)?.int32(at: 0) == 2) +} + +@Test +func rowArrayWriterRejects() throws { + var writer = RowArrayWriter(element: field(.int64), buffer: ByteBuffer()) + #expect(throws: ForyError.encodingError("array of -1 elements of 8 bytes each cannot be written")) { + try writer.reset(count: -1) + } + let tooMany = (Int(Int32.max) - 15) / 8 + 1 + #expect(throws: ForyError.encodingError("array of \(tooMany) elements of 8 bytes each cannot be written")) { + try writer.reset(count: tooMany) + } + #expect(writer.buffer.count == 0) +} From 962ef65ddd4eb76648ea68bf182cda477625c3c1 Mon Sep 17 00:00:00 2001 From: yash Date: Tue, 3 Nov 2026 12:12:11 +0530 Subject: [PATCH 09/19] swift: Add schema bytes and the schema hash. A row holds only values, so a reader in another language needs the schema too. toBytes writes each field as a header byte with the name encoding, name size, and nullable flag, then the packed name and the type, with lists, maps, and structs followed by their child fields. RowSchema(bytes:) reads that back, and hash folds the type ids into one number a reader can check before trusting a row. --- swift/Sources/Fory/Row/RowSchemaBytes.swift | 245 ++++++++++++++++++ .../Tests/ForyTests/RowSchemaBytesTests.swift | 201 ++++++++++++++ 2 files changed, 446 insertions(+) create mode 100644 swift/Sources/Fory/Row/RowSchemaBytes.swift create mode 100644 swift/Tests/ForyTests/RowSchemaBytesTests.swift diff --git a/swift/Sources/Fory/Row/RowSchemaBytes.swift b/swift/Sources/Fory/Row/RowSchemaBytes.swift new file mode 100644 index 0000000000..9a83430041 --- /dev/null +++ b/swift/Sources/Fory/Row/RowSchemaBytes.swift @@ -0,0 +1,245 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Schema bytes are a version byte and a field count, then each field in order. A field is a header +// byte, its packed name, and its type. The header holds the name encoding in bits 0 and 1, the name +// size minus one in bits 2 to 5 (15 means a varint with the size minus 15 follows), and the +// nullable flag in bit 6. A type is its id byte, followed by precision and scale for a decimal, the +// element field for a list, the key and value fields for a map, or a count and the fields for a +// struct. + +private let rowSchemaVersion: UInt8 = 1 +private let rowFieldNameSizeThreshold = 15 +// Deeper schemas are rejected so reading one cannot recurse without bound. +private let rowSchemaMaxDepth = 64 +// Header bits 0 and 1 hold a position in this list, not the encoding's raw value. +private let rowFieldNameEncodings: [MetaStringEncoding] = [.utf8, .allToLowerSpecial, .lowerUpperDigitSpecial] + +extension RowSchema { + /// The schema in the byte form every row format runtime reads. Throws if a field name is empty, + /// fields nest 64 or more levels deep, or a decimal precision or scale does not fit in a byte. + public func toBytes() throws -> [UInt8] { + let buffer = ByteBuffer() + buffer.writeUInt8(rowSchemaVersion) + buffer.writeVarUInt32(UInt32(fields.count)) + for field in fields { + try rowWriteSchemaField(field, depth: 0, to: buffer) + } + return buffer.storage + } + + public init(bytes: [UInt8]) throws { + let buffer = ByteBuffer(bytes: bytes) + let version = try buffer.readUInt8() + guard version == rowSchemaVersion else { + throw rowSchemaVersionUnsupported(version) + } + self.init(try rowReadSchemaFields(buffer, depth: 0)) + } + + /// A fingerprint of the field types, written before a row so the reader can tell it was written + /// with the same schema. Names and nullability do not change it. + public var hash: Int64 { + fields.reduce(17) { rowSchemaHash($0, $1.dataType) } + } +} + +private func rowWriteSchemaField(_ field: RowField, depth: Int, to buffer: ByteBuffer) throws { + guard depth < rowSchemaMaxDepth else { + throw rowSchemaTooDeepToWrite() + } + guard !field.name.isEmpty else { + throw rowSchemaEmptyFieldName() + } + let name = try MetaStringEncoder.fieldName.encode(field.name, allowedEncodings: rowFieldNameEncodings) + guard let encodingIndex = rowFieldNameEncodings.firstIndex(of: name.encoding) else { + preconditionFailure("field name encoder chose \(name.encoding), which was not allowed") + } + let nameSize = name.bytes.count + let longName = nameSize > rowFieldNameSizeThreshold + var header = UInt8(encodingIndex) + header |= UInt8(longName ? rowFieldNameSizeThreshold : nameSize - 1) << 2 + if field.nullable { + header |= 0x40 + } + buffer.writeUInt8(header) + if longName { + buffer.writeVarUInt32(UInt32(nameSize - rowFieldNameSizeThreshold)) + } + buffer.writeBytes(name.bytes) + try rowWriteSchemaType(field.dataType, depth: depth, to: buffer) +} + +private func rowWriteSchemaType(_ dataType: RowDataType, depth: Int, to buffer: ByteBuffer) throws { + buffer.writeUInt8(UInt8(dataType.typeId.rawValue)) + switch dataType { + case .decimal(let precision, let scale): + guard let precisionByte = UInt8(exactly: precision), let scaleByte = UInt8(exactly: scale) else { + throw rowSchemaDecimalOutOfRange(precision: precision, scale: scale) + } + buffer.writeUInt8(precisionByte) + buffer.writeUInt8(scaleByte) + case .list(let element): + try rowWriteSchemaField(element, depth: depth + 1, to: buffer) + case .map(let key, let value): + try rowWriteSchemaField(key, depth: depth + 1, to: buffer) + try rowWriteSchemaField(value, depth: depth + 1, to: buffer) + case .struct(let fields): + buffer.writeVarUInt32(UInt32(fields.count)) + for field in fields { + try rowWriteSchemaField(field, depth: depth + 1, to: buffer) + } + default: + break + } +} + +private func rowReadSchemaFields(_ buffer: ByteBuffer, depth: Int) throws -> [RowField] { + let count = try buffer.readVarUInt32() + // Every field takes at least one byte, so a larger count is wrong. Checking it first also keeps a + // forged count from reserving a huge array. + guard UInt64(count) <= UInt64(buffer.remaining) else { + throw rowSchemaCountTooLarge(count, remaining: buffer.remaining) + } + var fields: [RowField] = [] + fields.reserveCapacity(Int(count)) + for _ in 0.. RowField { + guard depth < rowSchemaMaxDepth else { + throw rowSchemaTooDeep() + } + let header = try buffer.readUInt8() + let encodingIndex = Int(header & 0x03) + guard encodingIndex < rowFieldNameEncodings.count else { + throw rowSchemaUnknownNameEncoding(encodingIndex) + } + var nameSize = UInt64((header >> 2) & 0x0F) + 1 + if nameSize == UInt64(rowFieldNameSizeThreshold) + 1 { + nameSize = UInt64(try buffer.readVarUInt32()) + UInt64(rowFieldNameSizeThreshold) + } + // readBytes checks the size against the bytes left before copying, and clamping keeps a huge + // size from trapping where Int is 32 bits. + let nameBytes = try buffer.readBytes(count: Int(clamping: nameSize)) + let name = try MetaStringDecoder.fieldName.decode(bytes: nameBytes, encoding: rowFieldNameEncodings[encodingIndex]) + let dataType = try rowReadSchemaType(buffer, depth: depth) + return RowField(name: name.value, dataType: dataType, nullable: header & 0x40 != 0) +} + +private func rowReadSchemaType(_ buffer: ByteBuffer, depth: Int) throws -> RowDataType { + let typeId = try buffer.readUInt8() + switch TypeId(rawValue: UInt32(typeId)) { + case .bool: return .bool + case .int8: return .int8 + case .int16: return .int16 + case .int32: return .int32 + case .int64: return .int64 + case .float16: return .float16 + case .float32: return .float32 + case .float64: return .float64 + case .string: return .string + case .binary: return .binary + case .date: return .date32 + case .timestamp: return .timestamp + case .duration: return .duration + case .decimal: + let precision = try buffer.readUInt8() + let scale = try buffer.readUInt8() + return .decimal(precision: Int(precision), scale: Int(scale)) + case .list: + return .list(try rowReadSchemaField(buffer, depth: depth + 1)) + case .map: + let key = try rowReadSchemaField(buffer, depth: depth + 1) + let value = try rowReadSchemaField(buffer, depth: depth + 1) + // Only the key and value types are kept; their names and nullability are always the standard ones. + return .mapOf(key.dataType, value.dataType) + case .structType: + return .struct(try rowReadSchemaFields(buffer, depth: depth + 1)) + default: + throw rowSchemaUnknownTypeId(typeId) + } +} + +// Folds `hash * 31 + typeId` over a type and then its children in order. When either step would +// overflow, the hash is shifted right by 2, keeping its sign, and the step is tried again. +private func rowSchemaHash(_ hash: Int64, _ dataType: RowDataType) -> Int64 { + let typeId = Int64(dataType.typeId.rawValue) + var hash = hash + while true { + let (product, multiplyOverflow) = hash.multipliedReportingOverflow(by: 31) + let (sum, addOverflow) = product.addingReportingOverflow(typeId) + if !multiplyOverflow && !addOverflow { + hash = sum + break + } + hash >>= 2 + } + switch dataType { + case .list(let element): + return rowSchemaHash(hash, element.dataType) + case .map(let key, let value): + return rowSchemaHash(rowSchemaHash(hash, key.dataType), value.dataType) + case .struct(let fields): + return fields.reduce(hash) { rowSchemaHash($0, $1.dataType) } + default: + return hash + } +} + +@inline(never) +private func rowSchemaTooDeepToWrite() -> ForyError { + .encodingError("row schema nests \(rowSchemaMaxDepth) or more levels deep") +} + +@inline(never) +private func rowSchemaEmptyFieldName() -> ForyError { + .encodingError("row schema field has an empty name") +} + +@inline(never) +private func rowSchemaDecimalOutOfRange(precision: Int, scale: Int) -> ForyError { + .encodingError("decimal(\(precision), \(scale)) precision and scale must each fit in one byte") +} + +@inline(never) +private func rowSchemaVersionUnsupported(_ version: UInt8) -> ForyError { + .invalidData("unsupported row schema version \(version), expected \(rowSchemaVersion)") +} + +@inline(never) +private func rowSchemaCountTooLarge(_ count: UInt32, remaining: Int) -> ForyError { + .invalidData("row schema declares \(count) fields but only \(remaining) bytes remain") +} + +@inline(never) +private func rowSchemaTooDeep() -> ForyError { + .invalidData("row schema nests \(rowSchemaMaxDepth) or more levels deep") +} + +@inline(never) +private func rowSchemaUnknownNameEncoding(_ index: Int) -> ForyError { + .invalidData("unknown row schema field name encoding \(index)") +} + +@inline(never) +private func rowSchemaUnknownTypeId(_ typeId: UInt8) -> ForyError { + .invalidData("unknown type id \(typeId) in row schema") +} diff --git a/swift/Tests/ForyTests/RowSchemaBytesTests.swift b/swift/Tests/ForyTests/RowSchemaBytesTests.swift new file mode 100644 index 0000000000..c0dd11b092 --- /dev/null +++ b/swift/Tests/ForyTests/RowSchemaBytesTests.swift @@ -0,0 +1,201 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation +import Testing +@testable import Fory + +private func field(_ name: String, _ dataType: RowDataType, nullable: Bool = true) -> RowField { + RowField(name: name, dataType: dataType, nullable: nullable) +} + +// Bytes and hashes that every row format runtime produces for these schemas. +private let simpleSchema = RowSchema([ + field("id", .int32), field("name", .string), field("score", .float64), field("active", .bool) +]) + +private let simpleBytes: [UInt8] = [ + 0x01, 0x04, 0x45, 0xa0, 0x60, 0x04, 0x49, 0x34, 0x0c, 0x20, 0x15, 0x4d, + 0xc8, 0x4e, 0x89, 0x00, 0x14, 0x4d, 0x00, 0x53, 0x45, 0x48, 0x01 +] + +private let nestedSchema = RowSchema([ + field("person", .struct([field("name", .string), field("age", .int32, nullable: false)])), + field("tags", .listOf(.string)), + field("attrs", .mapOf(.string, .int64)) +]) + +private let nestedBytes: [UInt8] = [ + 0x01, 0x03, 0x4d, 0x3c, 0x91, 0x93, 0x9a, 0x1b, 0x02, 0x49, 0x34, 0x0c, + 0x20, 0x15, 0x05, 0x00, 0xc4, 0x04, 0x49, 0x4c, 0x06, 0x90, 0x16, 0x49, + 0x22, 0x64, 0x60, 0x15, 0x4d, 0x82, 0x73, 0x8c, 0x80, 0x18, 0x05, 0x28, + 0x98, 0x15, 0x4d, 0xd4, 0x0b, 0xa1, 0x00, 0x06 +] + +private let specialSchema = RowSchema([ + field("a_very_long_field_name_exceeding_limit", .int64, nullable: false), + field("field2", .string), + field("$special_name", .date32), + field("CamelCase", .timestamp, nullable: false), + field("price", .decimal(precision: 10, scale: 2)) +]) + +private let specialBytes: [UInt8] = [ + 0x01, 0x05, 0x3d, 0x09, 0x03, 0x75, 0x24, 0x71, 0xb5, 0xb9, 0xa6, 0xd9, + 0x50, 0x45, 0x8f, 0x6d, 0x03, 0x09, 0xb2, 0x5c, 0x44, 0x20, 0xd0, 0xd3, + 0x6d, 0x68, 0x62, 0x26, 0x06, 0x52, 0x0a, 0x40, 0x85, 0x87, 0xb0, 0x15, + 0x61, 0xf2, 0x4f, 0x20, 0x90, 0x05, 0xed, 0xa0, 0x61, 0x00, 0x27, 0x1a, + 0x38, 0x01, 0x82, 0x16, 0xe0, 0x09, 0x08, 0x26, 0x4d, 0xbe, 0x28, 0x11, + 0x00, 0x28, 0x0a, 0x02 +] + +// The Foo schema from the cross-language row tests. +private let fooSchema = RowSchema([ + field("f1", .int32), + field("f2", .string), + field("f3", .listOf(.string)), + field("f4", .mapOf(.string, .int32)), + field("f5", .struct([field("f1", .int32), field("f2", .string)])) +]) + +private let fooBytes: [UInt8] = [ + 0x01, 0x05, + 0x46, 0x0b, 0xa8, 0x04, + 0x46, 0x0b, 0xb0, 0x15, + 0x46, 0x0b, 0xb8, 0x16, 0x49, 0x22, 0x64, 0x60, 0x15, + 0x46, 0x0b, 0xc0, 0x18, 0x05, 0x28, 0x98, 0x15, 0x4d, 0xd4, 0x0b, 0xa1, 0x00, 0x04, + 0x46, 0x0b, 0xc8, 0x1b, 0x02, 0x46, 0x0b, 0xa8, 0x04, 0x46, 0x0b, 0xb0, 0x15 +] + +private func nestedLists(_ levels: Int) -> RowDataType { + var dataType = RowDataType.int32 + for _ in 0.. [UInt8] { + var bytes: [UInt8] = [0x01, 0x01] + for _ in 0.. Date: Tue, 3 Nov 2026 16:24:13 +0530 Subject: [PATCH 10/19] swift: Add RowCodable with scalar and temporal conformances. RowCodable lets each Swift type say how it sits in a row: its row type, whether its field is nullable in the schema, its width, and how it writes and reads itself. Rows and arrays share one write surface and one read surface, so a single conformance serves a struct field or an array element. An optional reuses its wrapped type and stores nil as the null bit, and reading a null into a type that cannot be nil throws. --- swift/Sources/Fory/Row/RowCodable.swift | 338 ++++++++++++++++++++ swift/Tests/ForyTests/RowCodableTests.swift | 144 +++++++++ 2 files changed, 482 insertions(+) create mode 100644 swift/Sources/Fory/Row/RowCodable.swift create mode 100644 swift/Tests/ForyTests/RowCodableTests.swift diff --git a/swift/Sources/Fory/Row/RowCodable.swift b/swift/Sources/Fory/Row/RowCodable.swift new file mode 100644 index 0000000000..716fc494ad --- /dev/null +++ b/swift/Sources/Fory/Row/RowCodable.swift @@ -0,0 +1,338 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +/// The write methods ``RowWriter`` and ``RowArrayWriter`` share, so one ``RowCodable`` type can be +/// written as a struct field or as an array element. +public protocol RowValueWriter { + var buffer: ByteBuffer { get } + func setNullAt(_ index: Int) + func writeBool(_ value: Bool, at index: Int) + func writeInt8(_ value: Int8, at index: Int) + func writeInt16(_ value: Int16, at index: Int) + func writeInt32(_ value: Int32, at index: Int) + func writeInt64(_ value: Int64, at index: Int) + func writeFloat32(_ value: Float, at index: Int) + func writeFloat64(_ value: Double, at index: Int) + func writeLocalDate(_ value: LocalDate, at index: Int) throws + func writeTimestamp(_ value: Date, at index: Int) throws + func writeDuration(_ value: Duration, at index: Int) throws + func writeString(_ value: String, at index: Int) throws + func writeBytes(_ value: some ContiguousBytes, at index: Int) throws + func setOffsetAndSize(at index: Int, absoluteStart: Int, size: Int) throws +} + +/// The getters ``Row`` and ``RowArray`` share, so one ``RowCodable`` type can be read from a struct +/// field or from an array element. +public protocol RowValueReader { + func isNull(at index: Int) -> Bool + func bool(at index: Int) -> Bool? + func int8(at index: Int) -> Int8? + func int16(at index: Int) -> Int16? + func int32(at index: Int) -> Int32? + func int64(at index: Int) -> Int64? + func float32(at index: Int) -> Float? + func float64(at index: Int) -> Double? + func localDate(at index: Int) -> LocalDate? + func timestamp(at index: Int) -> Date? + func duration(at index: Int) -> Duration? + func string(at index: Int) throws -> String? + func binary(at index: Int) throws -> [UInt8]? + func row(at index: Int) throws -> Row? + func array(at index: Int) throws -> RowArray? + func map(at index: Int) throws -> RowMap? +} + +extension RowWriter: RowValueWriter {} +extension RowArrayWriter: RowValueWriter {} +extension Row: RowValueReader {} +extension RowArray: RowValueReader {} + +/// How one Swift type is stored in a row field or an array element. +public protocol RowCodable { + static var rowDataType: RowDataType { get } + /// Whether a field of this type is nullable in a schema built from Swift types. A type that + /// cannot hold nil may still be nullable, as `String` is; reading a null into it throws. + static var rowNullable: Bool { get } + /// The same as `rowDataType.byteWidth`, kept as a constant so writing never builds the data type. + static var rowFixedWidth: Int? { get } + static func writeRow(_ value: Self, to writer: W, at index: Int) throws + /// Throws when the slot is null and `Self` cannot hold nil. + static func readRow(from reader: R, at index: Int) throws -> Self +} + +extension RowWriter { + @inlinable + public func write(_ value: T, at index: Int) throws { + try T.writeRow(value, to: self, at: index) + } +} + +extension Row { + /// Reads field `index` as `T`. Read as an optional type to get nil for a null field. + @inlinable + public func read(_ type: T.Type = T.self, at index: Int) throws -> T { + try T.readRow(from: self, at: index) + } +} + +extension RowField { + /// The field a Swift type gets in a schema built from Swift types. + public static func inferred(_ name: String, _ type: T.Type) -> RowField { + RowField(name: name, dataType: T.rowDataType, nullable: T.rowNullable) + } +} + +extension Bool: RowCodable { + public static var rowDataType: RowDataType { .bool } + public static var rowNullable: Bool { false } + public static var rowFixedWidth: Int? { 1 } + + @inlinable + public static func writeRow(_ value: Bool, to writer: W, at index: Int) throws { + writer.writeBool(value, at: index) + } + + @inlinable + public static func readRow(from reader: R, at index: Int) throws -> Bool { + try rowNonNull(reader.bool(at: index)) + } +} + +extension Int8: RowCodable { + public static var rowDataType: RowDataType { .int8 } + public static var rowNullable: Bool { false } + public static var rowFixedWidth: Int? { 1 } + + @inlinable + public static func writeRow(_ value: Int8, to writer: W, at index: Int) throws { + writer.writeInt8(value, at: index) + } + + @inlinable + public static func readRow(from reader: R, at index: Int) throws -> Int8 { + try rowNonNull(reader.int8(at: index)) + } +} + +extension Int16: RowCodable { + public static var rowDataType: RowDataType { .int16 } + public static var rowNullable: Bool { false } + public static var rowFixedWidth: Int? { 2 } + + @inlinable + public static func writeRow(_ value: Int16, to writer: W, at index: Int) throws { + writer.writeInt16(value, at: index) + } + + @inlinable + public static func readRow(from reader: R, at index: Int) throws -> Int16 { + try rowNonNull(reader.int16(at: index)) + } +} + +extension Int32: RowCodable { + public static var rowDataType: RowDataType { .int32 } + public static var rowNullable: Bool { false } + public static var rowFixedWidth: Int? { 4 } + + @inlinable + public static func writeRow(_ value: Int32, to writer: W, at index: Int) throws { + writer.writeInt32(value, at: index) + } + + @inlinable + public static func readRow(from reader: R, at index: Int) throws -> Int32 { + try rowNonNull(reader.int32(at: index)) + } +} + +extension Int64: RowCodable { + public static var rowDataType: RowDataType { .int64 } + public static var rowNullable: Bool { false } + public static var rowFixedWidth: Int? { 8 } + + @inlinable + public static func writeRow(_ value: Int64, to writer: W, at index: Int) throws { + writer.writeInt64(value, at: index) + } + + @inlinable + public static func readRow(from reader: R, at index: Int) throws -> Int64 { + try rowNonNull(reader.int64(at: index)) + } +} + +// Int is stored as int64 on every platform. Where Int is 32 bits, a larger stored value throws. +extension Int: RowCodable { + public static var rowDataType: RowDataType { .int64 } + public static var rowNullable: Bool { false } + public static var rowFixedWidth: Int? { 8 } + + @inlinable + public static func writeRow(_ value: Int, to writer: W, at index: Int) throws { + writer.writeInt64(Int64(value), at: index) + } + + @inlinable + public static func readRow(from reader: R, at index: Int) throws -> Int { + let value = try rowNonNull(reader.int64(at: index)) + guard let narrowed = Int(exactly: value) else { + throw rowIntOutOfRange(value) + } + return narrowed + } +} + +extension Float: RowCodable { + public static var rowDataType: RowDataType { .float32 } + public static var rowNullable: Bool { false } + public static var rowFixedWidth: Int? { 4 } + + @inlinable + public static func writeRow(_ value: Float, to writer: W, at index: Int) throws { + writer.writeFloat32(value, at: index) + } + + @inlinable + public static func readRow(from reader: R, at index: Int) throws -> Float { + try rowNonNull(reader.float32(at: index)) + } +} + +extension Double: RowCodable { + public static var rowDataType: RowDataType { .float64 } + public static var rowNullable: Bool { false } + public static var rowFixedWidth: Int? { 8 } + + @inlinable + public static func writeRow(_ value: Double, to writer: W, at index: Int) throws { + writer.writeFloat64(value, at: index) + } + + @inlinable + public static func readRow(from reader: R, at index: Int) throws -> Double { + try rowNonNull(reader.float64(at: index)) + } +} + +extension String: RowCodable { + public static var rowDataType: RowDataType { .string } + public static var rowNullable: Bool { true } + public static var rowFixedWidth: Int? { nil } + + @inlinable + public static func writeRow(_ value: String, to writer: W, at index: Int) throws { + try writer.writeString(value, at: index) + } + + @inlinable + public static func readRow(from reader: R, at index: Int) throws -> String { + try rowNonNull(reader.string(at: index)) + } +} + +extension Date: RowCodable { + public static var rowDataType: RowDataType { .timestamp } + public static var rowNullable: Bool { true } + public static var rowFixedWidth: Int? { 8 } + + @inlinable + public static func writeRow(_ value: Date, to writer: W, at index: Int) throws { + try writer.writeTimestamp(value, at: index) + } + + @inlinable + public static func readRow(from reader: R, at index: Int) throws -> Date { + try rowNonNull(reader.timestamp(at: index)) + } +} + +extension LocalDate: RowCodable { + public static var rowDataType: RowDataType { .date32 } + public static var rowNullable: Bool { true } + public static var rowFixedWidth: Int? { 4 } + + @inlinable + public static func writeRow(_ value: LocalDate, to writer: W, at index: Int) throws { + try writer.writeLocalDate(value, at: index) + } + + @inlinable + public static func readRow(from reader: R, at index: Int) throws -> LocalDate { + try rowNonNull(reader.localDate(at: index)) + } +} + +extension Duration: RowCodable { + public static var rowDataType: RowDataType { .duration } + public static var rowNullable: Bool { true } + public static var rowFixedWidth: Int? { 8 } + + @inlinable + public static func writeRow(_ value: Duration, to writer: W, at index: Int) throws { + try writer.writeDuration(value, at: index) + } + + @inlinable + public static func readRow(from reader: R, at index: Int) throws -> Duration { + try rowNonNull(reader.duration(at: index)) + } +} + +// An optional uses its wrapped type's slot and width, and stores nil as a set null bit. +extension Optional: RowCodable where Wrapped: RowCodable { + public static var rowDataType: RowDataType { Wrapped.rowDataType } + public static var rowNullable: Bool { true } + public static var rowFixedWidth: Int? { Wrapped.rowFixedWidth } + + @inlinable + public static func writeRow(_ value: Wrapped?, to writer: W, at index: Int) throws { + guard let value else { + writer.setNullAt(index) + return + } + try Wrapped.writeRow(value, to: writer, at: index) + } + + @inlinable + public static func readRow(from reader: R, at index: Int) throws -> Wrapped? { + try reader.isNull(at: index) ? nil : Wrapped.readRow(from: reader, at: index) + } +} + +@inlinable +@inline(__always) +func rowNonNull(_ value: T?) throws -> T { + guard let value else { + throw rowNullValue(T.self) + } + return value +} + +@usableFromInline +@inline(never) +func rowNullValue(_ type: Any.Type) -> ForyError { + .invalidData("null row value cannot be read into non-optional \(type)") +} + +@usableFromInline +@inline(never) +func rowIntOutOfRange(_ value: Int64) -> ForyError { + .invalidData("int64 \(value) does not fit in Int") +} diff --git a/swift/Tests/ForyTests/RowCodableTests.swift b/swift/Tests/ForyTests/RowCodableTests.swift new file mode 100644 index 0000000000..2793eeee4b --- /dev/null +++ b/swift/Tests/ForyTests/RowCodableTests.swift @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation +import Testing +@testable import Fory + +private struct Scalars: Equatable { + var bool: Bool + var int8: Int8 + var int16: Int16 + var int32: Int32 + var int64: Int64 + var int: Int + var float: Float + var double: Double + var string: String + var date: Date + var localDate: LocalDate + var duration: Duration +} + +private let scalarSchema = RowSchema([ + .inferred("bool", Bool.self), .inferred("int8", Int8.self), .inferred("int16", Int16.self), + .inferred("int32", Int32.self), .inferred("int64", Int64.self), .inferred("int", Int.self), + .inferred("float", Float.self), .inferred("double", Double.self), .inferred("string", String.self), + .inferred("date", Date.self), .inferred("localDate", LocalDate.self), .inferred("duration", Duration.self) +]) + +private func roundTrip(_ value: Scalars) throws -> Scalars { + var writer = RowWriter(schema: scalarSchema) + writer.reset() + try writer.write(value.bool, at: 0) + try writer.write(value.int8, at: 1) + try writer.write(value.int16, at: 2) + try writer.write(value.int32, at: 3) + try writer.write(value.int64, at: 4) + try writer.write(value.int, at: 5) + try writer.write(value.float, at: 6) + try writer.write(value.double, at: 7) + try writer.write(value.string, at: 8) + try writer.write(value.date, at: 9) + try writer.write(value.localDate, at: 10) + try writer.write(value.duration, at: 11) + let row = try Row(schema: scalarSchema, bytes: writer.toBytes()) + return try Scalars( + bool: row.read(at: 0), int8: row.read(at: 1), int16: row.read(at: 2), int32: row.read(at: 3), + int64: row.read(at: 4), int: row.read(at: 5), float: row.read(at: 6), double: row.read(at: 7), + string: row.read(at: 8), date: row.read(at: 9), localDate: row.read(at: 10), duration: row.read(at: 11) + ) +} + +private let optionalSchema = RowSchema([ + .inferred("a", Int32?.self), .inferred("b", String?.self), .inferred("c", Date?.self) +]) + +private func optionalRow(_ number: Int32?, _ text: String?, _ date: Date?) throws -> Row { + var writer = RowWriter(schema: optionalSchema) + writer.reset() + try writer.write(number, at: 0) + try writer.write(text, at: 1) + try writer.write(date, at: 2) + return try Row(schema: optionalSchema, bytes: writer.toBytes()) +} + +@Test +func rowCodableScalars() throws { + let low = Scalars( + bool: false, int8: .min, int16: .min, int32: .min, int64: .min, int: .min, + float: -.greatestFiniteMagnitude, double: -.greatestFiniteMagnitude, string: "", + date: Date(timeIntervalSince1970: -86_400.5), localDate: LocalDate(epochDay: Int64(Int32.min)), + duration: .microseconds(Int64.min) + ) + let high = Scalars( + bool: true, int8: .max, int16: .max, int32: .max, int64: .max, int: .max, + float: .greatestFiniteMagnitude, double: .greatestFiniteMagnitude, string: "héllo, 世界", + date: Date(timeIntervalSinceReferenceDate: 721_692_800.25), localDate: LocalDate(epochDay: Int64(Int32.max)), + duration: .microseconds(Int64.max) + ) + #expect(try roundTrip(low) == low) + #expect(try roundTrip(high) == high) +} + +@Test +func rowCodableOptionalNull() throws { + let nulls = try optionalRow(nil, nil, nil) + for index in 0..<3 { + #expect(nulls.isNull(at: index)) + } + #expect(try nulls.read(Int32?.self, at: 0) == nil) + #expect(try nulls.read(String?.self, at: 1) == nil) + #expect(try nulls.read(Date?.self, at: 2) == nil) + + let date = Date(timeIntervalSinceReferenceDate: 0.5) + let values = try optionalRow(7, "x", date) + #expect(try values.read(Int32?.self, at: 0) == 7) + #expect(try values.read(String?.self, at: 1) == "x") + #expect(try values.read(Date?.self, at: 2) == date) +} + +@Test +func rowCodableNullIntoValue() throws { + let nulls = try optionalRow(nil, nil, nil) + #expect(throws: ForyError.invalidData("null row value cannot be read into non-optional Int32")) { + try nulls.read(Int32.self, at: 0) + } + #expect(throws: ForyError.invalidData("null row value cannot be read into non-optional String")) { + try nulls.read(String.self, at: 1) + } +} + +@Test +func rowCodableInferredFields() { + #expect(RowField.inferred("a", Int32.self) == RowField(name: "a", dataType: .int32, nullable: false)) + #expect(RowField.inferred("a", Int32?.self) == RowField(name: "a", dataType: .int32, nullable: true)) + #expect(RowField.inferred("a", String.self) == RowField(name: "a", dataType: .string, nullable: true)) + #expect(RowField.inferred("a", Int.self) == RowField(name: "a", dataType: .int64, nullable: false)) + #expect(RowField.inferred("a", Date.self) == RowField(name: "a", dataType: .timestamp, nullable: true)) + #expect(RowField.inferred("a", LocalDate.self) == RowField(name: "a", dataType: .date32, nullable: true)) + #expect(RowField.inferred("a", Duration.self) == RowField(name: "a", dataType: .duration, nullable: true)) + + // An array of Int32? still packs 4-byte elements, so the width must follow the wrapped type. + let types: [any RowCodable.Type] = [ + Bool.self, Int8.self, Int16.self, Int32.self, Int64.self, Int.self, Float.self, Double.self, + String.self, Date.self, LocalDate.self, Duration.self, Int16?.self, String?.self + ] + for type in types { + #expect(type.rowFixedWidth == type.rowDataType.byteWidth, "\(type)") + } +} From a3a4b95026c0994bfb99e93594bed7d8ee9be2b7 Mon Sep 17 00:00:00 2001 From: yash Date: Wed, 4 Nov 2026 23:23:11 +0530 Subject: [PATCH 11/19] swift: Add collection conformances and RowEncoder. Arrays and sets are written as a child array after the slots, and a dictionary as its keys array followed by its values array, each element using its own RowCodable rule, so collections nest freely. Data takes the same list shape but copies its bytes in one go, string enums opt in with an empty conformance, and a RowStruct is a child row. RowEncoder puts the 8-byte schema hash in front of the row and checks it before decoding. --- swift/Sources/Fory/Row/Row.swift | 10 + swift/Sources/Fory/Row/RowCodable.swift | 239 ++++++++++++++++++++ swift/Sources/Fory/Row/RowWriter.swift | 12 + swift/Tests/ForyTests/RowEncoderTests.swift | 172 ++++++++++++++ 4 files changed, 433 insertions(+) create mode 100644 swift/Tests/ForyTests/RowEncoderTests.swift diff --git a/swift/Sources/Fory/Row/Row.swift b/swift/Sources/Fory/Row/Row.swift index 0ebb4762ac..d251c5e3ab 100644 --- a/swift/Sources/Fory/Row/Row.swift +++ b/swift/Sources/Fory/Row/Row.swift @@ -292,6 +292,16 @@ public struct RowArray { Array(bytes[base..<(base + sizeInBytes)]) } + // Copies every element of an int8 array at once, or returns nil when any element is null. + func int8Elements() -> Data? { + precondition(elementWidth == 1, "array element type is \(element.dataType), not int8") + let hasNull = bytes.withUnsafeBytes { raw in + (0..(_ value: [Element], to writer: W, at index: Int) throws { + let array = try rowAppendArray(value, to: writer.buffer) + try writer.setOffsetAndSize(at: index, absoluteStart: array.base, size: array.size) + } + + @inlinable + public static func readRow(from reader: R, at index: Int) throws -> [Element] { + guard let array = try reader.array(at: index) else { + throw rowNullValue(Self.self) + } + var elements: [Element] = [] + elements.reserveCapacity(array.elementCount) + for elementIndex in 0..(_ value: Set, to writer: W, at index: Int) throws { + let array = try rowAppendArray(value, to: writer.buffer) + try writer.setOffsetAndSize(at: index, absoluteStart: array.base, size: array.size) + } + + @inlinable + public static func readRow(from reader: R, at index: Int) throws -> Set { + guard let array = try reader.array(at: index) else { + throw rowNullValue(Self.self) + } + var elements = Set(minimumCapacity: array.elementCount) + for elementIndex in 0..(_ value: [Key: Value], to writer: W, at index: Int) throws { + var map = RowMapWriter(buffer: writer.buffer) + map.reset() + _ = try rowAppendArray(value.keys, to: writer.buffer) + map.finishKeys() + _ = try rowAppendArray(value.values, to: writer.buffer) + try writer.setOffsetAndSize(at: index, absoluteStart: map.base, size: map.size) + } + + @inlinable + public static func readRow(from reader: R, at index: Int) throws -> [Key: Value] { + guard let map = try reader.map(at: index) else { + throw rowNullValue(Self.self) + } + var entries = [Key: Value](minimumCapacity: map.count) + for entryIndex in 0..(_ value: Data, to writer: W, at index: Int) throws { + var array = RowArrayWriter(elementWidth: 1, buffer: writer.buffer) + try array.reset(count: value.count) + array.writeInt8Elements(value) + try writer.setOffsetAndSize(at: index, absoluteStart: array.base, size: array.size) + } + + public static func readRow(from reader: R, at index: Int) throws -> Data { + guard let array = try reader.array(at: index) else { + throw rowNullValue(Data.self) + } + guard let bytes = array.int8Elements() else { + throw rowNullValue(Int8.self) + } + return bytes + } +} + +/// Stores an enum as the string of its raw value. Opt in with an empty `extension MyEnum: RowCodable {}`. +extension RowCodable where Self: RawRepresentable, Self.RawValue == String { + public static var rowDataType: RowDataType { .string } + public static var rowNullable: Bool { true } + public static var rowFixedWidth: Int? { nil } + + @inlinable + public static func writeRow(_ value: Self, to writer: W, at index: Int) throws { + try writer.writeString(value.rawValue, at: index) + } + + @inlinable + public static func readRow(from reader: R, at index: Int) throws -> Self { + guard let name = try reader.string(at: index) else { + throw rowNullValue(Self.self) + } + guard let value = Self(rawValue: name) else { + throw rowUnknownCase(name, of: Self.self) + } + return value + } +} + +/// A struct stored as its own row. `@ForyRow` adopts it. +public protocol RowStruct: RowCodable { + static var rowSchema: RowSchema { get } + static func writeRowFields(_ value: Self, to writer: RowWriter) throws + static func readRowFields(from row: Row) throws -> Self +} + +// A struct field holds a complete child row written after the parent's slots. +extension RowStruct { + public static var rowDataType: RowDataType { .struct(rowSchema.fields) } + public static var rowNullable: Bool { true } + public static var rowFixedWidth: Int? { nil } + + @inlinable + public static func writeRow(_ value: Self, to writer: W, at index: Int) throws { + var child = RowWriter(schema: rowSchema, buffer: writer.buffer) + child.reset() + try writeRowFields(value, to: child) + try writer.setOffsetAndSize(at: index, absoluteStart: child.base, size: child.size) + } + + @inlinable + public static func readRow(from reader: R, at index: Int) throws -> Self { + guard let row = try reader.row(at: index) else { + throw rowNullValue(Self.self) + } + return try readRowFields(from: row) + } +} + +/// Converts a ``RowStruct`` to and from row bytes. It reuses one buffer, so use an encoder from one +/// thread at a time. +public final class RowEncoder { + public let schema: RowSchema + public let schemaHash: Int64 + private let buffer = ByteBuffer() + + public init() { + schema = T.rowSchema + schemaHash = schema.hash + } + + public func toRow(_ value: T) throws -> Data { + buffer.clear() + try writeRow(value) + return buffer.toData() + } + + public func fromRow(_ data: Data) throws -> T { + try T.readRowFields(from: Row(schema: schema, bytes: Array(data))) + } + + /// The schema hash as 8 little-endian bytes, then the row. + public func encode(_ value: T) throws -> Data { + buffer.clear() + buffer.writeInt64(schemaHash) + try writeRow(value) + return buffer.toData() + } + + /// Throws if the data is shorter than the hash or was written with a different schema. + public func decode(_ data: Data) throws -> T { + guard data.count >= 8 else { + throw rowEncodedTooShort(data.count) + } + let bytes = Array(data) + let hash = Int64(littleEndian: bytes.withUnsafeBytes { $0.loadUnaligned(as: Int64.self) }) + guard hash == schemaHash else { + throw rowSchemaHashMismatch(found: hash, expected: schemaHash) + } + return try T.readRowFields(from: Row(fields: schema.fields, bytes: bytes, base: 8, count: bytes.count - 8)) + } + + private func writeRow(_ value: T) throws { + var writer = RowWriter(schema: schema, buffer: buffer) + writer.reset() + try T.writeRowFields(value, to: writer) + } +} + +// Appends `elements` as an array at the end of the buffer. The returned writer's base and size are +// what the parent slot points at. +@inlinable +func rowAppendArray(_ elements: C, to buffer: ByteBuffer) throws -> RowArrayWriter +where C.Element: RowCodable { + var array = RowArrayWriter(elementWidth: C.Element.rowFixedWidth ?? 8, buffer: buffer) + try array.reset(count: elements.count) + for (elementIndex, element) in elements.enumerated() { + try C.Element.writeRow(element, to: array, at: elementIndex) + } + return array +} + @inlinable @inline(__always) func rowNonNull(_ value: T?) throws -> T { @@ -336,3 +559,19 @@ func rowNullValue(_ type: Any.Type) -> ForyError { func rowIntOutOfRange(_ value: Int64) -> ForyError { .invalidData("int64 \(value) does not fit in Int") } + +@usableFromInline +@inline(never) +func rowUnknownCase(_ name: String, of type: Any.Type) -> ForyError { + .invalidData("unknown case \"\(name)\" for \(type)") +} + +@inline(never) +private func rowEncodedTooShort(_ count: Int) -> ForyError { + .invalidData("encoded row of \(count) bytes is shorter than its 8-byte schema hash") +} + +@inline(never) +private func rowSchemaHashMismatch(found: Int64, expected: Int64) -> ForyError { + .invalidData("row schema hash \(found) does not match the expected \(expected)") +} diff --git a/swift/Sources/Fory/Row/RowWriter.swift b/swift/Sources/Fory/Row/RowWriter.swift index 5bf36d1887..cbd220759f 100644 --- a/swift/Sources/Fory/Row/RowWriter.swift +++ b/swift/Sources/Fory/Row/RowWriter.swift @@ -146,6 +146,7 @@ public struct RowArrayWriter { self.init(elementWidth: element.dataType.byteWidth ?? 8, buffer: buffer) } + @usableFromInline init(elementWidth: Int, buffer: ByteBuffer) { self.elementWidth = elementWidth self.buffer = buffer @@ -240,6 +241,17 @@ public struct RowArrayWriter { buffer.copyBytes(start: base, end: buffer.count) } + // Fills every element of an int8 array at once, one byte each. + func writeInt8Elements(_ bytes: Data) { + precondition(elementWidth == 1 && bytes.count == elementCount, "\(bytes.count) bytes do not fill the array") + let start = base + headerSize + buffer.storage.withUnsafeMutableBytes { raw in + bytes.withUnsafeBytes { source in + UnsafeMutableRawBufferPointer(rebasing: raw[start..<(start + source.count)]).copyMemory(from: source) + } + } + } + private func elementOffset(_ index: Int) -> Int { precondition(index >= 0 && index < elementCount, "element index \(index) is outside 0..<\(elementCount)") return base + headerSize + index * elementWidth diff --git a/swift/Tests/ForyTests/RowEncoderTests.swift b/swift/Tests/ForyTests/RowEncoderTests.swift new file mode 100644 index 0000000000..eeb2ecbe83 --- /dev/null +++ b/swift/Tests/ForyTests/RowEncoderTests.swift @@ -0,0 +1,172 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation +import Testing +@testable import Fory + +private enum Color: String, RowCodable { + case red + case green +} + +private struct Point: RowStruct, Equatable { + var number: Int32 + var label: String? + + static let rowSchema = RowSchema([.inferred("label", String?.self), .inferred("number", Int32.self)]) + + static func writeRowFields(_ value: Point, to writer: RowWriter) throws { + try writer.write(value.label, at: 0) + try writer.write(value.number, at: 1) + } + + static func readRowFields(from row: Row) throws -> Point { + try Point(number: row.read(at: 1), label: row.read(at: 0)) + } +} + +private struct Bag: RowStruct, Equatable { + var names: [String?] + var ids: Set + var scores: [String: Int32?] + var blob: Data + var points: [Point] + var origin: Point? + var color: Color + + static let rowSchema = RowSchema([ + .inferred("blob", Data.self), .inferred("color", Color.self), .inferred("ids", Set.self), + .inferred("names", [String?].self), .inferred("origin", Point?.self), .inferred("points", [Point].self), + .inferred("scores", [String: Int32?].self) + ]) + + static func writeRowFields(_ value: Bag, to writer: RowWriter) throws { + try writer.write(value.blob, at: 0) + try writer.write(value.color, at: 1) + try writer.write(value.ids, at: 2) + try writer.write(value.names, at: 3) + try writer.write(value.origin, at: 4) + try writer.write(value.points, at: 5) + try writer.write(value.scores, at: 6) + } + + static func readRowFields(from row: Row) throws -> Bag { + try Bag( + names: row.read(at: 3), ids: row.read(at: 2), scores: row.read(at: 6), blob: row.read(at: 0), + points: row.read(at: 5), origin: row.read(at: 4), color: row.read(at: 1) + ) + } +} + +// One field of type T, to compare the bytes two Swift types produce for the same row type. +private struct Single: RowStruct { + var value: T + + static var rowSchema: RowSchema { RowSchema([.inferred("value", T.self)]) } + + static func writeRowFields(_ value: Single, to writer: RowWriter) throws { + try writer.write(value.value, at: 0) + } + + static func readRowFields(from row: Row) throws -> Single { + try Single(value: row.read(at: 0)) + } +} + +private let fullBag = Bag( + names: ["ab", nil, "xyz"], ids: [3, 1, 2], scores: ["k1": 1, "k2": nil], blob: Data([0, 1, 0xff, 0x7f, 0x80]), + points: [Point(number: 1, label: "a"), Point(number: 2, label: nil)], origin: Point(number: 5, label: "o"), color: .green +) + +private let emptyBag = Bag( + names: [], ids: [], scores: [:], blob: Data(), points: [], origin: nil, color: .red +) + +@Test +func rowEncoderFraming() throws { + let encoder = RowEncoder() + #expect(encoder.schemaHash == Bag.rowSchema.hash) + let encoded = try encoder.encode(fullBag) + let hash = withUnsafeBytes(of: encoder.schemaHash.littleEndian) { Data($0) } + #expect(encoded.prefix(8) == hash) + #expect(encoded.dropFirst(8) == (try encoder.toRow(fullBag))) + #expect(try encoder.decode(encoded) == fullBag) + + #expect(throws: ForyError.invalidData("encoded row of 7 bytes is shorter than its 8-byte schema hash")) { + try encoder.decode(encoded.prefix(7)) + } + var flipped = encoded + flipped[0] ^= 0x01 + let found = encoder.schemaHash ^ 0x01 + #expect(throws: ForyError.invalidData("row schema hash \(found) does not match the expected \(encoder.schemaHash)")) { + try encoder.decode(flipped) + } +} + +@Test +func rowEncoderCollections() throws { + let encoder = RowEncoder() + #expect(try encoder.fromRow(encoder.toRow(fullBag)) == fullBag) + #expect(try encoder.fromRow(encoder.toRow(emptyBag)) == emptyBag) + + // An empty map is its 8-byte keys size, then an empty keys array and an empty values array. + // Reading the map slot as binary gives those raw bytes. + let emptyMap = try RowEncoder>().toRow(Single(value: [:])) + let mapBytes = try Row(schema: Single<[String: Int32]>.rowSchema, bytes: Array(emptyMap)).binary(at: 0) + #expect(mapBytes == [0x08, 0, 0, 0, 0, 0, 0, 0] + [UInt8](repeating: 0, count: 16)) + + // Data copies its bytes in one go but must match an element by element [Int8] list. + let bytes = Data([0, 1, 0xff, 0x7f, 0x80]) + let int8s = bytes.map { Int8(bitPattern: $0) } + #expect(Single.rowSchema == Single<[Int8]>.rowSchema) + #expect(try RowEncoder>().toRow(Single(value: bytes)) == RowEncoder>().toRow(Single(value: int8s))) +} + +@Test +func rowEncoderDataNullElement() throws { + let schema = Single.rowSchema + var writer = RowWriter(schema: schema) + writer.reset() + var array = RowArrayWriter(element: RowField(name: "item", dataType: .int8, nullable: false), buffer: writer.buffer) + try array.reset(count: 2) + array.writeInt8(1, at: 0) + array.setNullAt(1) + try writer.setOffsetAndSize(at: 0, absoluteStart: array.base, size: array.size) + + #expect(throws: ForyError.invalidData("null row value cannot be read into non-optional Int8")) { + try RowEncoder>().fromRow(Data(writer.toBytes())) + } +} + +@Test +func rowEncoderEnum() throws { + let encoder = RowEncoder>() + #expect(encoder.schema.fields[0].dataType == .string) + #expect(try encoder.fromRow(encoder.toRow(Single(value: .green))).value == .green) + + let unknown = try RowEncoder>().toRow(Single(value: "blue")) + #expect(throws: ForyError.invalidData("unknown case \"blue\" for Color")) { + try encoder.fromRow(unknown) + } +} + +@Test +func rowEncoderDeterministic() throws { + let encoder = RowEncoder() + #expect(try encoder.toRow(fullBag) == encoder.toRow(fullBag)) +} From 7d90f34903787f333285c81461df2c5f94378beb Mon Sep 17 00:00:00 2001 From: yash Date: Tue, 10 Nov 2026 15:02:00 +0530 Subject: [PATCH 12/19] swift: Add the @ForyRow macro. @ForyRow reads a struct's stored properties and writes its RowStruct members: a schema with the fields sorted by name and renamed to lower underscore form, so every runtime puts them in the same slots under the same names, a writer that stores each field in its slot, and a reader that calls the memberwise initializer. Types a row cannot hold, such as unsigned integers, nested optionals, tuples, and Date map keys, are reported on the property itself. --- swift/Sources/Fory/MacroDeclarations.swift | 5 + swift/Sources/ForyMacro/ForyObjectMacro.swift | 5 +- swift/Sources/ForyMacro/ForyRowMacro.swift | 262 ++++++++++++++ swift/Tests/ForyTests/RowMacroTests.swift | 339 ++++++++++++++++++ 4 files changed, 609 insertions(+), 2 deletions(-) create mode 100644 swift/Sources/ForyMacro/ForyRowMacro.swift create mode 100644 swift/Tests/ForyTests/RowMacroTests.swift diff --git a/swift/Sources/Fory/MacroDeclarations.swift b/swift/Sources/Fory/MacroDeclarations.swift index 3d80b62731..e96ec45667 100644 --- a/swift/Sources/Fory/MacroDeclarations.swift +++ b/swift/Sources/Fory/MacroDeclarations.swift @@ -284,3 +284,8 @@ public macro ForyCase( @attached(peer) public macro ForyUnknownCase() = #externalMacro(module: "ForyMacro", type: "ForyUnknownCaseMacro") + +@attached(member, names: named(rowSchema), named(writeRowFields), named(readRowFields)) +@attached(extension, conformances: RowStruct) +/// Generates row format encoding for a struct's stored properties. +public macro ForyRow() = #externalMacro(module: "ForyMacro", type: "ForyRowMacro") diff --git a/swift/Sources/ForyMacro/ForyObjectMacro.swift b/swift/Sources/ForyMacro/ForyObjectMacro.swift index e7c805f21a..2e64f97646 100644 --- a/swift/Sources/ForyMacro/ForyObjectMacro.swift +++ b/swift/Sources/ForyMacro/ForyObjectMacro.swift @@ -36,7 +36,8 @@ struct ForySwiftPlugin: CompilerPlugin { SetFieldMacro.self, MapFieldMacro.self, ForyCaseMacro.self, - ForyUnknownCaseMacro.self + ForyUnknownCaseMacro.self, + ForyRowMacro.self ] } @@ -344,7 +345,7 @@ public struct ForyUnknownCaseMacro: PeerMacro { } } -private func serializerMemberAccessPrefix(_ declaration: some DeclGroupSyntax) -> String { +func serializerMemberAccessPrefix(_ declaration: some DeclGroupSyntax) -> String { let isPublicType = declaration.modifiers.contains(where: { modifier in modifier.name.tokenKind == .keyword(.public) || modifier.name.tokenKind == .keyword(.open) }) diff --git a/swift/Sources/ForyMacro/ForyRowMacro.swift b/swift/Sources/ForyMacro/ForyRowMacro.swift new file mode 100644 index 0000000000..32dcf7f606 --- /dev/null +++ b/swift/Sources/ForyMacro/ForyRowMacro.swift @@ -0,0 +1,262 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import SwiftDiagnostics +import SwiftSyntax +import SwiftSyntaxMacros + +// Adds `rowSchema`, `writeRowFields`, and `readRowFields` to a struct and makes it a `RowStruct`. +// Fields go into the schema sorted by name and renamed to lower underscore form, so every runtime +// agrees on the slot order and the field names. Decoding calls the memberwise initializer. +public struct ForyRowMacro: MemberMacro, ExtensionMacro { + public static func expansion( + of _: AttributeSyntax, + providingMembersOf declaration: some DeclGroupSyntax, + conformingTo _: [TypeSyntax], + in _: some MacroExpansionContext + ) throws -> [DeclSyntax] { + let fields = try parseRowFields(declaration) + let access = serializerMemberAccessPrefix(declaration) + let sorted = fields.indices.sorted { fields[$0].name.utf16.lexicographicallyPrecedes(fields[$1].name.utf16) } + var ordinals = [Int](repeating: 0, count: fields.count) + for (ordinal, fieldIndex) in sorted.enumerated() { + ordinals[fieldIndex] = ordinal + } + + let schemaFields = sorted.map { index in + " RowField.inferred(\"\(rowFieldName(fields[index].name))\", (\(fields[index].typeText)).self)" + } + let schema = schemaFields.isEmpty ? "RowSchema([])" : "RowSchema([\n\(schemaFields.joined(separator: ",\n"))\n])" + let writes = sorted.enumerated().map { ordinal, index in + " try writer.write(value.\(fields[index].codeName), at: \(ordinal))" + } + let arguments = fields.indices.map { index in + " \(fields[index].codeName): row.read(at: \(ordinals[index]))" + } + // With no fields there is nothing that throws, and a bare `try` would warn. + let construction = arguments.isEmpty ? "Self()" : "try Self(\n\(arguments.joined(separator: ",\n"))\n )" + + return [ + "\(raw: access)static let rowSchema = \(raw: schema)", + """ + \(raw: access)static func writeRowFields(_ value: Self, to writer: RowWriter) throws { + \(raw: writes.joined(separator: "\n")) + } + """, + """ + \(raw: access)static func readRowFields(from row: Row) throws -> Self { + \(raw: construction) + } + """ + ] + } + + public static func expansion( + of _: AttributeSyntax, + attachedTo declaration: some DeclGroupSyntax, + providingExtensionsOf type: some TypeSyntaxProtocol, + conformingTo protocols: [TypeSyntax], + in _: some MacroExpansionContext + ) throws -> [ExtensionDeclSyntax] { + // The member expansion reports what is wrong. Adding the conformance anyway would only add + // errors about missing members on top. + guard !protocols.isEmpty, (try? parseRowFields(declaration)) != nil else { + return [] + } + return [try ExtensionDeclSyntax("extension \(type.trimmed): RowStruct {}")] + } +} + +// Each ASCII capital letter becomes an underscore followed by its lowercase letter, and every other +// character stays, so `userID` becomes `user_i_d` and `URLValue` becomes `_u_r_l_value`. +func rowFieldName(_ name: String) -> String { + var result = String.UnicodeScalarView() + for scalar in name.unicodeScalars { + if scalar.isASCII, let ascii = UInt8(exactly: scalar.value), ascii >= 0x41, ascii <= 0x5A { + result.append("_") + result.append(Unicode.Scalar(ascii + 0x20)) + } else { + result.append(scalar) + } + } + return String(result) +} + +private struct RowMacroField { + // Without backticks, for sorting and the schema name. + let name: String + // As written, so a keyword such as `default` keeps its backticks in generated code. + let codeName: String + let typeText: String +} + +private let rowUnsignedTypes: Set = ["UInt", "UInt8", "UInt16", "UInt32", "UInt64"] + +private func parseRowFields(_ declaration: some DeclGroupSyntax) throws -> [RowMacroField] { + guard let structDecl = declaration.as(StructDeclSyntax.self) else { + throw DiagnosticsError(diagnostics: [rowDiagnostic(declaration.introducer, "@ForyRow can only be applied to a struct")]) + } + if let generics = structDecl.genericParameterClause { + throw DiagnosticsError(diagnostics: [rowDiagnostic(generics, "@ForyRow does not support generic structs")]) + } + let structName = structDecl.name.text + var fields: [RowMacroField] = [] + var diagnostics: [Diagnostic] = [] + for member in structDecl.memberBlock.members { + guard let variable = member.decl.as(VariableDeclSyntax.self) else { + continue + } + let skipped: [TokenKind] = [.keyword(.static), .keyword(.class), .keyword(.lazy)] + if variable.modifiers.contains(where: { skipped.contains($0.name.tokenKind) }) { + continue + } + let isLet = variable.bindingSpecifier.tokenKind == .keyword(.let) + for binding in variable.bindings { + guard let pattern = binding.pattern.as(IdentifierPatternSyntax.self) else { + let text = binding.pattern.trimmedDescription + diagnostics.append( + rowDiagnostic(binding.pattern, "@ForyRow does not support the tuple pattern '\(text)'; declare each property on its own") + ) + continue + } + guard rowIsStored(binding.accessorBlock) else { + continue + } + let name = pattern.identifier.identifier?.name ?? pattern.identifier.text + if let attribute = variable.attributes.first { + diagnostics.append( + rowDiagnostic(attribute, "@ForyRow field '\(name)' uses an attribute or property wrapper, which row encoding does not support") + ) + continue + } + // The memberwise initializer has no parameter for such a property, so decoding could not set it. + if isLet, binding.initializer != nil { + diagnostics.append(rowDiagnostic(binding, "@ForyRow field '\(name)' is a let with a default value, so decoding cannot set it; use var")) + continue + } + guard let type = binding.typeAnnotation?.type else { + diagnostics.append(rowDiagnostic(binding, "@ForyRow field '\(name)' needs its own type annotation")) + continue + } + if let problem = rowTypeProblem(type, structName: structName) { + diagnostics.append(rowDiagnostic(binding, "@ForyRow field '\(name)' \(problem)")) + continue + } + fields.append(RowMacroField(name: name, codeName: pattern.identifier.text, typeText: type.trimmedDescription)) + } + } + if !diagnostics.isEmpty { + throw DiagnosticsError(diagnostics: diagnostics) + } + return fields +} + +// A property is stored when it has no accessors, or only willSet and didSet observers. +private func rowIsStored(_ accessorBlock: AccessorBlockSyntax?) -> Bool { + guard let accessorBlock else { + return true + } + guard case .accessors(let accessors) = accessorBlock.accessors else { + return false + } + let observers: [TokenKind] = [.keyword(.willSet), .keyword(.didSet)] + return accessors.allSatisfy { observers.contains($0.accessorSpecifier.tokenKind) } +} + +// Returns why `type` cannot be a row field, or nil. Anything not rejected here must still conform to +// RowCodable, which the compiler checks. +private func rowTypeProblem(_ type: TypeSyntax, structName: String) -> String? { + if let optional = type.as(OptionalTypeSyntax.self) { + if rowIsOptional(optional.wrappedType) { + return "uses the nested optional '\(type.trimmedDescription)'" + } + return rowTypeProblem(optional.wrappedType, structName: structName) + } + if type.is(ImplicitlyUnwrappedOptionalTypeSyntax.self) { + return "uses the implicitly unwrapped optional '\(type.trimmedDescription)'; use '?' instead" + } + if let array = type.as(ArrayTypeSyntax.self) { + return rowTypeProblem(array.element, structName: structName) + } + if let dictionary = type.as(DictionaryTypeSyntax.self) { + return rowMapProblem(key: dictionary.key, value: dictionary.value, structName: structName) + } + if let tuple = type.as(TupleTypeSyntax.self) { + // `(Int32)` is only parentheses around one type. + if tuple.elements.count == 1, let only = tuple.elements.first, only.firstName == nil { + return rowTypeProblem(only.type, structName: structName) + } + return "uses the tuple type '\(type.trimmedDescription)', which has no row type" + } + if let identifier = type.as(IdentifierTypeSyntax.self) { + let name = identifier.name.text + if name == structName || name == "Self" { + return "refers to '\(structName)' itself, which a row cannot hold" + } + return rowNamedTypeProblem(name, identifier.genericArgumentClause, whole: type, structName: structName) + } + if let member = type.as(MemberTypeSyntax.self) { + return rowNamedTypeProblem(member.name.text, member.genericArgumentClause, whole: type, structName: structName) + } + return "uses the type '\(type.trimmedDescription)', which has no row type" +} + +private func rowNamedTypeProblem( + _ name: String, + _ generics: GenericArgumentClauseSyntax?, + whole: TypeSyntax, + structName: String +) -> String? { + if rowUnsignedTypes.contains(name) { + return "uses unsigned type '\(name)'; use Data for bytes or a signed integer type" + } + let arguments = generics?.arguments.map(\.argument) ?? [] + if name == "Optional", let wrapped = arguments.first, rowIsOptional(wrapped) { + return "uses the nested optional '\(whole.trimmedDescription)'" + } + if name == "Dictionary", arguments.count == 2 { + return rowMapProblem(key: arguments[0], value: arguments[1], structName: structName) + } + for argument in arguments { + if let problem = rowTypeProblem(argument, structName: structName) { + return problem + } + } + return nil +} + +// Map keys are never null, and a Date or Duration key is rounded to whole microseconds when written, +// so two different keys could come back equal. +private func rowMapProblem(key: TypeSyntax, value: TypeSyntax, structName: String) -> String? { + if rowIsOptional(key) { + return "uses the optional map key '\(key.trimmedDescription)'; map keys are never null" + } + let keyName = key.as(IdentifierTypeSyntax.self)?.name.text ?? key.as(MemberTypeSyntax.self)?.name.text + if let keyName, keyName == "Date" || keyName == "Duration" { + return "uses '\(keyName)' as a map key, which is not supported" + } + return rowTypeProblem(key, structName: structName) ?? rowTypeProblem(value, structName: structName) +} + +private func rowIsOptional(_ type: TypeSyntax) -> Bool { + type.is(OptionalTypeSyntax.self) || type.is(ImplicitlyUnwrappedOptionalTypeSyntax.self) + || type.as(IdentifierTypeSyntax.self)?.name.text == "Optional" +} + +private func rowDiagnostic(_ node: some SyntaxProtocol, _ message: String) -> Diagnostic { + Diagnostic(node: node, message: MacroExpansionErrorMessage(message)) +} diff --git a/swift/Tests/ForyTests/RowMacroTests.swift b/swift/Tests/ForyTests/RowMacroTests.swift new file mode 100644 index 0000000000..be518bae6c --- /dev/null +++ b/swift/Tests/ForyTests/RowMacroTests.swift @@ -0,0 +1,339 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import SwiftSyntaxMacroExpansion +import SwiftSyntaxMacros +import SwiftSyntaxMacrosTestSupport +import Testing +@testable import ForyMacro + +private let rowMacroSpecs = ["ForyRow": MacroSpec(type: ForyRowMacro.self, conformances: ["RowStruct"])] + +private func assertRowExpansion(_ source: String, _ expandedSource: String) { + assertMacroExpansion(source, expandedSource: expandedSource, macroSpecs: rowMacroSpecs) +} + +// A rejected declaration expands to itself, without the attribute and with nothing added. +private func assertRowRejects(_ source: String, _ expandedSource: String, _ message: String, line: Int, column: Int) { + assertMacroExpansion( + source, + expandedSource: expandedSource, + diagnostics: [DiagnosticSpec(message: message, line: line, column: column)], + macroSpecs: rowMacroSpecs + ) +} + +@Test +func rowMacroExpansion() { + assertRowExpansion( + """ + @ForyRow + struct Profile { + var userID: Int64 + var age: Int32? + } + """, + """ + struct Profile { + var userID: Int64 + var age: Int32? + + static let rowSchema = RowSchema([ + RowField.inferred("age", (Int32?).self), + RowField.inferred("user_i_d", (Int64).self) + ]) + + static func writeRowFields(_ value: Self, to writer: RowWriter) throws { + try writer.write(value.age, at: 0) + try writer.write(value.userID, at: 1) + } + + static func readRowFields(from row: Row) throws -> Self { + try Self( + userID: row.read(at: 1), + age: row.read(at: 0) + ) + } + } + + extension Profile: RowStruct { + } + """ + ) +} + +@Test +func rowMacroPublic() { + assertRowExpansion( + """ + @ForyRow + public struct Empty { + } + """, + """ + public struct Empty { + + public static let rowSchema = RowSchema([]) + + public static func writeRowFields(_ value: Self, to writer: RowWriter) throws { + + } + + public static func readRowFields(from row: Row) throws -> Self { + Self() + } + } + + extension Empty: RowStruct { + } + """ + ) +} + +@Test +func rowMacroSkips() { + assertRowExpansion( + """ + @ForyRow + struct Counter { + static var shared: Int32 = 0 + lazy var cache: Int32 = 0 + var doubled: Int32 { count * 2 } + var label: String { + get { "" } + set {} + } + var count: Int32 { + didSet {} + } + } + """, + """ + struct Counter { + static var shared: Int32 = 0 + lazy var cache: Int32 = 0 + var doubled: Int32 { count * 2 } + var label: String { + get { "" } + set {} + } + var count: Int32 { + didSet {} + } + + static let rowSchema = RowSchema([ + RowField.inferred("count", (Int32).self) + ]) + + static func writeRowFields(_ value: Self, to writer: RowWriter) throws { + try writer.write(value.count, at: 0) + } + + static func readRowFields(from row: Row) throws -> Self { + try Self( + count: row.read(at: 0) + ) + } + } + + extension Counter: RowStruct { + } + """ + ) +} + +@Test +func rowMacroFieldName() { + let cases = [ + ("userID", "user_i_d"), ("httpURL", "http_u_r_l"), ("f1", "f1"), ("field2Name", "field2_name"), + ("URLValue", "_u_r_l_value"), ("x", "x"), ("already", "already") + ] + for (name, expected) in cases { + #expect(rowFieldName(name) == expected, "\(name)") + } +} + +@Test +func rowMacroOrder() { + assertRowExpansion( + """ + @ForyRow + struct Names { + var x: Int8 + var userID: Int8 + var URLValue: Int8 + var already: Int8 + } + """, + """ + struct Names { + var x: Int8 + var userID: Int8 + var URLValue: Int8 + var already: Int8 + + static let rowSchema = RowSchema([ + RowField.inferred("_u_r_l_value", (Int8).self), + RowField.inferred("already", (Int8).self), + RowField.inferred("user_i_d", (Int8).self), + RowField.inferred("x", (Int8).self) + ]) + + static func writeRowFields(_ value: Self, to writer: RowWriter) throws { + try writer.write(value.URLValue, at: 0) + try writer.write(value.already, at: 1) + try writer.write(value.userID, at: 2) + try writer.write(value.x, at: 3) + } + + static func readRowFields(from row: Row) throws -> Self { + try Self( + x: row.read(at: 3), + userID: row.read(at: 2), + URLValue: row.read(at: 0), + already: row.read(at: 1) + ) + } + } + + extension Names: RowStruct { + } + """ + ) +} + +@Test +func rowMacroBacktick() { + assertRowExpansion( + """ + @ForyRow + struct Option { + var `default`: Int32 + } + """, + """ + struct Option { + var `default`: Int32 + + static let rowSchema = RowSchema([ + RowField.inferred("default", (Int32).self) + ]) + + static func writeRowFields(_ value: Self, to writer: RowWriter) throws { + try writer.write(value.`default`, at: 0) + } + + static func readRowFields(from row: Row) throws -> Self { + try Self( + `default`: row.read(at: 0) + ) + } + } + + extension Option: RowStruct { + } + """ + ) +} + +@Test +func rowMacroRejectsFieldTypes() { + let cases: [(declaration: String, message: String)] = [ + ("var count: UInt32", "@ForyRow field 'count' uses unsigned type 'UInt32'; use Data for bytes or a signed integer type"), + ("var ids: [UInt8]", "@ForyRow field 'ids' uses unsigned type 'UInt8'; use Data for bytes or a signed integer type"), + ("var value: Int??", "@ForyRow field 'value' uses the nested optional 'Int??'"), + ("var pair: (Int32, Int32)", "@ForyRow field 'pair' uses the tuple type '(Int32, Int32)', which has no row type"), + ("var times: [Date: Int32]", "@ForyRow field 'times' uses 'Date' as a map key, which is not supported"), + ("var names: [String?: Int32]", "@ForyRow field 'names' uses the optional map key 'String?'; map keys are never null"), + ("let version: Int32 = 1", "@ForyRow field 'version' is a let with a default value, so decoding cannot set it; use var"), + ("var count = 0", "@ForyRow field 'count' needs its own type annotation"), + ("var next: [Bad]", "@ForyRow field 'next' refers to 'Bad' itself, which a row cannot hold") + ] + for (declaration, message) in cases { + assertRowRejects( + "@ForyRow\nstruct Bad {\n \(declaration)\n}", + "struct Bad {\n \(declaration)\n}", + message, + line: 3, + column: 9 + ) + } +} + +@Test +func rowMacroRejectsDeclarations() { + assertRowRejects( + "@ForyRow\nstruct Bad {\n var (a, b): (Int32, Int32)\n}", + "struct Bad {\n var (a, b): (Int32, Int32)\n}", + "@ForyRow does not support the tuple pattern '(a, b)'; declare each property on its own", + line: 3, + column: 9 + ) + assertRowRejects( + "@ForyRow\nstruct Bad {\n @Wrapped var name: String\n}", + "struct Bad {\n @Wrapped var name: String\n}", + "@ForyRow field 'name' uses an attribute or property wrapper, which row encoding does not support", + line: 3, + column: 5 + ) + // In `var a, b: Int32` only `b` carries the annotation. + assertRowRejects( + "@ForyRow\nstruct Bad {\n var a, b: Int32\n}", + "struct Bad {\n var a, b: Int32\n}", + "@ForyRow field 'a' needs its own type annotation", + line: 3, + column: 9 + ) + assertRowRejects( + "@ForyRow\nenum Bad {\n case one\n}", + "enum Bad {\n case one\n}", + "@ForyRow can only be applied to a struct", + line: 2, + column: 1 + ) + assertRowRejects( + "@ForyRow\nclass Bad {\n}", + "class Bad {\n}", + "@ForyRow can only be applied to a struct", + line: 2, + column: 1 + ) + assertRowRejects( + "@ForyRow\nstruct Bad {\n}", + "struct Bad {\n}", + "@ForyRow does not support generic structs", + line: 2, + column: 11 + ) +} + +@Test +func rowMacroReportsEveryField() { + assertMacroExpansion( + "@ForyRow\nstruct Bad {\n var a: UInt8\n var b: Int??\n}", + expandedSource: "struct Bad {\n var a: UInt8\n var b: Int??\n}", + diagnostics: [ + DiagnosticSpec( + message: "@ForyRow field 'a' uses unsigned type 'UInt8'; use Data for bytes or a signed integer type", + line: 3, + column: 9 + ), + DiagnosticSpec(message: "@ForyRow field 'b' uses the nested optional 'Int??'", line: 4, column: 9) + ], + macroSpecs: rowMacroSpecs + ) +} From 3cf2ff7006271f4f483cf4364c7a5005118031dc Mon Sep 17 00:00:00 2001 From: yash Date: Wed, 11 Nov 2026 23:10:00 +0530 Subject: [PATCH 13/19] swift: Test @ForyRow encoding end to end. The fixtures are rows another runtime wrote for the shared cross-language models, so these tests catch layout drift before the interop runs. --- swift/Tests/ForyTests/RowGoldenData.swift | 88 ++++++ .../ForyTests/RowMacroEncodingTests.swift | 295 ++++++++++++++++++ .../Tests/ForyTests/RowSchemaBytesTests.swift | 13 +- 3 files changed, 385 insertions(+), 11 deletions(-) create mode 100644 swift/Tests/ForyTests/RowGoldenData.swift create mode 100644 swift/Tests/ForyTests/RowMacroEncodingTests.swift diff --git a/swift/Tests/ForyTests/RowGoldenData.swift b/swift/Tests/ForyTests/RowGoldenData.swift new file mode 100644 index 0000000000..0f00911ea1 --- /dev/null +++ b/swift/Tests/ForyTests/RowGoldenData.swift @@ -0,0 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Bytes written by the reference row encoder in java/fory-format for the cross-language test models +// CrossLanguageTest.Foo, CrossLanguageTest.A (StringMap here), and GoCrossLanguageTest.Blob, with +// their create() values. Regenerate them from those models; never edit them by hand. + +// The Foo row without the schema hash. +let goldenFooRow: [UInt8] = [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, + 0x38, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0xa0, 0x00, 0x00, 0x00, + 0x70, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x10, 0x01, 0x00, 0x00, + 0x73, 0x74, 0x72, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, + 0x73, 0x74, 0x72, 0x31, 0x00, 0x00, 0x00, 0x00, 0x73, 0x74, 0x72, 0x32, + 0x00, 0x00, 0x00, 0x00, 0x70, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x48, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x50, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x58, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x60, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x68, 0x00, 0x00, 0x00, 0x6b, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x6b, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6b, 0x33, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x6b, 0x34, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x6b, 0x35, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6b, 0x36, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x05, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x03, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x73, 0x74, 0x72, 0x00, + 0x00, 0x00, 0x00, 0x00 +] + +// The Foo schema, one line per field. +let goldenFooSchema: [UInt8] = [ + 0x01, 0x05, + 0x46, 0x0b, 0xa8, 0x04, + 0x46, 0x0b, 0xb0, 0x15, + 0x46, 0x0b, 0xb8, 0x16, 0x49, 0x22, 0x64, 0x60, 0x15, + 0x46, 0x0b, 0xc0, 0x18, 0x05, 0x28, 0x98, 0x15, 0x4d, 0xd4, 0x0b, 0xa1, 0x00, 0x04, + 0x46, 0x0b, 0xc8, 0x1b, 0x02, 0x46, 0x0b, 0xa8, 0x04, 0x46, 0x0b, 0xb0, 0x15 +] + +// The StringMap schema hash followed by its row, as an encoder writes them. +let goldenStringMapEncoded: [UInt8] = [ + 0xc5, 0xbd, 0xf1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x88, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x28, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, + 0x03, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x6b, 0x31, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x69, 0x70, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x70, 0x69, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, + 0x30, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, + 0x76, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x2e, 0x30, 0x2e, + 0x30, 0x2e, 0x30, 0x00, 0x31, 0x32, 0x33, 0x34, 0x35, 0x00, 0x00, 0x00 +] + +// The Blob schema hash followed by its row, as an encoder writes them. +let goldenBlobEncoded: [UInt8] = [ + 0x38, 0x0d, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, + 0x05, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x01, 0xff, 0x7f, 0x80, 0x00, 0x00, 0x00, 0x62, 0x79, 0x74, 0x65, + 0x73, 0x00, 0x00, 0x00 +] diff --git a/swift/Tests/ForyTests/RowMacroEncodingTests.swift b/swift/Tests/ForyTests/RowMacroEncodingTests.swift new file mode 100644 index 0000000000..31f52b77f8 --- /dev/null +++ b/swift/Tests/ForyTests/RowMacroEncodingTests.swift @@ -0,0 +1,295 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation +import Fory +import Testing + +// The models every row format runtime shares in its cross-language tests. + +@ForyRow +private struct Bar: Equatable { + var f1: Int32? + var f2: String? +} + +@ForyRow +private struct Foo: Equatable { + var f1: Int32? + var f2: String? + var f3: [String?]? + var f4: [String: Int32?]? + var f5: Bar? +} + +@ForyRow +private struct StringMap: Equatable { + var f1: Int32? + var f2: [String: String]? +} + +@ForyRow +private struct Blob: Equatable { + var f1: Data? + var f2: String? +} + +private let fooValue = Foo( + f1: 1, f2: "str", f3: ["str1", nil, "str2"], f4: ["k1": 1, "k2": 2, "k3": 3, "k4": 4, "k5": 5, "k6": 6], + f5: Bar(f1: 1, f2: "str") +) +private let stringMapValue = StringMap(f1: 1, f2: ["pid": "12345", "ip": "0.0.0.0", "k1": "v1"]) +private let blobValue = Blob(f1: Data([0, 1, 0xff, 0x7f, 0x80]), f2: "bytes") + +// Field counts around the bitmap's 64-bit word boundary. + +@ForyRow +private struct Empty: Equatable {} + +@ForyRow +private struct Wide63: Equatable { + var f00: Int64 = 0, f01: Int64 = 0, f02: Int64 = 0, f03: Int64 = 0, f04: Int64 = 0, f05: Int64 = 0, f06: Int64 = 0, f07: Int64 = 0 + var f08: Int64 = 0, f09: Int64 = 0, f10: Int64 = 0, f11: Int64 = 0, f12: Int64 = 0, f13: Int64 = 0, f14: Int64 = 0, f15: Int64 = 0 + var f16: Int64 = 0, f17: Int64 = 0, f18: Int64 = 0, f19: Int64 = 0, f20: Int64 = 0, f21: Int64 = 0, f22: Int64 = 0, f23: Int64 = 0 + var f24: Int64 = 0, f25: Int64 = 0, f26: Int64 = 0, f27: Int64 = 0, f28: Int64 = 0, f29: Int64 = 0, f30: Int64 = 0, f31: Int64 = 0 + var f32: Int64 = 0, f33: Int64 = 0, f34: Int64 = 0, f35: Int64 = 0, f36: Int64 = 0, f37: Int64 = 0, f38: Int64 = 0, f39: Int64 = 0 + var f40: Int64 = 0, f41: Int64 = 0, f42: Int64 = 0, f43: Int64 = 0, f44: Int64 = 0, f45: Int64 = 0, f46: Int64 = 0, f47: Int64 = 0 + var f48: Int64 = 0, f49: Int64 = 0, f50: Int64 = 0, f51: Int64 = 0, f52: Int64 = 0, f53: Int64 = 0, f54: Int64 = 0, f55: Int64 = 0 + var f56: Int64 = 0, f57: Int64 = 0, f58: Int64 = 0, f59: Int64 = 0, f60: Int64 = 0, f61: Int64 = 0, f62: Int64 = 0 +} + +@ForyRow +private struct Wide64: Equatable { + var f00: Int64 = 0, f01: Int64 = 0, f02: Int64 = 0, f03: Int64 = 0, f04: Int64 = 0, f05: Int64 = 0, f06: Int64 = 0, f07: Int64 = 0 + var f08: Int64 = 0, f09: Int64 = 0, f10: Int64 = 0, f11: Int64 = 0, f12: Int64 = 0, f13: Int64 = 0, f14: Int64 = 0, f15: Int64 = 0 + var f16: Int64 = 0, f17: Int64 = 0, f18: Int64 = 0, f19: Int64 = 0, f20: Int64 = 0, f21: Int64 = 0, f22: Int64 = 0, f23: Int64 = 0 + var f24: Int64 = 0, f25: Int64 = 0, f26: Int64 = 0, f27: Int64 = 0, f28: Int64 = 0, f29: Int64 = 0, f30: Int64 = 0, f31: Int64 = 0 + var f32: Int64 = 0, f33: Int64 = 0, f34: Int64 = 0, f35: Int64 = 0, f36: Int64 = 0, f37: Int64 = 0, f38: Int64 = 0, f39: Int64 = 0 + var f40: Int64 = 0, f41: Int64 = 0, f42: Int64 = 0, f43: Int64 = 0, f44: Int64 = 0, f45: Int64 = 0, f46: Int64 = 0, f47: Int64 = 0 + var f48: Int64 = 0, f49: Int64 = 0, f50: Int64 = 0, f51: Int64 = 0, f52: Int64 = 0, f53: Int64 = 0, f54: Int64 = 0, f55: Int64 = 0 + var f56: Int64 = 0, f57: Int64 = 0, f58: Int64 = 0, f59: Int64 = 0, f60: Int64 = 0, f61: Int64 = 0, f62: Int64 = 0, f63: Int64 = 0 +} + +@ForyRow +private struct Wide65: Equatable { + var f00: Int64 = 0, f01: Int64 = 0, f02: Int64 = 0, f03: Int64 = 0, f04: Int64 = 0, f05: Int64 = 0, f06: Int64 = 0, f07: Int64 = 0 + var f08: Int64 = 0, f09: Int64 = 0, f10: Int64 = 0, f11: Int64 = 0, f12: Int64 = 0, f13: Int64 = 0, f14: Int64 = 0, f15: Int64 = 0 + var f16: Int64 = 0, f17: Int64 = 0, f18: Int64 = 0, f19: Int64 = 0, f20: Int64 = 0, f21: Int64 = 0, f22: Int64 = 0, f23: Int64 = 0 + var f24: Int64 = 0, f25: Int64 = 0, f26: Int64 = 0, f27: Int64 = 0, f28: Int64 = 0, f29: Int64 = 0, f30: Int64 = 0, f31: Int64 = 0 + var f32: Int64 = 0, f33: Int64 = 0, f34: Int64 = 0, f35: Int64 = 0, f36: Int64 = 0, f37: Int64 = 0, f38: Int64 = 0, f39: Int64 = 0 + var f40: Int64 = 0, f41: Int64 = 0, f42: Int64 = 0, f43: Int64 = 0, f44: Int64 = 0, f45: Int64 = 0, f46: Int64 = 0, f47: Int64 = 0 + var f48: Int64 = 0, f49: Int64 = 0, f50: Int64 = 0, f51: Int64 = 0, f52: Int64 = 0, f53: Int64 = 0, f54: Int64 = 0, f55: Int64 = 0 + var f56: Int64 = 0, f57: Int64 = 0, f58: Int64 = 0, f59: Int64 = 0, f60: Int64 = 0, f61: Int64 = 0, f62: Int64 = 0, f63: Int64 = 0 + var f64: Int64 = 0 +} + +private enum Color: String, RowCodable { + case red + case green +} + +@ForyRow +private struct AllNullable: Equatable { + var flag: Bool? + var tiny: Int8? + var short: Int16? + var int: Int32? + var long: Int64? + var word: Int? + var real: Float? + var double: Double? + var text: String? + var time: Date? + var day: LocalDate? + var span: Duration? + var blob: Data? + var list: [Int32]? + var tags: Set? + var counts: [String: Int32]? + var bar: Bar? + var color: Color? +} + +@ForyRow +private struct Nested: Equatable { + var groups: [String: [Bar?]]? +} + +@ForyRow +private struct Names: Equatable { + var userID: Int8 + var httpURL: Int8 + var f1: Int8 + var field2Name: Int8 + var URLValue: Int8 + // swiftlint:disable:next identifier_name + var x: Int8 + var already: Int8 +} + +@ForyRow +private struct Secret { + private var token: String + var id: Int32 +} + +// An initializer in the struct body would remove the memberwise initializer that decoding calls. +extension Secret { + init(id: Int32, secret: String) { + self.init(token: secret, id: id) + } + + var revealed: String { token } +} + +@ForyRow +private struct OptionalName { + var name: String? +} + +@ForyRow +private struct RequiredName { + var name: String +} + +private func checkRoundTrip(_ value: T, size: Int) throws { + let encoder = RowEncoder() + let row = try encoder.toRow(value) + #expect(row.count == size, "\(T.self)") + #expect(try encoder.fromRow(row) == value, "\(T.self)") +} + +@Test +func rowMacroSchemaGolden() throws { + #expect(try Foo.rowSchema.toBytes() == goldenFooSchema) + #expect(Foo.rowSchema.hash == 14_057_974_700_829_126) + #expect(StringMap.rowSchema.hash == 15_842_757) + #expect(Blob.rowSchema.hash == 527_672) +} + +@Test +func rowMacroDecodeGolden() throws { + let fooEncoder = RowEncoder() + #expect(try fooEncoder.fromRow(Data(goldenFooRow)) == fooValue) + #expect(try fooEncoder.decode(fooEncoder.encode(fooValue)) == fooValue) + let stringMapEncoder = RowEncoder() + #expect(try stringMapEncoder.decode(Data(goldenStringMapEncoded)) == stringMapValue) + #expect(try stringMapEncoder.decode(stringMapEncoder.encode(stringMapValue)) == stringMapValue) + #expect(try RowEncoder().decode(Data(goldenBlobEncoded)) == blobValue) +} + +// Blob holds no map, so its bytes do not depend on dictionary order and must match exactly. +@Test +func rowMacroBlobBytes() throws { + #expect(try RowEncoder().encode(blobValue) == Data(goldenBlobEncoded)) +} + +@Test +func rowMacroFieldCounts() throws { + try checkRoundTrip(Empty(), size: 0) + var wide63 = Wide63() + wide63.f00 = 1 + wide63.f62 = .min + try checkRoundTrip(wide63, size: 8 + 63 * 8) + var wide64 = Wide64() + wide64.f00 = 1 + wide64.f63 = .min + try checkRoundTrip(wide64, size: 8 + 64 * 8) + var wide65 = Wide65() + wide65.f00 = 1 + wide65.f64 = .min + try checkRoundTrip(wide65, size: 16 + 65 * 8) +} + +@Test +func rowMacroAllNull() throws { + let encoder = RowEncoder() + let row = try encoder.toRow(AllNullable()) + // Eighteen fields set the low 18 bitmap bits, and every slot stays zero. + #expect(Array(row) == [0xff, 0xff, 0x03] + [UInt8](repeating: 0, count: 5 + 18 * 8)) + #expect(try encoder.fromRow(row) == AllNullable()) +} + +@Test +func rowMacroAllValues() throws { + let full = AllNullable( + flag: true, tiny: -8, short: -16, int: 32, long: .min, word: .max, real: -0.0, double: .nan, text: "text", + time: Date(timeIntervalSince1970: 1_700_000_000), day: LocalDate(epochDay: 19_000), span: .seconds(-90), + blob: Data([1, 2]), list: [1, 2, 3], tags: ["a", "b"], counts: ["k": 1], bar: Bar(f1: nil, f2: "bar"), color: .green + ) + let encoder = RowEncoder() + var decoded = try encoder.fromRow(encoder.toRow(full)) + #expect(decoded.real?.bitPattern == full.real?.bitPattern) + #expect(decoded.double?.bitPattern == full.double?.bitPattern) + // NaN never equals itself, so the floats are compared by bit pattern above and cleared here. + var expected = full + (expected.real, expected.double, decoded.real, decoded.double) = (nil, nil, nil, nil) + #expect(decoded == expected) +} + +@Test +func rowMacroNestedInListInMap() throws { + let encoder = RowEncoder() + let values = [ + Nested(groups: ["a": [Bar(f1: 1, f2: "x"), nil, Bar(f1: nil, f2: nil)], "b": []]), + Nested(groups: [:]), + Nested(groups: nil) + ] + for value in values { + #expect(try encoder.decode(encoder.encode(value)) == value) + } +} + +@Test +func rowMacroPrivateProperty() throws { + let encoder = RowEncoder() + #expect(encoder.schema.fields.map(\.name) == ["id", "token"]) + let decoded = try encoder.decode(encoder.encode(Secret(id: 7, secret: "hidden"))) + #expect(decoded.id == 7) + #expect(decoded.revealed == "hidden") +} + +@Test +func rowMacroNames() throws { + let encoder = RowEncoder() + let expected = ["_u_r_l_value", "already", "f1", "field2_name", "http_u_r_l", "user_i_d", "x"] + #expect(encoder.schema.fields.map(\.name) == expected) + let names = Names(userID: 1, httpURL: 2, f1: 3, field2Name: 4, URLValue: 5, x: 6, already: 7) + let row = try encoder.toRow(names) + #expect(try encoder.fromRow(row) == names) + // Slots follow the sorted names, so URLValue comes first and x last. + let view = try Row(schema: encoder.schema, bytes: Array(row)) + #expect(try view.read(Int8.self, at: 0) == 5) + #expect(try view.read(Int8.self, at: 6) == 6) +} + +@Test +func rowMacroDeterministic() throws { + let encoder = RowEncoder() + #expect(try encoder.encode(fooValue) == encoder.encode(fooValue)) +} + +// A row schema marks String nullable whether or not the property is optional, so both structs share +// one schema, and a null written by one cannot be read into the other. +@Test +func rowMacroNullIntoValue() throws { + #expect(OptionalName.rowSchema == RequiredName.rowSchema) + let row = try RowEncoder().toRow(OptionalName(name: nil)) + #expect(throws: ForyError.invalidData("null row value cannot be read into non-optional String")) { + try RowEncoder().fromRow(row) + } +} diff --git a/swift/Tests/ForyTests/RowSchemaBytesTests.swift b/swift/Tests/ForyTests/RowSchemaBytesTests.swift index c0dd11b092..7da47950f8 100644 --- a/swift/Tests/ForyTests/RowSchemaBytesTests.swift +++ b/swift/Tests/ForyTests/RowSchemaBytesTests.swift @@ -72,15 +72,6 @@ private let fooSchema = RowSchema([ field("f5", .struct([field("f1", .int32), field("f2", .string)])) ]) -private let fooBytes: [UInt8] = [ - 0x01, 0x05, - 0x46, 0x0b, 0xa8, 0x04, - 0x46, 0x0b, 0xb0, 0x15, - 0x46, 0x0b, 0xb8, 0x16, 0x49, 0x22, 0x64, 0x60, 0x15, - 0x46, 0x0b, 0xc0, 0x18, 0x05, 0x28, 0x98, 0x15, 0x4d, 0xd4, 0x0b, 0xa1, 0x00, 0x04, - 0x46, 0x0b, 0xc8, 0x1b, 0x02, 0x46, 0x0b, 0xa8, 0x04, 0x46, 0x0b, 0xb0, 0x15 -] - private func nestedLists(_ levels: Int) -> RowDataType { var dataType = RowDataType.int32 for _ in 0.. Date: Thu, 12 Nov 2026 10:14:00 +0530 Subject: [PATCH 14/19] swift: Rename a one-letter test variable. SwiftLint wants identifiers of at least two characters, and the point helper in the array writer tests used x. --- swift/Tests/ForyTests/RowArrayWriterTests.swift | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/swift/Tests/ForyTests/RowArrayWriterTests.swift b/swift/Tests/ForyTests/RowArrayWriterTests.swift index 60e1467773..cf62cf2853 100644 --- a/swift/Tests/ForyTests/RowArrayWriterTests.swift +++ b/swift/Tests/ForyTests/RowArrayWriterTests.swift @@ -25,10 +25,10 @@ private func field(_ dataType: RowDataType) -> RowField { RowField(name: "item", dataType: dataType, nullable: true) } -private func writePoint(_ x: Int32, into buffer: ByteBuffer) -> (start: Int, size: Int) { +private func writePoint(_ value: Int32, into buffer: ByteBuffer) -> (start: Int, size: Int) { var point = RowWriter(fields: pointFields, buffer: buffer) point.reset() - point.writeInt32(x, at: 0) + point.writeInt32(value, at: 0) return (point.base, point.size) } @@ -203,8 +203,8 @@ func rowWriterNestedInListInMap() throws { try values.reset(count: 1) var list = RowArrayWriter(element: field(.struct(pointFields)), buffer: buffer) try list.reset(count: 2) - for (index, x) in [Int32(1), 2].enumerated() { - let point = writePoint(x, into: buffer) + for (index, value) in [Int32(1), 2].enumerated() { + let point = writePoint(value, into: buffer) try list.setOffsetAndSize(at: index, absoluteStart: point.start, size: point.size) } try values.setOffsetAndSize(at: 0, absoluteStart: list.base, size: list.size) From 84adbf0c207da23731f535fd416197cd24f5aa56 Mon Sep 17 00:00:00 2001 From: yash Date: Thu, 12 Nov 2026 14:15:00 +0530 Subject: [PATCH 15/19] swift: Add a row format peer for the Java interop tests. Each case checks the decoded value and replays the row through the writers in read order, so maps must match the Java bytes exactly even though Swift dictionary order is unspecified. --- swift/Package.swift | 5 + swift/Tests/ForyRowXlangPeer/main.swift | 227 ++++++++++++++++++++++++ 2 files changed, 232 insertions(+) create mode 100644 swift/Tests/ForyRowXlangPeer/main.swift diff --git a/swift/Package.swift b/swift/Package.swift index 4f66d4c650..1e34a13fa3 100644 --- a/swift/Package.swift +++ b/swift/Package.swift @@ -66,6 +66,11 @@ let package = Package( dependencies: ["Fory"], path: "Tests/ForyXlangTests" ), + .executableTarget( + name: "ForyRowXlangPeer", + dependencies: ["Fory"], + path: "Tests/ForyRowXlangPeer" + ), .testTarget( name: "ForyTests", dependencies: [ diff --git a/swift/Tests/ForyRowXlangPeer/main.swift b/swift/Tests/ForyRowXlangPeer/main.swift new file mode 100644 index 0000000000..d48b843698 --- /dev/null +++ b/swift/Tests/ForyRowXlangPeer/main.swift @@ -0,0 +1,227 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation +import Fory + +// Row format peer driven by org.apache.fory.format.SwiftCrossLanguageTest. Each case reads the bytes +// Java wrote, checks the value and the exact layout, and overwrites the file with its own encoding +// for Java to read back. + +// The models are internal rather than private: a private type prints as +// "(unknown context at $...).Bar" in failure messages, which hides which value differed. + +@ForyRow +struct Bar: Equatable { + var f1: Int32? + var f2: String? +} + +@ForyRow +struct Foo: Equatable { + var f1: Int32? + var f2: String? + var f3: [String?]? + var f4: [String: Int32?]? + var f5: Bar? +} + +// CrossLanguageTest.A. +@ForyRow +struct StringMap: Equatable { + var f1: Int32? + var f2: [String: String]? +} + +@ForyRow +struct Blob: Equatable { + var f1: Data? + var f2: String? +} + +// Data prints only its byte count ("5 bytes"), so a failure message would show the same text for the +// value read and the value expected. Printing the bytes shows which one differs. +extension Blob: CustomStringConvertible { + var description: String { + "Blob(f1: \(f1.map { Array($0).description } ?? "nil"), f2: \(f2 ?? "nil"))" + } +} + +private let expectedFoo = Foo( + f1: 1, f2: "str", f3: ["str1", nil, "str2"], f4: ["k1": 1, "k2": 2, "k3": 3, "k4": 4, "k5": 5, "k6": 6], + f5: Bar(f1: 1, f2: "str") +) +private let expectedStringMap = StringMap(f1: 1, f2: ["pid": "12345", "ip": "0.0.0.0", "k1": "v1"]) +private let expectedBlob = Blob(f1: Data([0, 1, 0xff, 0x7f, 0x80]), f2: "bytes") + +private struct PeerFailure: Error, CustomStringConvertible { + let description: String +} + +private func check(_ condition: Bool, _ message: @autoclosure () -> String) throws { + if !condition { + throw PeerFailure(description: message()) + } +} + +private func present(_ value: T?) throws -> T { + guard let value else { + throw PeerFailure(description: "a slot that is not null has no value") + } + return value +} + +// Writes the row again through the low-level writers, copying map entries in the order they were +// read. A dictionary has no fixed order, so this is the only way to require the exact bytes of a +// row that holds a map. +private func replay(_ bytes: [UInt8], schema: RowSchema) throws -> [UInt8] { + let row = try Row(schema: schema, bytes: bytes) + var writer = RowWriter(schema: schema) + writer.reset() + for (index, field) in schema.fields.enumerated() { + try copy(from: row, index, field, to: writer) + } + return writer.toBytes() +} + +private func copy(from reader: some RowValueReader, _ index: Int, _ field: RowField, to writer: some RowValueWriter) throws { + if reader.isNull(at: index) { + writer.setNullAt(index) + return + } + switch field.dataType { + case .bool: writer.writeBool(try present(reader.bool(at: index)), at: index) + case .int8: writer.writeInt8(try present(reader.int8(at: index)), at: index) + case .int16: writer.writeInt16(try present(reader.int16(at: index)), at: index) + case .int32: writer.writeInt32(try present(reader.int32(at: index)), at: index) + case .int64: writer.writeInt64(try present(reader.int64(at: index)), at: index) + case .float32: writer.writeFloat32(try present(reader.float32(at: index)), at: index) + case .float64: writer.writeFloat64(try present(reader.float64(at: index)), at: index) + case .date32: try writer.writeLocalDate(present(reader.localDate(at: index)), at: index) + case .timestamp: try writer.writeTimestamp(present(reader.timestamp(at: index)), at: index) + case .duration: try writer.writeDuration(present(reader.duration(at: index)), at: index) + case .string: try writer.writeString(present(reader.string(at: index)), at: index) + case .binary: try writer.writeBytes(present(reader.binary(at: index)), at: index) + case .list: + let array = try copy(present(reader.array(at: index)), to: writer.buffer) + try writer.setOffsetAndSize(at: index, absoluteStart: array.base, size: array.size) + case .map: + let source = try present(reader.map(at: index)) + var map = RowMapWriter(buffer: writer.buffer) + map.reset() + _ = try copy(source.keys, to: writer.buffer) + map.finishKeys() + _ = try copy(source.values, to: writer.buffer) + try writer.setOffsetAndSize(at: index, absoluteStart: map.base, size: map.size) + case .struct(let fields): + let source = try present(reader.row(at: index)) + var child = RowWriter(schema: RowSchema(fields), buffer: writer.buffer) + child.reset() + for (childIndex, childField) in fields.enumerated() { + try copy(from: source, childIndex, childField, to: child) + } + try writer.setOffsetAndSize(at: index, absoluteStart: child.base, size: child.size) + case .float16, .decimal: + throw PeerFailure(description: "replay does not support \(field.dataType)") + } +} + +private func copy(_ source: RowArray, to buffer: ByteBuffer) throws -> RowArrayWriter { + var array = RowArrayWriter(element: source.element, buffer: buffer) + try array.reset(count: source.elementCount) + for index in 0.. Data { + try Data(contentsOf: URL(fileURLWithPath: path)) +} + +private func writeFile(_ path: String, _ data: Data) throws { + try data.write(to: URL(fileURLWithPath: path)) +} + +private func testByteArrayCarrier(_ dataFile: String) throws { + let encoder = RowEncoder() + let data = try readFile(dataFile) + let decoded = try encoder.decode(data) + try check(decoded == expectedBlob, "decoded \(decoded), expected \(expectedBlob)") + let encoded = try encoder.encode(expectedBlob) + try check(encoded == data, "encoded Blob bytes differ from Java's") + try writeFile(dataFile, encoded) +} + +private func testMapEncoder(_ dataFile: String) throws { + let encoder = RowEncoder() + let data = try readFile(dataFile) + let decoded = try encoder.decode(data) + try check(decoded == expectedStringMap, "decoded \(decoded), expected \(expectedStringMap)") + let hash = withUnsafeBytes(of: encoder.schemaHash.littleEndian) { Data($0) } + try check(data.prefix(8) == hash, "schema hash prefix differs from \(encoder.schemaHash)") + let rowBytes = Array(data.dropFirst(8)) + try check(try replay(rowBytes, schema: encoder.schema) == rowBytes, "replayed StringMap row differs from Java's") + try writeFile(dataFile, encoder.encode(expectedStringMap)) +} + +private func testSerializationWithoutSchema(_ dataFile: String) throws { + let encoder = RowEncoder() + let data = try readFile(dataFile) + let decoded = try encoder.fromRow(data) + try check(decoded == expectedFoo, "decoded \(decoded), expected \(expectedFoo)") + let rowBytes = Array(data) + try check(try replay(rowBytes, schema: encoder.schema) == rowBytes, "replayed Foo row differs from Java's") + try writeFile(dataFile, encoder.toRow(expectedFoo)) +} + +private func testSerializationWithSchema(_ schemaFile: String, _ dataFile: String) throws { + let schemaBytes = Array(try readFile(schemaFile)) + let parsed = try RowSchema(bytes: schemaBytes) + try check(parsed == Foo.rowSchema, "Java schema \(parsed), inferred schema \(Foo.rowSchema)") + try check(parsed.hash == Foo.rowSchema.hash, "schema hash \(parsed.hash), encoder hash \(Foo.rowSchema.hash)") + try check(try Foo.rowSchema.toBytes() == schemaBytes, "re-encoded schema bytes differ from Java's") + try testSerializationWithoutSchema(dataFile) +} + +private func run(_ arguments: [String]) throws { + guard arguments.count >= 3 else { + throw PeerFailure(description: "usage: ForyRowXlangPeer ...") + } + switch arguments[1] { + case "test_byte_array_carrier": + try testByteArrayCarrier(arguments[2]) + case "test_map_encoder": + try testMapEncoder(arguments[2]) + case "test_serialization_without_schema": + try testSerializationWithoutSchema(arguments[2]) + case "test_serialization_with_schema": + guard arguments.count >= 4 else { + throw PeerFailure(description: "test_serialization_with_schema needs ") + } + try testSerializationWithSchema(arguments[2], arguments[3]) + default: + throw PeerFailure(description: "unknown test case \"\(arguments[1])\"") + } +} + +do { + try run(CommandLine.arguments) +} catch { + FileHandle.standardError.write(Data("row_xlang: \(error)\n".utf8)) + exit(1) +} From 3f9db941f670d812a42676bb4a1c54645e1f7960 Mon Sep 17 00:00:00 2001 From: yash Date: Thu, 12 Nov 2026 16:19:02 +0530 Subject: [PATCH 16/19] java: Add SwiftCrossLanguageTest. It runs the row format cases of GoCrossLanguageTest against the Swift peer, and the byte array case also checks that Swift wrote Java's bytes exactly. --- .../fory/format/SwiftCrossLanguageTest.java | 173 ++++++++++++++++++ 1 file changed, 173 insertions(+) create mode 100644 java/fory-format/src/test/java/org/apache/fory/format/SwiftCrossLanguageTest.java diff --git a/java/fory-format/src/test/java/org/apache/fory/format/SwiftCrossLanguageTest.java b/java/fory-format/src/test/java/org/apache/fory/format/SwiftCrossLanguageTest.java new file mode 100644 index 0000000000..7e581c0faf --- /dev/null +++ b/java/fory-format/src/test/java/org/apache/fory/format/SwiftCrossLanguageTest.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format; + +import com.google.common.collect.ImmutableMap; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import org.apache.fory.format.GoCrossLanguageTest.Blob; +import org.apache.fory.format.encoder.Encoders; +import org.apache.fory.format.encoder.RowEncoder; +import org.apache.fory.format.row.binary.BinaryRow; +import org.apache.fory.format.type.DataTypes; +import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.memory.MemoryUtils; +import org.apache.fory.test.TestUtils; +import org.testng.Assert; +import org.testng.SkipException; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +/** + * Row format cross-language tests against the Swift peer built from {@code + * swift/Tests/ForyRowXlangPeer}. Data shapes are shared with {@link CrossLanguageTest} and {@link + * GoCrossLanguageTest}. Setting {@code FORY_SWIFT_JAVA_CI=1} opts into the suite; once opted in, a + * missing Swift toolchain or a peer build failure fails the tests instead of skipping them. Setting + * {@code FORY_SWIFT_PEER_PREBUILT=1} uses an already built release peer. + */ +@Test +public class SwiftCrossLanguageTest { + private static final String SWIFT_EXECUTABLE = "swift"; + private static final String SWIFT_PEER_TARGET = "ForyRowXlangPeer"; + private static final File SWIFT_WORK_DIR = new File("../../swift"); + private static final Path SWIFT_PEER_BINARY_PATH = + SWIFT_WORK_DIR.toPath().resolve(".build/release").resolve(SWIFT_PEER_TARGET); + // Fixed dictionary order in the peer, so a failure reproduces with the same bytes. + private static final Map SWIFT_PEER_ENV = + ImmutableMap.of("SWIFT_DETERMINISTIC_HASHING", "1", "ENABLE_FORY_DEBUG_OUTPUT", "1"); + private String swiftPeerBinaryPath; + + @BeforeClass + public void ensureSwiftPeerReady() { + if (!"1".equals(System.getenv("FORY_SWIFT_JAVA_CI"))) { + throw new SkipException("Skipping SwiftCrossLanguageTest: FORY_SWIFT_JAVA_CI not set to 1"); + } + try { + Process process = new ProcessBuilder(SWIFT_EXECUTABLE, "--version").start(); + Assert.assertEquals(process.waitFor(), 0, "swift is required when FORY_SWIFT_JAVA_CI=1"); + } catch (IOException e) { + throw new AssertionError("swift is required when FORY_SWIFT_JAVA_CI=1", e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError("interrupted while probing the swift toolchain", e); + } + if (!"1".equals(System.getenv("FORY_SWIFT_PEER_PREBUILT"))) { + Assert.assertTrue(buildSwiftPeer(), "failed to build the Swift row format peer"); + } + Assert.assertTrue( + Files.isRegularFile(SWIFT_PEER_BINARY_PATH), + "Swift row format peer not found at " + SWIFT_PEER_BINARY_PATH.toAbsolutePath()); + Assert.assertTrue( + Files.isExecutable(SWIFT_PEER_BINARY_PATH), + "Swift row format peer is not executable: " + SWIFT_PEER_BINARY_PATH.toAbsolutePath()); + swiftPeerBinaryPath = SWIFT_PEER_BINARY_PATH.toAbsolutePath().toString(); + } + + public void testByteArrayCarrier() throws IOException { + Blob blob = Blob.create(); + RowEncoder encoder = Encoders.bean(Blob.class); + Path dataFile = createTempFile("row_swift_blob"); + Files.write(dataFile, encoder.encode(blob)); + Assert.assertTrue(runSwiftPeer("test_byte_array_carrier", dataFile)); + byte[] swiftBytes = Files.readAllBytes(dataFile); + Assert.assertEquals(encoder.decode(swiftBytes), blob); + // Blob holds no map, so the Swift encoding has one possible layout and must equal Java's. + Assert.assertEquals(swiftBytes, encoder.encode(blob)); + } + + public void testMapEncoder() throws IOException { + CrossLanguageTest.A a = CrossLanguageTest.A.create(); + RowEncoder encoder = Encoders.bean(CrossLanguageTest.A.class); + Path dataFile = createTempFile("row_swift_map"); + Files.write(dataFile, encoder.encode(a)); + Assert.assertTrue(runSwiftPeer("test_map_encoder", dataFile)); + Assert.assertEquals(encoder.decode(Files.readAllBytes(dataFile)), a); + } + + public void testSerializationWithoutSchema() throws IOException { + CrossLanguageTest.Foo foo = CrossLanguageTest.Foo.create(); + RowEncoder encoder = Encoders.bean(CrossLanguageTest.Foo.class); + Path dataFile = createTempFile("row_swift_foo"); + Files.write(dataFile, encoder.toRow(foo).toBytes()); + Assert.assertTrue(runSwiftPeer("test_serialization_without_schema", dataFile)); + Assert.assertEquals(readFoo(encoder, dataFile), foo); + } + + public void testSerializationWithSchema() throws IOException { + CrossLanguageTest.Foo foo = CrossLanguageTest.Foo.create(); + RowEncoder encoder = Encoders.bean(CrossLanguageTest.Foo.class); + Path dataFile = createTempFile("row_swift_foo"); + Path schemaFile = createTempFile("row_swift_foo_schema"); + BinaryRow row = encoder.toRow(foo); + Files.write(dataFile, row.toBytes()); + Files.write(schemaFile, DataTypes.serializeSchema(row.getSchema())); + Assert.assertTrue(runSwiftPeer("test_serialization_with_schema", schemaFile, dataFile)); + Assert.assertEquals(readFoo(encoder, dataFile), foo); + } + + private static boolean buildSwiftPeer() { + Map env = ImmutableMap.of("ENABLE_FORY_DEBUG_OUTPUT", "1"); + List fastBuildCommand = + Arrays.asList( + SWIFT_EXECUTABLE, + "build", + "-c", + "release", + "--disable-automatic-resolution", + "--product", + SWIFT_PEER_TARGET); + if (TestUtils.executeCommand(fastBuildCommand, 600, env, SWIFT_WORK_DIR)) { + return true; + } + List fallbackBuildCommand = + Arrays.asList(SWIFT_EXECUTABLE, "build", "-c", "release", "--product", SWIFT_PEER_TARGET); + return TestUtils.executeCommand(fallbackBuildCommand, 600, env, SWIFT_WORK_DIR); + } + + private static CrossLanguageTest.Foo readFoo( + RowEncoder encoder, Path dataFile) throws IOException { + MemoryBuffer buffer = MemoryUtils.wrap(Files.readAllBytes(dataFile)); + BinaryRow row = new BinaryRow(encoder.schema()); + row.pointTo(buffer, 0, buffer.size()); + return encoder.fromRow(row); + } + + private static Path createTempFile(String prefix) throws IOException { + Path file = Files.createTempFile(prefix, "data"); + file.toFile().deleteOnExit(); + return file; + } + + private boolean runSwiftPeer(String caseName, Path... files) { + List command = new ArrayList<>(); + command.add(swiftPeerBinaryPath); + command.add(caseName); + for (Path file : files) { + command.add(file.toAbsolutePath().toString()); + } + return TestUtils.executeCommand(command, 30, SWIFT_PEER_ENV, SWIFT_WORK_DIR); + } +} From 980b76d498ad7e99f5011d02faafcad2e31c29a9 Mon Sep 17 00:00:00 2001 From: yash Date: Thu, 12 Nov 2026 17:09:16 +0530 Subject: [PATCH 17/19] ci: Run the Swift row format interop test. The Swift xlang job now also builds the row format peer and runs SwiftCrossLanguageTest after SwiftXlangTest, as the Go job does. --- .github/workflows/ci.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d2b550ea6..9076fb66d7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -560,18 +560,19 @@ jobs: ~/.swiftpm ~/Library/Caches/org.swift.swiftpm swift/.build - key: ${{ runner.os }}-${{ runner.arch }}-swiftpm-xlang-release-${{ steps.swift-package-cache.outputs.toolchain }}-${{ hashFiles('swift/Package.resolved', 'swift/Package.swift', 'swift/Sources/**', 'swift/Tests/ForyXlangTests/**') }} + key: ${{ runner.os }}-${{ runner.arch }}-swiftpm-xlang-release-${{ steps.swift-package-cache.outputs.toolchain }}-${{ hashFiles('swift/Package.resolved', 'swift/Package.swift', 'swift/Sources/**', 'swift/Tests/ForyXlangTests/**', 'swift/Tests/ForyRowXlangPeer/**') }} restore-keys: | ${{ runner.os }}-${{ runner.arch }}-swiftpm-xlang-release-${{ steps.swift-package-cache.outputs.toolchain }}- - - name: Prebuild Swift xlang peer for cache reuse + - name: Prebuild Swift xlang peers for cache reuse run: | cd swift - if [[ "${{ steps.swift-build-cache.outputs.cache-hit }}" == "true" && -x .build/release/ForyXlangTests ]]; then - echo "Using cached Swift xlang peer at swift/.build/release/ForyXlangTests" + if [[ "${{ steps.swift-build-cache.outputs.cache-hit }}" == "true" && -x .build/release/ForyXlangTests && -x .build/release/ForyRowXlangPeer ]]; then + echo "Using cached Swift xlang peers at swift/.build/release/ForyXlangTests and swift/.build/release/ForyRowXlangPeer" echo "FORY_SWIFT_PEER_PREBUILT=1" >> "$GITHUB_ENV" exit 0 fi swift build -c release --disable-automatic-resolution --product ForyXlangTests + swift build -c release --disable-automatic-resolution --product ForyRowXlangPeer echo "FORY_SWIFT_PEER_PREBUILT=1" >> "$GITHUB_ENV" - name: Run Swift Xlang Test env: @@ -582,6 +583,8 @@ jobs: mvn -T16 --no-transfer-progress clean install -DskipTests -Dmaven.javadoc.skip=true -Dmaven.source.skip=true cd fory-core mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.xlang.SwiftXlangTest + cd ../fory-format + mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.format.SwiftCrossLanguageTest swift_idl: name: Swift IDL Tests From f3c79cdc220f2495eb00fe6c17405b740f97aadf Mon Sep 17 00:00:00 2001 From: yash Date: Thu, 12 Nov 2026 20:24:33 +0530 Subject: [PATCH 18/19] docs: Add the Swift row format guide. Covers the @ForyRow macro, RowEncoder, the read views, the type mapping, and hand-written rows, and lists Swift wherever the docs name the Standard Row languages. --- README.md | 3 +- docs/introduction/choose-a-format.md | 12 +- docs/introduction/index.md | 2 +- docs/introduction/support-matrix.md | 2 +- docs/row-format/go.md | 2 +- docs/row-format/index.md | 15 +- docs/row-format/swift.md | 274 ++++++++++++++++++++++++++ docs/row-format/troubleshooting.md | 4 +- docs/specification/row_format_spec.md | 10 +- 9 files changed, 300 insertions(+), 24 deletions(-) create mode 100644 docs/row-format/swift.md diff --git a/README.md b/README.md index 22dc0110d8..934f3ec7d3 100644 --- a/README.md +++ b/README.md @@ -844,7 +844,8 @@ deserialization, see the [Java row-format guide](docs/row-format/java.md), [Python row-format guide](docs/row-format/python.md), [C++ row-format guide](docs/row-format/cpp.md), [Rust row-format guide](docs/row-format/rust.md), -[Go row-format guide](docs/row-format/go.md), and +[Go row-format guide](docs/row-format/go.md), +[Swift row-format guide](docs/row-format/swift.md), and [row-format specification](docs/specification/row_format_spec.md). ## Fory JSON diff --git a/docs/introduction/choose-a-format.md b/docs/introduction/choose-a-format.md index 44bd117613..53b744647f 100644 --- a/docs/introduction/choose-a-format.md +++ b/docs/introduction/choose-a-format.md @@ -19,12 +19,12 @@ license: | limitations under the License. --- -| Format or mode | Data model | Use it when | Interoperability | Start here | -| -------------- | ----------------------------------- | ------------------------------------------------------------- | --------------------------------------------------------------------------- | -------------------------------------------------------- | -| Xlang binary | Portable object graphs | Data crosses language boundaries | Shared wire format across supported Fory implementations | [Cross-language guide](../object-serialization/xlang.md) | -| Native binary | Language-native object graphs | Producer and consumer use one Fory implementation family | One Fory implementation family only | [Object Serialization](../object-serialization/index.md) | -| Row Format | Random-access binary rows | You need random field access or analytics-style partial reads | Standard Row is shared by Java, Python, C++, and Rust; Compact is Java-only | [Row Format guide](../row-format/index.md) | -| Fory JSON | Standard JSON mapped to JVM objects | Java, Kotlin, or Scala applications need standard JSON | Standard JSON text | [Fory JSON guide](../json/index.md) | +| Format or mode | Data model | Use it when | Interoperability | Start here | +| -------------- | ----------------------------------- | ------------------------------------------------------------- | ---------------------------------------------------------------------------------- | -------------------------------------------------------- | +| Xlang binary | Portable object graphs | Data crosses language boundaries | Shared wire format across supported Fory implementations | [Cross-language guide](../object-serialization/xlang.md) | +| Native binary | Language-native object graphs | Producer and consumer use one Fory implementation family | One Fory implementation family only | [Object Serialization](../object-serialization/index.md) | +| Row Format | Random-access binary rows | You need random field access or analytics-style partial reads | Standard Row is shared by Java, Python, C++, Rust, and Swift; Compact is Java-only | [Row Format guide](../row-format/index.md) | +| Fory JSON | Standard JSON mapped to JVM objects | Java, Kotlin, or Scala applications need standard JSON | Standard JSON text | [Fory JSON guide](../json/index.md) | Xlang and native are sibling modes of Object Serialization. Use them when the receiver needs to reconstruct an object graph. Row Format and Fory JSON are separate formats, not additional Object diff --git a/docs/introduction/index.md b/docs/introduction/index.md index 74d4b16741..052bb6f478 100644 --- a/docs/introduction/index.md +++ b/docs/introduction/index.md @@ -113,7 +113,7 @@ optimized for analytics and partial-read workloads: - **Zero-copy random access**: Read fields, arrays, and nested values without rebuilding whole objects. - **Partial operations**: Read only the values needed for a query or pipeline stage. - **Apache Arrow integration**: Convert to columnar data for analytics pipelines. -- **Multi-language support**: Exchange Standard Row Format data across Java, Python, C++, and Rust. +- **Multi-language support**: Exchange Standard Row Format data across Java, Python, C++, Rust, and Swift. ### Optimized Fory Implementations diff --git a/docs/introduction/support-matrix.md b/docs/introduction/support-matrix.md index d8fcbdb138..53c9051608 100644 --- a/docs/introduction/support-matrix.md +++ b/docs/introduction/support-matrix.md @@ -26,7 +26,7 @@ does not imply support for every Fory capability. | --------------------------- | ---------------------------------------------------------------------------------- | -------------------------------------------------------- | | Xlang object serialization | Java, Python, C++, Go, Rust, JavaScript/TypeScript, C#, Swift, Dart, Scala, Kotlin | One shared xlang wire format | | Native object serialization | Java, Python, C++, Go, Rust, Scala, Kotlin | One Fory implementation family only | -| Standard Row Format | Java, Python, C++, Rust | Shared Standard Row layout | +| Standard Row Format | Java, Python, C++, Rust, Swift | Shared Standard Row layout | | Compact Row Format | Java | Java-only compact layout | | Fory JSON | Java, Kotlin, Scala | Standard JSON text | | Fory compiler output | Java, Python, C++, Go, Rust, JavaScript/TypeScript, C#, Swift, Dart, Scala, Kotlin | Generated models use supported Fory APIs | diff --git a/docs/row-format/go.md b/docs/row-format/go.md index 5ef96b1ade..3421cc91f3 100644 --- a/docs/row-format/go.md +++ b/docs/row-format/go.md @@ -257,5 +257,5 @@ underlying bytes are not modified. ## Related Topics - [Basic Serialization](../object-serialization/go/basic-serialization.md) - Object graph serialization -- [Standard Row Format](index.md#standard-row) - Shared layout for Java, Python, C++, Rust, and Go +- [Standard Row Format](index.md#standard-row) - Shared layout for Java, Python, C++, Rust, Go, and Swift - [Row Format Specification](../specification/row_format_spec.md) - Protocol details diff --git a/docs/row-format/index.md b/docs/row-format/index.md index ac41122a90..dcc0b0a67f 100644 --- a/docs/row-format/index.md +++ b/docs/row-format/index.md @@ -30,20 +30,20 @@ reconstruction as its primary access pattern. ## Choose a Layout -| Layout | Language support | Compatibility | -| ------------ | --------------------------- | -------------------------------- | -| Standard Row | Java, Python, C++, Rust, Go | Shared Standard Row layout | -| Compact Row | Java | Java-only, space-oriented layout | +| Layout | Language support | Compatibility | +| ------------ | ---------------------------------- | -------------------------------- | +| Standard Row | Java, Python, C++, Rust, Go, Swift | Shared Standard Row layout | +| Compact Row | Java | Java-only, space-oriented layout | ## Standard Row -Standard Row is the interoperable layout for Java, Python, C++, Rust, and Go. +Standard Row is the interoperable layout for Java, Python, C++, Rust, Go, and Swift. ### Features - **Zero-copy random access**: Read selected fields directly from encoded data. - **Partial deserialization**: Reconstruct only the values an application needs. -- **Cross-language compatibility**: Share Standard Row bytes between Java, Python, C++, Rust, and Go. +- **Cross-language compatibility**: Share Standard Row bytes between Java, Python, C++, Rust, Go, and Swift. - **Apache Arrow integration**: Convert rows to Arrow data in Java and Python. ### Layout @@ -62,6 +62,7 @@ layout, alignment rules, type table, and endianness are defined by the | C++ | Compatible | [C++](cpp.md) | Native row readers and writers | | Rust | Compatible | [Rust](rust.md) | Borrowed struct, array, and map views | | Go | Compatible | [Go](go.md) | Reflection struct encoder; zero-copy row views | +| Swift | Compatible | [Swift](swift.md) | Macro-generated struct encoder; zero-copy row views | Use the language guides for installation, schema construction, encoding, random access, partial reads, and language-specific integrations. @@ -94,7 +95,7 @@ Reuse the encoder within one thread. Create separate encoders for concurrent thr - Fixed-size nested structs can be stored inline. Choose Compact Row only when every reader is Java and the space reduction justifies the -Java-specific layout. Use Standard Row for Java, Python, C++, Rust, and Go interchange. +Java-specific layout. Use Standard Row for Java, Python, C++, Rust, Go, and Swift interchange. See the [Row Format specification](../specification/row_format_spec.md) for the exact Standard and Compact binary layouts. diff --git a/docs/row-format/swift.md b/docs/row-format/swift.md new file mode 100644 index 0000000000..f4633e699a --- /dev/null +++ b/docs/row-format/swift.md @@ -0,0 +1,274 @@ +--- +title: Swift Standard Row Format +sidebar_position: 8 +id: swift +license: | + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--- + +Apache Fory™ Swift implements the Standard Row Format used by Java, C++, Python, Rust, and Go in +the `Fory` module. It provides a `@ForyRow` macro that generates a struct encoder, read-only views +for random field access, and the cross-language schema encoding. Add the `Fory` package as described +in the [Swift guide](../object-serialization/swift/index.md). + +## Overview + +Use Row Format when readers need selected fields or collection elements rather than a fully +reconstructed value. Readers are views over the encoded bytes: reading one field costs a bitmap +test and a slot lookup, and other fields are never touched. + +Row Format is a trusted in-memory format. Decode only bytes produced by a Fory row writer for the +same schema, from a source you trust. Malformed bytes surface as thrown errors, but the format does +not defend against hostile input the way object serialization does. + +## When to Use Row Format + +- Analytics workloads with selective field access +- Large datasets where only a subset of fields is needed +- Memory-mapped or shared data read by several languages +- High-throughput pipelines that exchange Standard Row bytes with Java, C++, Python, Rust, or Go + +## Basic Usage + +```swift +import Fory + +@ForyRow +struct UserProfile { + var id: Int64 + var username: String + var email: String? + var scores: [Int32] + var preferences: [String: String] + var isActive: Bool +} + +let encoder = RowEncoder() +let profile = UserProfile( + id: 12345, + username: "alice", + email: "alice@example.com", + scores: [95, 87, 92, 88], + preferences: ["theme": "dark", "language": "en"], + isActive: true +) + +let rowBytes = try encoder.toRow(profile) + +// Random access without decoding the whole struct. +let schema = encoder.schema +let row = try Row(schema: schema, bytes: Array(rowBytes)) +print(try row.string(at: schema.fieldIndex(of: "username")!) ?? "") // alice +print(try row.array(at: schema.fieldIndex(of: "scores")!)?.int32(at: 1) ?? 0) // 87 +print(row.isNull(at: schema.fieldIndex(of: "email")!)) // false + +// Full decode when the whole value is needed. +let decoded = try encoder.fromRow(rowBytes) +print(decoded.preferences["theme"] ?? "") // dark +``` + +`@ForyRow` builds the schema from the struct's stored properties at compile time. +`RowEncoder` reuses one write buffer; create one encoder per struct type and reuse it. + +## Rows and Framed Messages + +`toRow` and `fromRow` work with bare row bytes, which is what the other languages' `toRow` and +`BinaryRow.pointTo` exchange. `encode` and `decode` add the framing used by the Java and Python +row encoders: an 8-byte little-endian schema hash followed by the row. + +```swift +let framed = try encoder.encode(profile) // hash + row +let fromFrame = try encoder.decode(framed) // verifies the hash, then decodes +``` + +The hash is a type-shape fingerprint: it covers the field types and nothing else, so `decode` +rejects a writer whose field types differ but cannot detect renamed fields, changed nullability, +or reordered fields of the same type. Share the schema bytes (see +[Schema Exchange](#schema-exchange)) when peers must agree on more than the type shape. + +`decode` and `fromRow` throw `ForyError` for truncated or inconsistent bytes. Decoded values never +refer to the input, so the input can be reused immediately. + +## Zero-Copy Reading + +`Row(schema:bytes:)` creates a view over a row. `row(at:)`, `array(at:)`, and `map(at:)` return +views over the nested bytes without copying them. `string(at:)` and `binary(at:)` copy the value +out. A view holds its own reference to the byte array, so changing your array afterwards does not +change what the view reads. + +```swift +let scores = try row.array(at: schema.fieldIndex(of: "scores")!)! +for index in 0..` | Standard array | Yes | +| `[K: V]` | Standard map | Yes | +| `enum E: String` with `RowCodable` | Variable-width UTF-8 raw value | Yes | +| `@ForyRow` struct | Nested Standard Row | Yes | +| `T?` for any supported `T` | Same encoding as `T` | Yes | + +`Data` matches Java's `byte[]`, which Java also infers as a list of int8, and encodes the same as +`[Int8]`. The row format's binary type is available only to hand-built schemas through +`RowWriter.writeBytes` and `Row.binary(at:)`. + +`Date` is rounded to the nearest microsecond. Dates between roughly the years 1729 and 2273 come +back exactly to the microsecond; further away, a `Date` cannot hold every microsecond. `Duration` +is truncated toward zero to whole microseconds. Encoding throws when a `Date` or `Duration` does +not fit in a 64-bit number of microseconds, or a `LocalDate` does not fit in 32-bit epoch days. +Decoding an `Int` throws if the stored value does not fit the platform's `Int`. + +A `Set` is stored as a list, so element order is not kept. An enum stores its raw value; opt in +with an empty `extension MyEnum: RowCodable {}`. Decoding a raw value that names no case throws. + +Dictionaries and sets have no fixed order, so the bytes of a row that holds one can differ between +runs of a program, while the decoded values are always equal. Set the environment variable +`SWIFT_DETERMINISTIC_HASHING=1` when tests need the same bytes on every run. + +Map keys must be `Hashable` and are never null. `Date` and `Duration` keys are rejected because +they are rounded when written, so two different keys could decode as one. + +Unsupported: unsigned integers, tuples, nested optionals, classes, generic structs, recursive +types, `float16`, and `decimal`. + +## Schema Exchange + +`toBytes()` and `init(bytes:)` implement the cross-language schema encoding shared with Java's +`SchemaEncoder`, Python's `Schema.to_bytes`/`from_bytes`, and Go's +`SchemaToBytes`/`SchemaFromBytes`. `hash` computes the same type-shape hash used by `encode`. + +```swift +let schemaBytes = try encoder.schema.toBytes() +let received = try RowSchema(bytes: schemaBytes) +print(received == encoder.schema) // true +print(received.hash == encoder.schemaHash) // true +``` + +A Java bean and a Swift struct with equivalent fields produce identical schema bytes when their +nullability matches: use optional properties for Java boxed types (`Integer`, `String` in lists) +and non-optional properties for Java primitives. + +## Writing Rows by Hand + +`RowWriter`, `RowArrayWriter`, and `RowMapWriter` write rows for a schema you construct yourself, +without a Swift struct. They share one `ByteBuffer`; a nested value is written at the end of the +buffer and then attached to its parent slot with `setOffsetAndSize`. + +```swift +let item = RowField(name: "item", dataType: .string, nullable: true) +let schema = RowSchema([ + RowField(name: "id", dataType: .int64, nullable: false), + RowField(name: "tags", dataType: .list(item), nullable: true) +]) +var writer = RowWriter(schema: schema) +writer.reset() +writer.writeInt64(7, at: 0) + +var tags = RowArrayWriter(element: item, buffer: writer.buffer) +try tags.reset(count: 2) +try tags.writeString("swift", at: 0) +tags.setNullAt(1) +try writer.setOffsetAndSize(at: 1, absoluteStart: tags.base, size: tags.size) + +let rowBytes = writer.toBytes() +``` + +For a map, call `reset()` on a `RowMapWriter`, write the keys with one `RowArrayWriter`, call +`finishKeys()`, write the values with a second `RowArrayWriter`, then attach the map to its parent +slot with `setOffsetAndSize`, passing the map writer's `base` and `size`. Each `reset()` starts a +new row at the end of the buffer; call `writer.buffer.clear()` first to reuse the buffer for a new +top-level row. + +## Thread Safety + +A `RowEncoder` owns a reusable write buffer and is not safe for concurrent use; create one encoder +per thread or task, or guard it with a lock. Writers follow the same rule. `Row`, `RowArray`, and +`RowMap` views only read. + +## Related Topics + +- [Basic Serialization](../object-serialization/swift/basic-serialization.md) - Object graph serialization +- [Standard Row Format](index.md#standard-row) - Shared layout for Java, Python, C++, Rust, Go, and Swift +- [Row Format Specification](../specification/row_format_spec.md) - Protocol details diff --git a/docs/row-format/troubleshooting.md b/docs/row-format/troubleshooting.md index 34f51d1c6d..b302c3cd59 100644 --- a/docs/row-format/troubleshooting.md +++ b/docs/row-format/troubleshooting.md @@ -21,8 +21,8 @@ license: | ## A Standard Row peer cannot read Compact Row bytes -Compact Row is a Java-only row family. Use Standard Row on Java, Python, C++, Rust, and Go for shared -bytes. +Compact Row is a Java-only row family. Use Standard Row on Java, Python, C++, Rust, Go, and Swift +for shared bytes. ## A field lookup fails diff --git a/docs/specification/row_format_spec.md b/docs/specification/row_format_spec.md index a192e35f1e..1a7ea1b552 100644 --- a/docs/specification/row_format_spec.md +++ b/docs/specification/row_format_spec.md @@ -26,14 +26,14 @@ Apache Fory Row Format is a cache-friendly, random-access binary format designed - **Random Field Access**: Read individual fields without deserializing the entire row - **Zero-Copy Operations**: Direct memory access without data transformation - **Cache-Friendly Layout**: Optimized memory layout for CPU cache efficiency -- **Cross-Language Support**: Consistent binary format across Java, C++, Python, Rust, and Go +- **Cross-Language Support**: Consistent binary format across Java, C++, Python, Rust, Go, and Swift Fory provides two row format variants: -| Format | Languages | Use Case | -| --------------- | --------------------------- | ------------------------------ | -| Standard Format | Java, C++, Python, Rust, Go | Cross-language compatibility | -| Compact Format | Java only | Space efficiency, smaller rows | +| Format | Languages | Use Case | +| --------------- | ---------------------------------- | ------------------------------ | +| Standard Format | Java, C++, Python, Rust, Go, Swift | Cross-language compatibility | +| Compact Format | Java only | Space efficiency, smaller rows | ## Format Comparison From 03e497ac79e4271b9b126b761784cc280c4aad5a Mon Sep 17 00:00:00 2001 From: yash Date: Thu, 12 Nov 2026 23:58:47 +0530 Subject: [PATCH 19/19] swift: Format the row reader and writer tests. Apply swift-format to the row reader and writer tests so the Swift format check in CI passes. --- .../Tests/ForyTests/RowArrayReaderTests.swift | 34 +++++----- .../Tests/ForyTests/RowArrayWriterTests.swift | 64 ++++++++++--------- swift/Tests/ForyTests/RowReaderTests.swift | 34 +++++----- swift/Tests/ForyTests/RowWriterTests.swift | 41 ++++++------ 4 files changed, 90 insertions(+), 83 deletions(-) diff --git a/swift/Tests/ForyTests/RowArrayReaderTests.swift b/swift/Tests/ForyTests/RowArrayReaderTests.swift index 8c5174ecac..1d38c0e7c1 100644 --- a/swift/Tests/ForyTests/RowArrayReaderTests.swift +++ b/swift/Tests/ForyTests/RowArrayReaderTests.swift @@ -33,13 +33,13 @@ private func rowArray(_ bytes: [UInt8], of dataType: RowDataType) throws -> RowA @Test func rowArrayStrings() throws { let bytes: [UInt8] = [ - 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 3 elements - 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // bitmap: element 1 is null - 0x02, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, // element 0: size 2, offset 0x28 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // element 1: null - 0x03, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, // element 2: size 3, offset 0x30 - 0x61, 0x62, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // "ab" - 0x78, 0x79, 0x7A, 0x00, 0x00, 0x00, 0x00, 0x00 // "xyz" + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 3 elements + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // bitmap: element 1 is null + 0x02, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, // element 0: size 2, offset 0x28 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // element 1: null + 0x03, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, // element 2: size 3, offset 0x30 + 0x61, 0x62, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // "ab" + 0x78, 0x79, 0x7A, 0x00, 0x00, 0x00, 0x00, 0x00 // "xyz" ] let array = try #require(try rowHolding(bytes, as: .listOf(.string)).array(at: 0)) #expect(array.elementCount == 3) @@ -105,16 +105,16 @@ func rowArrayWidths() throws { @Test func rowMapLayout() throws { let bytes: [UInt8] = [ - 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // keys array is 0x30 bytes - 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // keys: 2 elements - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // keys bitmap - 0x02, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, // key 0: size 2, offset 0x20 from the keys array - 0x02, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, // key 1: size 2, offset 0x28 from the keys array - 0x6B, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // "k1" - 0x6B, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // "k2" - 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // values: 2 elements - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // values bitmap - 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00 // int32 1 and 2 + 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // keys array is 0x30 bytes + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // keys: 2 elements + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // keys bitmap + 0x02, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, // key 0: size 2, offset 0x20 from the keys array + 0x02, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, // key 1: size 2, offset 0x28 from the keys array + 0x6B, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // "k1" + 0x6B, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // "k2" + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // values: 2 elements + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // values bitmap + 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00 // int32 1 and 2 ] let map = try #require(try rowHolding(bytes, as: .mapOf(.string, .int32)).map(at: 0)) #expect(map.count == 2) diff --git a/swift/Tests/ForyTests/RowArrayWriterTests.swift b/swift/Tests/ForyTests/RowArrayWriterTests.swift index cf62cf2853..4179b7dd90 100644 --- a/swift/Tests/ForyTests/RowArrayWriterTests.swift +++ b/swift/Tests/ForyTests/RowArrayWriterTests.swift @@ -39,15 +39,16 @@ func rowArrayWriterStrings() throws { try writer.writeString("ab", at: 0) writer.setNullAt(1) try writer.writeString("xyz", at: 2) - #expect(writer.toBytes() == [ - 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 3 elements - 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // bitmap: element 1 is null - 0x02, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, // element 0: size 2, offset 0x28 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // element 1: null - 0x03, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, // element 2: size 3, offset 0x30 - 0x61, 0x62, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // "ab" - 0x78, 0x79, 0x7A, 0x00, 0x00, 0x00, 0x00, 0x00 // "xyz" - ]) + #expect( + writer.toBytes() == [ + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 3 elements + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // bitmap: element 1 is null + 0x02, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, // element 0: size 2, offset 0x28 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // element 1: null + 0x03, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, // element 2: size 3, offset 0x30 + 0x61, 0x62, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // "ab" + 0x78, 0x79, 0x7A, 0x00, 0x00, 0x00, 0x00, 0x00 // "xyz" + ]) } @Test @@ -65,18 +66,19 @@ func rowMapWriterLayout() throws { values.writeInt32(1, at: 0) values.writeInt32(2, at: 1) #expect(map.size == 80) - #expect(buffer.copyBytes(start: 0, end: buffer.count) == [ - 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // keys array is 48 bytes - 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 2 keys - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // key bitmap - 0x02, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, // key 0: offset counted from the keys array - 0x02, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, // key 1 - 0x6B, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // "k1" - 0x6B, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // "k2" - 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 2 values - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // value bitmap - 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00 // 1 and 2 - ]) + #expect( + buffer.copyBytes(start: 0, end: buffer.count) == [ + 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // keys array is 48 bytes + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 2 keys + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // key bitmap + 0x02, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, // key 0: offset counted from the keys array + 0x02, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, // key 1 + 0x6B, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // "k1" + 0x6B, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // "k2" + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 2 values + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // value bitmap + 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00 // 1 and 2 + ]) } @Test @@ -90,13 +92,14 @@ func rowWriterNested() throws { parent.writeInt32(1, at: 0) let child = writePoint(5, into: parent.buffer) try parent.setOffsetAndSize(at: 1, absoluteStart: child.start, size: child.size) - #expect(parent.toBytes() == [ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // parent bitmap - 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // id = 1 - 0x10, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, // child: size 16, offset 0x18 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // child bitmap - 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // x = 5 - ]) + #expect( + parent.toBytes() == [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // parent bitmap + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // id = 1 + 0x10, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, // child: size 16, offset 0x18 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // child bitmap + 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // x = 5 + ]) } @Test @@ -150,8 +153,9 @@ func rowMapWriterEmpty() throws { map.finishKeys() var values = RowArrayWriter(element: field(.int32), buffer: buffer) try values.reset(count: 0) - #expect(buffer.copyBytes(start: 0, end: buffer.count) - == [0x08, 0, 0, 0, 0, 0, 0, 0] + [UInt8](repeating: 0, count: 16)) + #expect( + buffer.copyBytes(start: 0, end: buffer.count) + == [0x08, 0, 0, 0, 0, 0, 0, 0] + [UInt8](repeating: 0, count: 16)) } @Test diff --git a/swift/Tests/ForyTests/RowReaderTests.swift b/swift/Tests/ForyTests/RowReaderTests.swift index 2f1e40d0fd..00170d0c7b 100644 --- a/swift/Tests/ForyTests/RowReaderTests.swift +++ b/swift/Tests/ForyTests/RowReaderTests.swift @@ -34,11 +34,11 @@ func rowReaderLayout() throws { RowField(name: "c", dataType: .string, nullable: true) ]) let bytes: [UInt8] = [ - 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // bitmap: a is null - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // slot a - 0x2C, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // slot b = 300 - 0x02, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, // slot c: size 2, offset 0x20 - 0x68, 0x69, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // "hi" + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // bitmap: a is null + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // slot a + 0x2C, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // slot b = 300 + 0x02, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, // slot c: size 2, offset 0x20 + 0x68, 0x69, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // "hi" ] let row = try Row(schema: schema, bytes: bytes) #expect(row.fieldCount == 3) @@ -74,11 +74,11 @@ func rowReaderNested() throws { RowField(name: "child", dataType: .struct([RowField(name: "x", dataType: .int32, nullable: false)]), nullable: true) ]) let bytes: [UInt8] = [ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // parent bitmap - 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // slot id = 1 - 0x10, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, // slot child: size 16, offset 0x18 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // child bitmap - 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // child slot x = 5 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // parent bitmap + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // slot id = 1 + 0x10, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, // slot child: size 16, offset 0x18 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // child bitmap + 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // child slot x = 5 ] let row = try Row(schema: schema, bytes: bytes) #expect(row.int32(at: 0) == 1) @@ -96,10 +96,10 @@ func rowReaderNullSkipsData() throws { RowField(name: "e", dataType: .string, nullable: true) ]) let bytes: [UInt8] = [ - 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // s and b are null - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // slot s: garbage - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // slot b: garbage - 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00 // slot e: empty, not null + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // s and b are null + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // slot s: garbage + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // slot b: garbage + 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00 // slot e: empty, not null ] let row = try Row(schema: schema, bytes: bytes) #expect(try row.string(at: 0) == nil) @@ -146,9 +146,9 @@ func rowReaderTemporal() throws { ]) let bytes: [UInt8] = [ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // ts = -1 microsecond - 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, // d = -1 day - 0x24, 0xFA, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF // dur = -1500 microseconds + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // ts = -1 microsecond + 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, // d = -1 day + 0x24, 0xFA, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF // dur = -1500 microseconds ] let row = try Row(schema: schema, bytes: bytes) // Counted back from 2001 in whole microseconds, so the check itself adds no rounding. diff --git a/swift/Tests/ForyTests/RowWriterTests.swift b/swift/Tests/ForyTests/RowWriterTests.swift index 38c184c17d..7154eaad74 100644 --- a/swift/Tests/ForyTests/RowWriterTests.swift +++ b/swift/Tests/ForyTests/RowWriterTests.swift @@ -26,11 +26,11 @@ private let rowSection31Schema = RowSchema([ ]) private let rowSection31Bytes: [UInt8] = [ - 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // bitmap: a is null - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // slot a - 0x2C, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // slot b = 300 - 0x02, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, // slot c: size 2, offset 0x20 - 0x68, 0x69, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // "hi" + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // bitmap: a is null + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // slot a + 0x2C, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // slot b = 300 + 0x02, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, // slot c: size 2, offset 0x20 + 0x68, 0x69, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // "hi" ] private func writeSection31Row(into buffer: ByteBuffer) throws -> RowWriter { @@ -65,12 +65,13 @@ func rowWriterGoLayout() throws { writer.reset() writer.writeInt64(0x0102_0304_0506_0708, at: 0) try writer.writeString("ab", at: 1) - #expect(writer.toBytes() == [ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x08, 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, - 0x02, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, - 0x61, 0x62, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 - ]) + #expect( + writer.toBytes() == [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x08, 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, + 0x02, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, + 0x61, 0x62, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + ]) } @Test @@ -151,10 +152,11 @@ func rowWriterEmptyString() throws { var writer = RowWriter(schema: RowSchema([RowField(name: "s", dataType: .string, nullable: true)])) writer.reset() try writer.writeString("", at: 0) - #expect(writer.toBytes() == [ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00 // size 0, offset 0x10 (the end of the slots) - ]) + #expect( + writer.toBytes() == [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00 // size 0, offset 0x10 (the end of the slots) + ]) } @Test @@ -162,10 +164,11 @@ func rowWriterMultiByteUTF8() throws { var writer = RowWriter(schema: RowSchema([RowField(name: "s", dataType: .string, nullable: true)])) writer.reset() try writer.writeString("héllo", at: 0) - #expect(Array(writer.toBytes()[8...]) == [ - 0x06, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, - 0x68, 0xC3, 0xA9, 0x6C, 0x6C, 0x6F, 0x00, 0x00 - ]) + #expect( + Array(writer.toBytes()[8...]) == [ + 0x06, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x68, 0xC3, 0xA9, 0x6C, 0x6C, 0x6F, 0x00, 0x00 + ]) } @Test