diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c83f87ddca..db8d68366a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -562,18 +562,19 @@ jobs: ~/.swiftpm ~/Library/Caches/org.swift.swiftpm swift/.build - key: ${{ runner.os }}-${{ runner.arch }}-swiftpm-xlang-release-${{ steps.swift-package-cache.outputs.toolchain }}-${{ hashFiles('swift/Package.resolved', 'swift/Package.swift', 'swift/Sources/**', 'swift/Tests/ForyXlangTests/**') }} + key: ${{ runner.os }}-${{ runner.arch }}-swiftpm-xlang-release-${{ steps.swift-package-cache.outputs.toolchain }}-${{ hashFiles('swift/Package.resolved', 'swift/Package.swift', 'swift/Sources/**', 'swift/Tests/ForyXlangTests/**', 'swift/Tests/ForyRowXlangPeer/**') }} restore-keys: | ${{ runner.os }}-${{ runner.arch }}-swiftpm-xlang-release-${{ steps.swift-package-cache.outputs.toolchain }}- - - name: Prebuild Swift xlang peer for cache reuse + - name: Prebuild Swift xlang peers for cache reuse run: | cd swift - if [[ "${{ steps.swift-build-cache.outputs.cache-hit }}" == "true" && -x .build/release/ForyXlangTests ]]; then - echo "Using cached Swift xlang peer at swift/.build/release/ForyXlangTests" + if [[ "${{ steps.swift-build-cache.outputs.cache-hit }}" == "true" && -x .build/release/ForyXlangTests && -x .build/release/ForyRowXlangPeer ]]; then + echo "Using cached Swift xlang peers at swift/.build/release/ForyXlangTests and swift/.build/release/ForyRowXlangPeer" echo "FORY_SWIFT_PEER_PREBUILT=1" >> "$GITHUB_ENV" exit 0 fi swift build -c release --disable-automatic-resolution --product ForyXlangTests + swift build -c release --disable-automatic-resolution --product ForyRowXlangPeer echo "FORY_SWIFT_PEER_PREBUILT=1" >> "$GITHUB_ENV" - name: Run Swift Xlang Test env: @@ -584,6 +585,8 @@ jobs: mvn -T16 --no-transfer-progress clean install -DskipTests -Dmaven.javadoc.skip=true -Dmaven.source.skip=true cd fory-core mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.xlang.SwiftXlangTest + cd ../fory-format + mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.format.SwiftCrossLanguageTest swift_idl: name: Swift IDL Tests diff --git a/README.md b/README.md index 22dc0110d8..934f3ec7d3 100644 --- a/README.md +++ b/README.md @@ -844,7 +844,8 @@ deserialization, see the [Java row-format guide](docs/row-format/java.md), [Python row-format guide](docs/row-format/python.md), [C++ row-format guide](docs/row-format/cpp.md), [Rust row-format guide](docs/row-format/rust.md), -[Go row-format guide](docs/row-format/go.md), and +[Go row-format guide](docs/row-format/go.md), +[Swift row-format guide](docs/row-format/swift.md), and [row-format specification](docs/specification/row_format_spec.md). ## Fory JSON diff --git a/docs/introduction/choose-a-format.md b/docs/introduction/choose-a-format.md index 44bd117613..53b744647f 100644 --- a/docs/introduction/choose-a-format.md +++ b/docs/introduction/choose-a-format.md @@ -19,12 +19,12 @@ license: | limitations under the License. --- -| Format or mode | Data model | Use it when | Interoperability | Start here | -| -------------- | ----------------------------------- | ------------------------------------------------------------- | --------------------------------------------------------------------------- | -------------------------------------------------------- | -| Xlang binary | Portable object graphs | Data crosses language boundaries | Shared wire format across supported Fory implementations | [Cross-language guide](../object-serialization/xlang.md) | -| Native binary | Language-native object graphs | Producer and consumer use one Fory implementation family | One Fory implementation family only | [Object Serialization](../object-serialization/index.md) | -| Row Format | Random-access binary rows | You need random field access or analytics-style partial reads | Standard Row is shared by Java, Python, C++, and Rust; Compact is Java-only | [Row Format guide](../row-format/index.md) | -| Fory JSON | Standard JSON mapped to JVM objects | Java, Kotlin, or Scala applications need standard JSON | Standard JSON text | [Fory JSON guide](../json/index.md) | +| Format or mode | Data model | Use it when | Interoperability | Start here | +| -------------- | ----------------------------------- | ------------------------------------------------------------- | ---------------------------------------------------------------------------------- | -------------------------------------------------------- | +| Xlang binary | Portable object graphs | Data crosses language boundaries | Shared wire format across supported Fory implementations | [Cross-language guide](../object-serialization/xlang.md) | +| Native binary | Language-native object graphs | Producer and consumer use one Fory implementation family | One Fory implementation family only | [Object Serialization](../object-serialization/index.md) | +| Row Format | Random-access binary rows | You need random field access or analytics-style partial reads | Standard Row is shared by Java, Python, C++, Rust, and Swift; Compact is Java-only | [Row Format guide](../row-format/index.md) | +| Fory JSON | Standard JSON mapped to JVM objects | Java, Kotlin, or Scala applications need standard JSON | Standard JSON text | [Fory JSON guide](../json/index.md) | Xlang and native are sibling modes of Object Serialization. Use them when the receiver needs to reconstruct an object graph. Row Format and Fory JSON are separate formats, not additional Object diff --git a/docs/introduction/index.md b/docs/introduction/index.md index 74d4b16741..052bb6f478 100644 --- a/docs/introduction/index.md +++ b/docs/introduction/index.md @@ -113,7 +113,7 @@ optimized for analytics and partial-read workloads: - **Zero-copy random access**: Read fields, arrays, and nested values without rebuilding whole objects. - **Partial operations**: Read only the values needed for a query or pipeline stage. - **Apache Arrow integration**: Convert to columnar data for analytics pipelines. -- **Multi-language support**: Exchange Standard Row Format data across Java, Python, C++, and Rust. +- **Multi-language support**: Exchange Standard Row Format data across Java, Python, C++, Rust, and Swift. ### Optimized Fory Implementations diff --git a/docs/introduction/support-matrix.md b/docs/introduction/support-matrix.md index d8fcbdb138..53c9051608 100644 --- a/docs/introduction/support-matrix.md +++ b/docs/introduction/support-matrix.md @@ -26,7 +26,7 @@ does not imply support for every Fory capability. | --------------------------- | ---------------------------------------------------------------------------------- | -------------------------------------------------------- | | Xlang object serialization | Java, Python, C++, Go, Rust, JavaScript/TypeScript, C#, Swift, Dart, Scala, Kotlin | One shared xlang wire format | | Native object serialization | Java, Python, C++, Go, Rust, Scala, Kotlin | One Fory implementation family only | -| Standard Row Format | Java, Python, C++, Rust | Shared Standard Row layout | +| Standard Row Format | Java, Python, C++, Rust, Swift | Shared Standard Row layout | | Compact Row Format | Java | Java-only compact layout | | Fory JSON | Java, Kotlin, Scala | Standard JSON text | | Fory compiler output | Java, Python, C++, Go, Rust, JavaScript/TypeScript, C#, Swift, Dart, Scala, Kotlin | Generated models use supported Fory APIs | diff --git a/docs/row-format/go.md b/docs/row-format/go.md index 5ef96b1ade..3421cc91f3 100644 --- a/docs/row-format/go.md +++ b/docs/row-format/go.md @@ -257,5 +257,5 @@ underlying bytes are not modified. ## Related Topics - [Basic Serialization](../object-serialization/go/basic-serialization.md) - Object graph serialization -- [Standard Row Format](index.md#standard-row) - Shared layout for Java, Python, C++, Rust, and Go +- [Standard Row Format](index.md#standard-row) - Shared layout for Java, Python, C++, Rust, Go, and Swift - [Row Format Specification](../specification/row_format_spec.md) - Protocol details diff --git a/docs/row-format/index.md b/docs/row-format/index.md index ac41122a90..dcc0b0a67f 100644 --- a/docs/row-format/index.md +++ b/docs/row-format/index.md @@ -30,20 +30,20 @@ reconstruction as its primary access pattern. ## Choose a Layout -| Layout | Language support | Compatibility | -| ------------ | --------------------------- | -------------------------------- | -| Standard Row | Java, Python, C++, Rust, Go | Shared Standard Row layout | -| Compact Row | Java | Java-only, space-oriented layout | +| Layout | Language support | Compatibility | +| ------------ | ---------------------------------- | -------------------------------- | +| Standard Row | Java, Python, C++, Rust, Go, Swift | Shared Standard Row layout | +| Compact Row | Java | Java-only, space-oriented layout | ## Standard Row -Standard Row is the interoperable layout for Java, Python, C++, Rust, and Go. +Standard Row is the interoperable layout for Java, Python, C++, Rust, Go, and Swift. ### Features - **Zero-copy random access**: Read selected fields directly from encoded data. - **Partial deserialization**: Reconstruct only the values an application needs. -- **Cross-language compatibility**: Share Standard Row bytes between Java, Python, C++, Rust, and Go. +- **Cross-language compatibility**: Share Standard Row bytes between Java, Python, C++, Rust, Go, and Swift. - **Apache Arrow integration**: Convert rows to Arrow data in Java and Python. ### Layout @@ -62,6 +62,7 @@ layout, alignment rules, type table, and endianness are defined by the | C++ | Compatible | [C++](cpp.md) | Native row readers and writers | | Rust | Compatible | [Rust](rust.md) | Borrowed struct, array, and map views | | Go | Compatible | [Go](go.md) | Reflection struct encoder; zero-copy row views | +| Swift | Compatible | [Swift](swift.md) | Macro-generated struct encoder; zero-copy row views | Use the language guides for installation, schema construction, encoding, random access, partial reads, and language-specific integrations. @@ -94,7 +95,7 @@ Reuse the encoder within one thread. Create separate encoders for concurrent thr - Fixed-size nested structs can be stored inline. Choose Compact Row only when every reader is Java and the space reduction justifies the -Java-specific layout. Use Standard Row for Java, Python, C++, Rust, and Go interchange. +Java-specific layout. Use Standard Row for Java, Python, C++, Rust, Go, and Swift interchange. See the [Row Format specification](../specification/row_format_spec.md) for the exact Standard and Compact binary layouts. diff --git a/docs/row-format/swift.md b/docs/row-format/swift.md new file mode 100644 index 0000000000..f4633e699a --- /dev/null +++ b/docs/row-format/swift.md @@ -0,0 +1,274 @@ +--- +title: Swift Standard Row Format +sidebar_position: 8 +id: swift +license: | + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--- + +Apache Fory™ Swift implements the Standard Row Format used by Java, C++, Python, Rust, and Go in +the `Fory` module. It provides a `@ForyRow` macro that generates a struct encoder, read-only views +for random field access, and the cross-language schema encoding. Add the `Fory` package as described +in the [Swift guide](../object-serialization/swift/index.md). + +## Overview + +Use Row Format when readers need selected fields or collection elements rather than a fully +reconstructed value. Readers are views over the encoded bytes: reading one field costs a bitmap +test and a slot lookup, and other fields are never touched. + +Row Format is a trusted in-memory format. Decode only bytes produced by a Fory row writer for the +same schema, from a source you trust. Malformed bytes surface as thrown errors, but the format does +not defend against hostile input the way object serialization does. + +## When to Use Row Format + +- Analytics workloads with selective field access +- Large datasets where only a subset of fields is needed +- Memory-mapped or shared data read by several languages +- High-throughput pipelines that exchange Standard Row bytes with Java, C++, Python, Rust, or Go + +## Basic Usage + +```swift +import Fory + +@ForyRow +struct UserProfile { + var id: Int64 + var username: String + var email: String? + var scores: [Int32] + var preferences: [String: String] + var isActive: Bool +} + +let encoder = RowEncoder() +let profile = UserProfile( + id: 12345, + username: "alice", + email: "alice@example.com", + scores: [95, 87, 92, 88], + preferences: ["theme": "dark", "language": "en"], + isActive: true +) + +let rowBytes = try encoder.toRow(profile) + +// Random access without decoding the whole struct. +let schema = encoder.schema +let row = try Row(schema: schema, bytes: Array(rowBytes)) +print(try row.string(at: schema.fieldIndex(of: "username")!) ?? "") // alice +print(try row.array(at: schema.fieldIndex(of: "scores")!)?.int32(at: 1) ?? 0) // 87 +print(row.isNull(at: schema.fieldIndex(of: "email")!)) // false + +// Full decode when the whole value is needed. +let decoded = try encoder.fromRow(rowBytes) +print(decoded.preferences["theme"] ?? "") // dark +``` + +`@ForyRow` builds the schema from the struct's stored properties at compile time. +`RowEncoder` reuses one write buffer; create one encoder per struct type and reuse it. + +## Rows and Framed Messages + +`toRow` and `fromRow` work with bare row bytes, which is what the other languages' `toRow` and +`BinaryRow.pointTo` exchange. `encode` and `decode` add the framing used by the Java and Python +row encoders: an 8-byte little-endian schema hash followed by the row. + +```swift +let framed = try encoder.encode(profile) // hash + row +let fromFrame = try encoder.decode(framed) // verifies the hash, then decodes +``` + +The hash is a type-shape fingerprint: it covers the field types and nothing else, so `decode` +rejects a writer whose field types differ but cannot detect renamed fields, changed nullability, +or reordered fields of the same type. Share the schema bytes (see +[Schema Exchange](#schema-exchange)) when peers must agree on more than the type shape. + +`decode` and `fromRow` throw `ForyError` for truncated or inconsistent bytes. Decoded values never +refer to the input, so the input can be reused immediately. + +## Zero-Copy Reading + +`Row(schema:bytes:)` creates a view over a row. `row(at:)`, `array(at:)`, and `map(at:)` return +views over the nested bytes without copying them. `string(at:)` and `binary(at:)` copy the value +out. A view holds its own reference to the byte array, so changing your array afterwards does not +change what the view reads. + +```swift +let scores = try row.array(at: schema.fieldIndex(of: "scores")!)! +for index in 0..` | Standard array | Yes | +| `[K: V]` | Standard map | Yes | +| `enum E: String` with `RowCodable` | Variable-width UTF-8 raw value | Yes | +| `@ForyRow` struct | Nested Standard Row | Yes | +| `T?` for any supported `T` | Same encoding as `T` | Yes | + +`Data` matches Java's `byte[]`, which Java also infers as a list of int8, and encodes the same as +`[Int8]`. The row format's binary type is available only to hand-built schemas through +`RowWriter.writeBytes` and `Row.binary(at:)`. + +`Date` is rounded to the nearest microsecond. Dates between roughly the years 1729 and 2273 come +back exactly to the microsecond; further away, a `Date` cannot hold every microsecond. `Duration` +is truncated toward zero to whole microseconds. Encoding throws when a `Date` or `Duration` does +not fit in a 64-bit number of microseconds, or a `LocalDate` does not fit in 32-bit epoch days. +Decoding an `Int` throws if the stored value does not fit the platform's `Int`. + +A `Set` is stored as a list, so element order is not kept. An enum stores its raw value; opt in +with an empty `extension MyEnum: RowCodable {}`. Decoding a raw value that names no case throws. + +Dictionaries and sets have no fixed order, so the bytes of a row that holds one can differ between +runs of a program, while the decoded values are always equal. Set the environment variable +`SWIFT_DETERMINISTIC_HASHING=1` when tests need the same bytes on every run. + +Map keys must be `Hashable` and are never null. `Date` and `Duration` keys are rejected because +they are rounded when written, so two different keys could decode as one. + +Unsupported: unsigned integers, tuples, nested optionals, classes, generic structs, recursive +types, `float16`, and `decimal`. + +## Schema Exchange + +`toBytes()` and `init(bytes:)` implement the cross-language schema encoding shared with Java's +`SchemaEncoder`, Python's `Schema.to_bytes`/`from_bytes`, and Go's +`SchemaToBytes`/`SchemaFromBytes`. `hash` computes the same type-shape hash used by `encode`. + +```swift +let schemaBytes = try encoder.schema.toBytes() +let received = try RowSchema(bytes: schemaBytes) +print(received == encoder.schema) // true +print(received.hash == encoder.schemaHash) // true +``` + +A Java bean and a Swift struct with equivalent fields produce identical schema bytes when their +nullability matches: use optional properties for Java boxed types (`Integer`, `String` in lists) +and non-optional properties for Java primitives. + +## Writing Rows by Hand + +`RowWriter`, `RowArrayWriter`, and `RowMapWriter` write rows for a schema you construct yourself, +without a Swift struct. They share one `ByteBuffer`; a nested value is written at the end of the +buffer and then attached to its parent slot with `setOffsetAndSize`. + +```swift +let item = RowField(name: "item", dataType: .string, nullable: true) +let schema = RowSchema([ + RowField(name: "id", dataType: .int64, nullable: false), + RowField(name: "tags", dataType: .list(item), nullable: true) +]) +var writer = RowWriter(schema: schema) +writer.reset() +writer.writeInt64(7, at: 0) + +var tags = RowArrayWriter(element: item, buffer: writer.buffer) +try tags.reset(count: 2) +try tags.writeString("swift", at: 0) +tags.setNullAt(1) +try writer.setOffsetAndSize(at: 1, absoluteStart: tags.base, size: tags.size) + +let rowBytes = writer.toBytes() +``` + +For a map, call `reset()` on a `RowMapWriter`, write the keys with one `RowArrayWriter`, call +`finishKeys()`, write the values with a second `RowArrayWriter`, then attach the map to its parent +slot with `setOffsetAndSize`, passing the map writer's `base` and `size`. Each `reset()` starts a +new row at the end of the buffer; call `writer.buffer.clear()` first to reuse the buffer for a new +top-level row. + +## Thread Safety + +A `RowEncoder` owns a reusable write buffer and is not safe for concurrent use; create one encoder +per thread or task, or guard it with a lock. Writers follow the same rule. `Row`, `RowArray`, and +`RowMap` views only read. + +## Related Topics + +- [Basic Serialization](../object-serialization/swift/basic-serialization.md) - Object graph serialization +- [Standard Row Format](index.md#standard-row) - Shared layout for Java, Python, C++, Rust, Go, and Swift +- [Row Format Specification](../specification/row_format_spec.md) - Protocol details diff --git a/docs/row-format/troubleshooting.md b/docs/row-format/troubleshooting.md index 34f51d1c6d..b302c3cd59 100644 --- a/docs/row-format/troubleshooting.md +++ b/docs/row-format/troubleshooting.md @@ -21,8 +21,8 @@ license: | ## A Standard Row peer cannot read Compact Row bytes -Compact Row is a Java-only row family. Use Standard Row on Java, Python, C++, Rust, and Go for shared -bytes. +Compact Row is a Java-only row family. Use Standard Row on Java, Python, C++, Rust, Go, and Swift +for shared bytes. ## A field lookup fails diff --git a/docs/specification/row_format_spec.md b/docs/specification/row_format_spec.md index a192e35f1e..1a7ea1b552 100644 --- a/docs/specification/row_format_spec.md +++ b/docs/specification/row_format_spec.md @@ -26,14 +26,14 @@ Apache Fory Row Format is a cache-friendly, random-access binary format designed - **Random Field Access**: Read individual fields without deserializing the entire row - **Zero-Copy Operations**: Direct memory access without data transformation - **Cache-Friendly Layout**: Optimized memory layout for CPU cache efficiency -- **Cross-Language Support**: Consistent binary format across Java, C++, Python, Rust, and Go +- **Cross-Language Support**: Consistent binary format across Java, C++, Python, Rust, Go, and Swift Fory provides two row format variants: -| Format | Languages | Use Case | -| --------------- | --------------------------- | ------------------------------ | -| Standard Format | Java, C++, Python, Rust, Go | Cross-language compatibility | -| Compact Format | Java only | Space efficiency, smaller rows | +| Format | Languages | Use Case | +| --------------- | ---------------------------------- | ------------------------------ | +| Standard Format | Java, C++, Python, Rust, Go, Swift | Cross-language compatibility | +| Compact Format | Java only | Space efficiency, smaller rows | ## Format Comparison diff --git a/java/fory-format/src/test/java/org/apache/fory/format/SwiftCrossLanguageTest.java b/java/fory-format/src/test/java/org/apache/fory/format/SwiftCrossLanguageTest.java new file mode 100644 index 0000000000..7e581c0faf --- /dev/null +++ b/java/fory-format/src/test/java/org/apache/fory/format/SwiftCrossLanguageTest.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format; + +import com.google.common.collect.ImmutableMap; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import org.apache.fory.format.GoCrossLanguageTest.Blob; +import org.apache.fory.format.encoder.Encoders; +import org.apache.fory.format.encoder.RowEncoder; +import org.apache.fory.format.row.binary.BinaryRow; +import org.apache.fory.format.type.DataTypes; +import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.memory.MemoryUtils; +import org.apache.fory.test.TestUtils; +import org.testng.Assert; +import org.testng.SkipException; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +/** + * Row format cross-language tests against the Swift peer built from {@code + * swift/Tests/ForyRowXlangPeer}. Data shapes are shared with {@link CrossLanguageTest} and {@link + * GoCrossLanguageTest}. Setting {@code FORY_SWIFT_JAVA_CI=1} opts into the suite; once opted in, a + * missing Swift toolchain or a peer build failure fails the tests instead of skipping them. Setting + * {@code FORY_SWIFT_PEER_PREBUILT=1} uses an already built release peer. + */ +@Test +public class SwiftCrossLanguageTest { + private static final String SWIFT_EXECUTABLE = "swift"; + private static final String SWIFT_PEER_TARGET = "ForyRowXlangPeer"; + private static final File SWIFT_WORK_DIR = new File("../../swift"); + private static final Path SWIFT_PEER_BINARY_PATH = + SWIFT_WORK_DIR.toPath().resolve(".build/release").resolve(SWIFT_PEER_TARGET); + // Fixed dictionary order in the peer, so a failure reproduces with the same bytes. + private static final Map SWIFT_PEER_ENV = + ImmutableMap.of("SWIFT_DETERMINISTIC_HASHING", "1", "ENABLE_FORY_DEBUG_OUTPUT", "1"); + private String swiftPeerBinaryPath; + + @BeforeClass + public void ensureSwiftPeerReady() { + if (!"1".equals(System.getenv("FORY_SWIFT_JAVA_CI"))) { + throw new SkipException("Skipping SwiftCrossLanguageTest: FORY_SWIFT_JAVA_CI not set to 1"); + } + try { + Process process = new ProcessBuilder(SWIFT_EXECUTABLE, "--version").start(); + Assert.assertEquals(process.waitFor(), 0, "swift is required when FORY_SWIFT_JAVA_CI=1"); + } catch (IOException e) { + throw new AssertionError("swift is required when FORY_SWIFT_JAVA_CI=1", e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError("interrupted while probing the swift toolchain", e); + } + if (!"1".equals(System.getenv("FORY_SWIFT_PEER_PREBUILT"))) { + Assert.assertTrue(buildSwiftPeer(), "failed to build the Swift row format peer"); + } + Assert.assertTrue( + Files.isRegularFile(SWIFT_PEER_BINARY_PATH), + "Swift row format peer not found at " + SWIFT_PEER_BINARY_PATH.toAbsolutePath()); + Assert.assertTrue( + Files.isExecutable(SWIFT_PEER_BINARY_PATH), + "Swift row format peer is not executable: " + SWIFT_PEER_BINARY_PATH.toAbsolutePath()); + swiftPeerBinaryPath = SWIFT_PEER_BINARY_PATH.toAbsolutePath().toString(); + } + + public void testByteArrayCarrier() throws IOException { + Blob blob = Blob.create(); + RowEncoder encoder = Encoders.bean(Blob.class); + Path dataFile = createTempFile("row_swift_blob"); + Files.write(dataFile, encoder.encode(blob)); + Assert.assertTrue(runSwiftPeer("test_byte_array_carrier", dataFile)); + byte[] swiftBytes = Files.readAllBytes(dataFile); + Assert.assertEquals(encoder.decode(swiftBytes), blob); + // Blob holds no map, so the Swift encoding has one possible layout and must equal Java's. + Assert.assertEquals(swiftBytes, encoder.encode(blob)); + } + + public void testMapEncoder() throws IOException { + CrossLanguageTest.A a = CrossLanguageTest.A.create(); + RowEncoder encoder = Encoders.bean(CrossLanguageTest.A.class); + Path dataFile = createTempFile("row_swift_map"); + Files.write(dataFile, encoder.encode(a)); + Assert.assertTrue(runSwiftPeer("test_map_encoder", dataFile)); + Assert.assertEquals(encoder.decode(Files.readAllBytes(dataFile)), a); + } + + public void testSerializationWithoutSchema() throws IOException { + CrossLanguageTest.Foo foo = CrossLanguageTest.Foo.create(); + RowEncoder encoder = Encoders.bean(CrossLanguageTest.Foo.class); + Path dataFile = createTempFile("row_swift_foo"); + Files.write(dataFile, encoder.toRow(foo).toBytes()); + Assert.assertTrue(runSwiftPeer("test_serialization_without_schema", dataFile)); + Assert.assertEquals(readFoo(encoder, dataFile), foo); + } + + public void testSerializationWithSchema() throws IOException { + CrossLanguageTest.Foo foo = CrossLanguageTest.Foo.create(); + RowEncoder encoder = Encoders.bean(CrossLanguageTest.Foo.class); + Path dataFile = createTempFile("row_swift_foo"); + Path schemaFile = createTempFile("row_swift_foo_schema"); + BinaryRow row = encoder.toRow(foo); + Files.write(dataFile, row.toBytes()); + Files.write(schemaFile, DataTypes.serializeSchema(row.getSchema())); + Assert.assertTrue(runSwiftPeer("test_serialization_with_schema", schemaFile, dataFile)); + Assert.assertEquals(readFoo(encoder, dataFile), foo); + } + + private static boolean buildSwiftPeer() { + Map env = ImmutableMap.of("ENABLE_FORY_DEBUG_OUTPUT", "1"); + List fastBuildCommand = + Arrays.asList( + SWIFT_EXECUTABLE, + "build", + "-c", + "release", + "--disable-automatic-resolution", + "--product", + SWIFT_PEER_TARGET); + if (TestUtils.executeCommand(fastBuildCommand, 600, env, SWIFT_WORK_DIR)) { + return true; + } + List fallbackBuildCommand = + Arrays.asList(SWIFT_EXECUTABLE, "build", "-c", "release", "--product", SWIFT_PEER_TARGET); + return TestUtils.executeCommand(fallbackBuildCommand, 600, env, SWIFT_WORK_DIR); + } + + private static CrossLanguageTest.Foo readFoo( + RowEncoder encoder, Path dataFile) throws IOException { + MemoryBuffer buffer = MemoryUtils.wrap(Files.readAllBytes(dataFile)); + BinaryRow row = new BinaryRow(encoder.schema()); + row.pointTo(buffer, 0, buffer.size()); + return encoder.fromRow(row); + } + + private static Path createTempFile(String prefix) throws IOException { + Path file = Files.createTempFile(prefix, "data"); + file.toFile().deleteOnExit(); + return file; + } + + private boolean runSwiftPeer(String caseName, Path... files) { + List command = new ArrayList<>(); + command.add(swiftPeerBinaryPath); + command.add(caseName); + for (Path file : files) { + command.add(file.toAbsolutePath().toString()); + } + return TestUtils.executeCommand(command, 30, SWIFT_PEER_ENV, SWIFT_WORK_DIR); + } +} diff --git a/swift/Package.swift b/swift/Package.swift index 4f66d4c650..1e34a13fa3 100644 --- a/swift/Package.swift +++ b/swift/Package.swift @@ -66,6 +66,11 @@ let package = Package( dependencies: ["Fory"], path: "Tests/ForyXlangTests" ), + .executableTarget( + name: "ForyRowXlangPeer", + dependencies: ["Fory"], + path: "Tests/ForyRowXlangPeer" + ), .testTarget( name: "ForyTests", dependencies: [ diff --git a/swift/Sources/Fory/ByteBuffer.swift b/swift/Sources/Fory/ByteBuffer.swift index 0c82279830..d49130e71b 100644 --- a/swift/Sources/Fory/ByteBuffer.swift +++ b/swift/Sources/Fory/ByteBuffer.swift @@ -819,39 +819,7 @@ public final class ByteBuffer { guard let base = buffer.baseAddress else { return nil } - let utf8Base = base.advanced(by: start) - let utf8Bytes = UnsafeBufferPointer(start: utf8Base, count: count) - var index = 0 - var isASCII = true - // An ASCII byte has a clear high bit, so this mask checks eight bytes at once - // regardless of native byte order. The bounded loop makes the unaligned load safe. - while index <= count - MemoryLayout.size { - let word = UnsafeRawPointer(utf8Base.advanced(by: index)) - .loadUnaligned(as: UInt64.self) - if word & 0x8080_8080_8080_8080 != 0 { - isASCII = false - break - } - index += MemoryLayout.size - } - if isASCII { - while index < count { - if utf8Bytes[index] >= 0x80 { - isASCII = false - break - } - index += 1 - } - } - if isASCII { - // ASCII bytes are always valid UTF-8. - // swiftlint:disable:next optional_data_string_conversion - return String(decoding: utf8Bytes, as: UTF8.self) - } - if #available(macOS 15.0, iOS 18.0, tvOS 18.0, watchOS 11.0, visionOS 2.0, *) { - return String(validating: utf8Bytes, as: UTF8.self) - } - return String(bytes: utf8Bytes, encoding: .utf8) + return decodeValidatedUTF8(UnsafeBufferPointer(start: base.advanced(by: start), count: count)) } guard let decoded else { throw ForyError.invalidData("invalid UTF-8 sequence") @@ -875,3 +843,67 @@ public final class ByteBuffer { } } } + +extension ByteBuffer { + + // For row support, a writer first reserves the fixed slots and patches them in place later. + // This appends `count` zero bytes and returns the absolute offset of the first one. + @inlinable + @inline(__always) + internal func appendZeros(_ count: Int) -> Int { + let start = storage.count + storage.append(contentsOf: repeatElement(0, count: count)) + return start + } + + // Writes `value` at `offset` with the smallest byte first, replacing the bytes already there. + // Row writers use this to fill a slot they reserved earlier. The offset doesn't need to be + // aligned, but those bytes must already exist. + @inlinable + @inline(__always) + internal func storeLittleEndian(_ value: T, at offset: Int) { + storage.withUnsafeMutableBytes { raw in + raw.storeBytes(of: value.littleEndian, toByteOffset: offset, as: T.self) + } + } +} + +@inlinable +@inline(__always) +internal func decodeValidatedUTF8(_ utf8Bytes: UnsafeBufferPointer) -> String? { + guard let utf8Base = utf8Bytes.baseAddress else { + return "" + } + let count = utf8Bytes.count + var index = 0 + var isASCII = true + // An ASCII byte has a clear high bit, so this mask checks eight bytes at once + // regardless of native byte order. The bounded loop makes the unaligned load safe. + while index <= count - MemoryLayout.size { + let word = UnsafeRawPointer(utf8Base.advanced(by: index)) + .loadUnaligned(as: UInt64.self) + if word & 0x8080_8080_8080_8080 != 0 { + isASCII = false + break + } + index += MemoryLayout.size + } + if isASCII { + while index < count { + if utf8Bytes[index] >= 0x80 { + isASCII = false + break + } + index += 1 + } + } + if isASCII { + // ASCII bytes are always valid UTF-8. + // swiftlint:disable:next optional_data_string_conversion + return String(decoding: utf8Bytes, as: UTF8.self) + } + if #available(macOS 15.0, iOS 18.0, tvOS 18.0, watchOS 11.0, visionOS 2.0, *) { + return String(validating: utf8Bytes, as: UTF8.self) + } + return String(bytes: utf8Bytes, encoding: .utf8) +} diff --git a/swift/Sources/Fory/MacroDeclarations.swift b/swift/Sources/Fory/MacroDeclarations.swift index 3d80b62731..e96ec45667 100644 --- a/swift/Sources/Fory/MacroDeclarations.swift +++ b/swift/Sources/Fory/MacroDeclarations.swift @@ -284,3 +284,8 @@ public macro ForyCase( @attached(peer) public macro ForyUnknownCase() = #externalMacro(module: "ForyMacro", type: "ForyUnknownCaseMacro") + +@attached(member, names: named(rowSchema), named(writeRowFields), named(readRowFields)) +@attached(extension, conformances: RowStruct) +/// Generates row format encoding for a struct's stored properties. +public macro ForyRow() = #externalMacro(module: "ForyMacro", type: "ForyRowMacro") diff --git a/swift/Sources/Fory/Row/Row.swift b/swift/Sources/Fory/Row/Row.swift new file mode 100644 index 0000000000..d251c5e3ab --- /dev/null +++ b/swift/Sources/Fory/Row/Row.swift @@ -0,0 +1,401 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +/// A read-only view of one row's bytes. Each getter reads its field in place and returns nil when the +/// field is null. An index outside `0..= 0 && count >= 0 && count <= bytes.count - base, "row range is outside its buffer") + let bitmapWidth = rowBitmapWidth(fields.count) + let fixedSize = bitmapWidth + fields.count * 8 + guard count >= fixedSize else { + throw rowTooShort(count: count, fixedSize: fixedSize) + } + self.fields = fields + self.sizeInBytes = count + self.bytes = bytes + self.base = base + self.bitmapWidth = bitmapWidth + } + + public var fieldCount: Int { + fields.count + } + + public func isNull(at index: Int) -> Bool { + precondition(index >= 0 && index < fields.count, "field index \(index) is outside 0..<\(fields.count)") + return bytes.withUnsafeBytes { rowIsBitSet($0, bitmapOffset: base, index: index) } + } + + public func bool(at index: Int) -> Bool? { + isNull(at: index) ? nil : loadSlot(UInt8.self, at: index) != 0 + } + + public func int8(at index: Int) -> Int8? { + isNull(at: index) ? nil : Int8(bitPattern: loadSlot(UInt8.self, at: index)) + } + + public func int16(at index: Int) -> Int16? { + isNull(at: index) ? nil : Int16(bitPattern: loadSlot(UInt16.self, at: index)) + } + + public func int32(at index: Int) -> Int32? { + isNull(at: index) ? nil : Int32(bitPattern: loadSlot(UInt32.self, at: index)) + } + + public func int64(at index: Int) -> Int64? { + isNull(at: index) ? nil : Int64(bitPattern: loadSlot(UInt64.self, at: index)) + } + + public func float32(at index: Int) -> Float? { + isNull(at: index) ? nil : Float(bitPattern: loadSlot(UInt32.self, at: index)) + } + + public func float64(at index: Int) -> Double? { + isNull(at: index) ? nil : Double(bitPattern: loadSlot(UInt64.self, at: index)) + } + + public func localDate(at index: Int) -> LocalDate? { + int32(at: index).map { LocalDate(epochDay: Int64($0)) } + } + + public func timestamp(at index: Int) -> Date? { + int64(at: index).map(rowDate(micros:)) + } + + public func duration(at index: Int) -> Duration? { + int64(at: index).map { .microseconds($0) } + } + + /// Throws if the stored offset and size point outside this row, or the bytes are not valid UTF-8. + public func string(at index: Int) throws -> String? { + try valueRange(at: index).map { try rowString(bytes, in: $0) } + } + + /// Throws if the stored offset and size point outside this row. + public func binary(at index: Int) throws -> [UInt8]? { + guard let range = try valueRange(at: index) else { + return nil + } + return Array(bytes[range]) + } + + /// The field must have a struct type. + public func row(at index: Int) throws -> Row? { + guard case .struct(let childFields) = fields[index].dataType else { + preconditionFailure("field \(index) has type \(fields[index].dataType), not a struct") + } + guard let range = try valueRange(at: index) else { + return nil + } + return try Row(fields: childFields, bytes: bytes, base: range.lowerBound, count: range.count) + } + + /// The field must have a list type. + public func array(at index: Int) throws -> RowArray? { + guard case .list(let element) = fields[index].dataType else { + preconditionFailure("field \(index) has type \(fields[index].dataType), not a list") + } + guard let range = try valueRange(at: index) else { + return nil + } + return try RowArray(element: element, bytes: bytes, base: range.lowerBound, count: range.count) + } + + /// The field must have a map type. + public func map(at index: Int) throws -> RowMap? { + guard case .map(let key, let value) = fields[index].dataType else { + preconditionFailure("field \(index) has type \(fields[index].dataType), not a map") + } + guard let range = try valueRange(at: index) else { + return nil + } + return try RowMap(key: key, value: value, bytes: bytes, base: range.lowerBound, count: range.count) + } + + /// A copy of only this row's bytes, even when it sits inside a larger buffer. + public func toBytes() -> [UInt8] { + Array(bytes[base..<(base + sizeInBytes)]) + } + + @inline(__always) + private func loadSlot(_: T.Type, at index: Int) -> T { + rowLoad(T.self, bytes, at: base + bitmapWidth + index * 8) + } + + // The null bit is checked before the slot is read, because a null field's slot can hold anything. + private func valueRange(at index: Int) throws -> Range? { + if isNull(at: index) { + return nil + } + let ticket = rowDecodeOffsetAndSize(loadSlot(UInt64.self, at: index)) + return try rowCheckedRange(offset: ticket.offset, size: ticket.size, regionBase: base, regionCount: sizeInBytes) + } +} + +/// A read-only view of one array's bytes, with the same getters as ``Row`` taking element indexes. +public struct RowArray { + public let element: RowField + public let elementCount: Int + public let sizeInBytes: Int + private let bytes: [UInt8] + private let base: Int + private let headerSize: Int + private let elementWidth: Int + + // An array is an 8-byte element count, a null bitmap, then one slot per element. A fixed-width + // element takes only its own width, so three int8 values take three bytes, not three slots of 8. + init(element: RowField, bytes: [UInt8], base: Int, count: Int) throws { + precondition(base >= 0 && count >= 0 && count <= bytes.count - base, "array range is outside its buffer") + guard count >= 8 else { + throw rowArrayTooShort(count: count) + } + // Every element takes at least one byte, so a count above the byte count is rejected before + // any arithmetic that a forged count could overflow. + let declaredCount = rowLoad(UInt64.self, bytes, at: base) + guard declaredCount <= UInt64(count) else { + throw rowArrayCountTooLarge(elementCount: declaredCount, count: count) + } + let elementCount = Int(declaredCount) + let headerSize = 8 + rowBitmapWidth(elementCount) + let elementWidth = element.dataType.byteWidth ?? 8 + let (elementsSize, multiplyOverflow) = elementCount.multipliedReportingOverflow(by: elementWidth) + let (fixedSize, addOverflow) = headerSize.addingReportingOverflow(elementsSize) + guard !multiplyOverflow, !addOverflow, fixedSize <= count else { + throw rowArrayCountTooLarge(elementCount: declaredCount, count: count) + } + self.element = element + self.elementCount = elementCount + self.sizeInBytes = count + self.bytes = bytes + self.base = base + self.headerSize = headerSize + self.elementWidth = elementWidth + } + + public func isNull(at index: Int) -> Bool { + precondition(index >= 0 && index < elementCount, "element index \(index) is outside 0..<\(elementCount)") + return bytes.withUnsafeBytes { rowIsBitSet($0, bitmapOffset: base + 8, index: index) } + } + + public func bool(at index: Int) -> Bool? { + isNull(at: index) ? nil : loadSlot(UInt8.self, at: index) != 0 + } + + public func int8(at index: Int) -> Int8? { + isNull(at: index) ? nil : Int8(bitPattern: loadSlot(UInt8.self, at: index)) + } + + public func int16(at index: Int) -> Int16? { + isNull(at: index) ? nil : Int16(bitPattern: loadSlot(UInt16.self, at: index)) + } + + public func int32(at index: Int) -> Int32? { + isNull(at: index) ? nil : Int32(bitPattern: loadSlot(UInt32.self, at: index)) + } + + public func int64(at index: Int) -> Int64? { + isNull(at: index) ? nil : Int64(bitPattern: loadSlot(UInt64.self, at: index)) + } + + public func float32(at index: Int) -> Float? { + isNull(at: index) ? nil : Float(bitPattern: loadSlot(UInt32.self, at: index)) + } + + public func float64(at index: Int) -> Double? { + isNull(at: index) ? nil : Double(bitPattern: loadSlot(UInt64.self, at: index)) + } + + public func localDate(at index: Int) -> LocalDate? { + int32(at: index).map { LocalDate(epochDay: Int64($0)) } + } + + public func timestamp(at index: Int) -> Date? { + int64(at: index).map(rowDate(micros:)) + } + + public func duration(at index: Int) -> Duration? { + int64(at: index).map { .microseconds($0) } + } + + public func string(at index: Int) throws -> String? { + try valueRange(at: index).map { try rowString(bytes, in: $0) } + } + + public func binary(at index: Int) throws -> [UInt8]? { + guard let range = try valueRange(at: index) else { + return nil + } + return Array(bytes[range]) + } + + public func row(at index: Int) throws -> Row? { + guard case .struct(let childFields) = element.dataType else { + preconditionFailure("array element type is \(element.dataType), not a struct") + } + guard let range = try valueRange(at: index) else { + return nil + } + return try Row(fields: childFields, bytes: bytes, base: range.lowerBound, count: range.count) + } + + public func array(at index: Int) throws -> RowArray? { + guard case .list(let childElement) = element.dataType else { + preconditionFailure("array element type is \(element.dataType), not a list") + } + guard let range = try valueRange(at: index) else { + return nil + } + return try RowArray(element: childElement, bytes: bytes, base: range.lowerBound, count: range.count) + } + + public func map(at index: Int) throws -> RowMap? { + guard case .map(let key, let value) = element.dataType else { + preconditionFailure("array element type is \(element.dataType), not a map") + } + guard let range = try valueRange(at: index) else { + return nil + } + return try RowMap(key: key, value: value, bytes: bytes, base: range.lowerBound, count: range.count) + } + + public func toBytes() -> [UInt8] { + Array(bytes[base..<(base + sizeInBytes)]) + } + + // Copies every element of an int8 array at once, or returns nil when any element is null. + func int8Elements() -> Data? { + precondition(elementWidth == 1, "array element type is \(element.dataType), not int8") + let hasNull = bytes.withUnsafeBytes { raw in + (0..(_: T.Type, at index: Int) -> T { + precondition(MemoryLayout.size <= elementWidth, "array element type is \(element.dataType)") + return rowLoad(T.self, bytes, at: base + headerSize + index * elementWidth) + } + + private func valueRange(at index: Int) throws -> Range? { + if isNull(at: index) { + return nil + } + let ticket = rowDecodeOffsetAndSize(loadSlot(UInt64.self, at: index)) + return try rowCheckedRange(offset: ticket.offset, size: ticket.size, regionBase: base, regionCount: sizeInBytes) + } +} + +/// A read-only view of one map: an array of keys and an array of values, paired by index. +public struct RowMap { + public let keys: RowArray + public let values: RowArray + + // A map is the byte size of its keys array, then the keys array, then the values array. Each + // array counts its offsets from its own start, not from the map's. + init(key: RowField, value: RowField, bytes: [UInt8], base: Int, count: Int) throws { + precondition(base >= 0 && count >= 0 && count <= bytes.count - base, "map range is outside its buffer") + guard count >= 8 else { + throw rowMapTooShort(count: count) + } + let declaredKeysSize = rowLoad(UInt64.self, bytes, at: base) + guard declaredKeysSize <= UInt64(count - 8) else { + throw rowMapKeysTooLarge(keysSize: declaredKeysSize, count: count) + } + let keysSize = Int(declaredKeysSize) + let keys = try RowArray(element: key, bytes: bytes, base: base + 8, count: keysSize) + let values = try RowArray(element: value, bytes: bytes, base: base + 8 + keysSize, count: count - 8 - keysSize) + // Readers walk both arrays with one index, so unequal counts would run past the shorter one. + guard keys.elementCount == values.elementCount else { + throw rowMapCountMismatch(keyCount: keys.elementCount, valueCount: values.elementCount) + } + self.keys = keys + self.values = values + } + + public var count: Int { + keys.elementCount + } +} + +// Reads the little-endian integer at an absolute position that the caller has already checked. +@inline(__always) +private func rowLoad(_: T.Type, _ bytes: [UInt8], at offset: Int) -> T { + bytes.withUnsafeBytes { T(littleEndian: $0.loadUnaligned(fromByteOffset: offset, as: T.self)) } +} + +private func rowString(_ bytes: [UInt8], in range: Range) throws -> String { + let decoded = bytes.withUnsafeBufferPointer { decodeValidatedUTF8(UnsafeBufferPointer(rebasing: $0[range])) } + guard let decoded else { + throw rowInvalidUTF8() + } + return decoded +} + +@inline(never) +func rowTooShort(count: Int, fixedSize: Int) -> ForyError { + .invalidData("row of \(count) bytes is shorter than its \(fixedSize)-byte fixed region") +} + +@inline(never) +func rowInvalidUTF8() -> ForyError { + .invalidData("invalid UTF-8 sequence") +} + +@inline(never) +func rowArrayTooShort(count: Int) -> ForyError { + .invalidData("array of \(count) bytes is shorter than its 8-byte element count") +} + +@inline(never) +func rowArrayCountTooLarge(elementCount: UInt64, count: Int) -> ForyError { + .invalidData("array declares \(elementCount) elements but holds only \(count) bytes") +} + +@inline(never) +func rowMapTooShort(count: Int) -> ForyError { + .invalidData("map of \(count) bytes is shorter than its 8-byte keys size") +} + +@inline(never) +func rowMapKeysTooLarge(keysSize: UInt64, count: Int) -> ForyError { + .invalidData("map keys array of \(keysSize) bytes does not fit in a \(count)-byte map") +} + +@inline(never) +func rowMapCountMismatch(keyCount: Int, valueCount: Int) -> ForyError { + .invalidData("map has \(keyCount) keys but \(valueCount) values") +} diff --git a/swift/Sources/Fory/Row/RowBitmap.swift b/swift/Sources/Fory/Row/RowBitmap.swift new file mode 100644 index 0000000000..fe15dbc616 --- /dev/null +++ b/swift/Sources/Fory/Row/RowBitmap.swift @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// The null bitmap keeps one bit per field or array element, and a set bit means the value is null. +// Bit `index` lives in byte `bitmapOffset + index / 8`, counting bits from the least significant end. + +// Returns the bitmap size in bytes for `count` fields or elements, rounded up to whole 8-byte +// words. `count` must not be negative. +@inline(__always) +func rowBitmapWidth(_ count: Int) -> Int { + ((count + 63) / 64) * 8 +} + +// Rounds `size` up to the next multiple of 8. +@inline(__always) +func rowRoundToWord(_ size: Int) -> Int { + (size + 7) & ~7 +} + +// Reports whether bit `index` is set in the bitmap that starts at `bitmapOffset`. +@inline(__always) +func rowIsBitSet(_ bytes: UnsafeRawBufferPointer, bitmapOffset: Int, index: Int) -> Bool { + bytes[bitmapOffset + (index >> 3)] & (UInt8(1) << UInt8(index & 7)) != 0 +} + +// Sets bit `index` in the bitmap that starts at `bitmapOffset`. +@inline(__always) +func rowSetBit(_ bytes: UnsafeMutableRawBufferPointer, bitmapOffset: Int, index: Int) { + bytes[bitmapOffset + (index >> 3)] |= UInt8(1) << UInt8(index & 7) +} diff --git a/swift/Sources/Fory/Row/RowCodable.swift b/swift/Sources/Fory/Row/RowCodable.swift new file mode 100644 index 0000000000..3e8148143f --- /dev/null +++ b/swift/Sources/Fory/Row/RowCodable.swift @@ -0,0 +1,577 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +/// The write methods ``RowWriter`` and ``RowArrayWriter`` share, so one ``RowCodable`` type can be +/// written as a struct field or as an array element. +public protocol RowValueWriter { + var buffer: ByteBuffer { get } + func setNullAt(_ index: Int) + func writeBool(_ value: Bool, at index: Int) + func writeInt8(_ value: Int8, at index: Int) + func writeInt16(_ value: Int16, at index: Int) + func writeInt32(_ value: Int32, at index: Int) + func writeInt64(_ value: Int64, at index: Int) + func writeFloat32(_ value: Float, at index: Int) + func writeFloat64(_ value: Double, at index: Int) + func writeLocalDate(_ value: LocalDate, at index: Int) throws + func writeTimestamp(_ value: Date, at index: Int) throws + func writeDuration(_ value: Duration, at index: Int) throws + func writeString(_ value: String, at index: Int) throws + func writeBytes(_ value: some ContiguousBytes, at index: Int) throws + func setOffsetAndSize(at index: Int, absoluteStart: Int, size: Int) throws +} + +/// The getters ``Row`` and ``RowArray`` share, so one ``RowCodable`` type can be read from a struct +/// field or from an array element. +public protocol RowValueReader { + func isNull(at index: Int) -> Bool + func bool(at index: Int) -> Bool? + func int8(at index: Int) -> Int8? + func int16(at index: Int) -> Int16? + func int32(at index: Int) -> Int32? + func int64(at index: Int) -> Int64? + func float32(at index: Int) -> Float? + func float64(at index: Int) -> Double? + func localDate(at index: Int) -> LocalDate? + func timestamp(at index: Int) -> Date? + func duration(at index: Int) -> Duration? + func string(at index: Int) throws -> String? + func binary(at index: Int) throws -> [UInt8]? + func row(at index: Int) throws -> Row? + func array(at index: Int) throws -> RowArray? + func map(at index: Int) throws -> RowMap? +} + +extension RowWriter: RowValueWriter {} +extension RowArrayWriter: RowValueWriter {} +extension Row: RowValueReader {} +extension RowArray: RowValueReader {} + +/// How one Swift type is stored in a row field or an array element. +public protocol RowCodable { + static var rowDataType: RowDataType { get } + /// Whether a field of this type is nullable in a schema built from Swift types. A type that + /// cannot hold nil may still be nullable, as `String` is; reading a null into it throws. + static var rowNullable: Bool { get } + /// The same as `rowDataType.byteWidth`, kept as a constant so writing never builds the data type. + static var rowFixedWidth: Int? { get } + static func writeRow(_ value: Self, to writer: W, at index: Int) throws + /// Throws when the slot is null and `Self` cannot hold nil. + static func readRow(from reader: R, at index: Int) throws -> Self +} + +extension RowWriter { + @inlinable + public func write(_ value: T, at index: Int) throws { + try T.writeRow(value, to: self, at: index) + } +} + +extension Row { + /// Reads field `index` as `T`. Read as an optional type to get nil for a null field. + @inlinable + public func read(_ type: T.Type = T.self, at index: Int) throws -> T { + try T.readRow(from: self, at: index) + } +} + +extension RowField { + /// The field a Swift type gets in a schema built from Swift types. + public static func inferred(_ name: String, _ type: T.Type) -> RowField { + RowField(name: name, dataType: T.rowDataType, nullable: T.rowNullable) + } +} + +extension Bool: RowCodable { + public static var rowDataType: RowDataType { .bool } + public static var rowNullable: Bool { false } + public static var rowFixedWidth: Int? { 1 } + + @inlinable + public static func writeRow(_ value: Bool, to writer: W, at index: Int) throws { + writer.writeBool(value, at: index) + } + + @inlinable + public static func readRow(from reader: R, at index: Int) throws -> Bool { + try rowNonNull(reader.bool(at: index)) + } +} + +extension Int8: RowCodable { + public static var rowDataType: RowDataType { .int8 } + public static var rowNullable: Bool { false } + public static var rowFixedWidth: Int? { 1 } + + @inlinable + public static func writeRow(_ value: Int8, to writer: W, at index: Int) throws { + writer.writeInt8(value, at: index) + } + + @inlinable + public static func readRow(from reader: R, at index: Int) throws -> Int8 { + try rowNonNull(reader.int8(at: index)) + } +} + +extension Int16: RowCodable { + public static var rowDataType: RowDataType { .int16 } + public static var rowNullable: Bool { false } + public static var rowFixedWidth: Int? { 2 } + + @inlinable + public static func writeRow(_ value: Int16, to writer: W, at index: Int) throws { + writer.writeInt16(value, at: index) + } + + @inlinable + public static func readRow(from reader: R, at index: Int) throws -> Int16 { + try rowNonNull(reader.int16(at: index)) + } +} + +extension Int32: RowCodable { + public static var rowDataType: RowDataType { .int32 } + public static var rowNullable: Bool { false } + public static var rowFixedWidth: Int? { 4 } + + @inlinable + public static func writeRow(_ value: Int32, to writer: W, at index: Int) throws { + writer.writeInt32(value, at: index) + } + + @inlinable + public static func readRow(from reader: R, at index: Int) throws -> Int32 { + try rowNonNull(reader.int32(at: index)) + } +} + +extension Int64: RowCodable { + public static var rowDataType: RowDataType { .int64 } + public static var rowNullable: Bool { false } + public static var rowFixedWidth: Int? { 8 } + + @inlinable + public static func writeRow(_ value: Int64, to writer: W, at index: Int) throws { + writer.writeInt64(value, at: index) + } + + @inlinable + public static func readRow(from reader: R, at index: Int) throws -> Int64 { + try rowNonNull(reader.int64(at: index)) + } +} + +// Int is stored as int64 on every platform. Where Int is 32 bits, a larger stored value throws. +extension Int: RowCodable { + public static var rowDataType: RowDataType { .int64 } + public static var rowNullable: Bool { false } + public static var rowFixedWidth: Int? { 8 } + + @inlinable + public static func writeRow(_ value: Int, to writer: W, at index: Int) throws { + writer.writeInt64(Int64(value), at: index) + } + + @inlinable + public static func readRow(from reader: R, at index: Int) throws -> Int { + let value = try rowNonNull(reader.int64(at: index)) + guard let narrowed = Int(exactly: value) else { + throw rowIntOutOfRange(value) + } + return narrowed + } +} + +extension Float: RowCodable { + public static var rowDataType: RowDataType { .float32 } + public static var rowNullable: Bool { false } + public static var rowFixedWidth: Int? { 4 } + + @inlinable + public static func writeRow(_ value: Float, to writer: W, at index: Int) throws { + writer.writeFloat32(value, at: index) + } + + @inlinable + public static func readRow(from reader: R, at index: Int) throws -> Float { + try rowNonNull(reader.float32(at: index)) + } +} + +extension Double: RowCodable { + public static var rowDataType: RowDataType { .float64 } + public static var rowNullable: Bool { false } + public static var rowFixedWidth: Int? { 8 } + + @inlinable + public static func writeRow(_ value: Double, to writer: W, at index: Int) throws { + writer.writeFloat64(value, at: index) + } + + @inlinable + public static func readRow(from reader: R, at index: Int) throws -> Double { + try rowNonNull(reader.float64(at: index)) + } +} + +extension String: RowCodable { + public static var rowDataType: RowDataType { .string } + public static var rowNullable: Bool { true } + public static var rowFixedWidth: Int? { nil } + + @inlinable + public static func writeRow(_ value: String, to writer: W, at index: Int) throws { + try writer.writeString(value, at: index) + } + + @inlinable + public static func readRow(from reader: R, at index: Int) throws -> String { + try rowNonNull(reader.string(at: index)) + } +} + +extension Date: RowCodable { + public static var rowDataType: RowDataType { .timestamp } + public static var rowNullable: Bool { true } + public static var rowFixedWidth: Int? { 8 } + + @inlinable + public static func writeRow(_ value: Date, to writer: W, at index: Int) throws { + try writer.writeTimestamp(value, at: index) + } + + @inlinable + public static func readRow(from reader: R, at index: Int) throws -> Date { + try rowNonNull(reader.timestamp(at: index)) + } +} + +extension LocalDate: RowCodable { + public static var rowDataType: RowDataType { .date32 } + public static var rowNullable: Bool { true } + public static var rowFixedWidth: Int? { 4 } + + @inlinable + public static func writeRow(_ value: LocalDate, to writer: W, at index: Int) throws { + try writer.writeLocalDate(value, at: index) + } + + @inlinable + public static func readRow(from reader: R, at index: Int) throws -> LocalDate { + try rowNonNull(reader.localDate(at: index)) + } +} + +extension Duration: RowCodable { + public static var rowDataType: RowDataType { .duration } + public static var rowNullable: Bool { true } + public static var rowFixedWidth: Int? { 8 } + + @inlinable + public static func writeRow(_ value: Duration, to writer: W, at index: Int) throws { + try writer.writeDuration(value, at: index) + } + + @inlinable + public static func readRow(from reader: R, at index: Int) throws -> Duration { + try rowNonNull(reader.duration(at: index)) + } +} + +// An optional uses its wrapped type's slot and width, and stores nil as a set null bit. +extension Optional: RowCodable where Wrapped: RowCodable { + public static var rowDataType: RowDataType { Wrapped.rowDataType } + public static var rowNullable: Bool { true } + public static var rowFixedWidth: Int? { Wrapped.rowFixedWidth } + + @inlinable + public static func writeRow(_ value: Wrapped?, to writer: W, at index: Int) throws { + guard let value else { + writer.setNullAt(index) + return + } + try Wrapped.writeRow(value, to: writer, at: index) + } + + @inlinable + public static func readRow(from reader: R, at index: Int) throws -> Wrapped? { + try reader.isNull(at: index) ? nil : Wrapped.readRow(from: reader, at: index) + } +} + +extension Array: RowCodable where Element: RowCodable { + public static var rowDataType: RowDataType { .list(.inferred("item", Element.self)) } + public static var rowNullable: Bool { true } + public static var rowFixedWidth: Int? { nil } + + @inlinable + public static func writeRow(_ value: [Element], to writer: W, at index: Int) throws { + let array = try rowAppendArray(value, to: writer.buffer) + try writer.setOffsetAndSize(at: index, absoluteStart: array.base, size: array.size) + } + + @inlinable + public static func readRow(from reader: R, at index: Int) throws -> [Element] { + guard let array = try reader.array(at: index) else { + throw rowNullValue(Self.self) + } + var elements: [Element] = [] + elements.reserveCapacity(array.elementCount) + for elementIndex in 0..(_ value: Set, to writer: W, at index: Int) throws { + let array = try rowAppendArray(value, to: writer.buffer) + try writer.setOffsetAndSize(at: index, absoluteStart: array.base, size: array.size) + } + + @inlinable + public static func readRow(from reader: R, at index: Int) throws -> Set { + guard let array = try reader.array(at: index) else { + throw rowNullValue(Self.self) + } + var elements = Set(minimumCapacity: array.elementCount) + for elementIndex in 0..(_ value: [Key: Value], to writer: W, at index: Int) throws { + var map = RowMapWriter(buffer: writer.buffer) + map.reset() + _ = try rowAppendArray(value.keys, to: writer.buffer) + map.finishKeys() + _ = try rowAppendArray(value.values, to: writer.buffer) + try writer.setOffsetAndSize(at: index, absoluteStart: map.base, size: map.size) + } + + @inlinable + public static func readRow(from reader: R, at index: Int) throws -> [Key: Value] { + guard let map = try reader.map(at: index) else { + throw rowNullValue(Self.self) + } + var entries = [Key: Value](minimumCapacity: map.count) + for entryIndex in 0..(_ value: Data, to writer: W, at index: Int) throws { + var array = RowArrayWriter(elementWidth: 1, buffer: writer.buffer) + try array.reset(count: value.count) + array.writeInt8Elements(value) + try writer.setOffsetAndSize(at: index, absoluteStart: array.base, size: array.size) + } + + public static func readRow(from reader: R, at index: Int) throws -> Data { + guard let array = try reader.array(at: index) else { + throw rowNullValue(Data.self) + } + guard let bytes = array.int8Elements() else { + throw rowNullValue(Int8.self) + } + return bytes + } +} + +/// Stores an enum as the string of its raw value. Opt in with an empty `extension MyEnum: RowCodable {}`. +extension RowCodable where Self: RawRepresentable, Self.RawValue == String { + public static var rowDataType: RowDataType { .string } + public static var rowNullable: Bool { true } + public static var rowFixedWidth: Int? { nil } + + @inlinable + public static func writeRow(_ value: Self, to writer: W, at index: Int) throws { + try writer.writeString(value.rawValue, at: index) + } + + @inlinable + public static func readRow(from reader: R, at index: Int) throws -> Self { + guard let name = try reader.string(at: index) else { + throw rowNullValue(Self.self) + } + guard let value = Self(rawValue: name) else { + throw rowUnknownCase(name, of: Self.self) + } + return value + } +} + +/// A struct stored as its own row. `@ForyRow` adopts it. +public protocol RowStruct: RowCodable { + static var rowSchema: RowSchema { get } + static func writeRowFields(_ value: Self, to writer: RowWriter) throws + static func readRowFields(from row: Row) throws -> Self +} + +// A struct field holds a complete child row written after the parent's slots. +extension RowStruct { + public static var rowDataType: RowDataType { .struct(rowSchema.fields) } + public static var rowNullable: Bool { true } + public static var rowFixedWidth: Int? { nil } + + @inlinable + public static func writeRow(_ value: Self, to writer: W, at index: Int) throws { + var child = RowWriter(schema: rowSchema, buffer: writer.buffer) + child.reset() + try writeRowFields(value, to: child) + try writer.setOffsetAndSize(at: index, absoluteStart: child.base, size: child.size) + } + + @inlinable + public static func readRow(from reader: R, at index: Int) throws -> Self { + guard let row = try reader.row(at: index) else { + throw rowNullValue(Self.self) + } + return try readRowFields(from: row) + } +} + +/// Converts a ``RowStruct`` to and from row bytes. It reuses one buffer, so use an encoder from one +/// thread at a time. +public final class RowEncoder { + public let schema: RowSchema + public let schemaHash: Int64 + private let buffer = ByteBuffer() + + public init() { + schema = T.rowSchema + schemaHash = schema.hash + } + + public func toRow(_ value: T) throws -> Data { + buffer.clear() + try writeRow(value) + return buffer.toData() + } + + public func fromRow(_ data: Data) throws -> T { + try T.readRowFields(from: Row(schema: schema, bytes: Array(data))) + } + + /// The schema hash as 8 little-endian bytes, then the row. + public func encode(_ value: T) throws -> Data { + buffer.clear() + buffer.writeInt64(schemaHash) + try writeRow(value) + return buffer.toData() + } + + /// Throws if the data is shorter than the hash or was written with a different schema. + public func decode(_ data: Data) throws -> T { + guard data.count >= 8 else { + throw rowEncodedTooShort(data.count) + } + let bytes = Array(data) + let hash = Int64(littleEndian: bytes.withUnsafeBytes { $0.loadUnaligned(as: Int64.self) }) + guard hash == schemaHash else { + throw rowSchemaHashMismatch(found: hash, expected: schemaHash) + } + return try T.readRowFields(from: Row(fields: schema.fields, bytes: bytes, base: 8, count: bytes.count - 8)) + } + + private func writeRow(_ value: T) throws { + var writer = RowWriter(schema: schema, buffer: buffer) + writer.reset() + try T.writeRowFields(value, to: writer) + } +} + +// Appends `elements` as an array at the end of the buffer. The returned writer's base and size are +// what the parent slot points at. +@inlinable +func rowAppendArray(_ elements: C, to buffer: ByteBuffer) throws -> RowArrayWriter +where C.Element: RowCodable { + var array = RowArrayWriter(elementWidth: C.Element.rowFixedWidth ?? 8, buffer: buffer) + try array.reset(count: elements.count) + for (elementIndex, element) in elements.enumerated() { + try C.Element.writeRow(element, to: array, at: elementIndex) + } + return array +} + +@inlinable +@inline(__always) +func rowNonNull(_ value: T?) throws -> T { + guard let value else { + throw rowNullValue(T.self) + } + return value +} + +@usableFromInline +@inline(never) +func rowNullValue(_ type: Any.Type) -> ForyError { + .invalidData("null row value cannot be read into non-optional \(type)") +} + +@usableFromInline +@inline(never) +func rowIntOutOfRange(_ value: Int64) -> ForyError { + .invalidData("int64 \(value) does not fit in Int") +} + +@usableFromInline +@inline(never) +func rowUnknownCase(_ name: String, of type: Any.Type) -> ForyError { + .invalidData("unknown case \"\(name)\" for \(type)") +} + +@inline(never) +private func rowEncodedTooShort(_ count: Int) -> ForyError { + .invalidData("encoded row of \(count) bytes is shorter than its 8-byte schema hash") +} + +@inline(never) +private func rowSchemaHashMismatch(found: Int64, expected: Int64) -> ForyError { + .invalidData("row schema hash \(found) does not match the expected \(expected)") +} diff --git a/swift/Sources/Fory/Row/RowSchema.swift b/swift/Sources/Fory/Row/RowSchema.swift new file mode 100644 index 0000000000..df9de2081b --- /dev/null +++ b/swift/Sources/Fory/Row/RowSchema.swift @@ -0,0 +1,159 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/// The type of a row field, an array element, or a map key or value. +public indirect enum RowDataType: Equatable, Sendable, CustomStringConvertible { + case bool + case int8 + case int16 + case int32 + case int64 + /// Only parsed from schemas; its values are not read or written yet. + case float16 + case float32 + case float64 + case string + case binary + case date32 + case timestamp + case duration + /// Only parsed from schemas; its values are not read or written yet. + case decimal(precision: Int, scale: Int) + case list(RowField) + case map(key: RowField, value: RowField) + case `struct`([RowField]) + + /// The cross-language type id written in schema bytes. + public var typeId: TypeId { + switch self { + case .bool: return .bool + case .int8: return .int8 + case .int16: return .int16 + case .int32: return .int32 + case .int64: return .int64 + case .float16: return .float16 + case .float32: return .float32 + case .float64: return .float64 + case .string: return .string + case .binary: return .binary + case .date32: return .date + case .timestamp: return .timestamp + case .duration: return .duration + case .decimal: return .decimal + case .list: return .list + case .map: return .map + case .struct: return .structType + } + } + + /// The size of one value in bytes, or nil when the value lives after the slots and is + /// reached through an offset and size. + public var byteWidth: Int? { + switch self { + case .bool, .int8: return 1 + case .int16, .float16: return 2 + case .int32, .float32, .date32: return 4 + case .int64, .float64, .timestamp, .duration: return 8 + case .string, .binary, .decimal, .list, .map, .struct: return nil + } + } + + /// A list whose element field is named `item` and can be null. + public static func listOf(_ element: RowDataType) -> RowDataType { + .list(RowField(name: "item", dataType: element, nullable: true)) + } + + /// A map whose key field is named `key` and is never null, and whose value field is named + /// `value` and can be null. + public static func mapOf(_ key: RowDataType, _ value: RowDataType) -> RowDataType { + .map( + key: RowField(name: "key", dataType: key, nullable: false), + value: RowField(name: "value", dataType: value, nullable: true) + ) + } + + public var description: String { + switch self { + case .bool: return "bool" + case .int8: return "int8" + case .int16: return "int16" + case .int32: return "int32" + case .int64: return "int64" + case .float16: return "float16" + case .float32: return "float32" + case .float64: return "float64" + case .string: return "string" + case .binary: return "binary" + case .date32: return "date32" + case .timestamp: return "timestamp" + case .duration: return "duration" + case .decimal(let precision, let scale): return "decimal(\(precision), \(scale))" + case .list(let element): return "list<\(element.dataType)>" + case .map(let key, let value): return "map<\(key.dataType), \(value.dataType)>" + case .struct(let fields): return "struct<\(rowFieldListDescription(fields))>" + } + } +} + +/// A named slot in a schema, or the element of a list, or the key or value of a map. +public struct RowField: Equatable, Sendable { + public var name: String + public var dataType: RowDataType + public var nullable: Bool + + public init(name: String, dataType: RowDataType, nullable: Bool) { + self.name = name + self.dataType = dataType + self.nullable = nullable + } +} + +/// The fields of a row in slot order. A row can only be read with the schema it was written with. +public struct RowSchema: Equatable, Sendable, CustomStringConvertible { + public let fields: [RowField] + // Built once from `fields`, so it never disagrees with them. + private let indexByName: [String: Int] + + public init(_ fields: [RowField]) { + self.fields = fields + var indexByName: [String: Int] = [:] + indexByName.reserveCapacity(fields.count) + for (index, field) in fields.enumerated() { + indexByName[field.name] = index + } + self.indexByName = indexByName + } + + public var fieldCount: Int { + fields.count + } + + /// The slot index of the field named `name`, or nil if there is none. When a name repeats, the + /// last field with that name wins. + public func fieldIndex(of name: String) -> Int? { + indexByName[name] + } + + public var description: String { + "schema<\(rowFieldListDescription(fields))>" + } +} + +// Shared by the struct and schema descriptions: "a: int32, b: string". +private func rowFieldListDescription(_ fields: [RowField]) -> String { + fields.map { "\($0.name): \($0.dataType)" }.joined(separator: ", ") +} diff --git a/swift/Sources/Fory/Row/RowSchemaBytes.swift b/swift/Sources/Fory/Row/RowSchemaBytes.swift new file mode 100644 index 0000000000..9a83430041 --- /dev/null +++ b/swift/Sources/Fory/Row/RowSchemaBytes.swift @@ -0,0 +1,245 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Schema bytes are a version byte and a field count, then each field in order. A field is a header +// byte, its packed name, and its type. The header holds the name encoding in bits 0 and 1, the name +// size minus one in bits 2 to 5 (15 means a varint with the size minus 15 follows), and the +// nullable flag in bit 6. A type is its id byte, followed by precision and scale for a decimal, the +// element field for a list, the key and value fields for a map, or a count and the fields for a +// struct. + +private let rowSchemaVersion: UInt8 = 1 +private let rowFieldNameSizeThreshold = 15 +// Deeper schemas are rejected so reading one cannot recurse without bound. +private let rowSchemaMaxDepth = 64 +// Header bits 0 and 1 hold a position in this list, not the encoding's raw value. +private let rowFieldNameEncodings: [MetaStringEncoding] = [.utf8, .allToLowerSpecial, .lowerUpperDigitSpecial] + +extension RowSchema { + /// The schema in the byte form every row format runtime reads. Throws if a field name is empty, + /// fields nest 64 or more levels deep, or a decimal precision or scale does not fit in a byte. + public func toBytes() throws -> [UInt8] { + let buffer = ByteBuffer() + buffer.writeUInt8(rowSchemaVersion) + buffer.writeVarUInt32(UInt32(fields.count)) + for field in fields { + try rowWriteSchemaField(field, depth: 0, to: buffer) + } + return buffer.storage + } + + public init(bytes: [UInt8]) throws { + let buffer = ByteBuffer(bytes: bytes) + let version = try buffer.readUInt8() + guard version == rowSchemaVersion else { + throw rowSchemaVersionUnsupported(version) + } + self.init(try rowReadSchemaFields(buffer, depth: 0)) + } + + /// A fingerprint of the field types, written before a row so the reader can tell it was written + /// with the same schema. Names and nullability do not change it. + public var hash: Int64 { + fields.reduce(17) { rowSchemaHash($0, $1.dataType) } + } +} + +private func rowWriteSchemaField(_ field: RowField, depth: Int, to buffer: ByteBuffer) throws { + guard depth < rowSchemaMaxDepth else { + throw rowSchemaTooDeepToWrite() + } + guard !field.name.isEmpty else { + throw rowSchemaEmptyFieldName() + } + let name = try MetaStringEncoder.fieldName.encode(field.name, allowedEncodings: rowFieldNameEncodings) + guard let encodingIndex = rowFieldNameEncodings.firstIndex(of: name.encoding) else { + preconditionFailure("field name encoder chose \(name.encoding), which was not allowed") + } + let nameSize = name.bytes.count + let longName = nameSize > rowFieldNameSizeThreshold + var header = UInt8(encodingIndex) + header |= UInt8(longName ? rowFieldNameSizeThreshold : nameSize - 1) << 2 + if field.nullable { + header |= 0x40 + } + buffer.writeUInt8(header) + if longName { + buffer.writeVarUInt32(UInt32(nameSize - rowFieldNameSizeThreshold)) + } + buffer.writeBytes(name.bytes) + try rowWriteSchemaType(field.dataType, depth: depth, to: buffer) +} + +private func rowWriteSchemaType(_ dataType: RowDataType, depth: Int, to buffer: ByteBuffer) throws { + buffer.writeUInt8(UInt8(dataType.typeId.rawValue)) + switch dataType { + case .decimal(let precision, let scale): + guard let precisionByte = UInt8(exactly: precision), let scaleByte = UInt8(exactly: scale) else { + throw rowSchemaDecimalOutOfRange(precision: precision, scale: scale) + } + buffer.writeUInt8(precisionByte) + buffer.writeUInt8(scaleByte) + case .list(let element): + try rowWriteSchemaField(element, depth: depth + 1, to: buffer) + case .map(let key, let value): + try rowWriteSchemaField(key, depth: depth + 1, to: buffer) + try rowWriteSchemaField(value, depth: depth + 1, to: buffer) + case .struct(let fields): + buffer.writeVarUInt32(UInt32(fields.count)) + for field in fields { + try rowWriteSchemaField(field, depth: depth + 1, to: buffer) + } + default: + break + } +} + +private func rowReadSchemaFields(_ buffer: ByteBuffer, depth: Int) throws -> [RowField] { + let count = try buffer.readVarUInt32() + // Every field takes at least one byte, so a larger count is wrong. Checking it first also keeps a + // forged count from reserving a huge array. + guard UInt64(count) <= UInt64(buffer.remaining) else { + throw rowSchemaCountTooLarge(count, remaining: buffer.remaining) + } + var fields: [RowField] = [] + fields.reserveCapacity(Int(count)) + for _ in 0.. RowField { + guard depth < rowSchemaMaxDepth else { + throw rowSchemaTooDeep() + } + let header = try buffer.readUInt8() + let encodingIndex = Int(header & 0x03) + guard encodingIndex < rowFieldNameEncodings.count else { + throw rowSchemaUnknownNameEncoding(encodingIndex) + } + var nameSize = UInt64((header >> 2) & 0x0F) + 1 + if nameSize == UInt64(rowFieldNameSizeThreshold) + 1 { + nameSize = UInt64(try buffer.readVarUInt32()) + UInt64(rowFieldNameSizeThreshold) + } + // readBytes checks the size against the bytes left before copying, and clamping keeps a huge + // size from trapping where Int is 32 bits. + let nameBytes = try buffer.readBytes(count: Int(clamping: nameSize)) + let name = try MetaStringDecoder.fieldName.decode(bytes: nameBytes, encoding: rowFieldNameEncodings[encodingIndex]) + let dataType = try rowReadSchemaType(buffer, depth: depth) + return RowField(name: name.value, dataType: dataType, nullable: header & 0x40 != 0) +} + +private func rowReadSchemaType(_ buffer: ByteBuffer, depth: Int) throws -> RowDataType { + let typeId = try buffer.readUInt8() + switch TypeId(rawValue: UInt32(typeId)) { + case .bool: return .bool + case .int8: return .int8 + case .int16: return .int16 + case .int32: return .int32 + case .int64: return .int64 + case .float16: return .float16 + case .float32: return .float32 + case .float64: return .float64 + case .string: return .string + case .binary: return .binary + case .date: return .date32 + case .timestamp: return .timestamp + case .duration: return .duration + case .decimal: + let precision = try buffer.readUInt8() + let scale = try buffer.readUInt8() + return .decimal(precision: Int(precision), scale: Int(scale)) + case .list: + return .list(try rowReadSchemaField(buffer, depth: depth + 1)) + case .map: + let key = try rowReadSchemaField(buffer, depth: depth + 1) + let value = try rowReadSchemaField(buffer, depth: depth + 1) + // Only the key and value types are kept; their names and nullability are always the standard ones. + return .mapOf(key.dataType, value.dataType) + case .structType: + return .struct(try rowReadSchemaFields(buffer, depth: depth + 1)) + default: + throw rowSchemaUnknownTypeId(typeId) + } +} + +// Folds `hash * 31 + typeId` over a type and then its children in order. When either step would +// overflow, the hash is shifted right by 2, keeping its sign, and the step is tried again. +private func rowSchemaHash(_ hash: Int64, _ dataType: RowDataType) -> Int64 { + let typeId = Int64(dataType.typeId.rawValue) + var hash = hash + while true { + let (product, multiplyOverflow) = hash.multipliedReportingOverflow(by: 31) + let (sum, addOverflow) = product.addingReportingOverflow(typeId) + if !multiplyOverflow && !addOverflow { + hash = sum + break + } + hash >>= 2 + } + switch dataType { + case .list(let element): + return rowSchemaHash(hash, element.dataType) + case .map(let key, let value): + return rowSchemaHash(rowSchemaHash(hash, key.dataType), value.dataType) + case .struct(let fields): + return fields.reduce(hash) { rowSchemaHash($0, $1.dataType) } + default: + return hash + } +} + +@inline(never) +private func rowSchemaTooDeepToWrite() -> ForyError { + .encodingError("row schema nests \(rowSchemaMaxDepth) or more levels deep") +} + +@inline(never) +private func rowSchemaEmptyFieldName() -> ForyError { + .encodingError("row schema field has an empty name") +} + +@inline(never) +private func rowSchemaDecimalOutOfRange(precision: Int, scale: Int) -> ForyError { + .encodingError("decimal(\(precision), \(scale)) precision and scale must each fit in one byte") +} + +@inline(never) +private func rowSchemaVersionUnsupported(_ version: UInt8) -> ForyError { + .invalidData("unsupported row schema version \(version), expected \(rowSchemaVersion)") +} + +@inline(never) +private func rowSchemaCountTooLarge(_ count: UInt32, remaining: Int) -> ForyError { + .invalidData("row schema declares \(count) fields but only \(remaining) bytes remain") +} + +@inline(never) +private func rowSchemaTooDeep() -> ForyError { + .invalidData("row schema nests \(rowSchemaMaxDepth) or more levels deep") +} + +@inline(never) +private func rowSchemaUnknownNameEncoding(_ index: Int) -> ForyError { + .invalidData("unknown row schema field name encoding \(index)") +} + +@inline(never) +private func rowSchemaUnknownTypeId(_ typeId: UInt8) -> ForyError { + .invalidData("unknown type id \(typeId) in row schema") +} diff --git a/swift/Sources/Fory/Row/RowSlot.swift b/swift/Sources/Fory/Row/RowSlot.swift new file mode 100644 index 0000000000..d6f8c746e6 --- /dev/null +++ b/swift/Sources/Fory/Row/RowSlot.swift @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +// A variable-width value such as a string, array, map, or nested struct is stored after the fixed +// slots. Its slot holds one UInt64 ticket: the offset from the start of the enclosing row or array +// in the upper 32 bits and the byte size in the lower 32 bits, so the size comes first in memory. + +// Packs an offset and a size into one slot value. Both must fit in 32 bits. +func rowEncodeOffsetAndSize(offset: Int, size: Int) throws -> UInt64 { + guard let offset32 = UInt32(exactly: offset), let size32 = UInt32(exactly: size) else { + throw rowValueTooLarge(offset: offset, size: size) + } + return UInt64(offset32) << 32 | UInt64(size32) +} + +// Splits a slot value back into its offset and size. They stay UInt32 so a large offset cannot +// trap on platforms where Int is 32 bits. +@inline(__always) +func rowDecodeOffsetAndSize(_ slot: UInt64) -> (offset: UInt32, size: UInt32) { + (UInt32(truncatingIfNeeded: slot >> 32), UInt32(truncatingIfNeeded: slot)) +} + +// Checks that `size` bytes at `offset` fit inside a region of `regionCount` bytes and returns their +// absolute range in the buffer. The sum is taken in UInt64, so it cannot overflow. +@inline(__always) +func rowCheckedRange(offset: UInt32, size: UInt32, regionBase: Int, regionCount: Int) throws -> Range { + guard UInt64(offset) + UInt64(size) <= UInt64(regionCount) else { + throw rowOutOfBounds(offset: offset, size: size, regionCount: regionCount) + } + let start = regionBase + Int(offset) + return start..<(start + Int(size)) +} + +// `micros` counts from 1970 but a Date counts from 2001, so the whole seconds are shifted as integers +// and the fraction is added once; going through timeIntervalSince1970 would round twice. Floor +// division keeps the fraction positive. +func rowDate(micros: Int64) -> Date { + var (seconds, remainder) = micros.quotientAndRemainder(dividingBy: 1_000_000) + if remainder < 0 { + seconds -= 1 + remainder += 1_000_000 + } + return Date(timeIntervalSinceReferenceDate: Double(seconds - 978_307_200) + Double(remainder) / 1e6) +} + +// The reverse of rowDate. The fraction is rounded to the nearest microsecond rather than floored, +// because the Double inside a Date made from `m` microseconds can land just below `m`. A fraction +// that rounds up to a whole second needs no carry: adding 1_000_000 to the scaled seconds is the same. +func rowTimestampMicros(_ date: Date) throws -> Int64 { + let interval = date.timeIntervalSinceReferenceDate + let wholeSeconds = interval.rounded(.down) + // Int64(exactly:) is nil for infinity and NaN as well as for values too large. + guard let seconds = Int64(exactly: wholeSeconds) else { + throw rowTimestampOutOfRange(interval) + } + let fraction = Int64(((interval - wholeSeconds) * 1_000_000).rounded(.toNearestOrAwayFromZero)) + let (unixSeconds, shiftOverflow) = seconds.addingReportingOverflow(978_307_200) + let (scaled, scaleOverflow) = unixSeconds.multipliedReportingOverflow(by: 1_000_000) + let (micros, addOverflow) = scaled.addingReportingOverflow(fraction) + guard !shiftOverflow, !scaleOverflow, !addOverflow else { + throw rowTimestampOutOfRange(interval) + } + return micros +} + +func rowEpochDay(_ date: LocalDate) throws -> Int32 { + guard let days = Int32(exactly: date.epochDay) else { + throw rowDateOutOfRange(date.epochDay) + } + return days +} + +// Both parts of `components` carry the same sign, so dividing each toward zero truncates the +// whole duration toward zero. +func rowDurationMicros(_ duration: Duration) throws -> Int64 { + let (seconds, attoseconds) = duration.components + let (scaled, scaleOverflow) = seconds.multipliedReportingOverflow(by: 1_000_000) + let (micros, addOverflow) = scaled.addingReportingOverflow(attoseconds / 1_000_000_000_000) + guard !scaleOverflow, !addOverflow else { + throw rowDurationOutOfRange(duration) + } + return micros +} + +@inline(never) +func rowDateOutOfRange(_ epochDay: Int64) -> ForyError { + .encodingError("date \(epochDay) days from 1970 does not fit in 32 bits") +} + +@inline(never) +func rowTimestampOutOfRange(_ interval: Double) -> ForyError { + .encodingError("timestamp \(interval) seconds from 2001 does not fit in 64-bit microseconds") +} + +@inline(never) +func rowDurationOutOfRange(_ duration: Duration) -> ForyError { + .encodingError("duration \(duration) does not fit in 64-bit microseconds") +} + +@inline(never) +func rowValueTooLarge(offset: Int, size: Int) -> ForyError { + .encodingError( + "row value at offset \(offset) with \(size) bytes does not fit the row format's 32-bit offset and size" + ) +} + +@inline(never) +func rowOutOfBounds(offset: UInt32, size: UInt32, regionCount: Int) -> ForyError { + .outOfBounds(cursor: Int(clamping: offset), need: Int(clamping: size), length: regionCount) +} diff --git a/swift/Sources/Fory/Row/RowWriter.swift b/swift/Sources/Fory/Row/RowWriter.swift new file mode 100644 index 0000000000..cbd220759f --- /dev/null +++ b/swift/Sources/Fory/Row/RowWriter.swift @@ -0,0 +1,305 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +/// Writes one row into a buffer: the null bitmap, one slot per field, then strings, bytes, and +/// nested values after the slots. Call `reset()` before each row. An index outside +/// `0.. [UInt8] { + buffer.copyBytes(start: base, end: buffer.count) + } + + private func slotOffset(_ index: Int) -> Int { + precondition(index >= 0 && index < fields.count, "field index \(index) is outside 0..<\(fields.count)") + return base + bitmapWidth + index * 8 + } +} + +/// Writes one array into a buffer: the element count, the null bitmap, each element at its natural +/// width, then strings, bytes, and nested values. It has the same write methods as ``RowWriter``, +/// taking element indexes. +public struct RowArrayWriter { + public let elementWidth: Int + public let buffer: ByteBuffer + public private(set) var base: Int + public private(set) var elementCount: Int + private var headerSize: Int + + public init(element: RowField, buffer: ByteBuffer) { + self.init(elementWidth: element.dataType.byteWidth ?? 8, buffer: buffer) + } + + @usableFromInline + init(elementWidth: Int, buffer: ByteBuffer) { + self.elementWidth = elementWidth + self.buffer = buffer + self.base = buffer.count + self.elementCount = 0 + self.headerSize = 8 + } + + /// Throws if `count` is negative or the elements would not fit in 2 GiB. + public mutating func reset(count: Int) throws { + // Dividing instead of multiplying keeps a huge count from overflowing before it is rejected. + guard count >= 0, count <= (Int(Int32.max) - 15) / elementWidth else { + throw rowArrayCountOutOfRange(count, elementWidth: elementWidth) + } + headerSize = 8 + rowBitmapWidth(count) + base = buffer.appendZeros(headerSize + rowRoundToWord(count * elementWidth)) + elementCount = count + buffer.storeLittleEndian(UInt64(count), at: base) + } + + public var size: Int { + buffer.count - base + } + + public func setNullAt(_ index: Int) { + let offset = elementOffset(index) + buffer.storage.withUnsafeMutableBytes { raw in + rowSetBit(raw, bitmapOffset: base + 8, index: index) + for byte in offset.. [UInt8] { + buffer.copyBytes(start: base, end: buffer.count) + } + + // Fills every element of an int8 array at once, one byte each. + func writeInt8Elements(_ bytes: Data) { + precondition(elementWidth == 1 && bytes.count == elementCount, "\(bytes.count) bytes do not fill the array") + let start = base + headerSize + buffer.storage.withUnsafeMutableBytes { raw in + bytes.withUnsafeBytes { source in + UnsafeMutableRawBufferPointer(rebasing: raw[start..<(start + source.count)]).copyMemory(from: source) + } + } + } + + private func elementOffset(_ index: Int) -> Int { + precondition(index >= 0 && index < elementCount, "element index \(index) is outside 0..<\(elementCount)") + return base + headerSize + index * elementWidth + } + + // A value wider than the element would overwrite its neighbors or run past the buffer. + private func store(_ value: T, at index: Int) { + precondition( + MemoryLayout.size <= elementWidth, + "\(MemoryLayout.size)-byte value written to a \(elementWidth)-byte element" + ) + buffer.storeLittleEndian(value, at: elementOffset(index)) + } +} + +/// Writes one map into a buffer: the size of the keys array, the keys array, then the values array. +/// Call `reset()`, write the keys with a ``RowArrayWriter``, call `finishKeys()`, then write the +/// values with another ``RowArrayWriter`` on the same buffer. +public struct RowMapWriter { + public let buffer: ByteBuffer + public private(set) var base: Int + + public init(buffer: ByteBuffer) { + self.buffer = buffer + self.base = buffer.count + } + + public mutating func reset() { + base = buffer.appendZeros(8) + } + + public func finishKeys() { + buffer.storeLittleEndian(UInt64(buffer.count - base - 8), at: base) + } + + public var size: Int { + buffer.count - base + } +} + +// Pads the value appended since `start` to a whole word and returns its unpadded size. +private func rowPadValue(_ buffer: ByteBuffer, from start: Int) -> Int { + let size = buffer.count - start + _ = buffer.appendZeros(rowRoundToWord(size) - size) + return size +} + +@inline(never) +func rowArrayCountOutOfRange(_ count: Int, elementWidth: Int) -> ForyError { + .encodingError("array of \(count) elements of \(elementWidth) bytes each cannot be written") +} diff --git a/swift/Sources/ForyMacro/ForyObjectMacro.swift b/swift/Sources/ForyMacro/ForyObjectMacro.swift index e7c805f21a..2e64f97646 100644 --- a/swift/Sources/ForyMacro/ForyObjectMacro.swift +++ b/swift/Sources/ForyMacro/ForyObjectMacro.swift @@ -36,7 +36,8 @@ struct ForySwiftPlugin: CompilerPlugin { SetFieldMacro.self, MapFieldMacro.self, ForyCaseMacro.self, - ForyUnknownCaseMacro.self + ForyUnknownCaseMacro.self, + ForyRowMacro.self ] } @@ -344,7 +345,7 @@ public struct ForyUnknownCaseMacro: PeerMacro { } } -private func serializerMemberAccessPrefix(_ declaration: some DeclGroupSyntax) -> String { +func serializerMemberAccessPrefix(_ declaration: some DeclGroupSyntax) -> String { let isPublicType = declaration.modifiers.contains(where: { modifier in modifier.name.tokenKind == .keyword(.public) || modifier.name.tokenKind == .keyword(.open) }) diff --git a/swift/Sources/ForyMacro/ForyRowMacro.swift b/swift/Sources/ForyMacro/ForyRowMacro.swift new file mode 100644 index 0000000000..32dcf7f606 --- /dev/null +++ b/swift/Sources/ForyMacro/ForyRowMacro.swift @@ -0,0 +1,262 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import SwiftDiagnostics +import SwiftSyntax +import SwiftSyntaxMacros + +// Adds `rowSchema`, `writeRowFields`, and `readRowFields` to a struct and makes it a `RowStruct`. +// Fields go into the schema sorted by name and renamed to lower underscore form, so every runtime +// agrees on the slot order and the field names. Decoding calls the memberwise initializer. +public struct ForyRowMacro: MemberMacro, ExtensionMacro { + public static func expansion( + of _: AttributeSyntax, + providingMembersOf declaration: some DeclGroupSyntax, + conformingTo _: [TypeSyntax], + in _: some MacroExpansionContext + ) throws -> [DeclSyntax] { + let fields = try parseRowFields(declaration) + let access = serializerMemberAccessPrefix(declaration) + let sorted = fields.indices.sorted { fields[$0].name.utf16.lexicographicallyPrecedes(fields[$1].name.utf16) } + var ordinals = [Int](repeating: 0, count: fields.count) + for (ordinal, fieldIndex) in sorted.enumerated() { + ordinals[fieldIndex] = ordinal + } + + let schemaFields = sorted.map { index in + " RowField.inferred(\"\(rowFieldName(fields[index].name))\", (\(fields[index].typeText)).self)" + } + let schema = schemaFields.isEmpty ? "RowSchema([])" : "RowSchema([\n\(schemaFields.joined(separator: ",\n"))\n])" + let writes = sorted.enumerated().map { ordinal, index in + " try writer.write(value.\(fields[index].codeName), at: \(ordinal))" + } + let arguments = fields.indices.map { index in + " \(fields[index].codeName): row.read(at: \(ordinals[index]))" + } + // With no fields there is nothing that throws, and a bare `try` would warn. + let construction = arguments.isEmpty ? "Self()" : "try Self(\n\(arguments.joined(separator: ",\n"))\n )" + + return [ + "\(raw: access)static let rowSchema = \(raw: schema)", + """ + \(raw: access)static func writeRowFields(_ value: Self, to writer: RowWriter) throws { + \(raw: writes.joined(separator: "\n")) + } + """, + """ + \(raw: access)static func readRowFields(from row: Row) throws -> Self { + \(raw: construction) + } + """ + ] + } + + public static func expansion( + of _: AttributeSyntax, + attachedTo declaration: some DeclGroupSyntax, + providingExtensionsOf type: some TypeSyntaxProtocol, + conformingTo protocols: [TypeSyntax], + in _: some MacroExpansionContext + ) throws -> [ExtensionDeclSyntax] { + // The member expansion reports what is wrong. Adding the conformance anyway would only add + // errors about missing members on top. + guard !protocols.isEmpty, (try? parseRowFields(declaration)) != nil else { + return [] + } + return [try ExtensionDeclSyntax("extension \(type.trimmed): RowStruct {}")] + } +} + +// Each ASCII capital letter becomes an underscore followed by its lowercase letter, and every other +// character stays, so `userID` becomes `user_i_d` and `URLValue` becomes `_u_r_l_value`. +func rowFieldName(_ name: String) -> String { + var result = String.UnicodeScalarView() + for scalar in name.unicodeScalars { + if scalar.isASCII, let ascii = UInt8(exactly: scalar.value), ascii >= 0x41, ascii <= 0x5A { + result.append("_") + result.append(Unicode.Scalar(ascii + 0x20)) + } else { + result.append(scalar) + } + } + return String(result) +} + +private struct RowMacroField { + // Without backticks, for sorting and the schema name. + let name: String + // As written, so a keyword such as `default` keeps its backticks in generated code. + let codeName: String + let typeText: String +} + +private let rowUnsignedTypes: Set = ["UInt", "UInt8", "UInt16", "UInt32", "UInt64"] + +private func parseRowFields(_ declaration: some DeclGroupSyntax) throws -> [RowMacroField] { + guard let structDecl = declaration.as(StructDeclSyntax.self) else { + throw DiagnosticsError(diagnostics: [rowDiagnostic(declaration.introducer, "@ForyRow can only be applied to a struct")]) + } + if let generics = structDecl.genericParameterClause { + throw DiagnosticsError(diagnostics: [rowDiagnostic(generics, "@ForyRow does not support generic structs")]) + } + let structName = structDecl.name.text + var fields: [RowMacroField] = [] + var diagnostics: [Diagnostic] = [] + for member in structDecl.memberBlock.members { + guard let variable = member.decl.as(VariableDeclSyntax.self) else { + continue + } + let skipped: [TokenKind] = [.keyword(.static), .keyword(.class), .keyword(.lazy)] + if variable.modifiers.contains(where: { skipped.contains($0.name.tokenKind) }) { + continue + } + let isLet = variable.bindingSpecifier.tokenKind == .keyword(.let) + for binding in variable.bindings { + guard let pattern = binding.pattern.as(IdentifierPatternSyntax.self) else { + let text = binding.pattern.trimmedDescription + diagnostics.append( + rowDiagnostic(binding.pattern, "@ForyRow does not support the tuple pattern '\(text)'; declare each property on its own") + ) + continue + } + guard rowIsStored(binding.accessorBlock) else { + continue + } + let name = pattern.identifier.identifier?.name ?? pattern.identifier.text + if let attribute = variable.attributes.first { + diagnostics.append( + rowDiagnostic(attribute, "@ForyRow field '\(name)' uses an attribute or property wrapper, which row encoding does not support") + ) + continue + } + // The memberwise initializer has no parameter for such a property, so decoding could not set it. + if isLet, binding.initializer != nil { + diagnostics.append(rowDiagnostic(binding, "@ForyRow field '\(name)' is a let with a default value, so decoding cannot set it; use var")) + continue + } + guard let type = binding.typeAnnotation?.type else { + diagnostics.append(rowDiagnostic(binding, "@ForyRow field '\(name)' needs its own type annotation")) + continue + } + if let problem = rowTypeProblem(type, structName: structName) { + diagnostics.append(rowDiagnostic(binding, "@ForyRow field '\(name)' \(problem)")) + continue + } + fields.append(RowMacroField(name: name, codeName: pattern.identifier.text, typeText: type.trimmedDescription)) + } + } + if !diagnostics.isEmpty { + throw DiagnosticsError(diagnostics: diagnostics) + } + return fields +} + +// A property is stored when it has no accessors, or only willSet and didSet observers. +private func rowIsStored(_ accessorBlock: AccessorBlockSyntax?) -> Bool { + guard let accessorBlock else { + return true + } + guard case .accessors(let accessors) = accessorBlock.accessors else { + return false + } + let observers: [TokenKind] = [.keyword(.willSet), .keyword(.didSet)] + return accessors.allSatisfy { observers.contains($0.accessorSpecifier.tokenKind) } +} + +// Returns why `type` cannot be a row field, or nil. Anything not rejected here must still conform to +// RowCodable, which the compiler checks. +private func rowTypeProblem(_ type: TypeSyntax, structName: String) -> String? { + if let optional = type.as(OptionalTypeSyntax.self) { + if rowIsOptional(optional.wrappedType) { + return "uses the nested optional '\(type.trimmedDescription)'" + } + return rowTypeProblem(optional.wrappedType, structName: structName) + } + if type.is(ImplicitlyUnwrappedOptionalTypeSyntax.self) { + return "uses the implicitly unwrapped optional '\(type.trimmedDescription)'; use '?' instead" + } + if let array = type.as(ArrayTypeSyntax.self) { + return rowTypeProblem(array.element, structName: structName) + } + if let dictionary = type.as(DictionaryTypeSyntax.self) { + return rowMapProblem(key: dictionary.key, value: dictionary.value, structName: structName) + } + if let tuple = type.as(TupleTypeSyntax.self) { + // `(Int32)` is only parentheses around one type. + if tuple.elements.count == 1, let only = tuple.elements.first, only.firstName == nil { + return rowTypeProblem(only.type, structName: structName) + } + return "uses the tuple type '\(type.trimmedDescription)', which has no row type" + } + if let identifier = type.as(IdentifierTypeSyntax.self) { + let name = identifier.name.text + if name == structName || name == "Self" { + return "refers to '\(structName)' itself, which a row cannot hold" + } + return rowNamedTypeProblem(name, identifier.genericArgumentClause, whole: type, structName: structName) + } + if let member = type.as(MemberTypeSyntax.self) { + return rowNamedTypeProblem(member.name.text, member.genericArgumentClause, whole: type, structName: structName) + } + return "uses the type '\(type.trimmedDescription)', which has no row type" +} + +private func rowNamedTypeProblem( + _ name: String, + _ generics: GenericArgumentClauseSyntax?, + whole: TypeSyntax, + structName: String +) -> String? { + if rowUnsignedTypes.contains(name) { + return "uses unsigned type '\(name)'; use Data for bytes or a signed integer type" + } + let arguments = generics?.arguments.map(\.argument) ?? [] + if name == "Optional", let wrapped = arguments.first, rowIsOptional(wrapped) { + return "uses the nested optional '\(whole.trimmedDescription)'" + } + if name == "Dictionary", arguments.count == 2 { + return rowMapProblem(key: arguments[0], value: arguments[1], structName: structName) + } + for argument in arguments { + if let problem = rowTypeProblem(argument, structName: structName) { + return problem + } + } + return nil +} + +// Map keys are never null, and a Date or Duration key is rounded to whole microseconds when written, +// so two different keys could come back equal. +private func rowMapProblem(key: TypeSyntax, value: TypeSyntax, structName: String) -> String? { + if rowIsOptional(key) { + return "uses the optional map key '\(key.trimmedDescription)'; map keys are never null" + } + let keyName = key.as(IdentifierTypeSyntax.self)?.name.text ?? key.as(MemberTypeSyntax.self)?.name.text + if let keyName, keyName == "Date" || keyName == "Duration" { + return "uses '\(keyName)' as a map key, which is not supported" + } + return rowTypeProblem(key, structName: structName) ?? rowTypeProblem(value, structName: structName) +} + +private func rowIsOptional(_ type: TypeSyntax) -> Bool { + type.is(OptionalTypeSyntax.self) || type.is(ImplicitlyUnwrappedOptionalTypeSyntax.self) + || type.as(IdentifierTypeSyntax.self)?.name.text == "Optional" +} + +private func rowDiagnostic(_ node: some SyntaxProtocol, _ message: String) -> Diagnostic { + Diagnostic(node: node, message: MacroExpansionErrorMessage(message)) +} diff --git a/swift/Tests/ForyRowXlangPeer/main.swift b/swift/Tests/ForyRowXlangPeer/main.swift new file mode 100644 index 0000000000..d48b843698 --- /dev/null +++ b/swift/Tests/ForyRowXlangPeer/main.swift @@ -0,0 +1,227 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation +import Fory + +// Row format peer driven by org.apache.fory.format.SwiftCrossLanguageTest. Each case reads the bytes +// Java wrote, checks the value and the exact layout, and overwrites the file with its own encoding +// for Java to read back. + +// The models are internal rather than private: a private type prints as +// "(unknown context at $...).Bar" in failure messages, which hides which value differed. + +@ForyRow +struct Bar: Equatable { + var f1: Int32? + var f2: String? +} + +@ForyRow +struct Foo: Equatable { + var f1: Int32? + var f2: String? + var f3: [String?]? + var f4: [String: Int32?]? + var f5: Bar? +} + +// CrossLanguageTest.A. +@ForyRow +struct StringMap: Equatable { + var f1: Int32? + var f2: [String: String]? +} + +@ForyRow +struct Blob: Equatable { + var f1: Data? + var f2: String? +} + +// Data prints only its byte count ("5 bytes"), so a failure message would show the same text for the +// value read and the value expected. Printing the bytes shows which one differs. +extension Blob: CustomStringConvertible { + var description: String { + "Blob(f1: \(f1.map { Array($0).description } ?? "nil"), f2: \(f2 ?? "nil"))" + } +} + +private let expectedFoo = Foo( + f1: 1, f2: "str", f3: ["str1", nil, "str2"], f4: ["k1": 1, "k2": 2, "k3": 3, "k4": 4, "k5": 5, "k6": 6], + f5: Bar(f1: 1, f2: "str") +) +private let expectedStringMap = StringMap(f1: 1, f2: ["pid": "12345", "ip": "0.0.0.0", "k1": "v1"]) +private let expectedBlob = Blob(f1: Data([0, 1, 0xff, 0x7f, 0x80]), f2: "bytes") + +private struct PeerFailure: Error, CustomStringConvertible { + let description: String +} + +private func check(_ condition: Bool, _ message: @autoclosure () -> String) throws { + if !condition { + throw PeerFailure(description: message()) + } +} + +private func present(_ value: T?) throws -> T { + guard let value else { + throw PeerFailure(description: "a slot that is not null has no value") + } + return value +} + +// Writes the row again through the low-level writers, copying map entries in the order they were +// read. A dictionary has no fixed order, so this is the only way to require the exact bytes of a +// row that holds a map. +private func replay(_ bytes: [UInt8], schema: RowSchema) throws -> [UInt8] { + let row = try Row(schema: schema, bytes: bytes) + var writer = RowWriter(schema: schema) + writer.reset() + for (index, field) in schema.fields.enumerated() { + try copy(from: row, index, field, to: writer) + } + return writer.toBytes() +} + +private func copy(from reader: some RowValueReader, _ index: Int, _ field: RowField, to writer: some RowValueWriter) throws { + if reader.isNull(at: index) { + writer.setNullAt(index) + return + } + switch field.dataType { + case .bool: writer.writeBool(try present(reader.bool(at: index)), at: index) + case .int8: writer.writeInt8(try present(reader.int8(at: index)), at: index) + case .int16: writer.writeInt16(try present(reader.int16(at: index)), at: index) + case .int32: writer.writeInt32(try present(reader.int32(at: index)), at: index) + case .int64: writer.writeInt64(try present(reader.int64(at: index)), at: index) + case .float32: writer.writeFloat32(try present(reader.float32(at: index)), at: index) + case .float64: writer.writeFloat64(try present(reader.float64(at: index)), at: index) + case .date32: try writer.writeLocalDate(present(reader.localDate(at: index)), at: index) + case .timestamp: try writer.writeTimestamp(present(reader.timestamp(at: index)), at: index) + case .duration: try writer.writeDuration(present(reader.duration(at: index)), at: index) + case .string: try writer.writeString(present(reader.string(at: index)), at: index) + case .binary: try writer.writeBytes(present(reader.binary(at: index)), at: index) + case .list: + let array = try copy(present(reader.array(at: index)), to: writer.buffer) + try writer.setOffsetAndSize(at: index, absoluteStart: array.base, size: array.size) + case .map: + let source = try present(reader.map(at: index)) + var map = RowMapWriter(buffer: writer.buffer) + map.reset() + _ = try copy(source.keys, to: writer.buffer) + map.finishKeys() + _ = try copy(source.values, to: writer.buffer) + try writer.setOffsetAndSize(at: index, absoluteStart: map.base, size: map.size) + case .struct(let fields): + let source = try present(reader.row(at: index)) + var child = RowWriter(schema: RowSchema(fields), buffer: writer.buffer) + child.reset() + for (childIndex, childField) in fields.enumerated() { + try copy(from: source, childIndex, childField, to: child) + } + try writer.setOffsetAndSize(at: index, absoluteStart: child.base, size: child.size) + case .float16, .decimal: + throw PeerFailure(description: "replay does not support \(field.dataType)") + } +} + +private func copy(_ source: RowArray, to buffer: ByteBuffer) throws -> RowArrayWriter { + var array = RowArrayWriter(element: source.element, buffer: buffer) + try array.reset(count: source.elementCount) + for index in 0.. Data { + try Data(contentsOf: URL(fileURLWithPath: path)) +} + +private func writeFile(_ path: String, _ data: Data) throws { + try data.write(to: URL(fileURLWithPath: path)) +} + +private func testByteArrayCarrier(_ dataFile: String) throws { + let encoder = RowEncoder() + let data = try readFile(dataFile) + let decoded = try encoder.decode(data) + try check(decoded == expectedBlob, "decoded \(decoded), expected \(expectedBlob)") + let encoded = try encoder.encode(expectedBlob) + try check(encoded == data, "encoded Blob bytes differ from Java's") + try writeFile(dataFile, encoded) +} + +private func testMapEncoder(_ dataFile: String) throws { + let encoder = RowEncoder() + let data = try readFile(dataFile) + let decoded = try encoder.decode(data) + try check(decoded == expectedStringMap, "decoded \(decoded), expected \(expectedStringMap)") + let hash = withUnsafeBytes(of: encoder.schemaHash.littleEndian) { Data($0) } + try check(data.prefix(8) == hash, "schema hash prefix differs from \(encoder.schemaHash)") + let rowBytes = Array(data.dropFirst(8)) + try check(try replay(rowBytes, schema: encoder.schema) == rowBytes, "replayed StringMap row differs from Java's") + try writeFile(dataFile, encoder.encode(expectedStringMap)) +} + +private func testSerializationWithoutSchema(_ dataFile: String) throws { + let encoder = RowEncoder() + let data = try readFile(dataFile) + let decoded = try encoder.fromRow(data) + try check(decoded == expectedFoo, "decoded \(decoded), expected \(expectedFoo)") + let rowBytes = Array(data) + try check(try replay(rowBytes, schema: encoder.schema) == rowBytes, "replayed Foo row differs from Java's") + try writeFile(dataFile, encoder.toRow(expectedFoo)) +} + +private func testSerializationWithSchema(_ schemaFile: String, _ dataFile: String) throws { + let schemaBytes = Array(try readFile(schemaFile)) + let parsed = try RowSchema(bytes: schemaBytes) + try check(parsed == Foo.rowSchema, "Java schema \(parsed), inferred schema \(Foo.rowSchema)") + try check(parsed.hash == Foo.rowSchema.hash, "schema hash \(parsed.hash), encoder hash \(Foo.rowSchema.hash)") + try check(try Foo.rowSchema.toBytes() == schemaBytes, "re-encoded schema bytes differ from Java's") + try testSerializationWithoutSchema(dataFile) +} + +private func run(_ arguments: [String]) throws { + guard arguments.count >= 3 else { + throw PeerFailure(description: "usage: ForyRowXlangPeer ...") + } + switch arguments[1] { + case "test_byte_array_carrier": + try testByteArrayCarrier(arguments[2]) + case "test_map_encoder": + try testMapEncoder(arguments[2]) + case "test_serialization_without_schema": + try testSerializationWithoutSchema(arguments[2]) + case "test_serialization_with_schema": + guard arguments.count >= 4 else { + throw PeerFailure(description: "test_serialization_with_schema needs ") + } + try testSerializationWithSchema(arguments[2], arguments[3]) + default: + throw PeerFailure(description: "unknown test case \"\(arguments[1])\"") + } +} + +do { + try run(CommandLine.arguments) +} catch { + FileHandle.standardError.write(Data("row_xlang: \(error)\n".utf8)) + exit(1) +} diff --git a/swift/Tests/ForyTests/ByteBufferTests.swift b/swift/Tests/ForyTests/ByteBufferTests.swift index 71ff055cd7..da87ed6704 100644 --- a/swift/Tests/ForyTests/ByteBufferTests.swift +++ b/swift/Tests/ForyTests/ByteBufferTests.swift @@ -283,3 +283,30 @@ func byteBufferRejectsInvalidVarintsAndUtf8() throws { #expect("\(error)".contains("invalid UTF-8")) } } + +@Test +func byteBufferAppendZeros() { + let buffer = ByteBuffer(bytes: [0x01, 0x02, 0x03]) + #expect(buffer.appendZeros(5) == 3) + #expect(buffer.storage == [0x01, 0x02, 0x03, 0, 0, 0, 0, 0]) + + buffer.writeBytes([0xFF, 0xFF]) + buffer.clear() + #expect(buffer.appendZeros(2) == 0) + #expect(buffer.storage == [0, 0]) +} + +@Test +func byteBufferStoreLittleEndian() { + let buffer = ByteBuffer(bytes: [UInt8](repeating: 0, count: 16)) + buffer.storeLittleEndian(UInt32(0x0102_0304), at: 3) + buffer.storeLittleEndian(UInt64(0x1122_3344_5566_7788), at: 7) + #expect( + buffer.storage == [ + 0x00, 0x00, 0x00, + 0x04, 0x03, 0x02, 0x01, + 0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11, + 0x00 + ] + ) +} diff --git a/swift/Tests/ForyTests/RowArrayReaderTests.swift b/swift/Tests/ForyTests/RowArrayReaderTests.swift new file mode 100644 index 0000000000..1d38c0e7c1 --- /dev/null +++ b/swift/Tests/ForyTests/RowArrayReaderTests.swift @@ -0,0 +1,159 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation +import Testing +@testable import Fory + +// A one-field row whose only slot points at `value`, so the value starts at byte 16 rather than 0. +private func rowHolding(_ value: [UInt8], as dataType: RowDataType) throws -> Row { + let slot = withUnsafeBytes(of: (UInt64(16) << 32 | UInt64(value.count)).littleEndian, Array.init) + let schema = RowSchema([RowField(name: "v", dataType: dataType, nullable: true)]) + return try Row(schema: schema, bytes: [UInt8](repeating: 0, count: 8) + slot + value) +} + +private func rowArray(_ bytes: [UInt8], of dataType: RowDataType) throws -> RowArray { + try RowArray(element: RowField(name: "item", dataType: dataType, nullable: true), bytes: bytes, base: 0, count: bytes.count) +} + +@Test +func rowArrayStrings() throws { + let bytes: [UInt8] = [ + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 3 elements + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // bitmap: element 1 is null + 0x02, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, // element 0: size 2, offset 0x28 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // element 1: null + 0x03, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, // element 2: size 3, offset 0x30 + 0x61, 0x62, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // "ab" + 0x78, 0x79, 0x7A, 0x00, 0x00, 0x00, 0x00, 0x00 // "xyz" + ] + let array = try #require(try rowHolding(bytes, as: .listOf(.string)).array(at: 0)) + #expect(array.elementCount == 3) + #expect(array.sizeInBytes == 56) + #expect(try array.string(at: 0) == "ab") + #expect(array.isNull(at: 1)) + #expect(try array.string(at: 1) == nil) + #expect(try array.string(at: 2) == "xyz") + #expect(array.toBytes() == bytes) +} + +@Test +func rowArrayEmpty() throws { + let array = try rowArray([UInt8](repeating: 0, count: 8), of: .int32) + #expect(array.elementCount == 0) + #expect(array.sizeInBytes == 8) +} + +@Test +func rowArrayCounts() throws { + for (count, headerSize, size) in [(64, 16, 80), (65, 24, 96)] { + var bytes = [UInt8](repeating: 0, count: size) + bytes[0] = UInt8(count) + bytes[8 + (count - 1) / 8] |= 1 << ((count - 1) % 8) + bytes[headerSize] = 7 + let array = try rowArray(bytes, of: .int8) + #expect(array.elementCount == count) + #expect(array.isNull(at: count - 1)) + #expect(!array.isNull(at: count - 2)) + #expect(array.int8(at: 0) == 7) + } +} + +@Test +func rowArrayWidths() throws { + let header: [UInt8] = [0x03, 0, 0, 0, 0, 0, 0, 0] + [UInt8](repeating: 0, count: 8) + + let int8s = try rowArray(header + [0x01, 0xFE, 0x7F, 0, 0, 0, 0, 0], of: .int8) + #expect(int8s.sizeInBytes == 24) + #expect((0..<3).map { int8s.int8(at: $0) } == [1, -2, 127]) + + let int16s = try rowArray(header + [0x02, 0x01, 0xFE, 0xFF, 0x03, 0x00, 0, 0], of: .int16) + #expect(int16s.sizeInBytes == 24) + #expect((0..<3).map { int16s.int16(at: $0) } == [0x0102, -2, 3]) + + let int32s = try rowArray( + header + [0x04, 0x03, 0x02, 0x01, 0xFE, 0xFF, 0xFF, 0xFF, 0x03, 0x00, 0x00, 0x00, 0, 0, 0, 0], + of: .int32 + ) + #expect(int32s.sizeInBytes == 32) + #expect((0..<3).map { int32s.int32(at: $0) } == [0x0102_0304, -2, 3]) + + let int64s = try rowArray( + header + [0x08, 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01] + + [0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF] + + [0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00], + of: .int64 + ) + #expect(int64s.sizeInBytes == 40) + #expect((0..<3).map { int64s.int64(at: $0) } == [0x0102_0304_0506_0708, -2, 3]) +} + +@Test +func rowMapLayout() throws { + let bytes: [UInt8] = [ + 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // keys array is 0x30 bytes + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // keys: 2 elements + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // keys bitmap + 0x02, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, // key 0: size 2, offset 0x20 from the keys array + 0x02, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, // key 1: size 2, offset 0x28 from the keys array + 0x6B, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // "k1" + 0x6B, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // "k2" + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // values: 2 elements + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // values bitmap + 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00 // int32 1 and 2 + ] + let map = try #require(try rowHolding(bytes, as: .mapOf(.string, .int32)).map(at: 0)) + #expect(map.count == 2) + #expect(try map.keys.string(at: 0) == "k1") + #expect(try map.keys.string(at: 1) == "k2") + #expect(map.values.int32(at: 0) == 1) + #expect(map.values.int32(at: 1) == 2) +} + +@Test +func rowArrayCorrupt() throws { + #expect(throws: ForyError.invalidData("array of 4 bytes is shorter than its 8-byte element count")) { + try rowArray([0, 0, 0, 0], of: .int8) + } + #expect(throws: ForyError.invalidData("array declares \(UInt64.max) elements but holds only 16 bytes")) { + try rowArray([UInt8](repeating: 0xFF, count: 8) + [UInt8](repeating: 0, count: 8), of: .int8) + } + // Four int64 elements need 16 + 32 bytes, but only 32 are there. + #expect(throws: ForyError.invalidData("array declares 4 elements but holds only 32 bytes")) { + try rowArray([0x04, 0, 0, 0, 0, 0, 0, 0] + [UInt8](repeating: 0, count: 24), of: .int64) + } + let pastEnd = try rowArray( + [0x01, 0, 0, 0, 0, 0, 0, 0] + [UInt8](repeating: 0, count: 8) + [0x01, 0, 0, 0, 0x18, 0, 0, 0], + of: .string + ) + #expect(throws: ForyError.outOfBounds(cursor: 0x18, need: 1, length: 24)) { + try pastEnd.string(at: 0) + } +} + +@Test +func rowMapCorrupt() throws { + let mapType = RowDataType.mapOf(.int32, .int32) + #expect(throws: ForyError.invalidData("map keys array of 16 bytes does not fit in a 16-byte map")) { + try rowHolding([0x10, 0, 0, 0, 0, 0, 0, 0] + [UInt8](repeating: 0, count: 8), as: mapType).map(at: 0) + } + let oneKey: [UInt8] = [0x01, 0, 0, 0, 0, 0, 0, 0] + [UInt8](repeating: 0, count: 16) + let noValues = [UInt8](repeating: 0, count: 8) + #expect(throws: ForyError.invalidData("map has 1 keys but 0 values")) { + try rowHolding([0x18, 0, 0, 0, 0, 0, 0, 0] + oneKey + noValues, as: mapType).map(at: 0) + } +} diff --git a/swift/Tests/ForyTests/RowArrayWriterTests.swift b/swift/Tests/ForyTests/RowArrayWriterTests.swift new file mode 100644 index 0000000000..4179b7dd90 --- /dev/null +++ b/swift/Tests/ForyTests/RowArrayWriterTests.swift @@ -0,0 +1,236 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation +import Testing +@testable import Fory + +private let pointFields = [RowField(name: "x", dataType: .int32, nullable: false)] + +private func field(_ dataType: RowDataType) -> RowField { + RowField(name: "item", dataType: dataType, nullable: true) +} + +private func writePoint(_ value: Int32, into buffer: ByteBuffer) -> (start: Int, size: Int) { + var point = RowWriter(fields: pointFields, buffer: buffer) + point.reset() + point.writeInt32(value, at: 0) + return (point.base, point.size) +} + +@Test +func rowArrayWriterStrings() throws { + var writer = RowArrayWriter(element: field(.string), buffer: ByteBuffer()) + try writer.reset(count: 3) + try writer.writeString("ab", at: 0) + writer.setNullAt(1) + try writer.writeString("xyz", at: 2) + #expect( + writer.toBytes() == [ + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 3 elements + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // bitmap: element 1 is null + 0x02, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, // element 0: size 2, offset 0x28 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // element 1: null + 0x03, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, // element 2: size 3, offset 0x30 + 0x61, 0x62, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // "ab" + 0x78, 0x79, 0x7A, 0x00, 0x00, 0x00, 0x00, 0x00 // "xyz" + ]) +} + +@Test +func rowMapWriterLayout() throws { + let buffer = ByteBuffer() + var map = RowMapWriter(buffer: buffer) + map.reset() + var keys = RowArrayWriter(element: field(.string), buffer: buffer) + try keys.reset(count: 2) + try keys.writeString("k1", at: 0) + try keys.writeString("k2", at: 1) + map.finishKeys() + var values = RowArrayWriter(element: field(.int32), buffer: buffer) + try values.reset(count: 2) + values.writeInt32(1, at: 0) + values.writeInt32(2, at: 1) + #expect(map.size == 80) + #expect( + buffer.copyBytes(start: 0, end: buffer.count) == [ + 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // keys array is 48 bytes + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 2 keys + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // key bitmap + 0x02, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, // key 0: offset counted from the keys array + 0x02, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, // key 1 + 0x6B, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // "k1" + 0x6B, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // "k2" + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 2 values + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // value bitmap + 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00 // 1 and 2 + ]) +} + +@Test +func rowWriterNested() throws { + let schema = RowSchema([ + RowField(name: "id", dataType: .int32, nullable: false), + RowField(name: "child", dataType: .struct(pointFields), nullable: true) + ]) + var parent = RowWriter(schema: schema) + parent.reset() + parent.writeInt32(1, at: 0) + let child = writePoint(5, into: parent.buffer) + try parent.setOffsetAndSize(at: 1, absoluteStart: child.start, size: child.size) + #expect( + parent.toBytes() == [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // parent bitmap + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // id = 1 + 0x10, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, // child: size 16, offset 0x18 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // child bitmap + 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // x = 5 + ]) +} + +@Test +func rowArrayWriterWidths() throws { + for (dataType, size) in [(RowDataType.int8, 24), (.int16, 24), (.int32, 32), (.int64, 40)] { + var writer = RowArrayWriter(element: field(dataType), buffer: ByteBuffer()) + try writer.reset(count: 3) + for index in 0..<3 { + switch dataType { + case .int8: writer.writeInt8(-1, at: index) + case .int16: writer.writeInt16(-1, at: index) + case .int32: writer.writeInt32(-1, at: index) + default: writer.writeInt64(-1, at: index) + } + } + let filled = 3 * writer.elementWidth + var expected: [UInt8] = [0x03, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + expected += [UInt8](repeating: 0xFF, count: filled) + expected += [UInt8](repeating: 0, count: size - 16 - filled) + #expect(writer.toBytes() == expected) + } +} + +@Test +func rowArrayWriterCounts() throws { + for (count, headerSize) in [(0, 8), (64, 16), (65, 24)] { + var writer = RowArrayWriter(element: field(.int64), buffer: ByteBuffer()) + try writer.reset(count: count) + #expect(writer.size == headerSize + count * 8) + if count == 0 { + #expect(writer.toBytes() == [UInt8](repeating: 0, count: 8)) + continue + } + writer.writeInt64(7, at: 0) + writer.setNullAt(count - 1) + let array = try RowArray(element: field(.int64), bytes: writer.toBytes(), base: 0, count: writer.size) + #expect(array.elementCount == count) + #expect(array.int64(at: 0) == 7) + #expect(array.isNull(at: count - 1)) + #expect(!array.isNull(at: count - 2)) + } +} + +@Test +func rowMapWriterEmpty() throws { + let buffer = ByteBuffer() + var map = RowMapWriter(buffer: buffer) + map.reset() + var keys = RowArrayWriter(element: field(.string), buffer: buffer) + try keys.reset(count: 0) + map.finishKeys() + var values = RowArrayWriter(element: field(.int32), buffer: buffer) + try values.reset(count: 0) + #expect( + buffer.copyBytes(start: 0, end: buffer.count) + == [0x08, 0, 0, 0, 0, 0, 0, 0] + [UInt8](repeating: 0, count: 16)) +} + +@Test +func rowWriterNullElements() throws { + let schema = RowSchema([ + RowField(name: "names", dataType: .listOf(.string), nullable: true), + RowField(name: "points", dataType: .listOf(.struct(pointFields)), nullable: true) + ]) + var writer = RowWriter(schema: schema) + writer.reset() + + var names = RowArrayWriter(element: field(.string), buffer: writer.buffer) + try names.reset(count: 2) + try names.writeString("a", at: 0) + names.setNullAt(1) + try writer.setOffsetAndSize(at: 0, absoluteStart: names.base, size: names.size) + + var points = RowArrayWriter(element: field(.struct(pointFields)), buffer: writer.buffer) + try points.reset(count: 2) + let point = writePoint(1, into: writer.buffer) + try points.setOffsetAndSize(at: 0, absoluteStart: point.start, size: point.size) + points.setNullAt(1) + try writer.setOffsetAndSize(at: 1, absoluteStart: points.base, size: points.size) + + let row = try Row(schema: schema, bytes: writer.toBytes()) + let readNames = try #require(try row.array(at: 0)) + #expect(try readNames.string(at: 0) == "a") + #expect(try readNames.string(at: 1) == nil) + let readPoints = try #require(try row.array(at: 1)) + #expect(try readPoints.row(at: 0)?.int32(at: 0) == 1) + #expect(try readPoints.row(at: 1) == nil) +} + +@Test +func rowWriterNestedInListInMap() throws { + let listType = RowDataType.listOf(.struct(pointFields)) + let schema = RowSchema([RowField(name: "m", dataType: .mapOf(.string, listType), nullable: true)]) + var writer = RowWriter(schema: schema) + writer.reset() + let buffer = writer.buffer + + var map = RowMapWriter(buffer: buffer) + map.reset() + var keys = RowArrayWriter(element: field(.string), buffer: buffer) + try keys.reset(count: 1) + try keys.writeString("k", at: 0) + map.finishKeys() + var values = RowArrayWriter(element: field(listType), buffer: buffer) + try values.reset(count: 1) + var list = RowArrayWriter(element: field(.struct(pointFields)), buffer: buffer) + try list.reset(count: 2) + for (index, value) in [Int32(1), 2].enumerated() { + let point = writePoint(value, into: buffer) + try list.setOffsetAndSize(at: index, absoluteStart: point.start, size: point.size) + } + try values.setOffsetAndSize(at: 0, absoluteStart: list.base, size: list.size) + try writer.setOffsetAndSize(at: 0, absoluteStart: map.base, size: map.size) + + let readMap = try #require(try Row(schema: schema, bytes: writer.toBytes()).map(at: 0)) + #expect(try readMap.keys.string(at: 0) == "k") + let readList = try #require(try readMap.values.array(at: 0)) + #expect(readList.elementCount == 2) + #expect(try readList.row(at: 0)?.int32(at: 0) == 1) + #expect(try readList.row(at: 1)?.int32(at: 0) == 2) +} + +@Test +func rowArrayWriterRejects() throws { + var writer = RowArrayWriter(element: field(.int64), buffer: ByteBuffer()) + #expect(throws: ForyError.encodingError("array of -1 elements of 8 bytes each cannot be written")) { + try writer.reset(count: -1) + } + let tooMany = (Int(Int32.max) - 15) / 8 + 1 + #expect(throws: ForyError.encodingError("array of \(tooMany) elements of 8 bytes each cannot be written")) { + try writer.reset(count: tooMany) + } + #expect(writer.buffer.count == 0) +} diff --git a/swift/Tests/ForyTests/RowBitmapTests.swift b/swift/Tests/ForyTests/RowBitmapTests.swift new file mode 100644 index 0000000000..537f18e1ec --- /dev/null +++ b/swift/Tests/ForyTests/RowBitmapTests.swift @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation +import Testing +@testable import Fory + +@Test +func rowBitmapWidths() { + let cases: [(count: Int, width: Int)] = [ + (0, 0), (1, 8), (63, 8), (64, 8), (65, 16), (128, 16), (129, 24) + ] + for (count, width) in cases { + #expect(rowBitmapWidth(count) == width) + } +} + +@Test +func rowBitmapWordRounding() { + let cases: [(size: Int, rounded: Int)] = [(0, 0), (1, 8), (7, 8), (8, 8), (9, 16)] + for (size, rounded) in cases { + #expect(rowRoundToWord(size) == rounded) + } +} + +@Test +func rowBitmapBits() { + let nullIndexes: Set = [0, 7, 8, 63, 64] + var bytes = [UInt8](repeating: 0, count: 24) + bytes.withUnsafeMutableBytes { raw in + for index in nullIndexes { + rowSetBit(raw, bitmapOffset: 8, index: index) + } + } + + var expected = [UInt8](repeating: 0, count: 24) + expected[8] = 0x81 + expected[9] = 0x01 + expected[15] = 0x80 + expected[16] = 0x01 + #expect(bytes == expected) + + bytes.withUnsafeBytes { raw in + for index in 0..<72 { + #expect(rowIsBitSet(raw, bitmapOffset: 8, index: index) == nullIndexes.contains(index)) + } + } +} diff --git a/swift/Tests/ForyTests/RowCodableTests.swift b/swift/Tests/ForyTests/RowCodableTests.swift new file mode 100644 index 0000000000..2793eeee4b --- /dev/null +++ b/swift/Tests/ForyTests/RowCodableTests.swift @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation +import Testing +@testable import Fory + +private struct Scalars: Equatable { + var bool: Bool + var int8: Int8 + var int16: Int16 + var int32: Int32 + var int64: Int64 + var int: Int + var float: Float + var double: Double + var string: String + var date: Date + var localDate: LocalDate + var duration: Duration +} + +private let scalarSchema = RowSchema([ + .inferred("bool", Bool.self), .inferred("int8", Int8.self), .inferred("int16", Int16.self), + .inferred("int32", Int32.self), .inferred("int64", Int64.self), .inferred("int", Int.self), + .inferred("float", Float.self), .inferred("double", Double.self), .inferred("string", String.self), + .inferred("date", Date.self), .inferred("localDate", LocalDate.self), .inferred("duration", Duration.self) +]) + +private func roundTrip(_ value: Scalars) throws -> Scalars { + var writer = RowWriter(schema: scalarSchema) + writer.reset() + try writer.write(value.bool, at: 0) + try writer.write(value.int8, at: 1) + try writer.write(value.int16, at: 2) + try writer.write(value.int32, at: 3) + try writer.write(value.int64, at: 4) + try writer.write(value.int, at: 5) + try writer.write(value.float, at: 6) + try writer.write(value.double, at: 7) + try writer.write(value.string, at: 8) + try writer.write(value.date, at: 9) + try writer.write(value.localDate, at: 10) + try writer.write(value.duration, at: 11) + let row = try Row(schema: scalarSchema, bytes: writer.toBytes()) + return try Scalars( + bool: row.read(at: 0), int8: row.read(at: 1), int16: row.read(at: 2), int32: row.read(at: 3), + int64: row.read(at: 4), int: row.read(at: 5), float: row.read(at: 6), double: row.read(at: 7), + string: row.read(at: 8), date: row.read(at: 9), localDate: row.read(at: 10), duration: row.read(at: 11) + ) +} + +private let optionalSchema = RowSchema([ + .inferred("a", Int32?.self), .inferred("b", String?.self), .inferred("c", Date?.self) +]) + +private func optionalRow(_ number: Int32?, _ text: String?, _ date: Date?) throws -> Row { + var writer = RowWriter(schema: optionalSchema) + writer.reset() + try writer.write(number, at: 0) + try writer.write(text, at: 1) + try writer.write(date, at: 2) + return try Row(schema: optionalSchema, bytes: writer.toBytes()) +} + +@Test +func rowCodableScalars() throws { + let low = Scalars( + bool: false, int8: .min, int16: .min, int32: .min, int64: .min, int: .min, + float: -.greatestFiniteMagnitude, double: -.greatestFiniteMagnitude, string: "", + date: Date(timeIntervalSince1970: -86_400.5), localDate: LocalDate(epochDay: Int64(Int32.min)), + duration: .microseconds(Int64.min) + ) + let high = Scalars( + bool: true, int8: .max, int16: .max, int32: .max, int64: .max, int: .max, + float: .greatestFiniteMagnitude, double: .greatestFiniteMagnitude, string: "héllo, 世界", + date: Date(timeIntervalSinceReferenceDate: 721_692_800.25), localDate: LocalDate(epochDay: Int64(Int32.max)), + duration: .microseconds(Int64.max) + ) + #expect(try roundTrip(low) == low) + #expect(try roundTrip(high) == high) +} + +@Test +func rowCodableOptionalNull() throws { + let nulls = try optionalRow(nil, nil, nil) + for index in 0..<3 { + #expect(nulls.isNull(at: index)) + } + #expect(try nulls.read(Int32?.self, at: 0) == nil) + #expect(try nulls.read(String?.self, at: 1) == nil) + #expect(try nulls.read(Date?.self, at: 2) == nil) + + let date = Date(timeIntervalSinceReferenceDate: 0.5) + let values = try optionalRow(7, "x", date) + #expect(try values.read(Int32?.self, at: 0) == 7) + #expect(try values.read(String?.self, at: 1) == "x") + #expect(try values.read(Date?.self, at: 2) == date) +} + +@Test +func rowCodableNullIntoValue() throws { + let nulls = try optionalRow(nil, nil, nil) + #expect(throws: ForyError.invalidData("null row value cannot be read into non-optional Int32")) { + try nulls.read(Int32.self, at: 0) + } + #expect(throws: ForyError.invalidData("null row value cannot be read into non-optional String")) { + try nulls.read(String.self, at: 1) + } +} + +@Test +func rowCodableInferredFields() { + #expect(RowField.inferred("a", Int32.self) == RowField(name: "a", dataType: .int32, nullable: false)) + #expect(RowField.inferred("a", Int32?.self) == RowField(name: "a", dataType: .int32, nullable: true)) + #expect(RowField.inferred("a", String.self) == RowField(name: "a", dataType: .string, nullable: true)) + #expect(RowField.inferred("a", Int.self) == RowField(name: "a", dataType: .int64, nullable: false)) + #expect(RowField.inferred("a", Date.self) == RowField(name: "a", dataType: .timestamp, nullable: true)) + #expect(RowField.inferred("a", LocalDate.self) == RowField(name: "a", dataType: .date32, nullable: true)) + #expect(RowField.inferred("a", Duration.self) == RowField(name: "a", dataType: .duration, nullable: true)) + + // An array of Int32? still packs 4-byte elements, so the width must follow the wrapped type. + let types: [any RowCodable.Type] = [ + Bool.self, Int8.self, Int16.self, Int32.self, Int64.self, Int.self, Float.self, Double.self, + String.self, Date.self, LocalDate.self, Duration.self, Int16?.self, String?.self + ] + for type in types { + #expect(type.rowFixedWidth == type.rowDataType.byteWidth, "\(type)") + } +} diff --git a/swift/Tests/ForyTests/RowEncoderTests.swift b/swift/Tests/ForyTests/RowEncoderTests.swift new file mode 100644 index 0000000000..eeb2ecbe83 --- /dev/null +++ b/swift/Tests/ForyTests/RowEncoderTests.swift @@ -0,0 +1,172 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation +import Testing +@testable import Fory + +private enum Color: String, RowCodable { + case red + case green +} + +private struct Point: RowStruct, Equatable { + var number: Int32 + var label: String? + + static let rowSchema = RowSchema([.inferred("label", String?.self), .inferred("number", Int32.self)]) + + static func writeRowFields(_ value: Point, to writer: RowWriter) throws { + try writer.write(value.label, at: 0) + try writer.write(value.number, at: 1) + } + + static func readRowFields(from row: Row) throws -> Point { + try Point(number: row.read(at: 1), label: row.read(at: 0)) + } +} + +private struct Bag: RowStruct, Equatable { + var names: [String?] + var ids: Set + var scores: [String: Int32?] + var blob: Data + var points: [Point] + var origin: Point? + var color: Color + + static let rowSchema = RowSchema([ + .inferred("blob", Data.self), .inferred("color", Color.self), .inferred("ids", Set.self), + .inferred("names", [String?].self), .inferred("origin", Point?.self), .inferred("points", [Point].self), + .inferred("scores", [String: Int32?].self) + ]) + + static func writeRowFields(_ value: Bag, to writer: RowWriter) throws { + try writer.write(value.blob, at: 0) + try writer.write(value.color, at: 1) + try writer.write(value.ids, at: 2) + try writer.write(value.names, at: 3) + try writer.write(value.origin, at: 4) + try writer.write(value.points, at: 5) + try writer.write(value.scores, at: 6) + } + + static func readRowFields(from row: Row) throws -> Bag { + try Bag( + names: row.read(at: 3), ids: row.read(at: 2), scores: row.read(at: 6), blob: row.read(at: 0), + points: row.read(at: 5), origin: row.read(at: 4), color: row.read(at: 1) + ) + } +} + +// One field of type T, to compare the bytes two Swift types produce for the same row type. +private struct Single: RowStruct { + var value: T + + static var rowSchema: RowSchema { RowSchema([.inferred("value", T.self)]) } + + static func writeRowFields(_ value: Single, to writer: RowWriter) throws { + try writer.write(value.value, at: 0) + } + + static func readRowFields(from row: Row) throws -> Single { + try Single(value: row.read(at: 0)) + } +} + +private let fullBag = Bag( + names: ["ab", nil, "xyz"], ids: [3, 1, 2], scores: ["k1": 1, "k2": nil], blob: Data([0, 1, 0xff, 0x7f, 0x80]), + points: [Point(number: 1, label: "a"), Point(number: 2, label: nil)], origin: Point(number: 5, label: "o"), color: .green +) + +private let emptyBag = Bag( + names: [], ids: [], scores: [:], blob: Data(), points: [], origin: nil, color: .red +) + +@Test +func rowEncoderFraming() throws { + let encoder = RowEncoder() + #expect(encoder.schemaHash == Bag.rowSchema.hash) + let encoded = try encoder.encode(fullBag) + let hash = withUnsafeBytes(of: encoder.schemaHash.littleEndian) { Data($0) } + #expect(encoded.prefix(8) == hash) + #expect(encoded.dropFirst(8) == (try encoder.toRow(fullBag))) + #expect(try encoder.decode(encoded) == fullBag) + + #expect(throws: ForyError.invalidData("encoded row of 7 bytes is shorter than its 8-byte schema hash")) { + try encoder.decode(encoded.prefix(7)) + } + var flipped = encoded + flipped[0] ^= 0x01 + let found = encoder.schemaHash ^ 0x01 + #expect(throws: ForyError.invalidData("row schema hash \(found) does not match the expected \(encoder.schemaHash)")) { + try encoder.decode(flipped) + } +} + +@Test +func rowEncoderCollections() throws { + let encoder = RowEncoder() + #expect(try encoder.fromRow(encoder.toRow(fullBag)) == fullBag) + #expect(try encoder.fromRow(encoder.toRow(emptyBag)) == emptyBag) + + // An empty map is its 8-byte keys size, then an empty keys array and an empty values array. + // Reading the map slot as binary gives those raw bytes. + let emptyMap = try RowEncoder>().toRow(Single(value: [:])) + let mapBytes = try Row(schema: Single<[String: Int32]>.rowSchema, bytes: Array(emptyMap)).binary(at: 0) + #expect(mapBytes == [0x08, 0, 0, 0, 0, 0, 0, 0] + [UInt8](repeating: 0, count: 16)) + + // Data copies its bytes in one go but must match an element by element [Int8] list. + let bytes = Data([0, 1, 0xff, 0x7f, 0x80]) + let int8s = bytes.map { Int8(bitPattern: $0) } + #expect(Single.rowSchema == Single<[Int8]>.rowSchema) + #expect(try RowEncoder>().toRow(Single(value: bytes)) == RowEncoder>().toRow(Single(value: int8s))) +} + +@Test +func rowEncoderDataNullElement() throws { + let schema = Single.rowSchema + var writer = RowWriter(schema: schema) + writer.reset() + var array = RowArrayWriter(element: RowField(name: "item", dataType: .int8, nullable: false), buffer: writer.buffer) + try array.reset(count: 2) + array.writeInt8(1, at: 0) + array.setNullAt(1) + try writer.setOffsetAndSize(at: 0, absoluteStart: array.base, size: array.size) + + #expect(throws: ForyError.invalidData("null row value cannot be read into non-optional Int8")) { + try RowEncoder>().fromRow(Data(writer.toBytes())) + } +} + +@Test +func rowEncoderEnum() throws { + let encoder = RowEncoder>() + #expect(encoder.schema.fields[0].dataType == .string) + #expect(try encoder.fromRow(encoder.toRow(Single(value: .green))).value == .green) + + let unknown = try RowEncoder>().toRow(Single(value: "blue")) + #expect(throws: ForyError.invalidData("unknown case \"blue\" for Color")) { + try encoder.fromRow(unknown) + } +} + +@Test +func rowEncoderDeterministic() throws { + let encoder = RowEncoder() + #expect(try encoder.toRow(fullBag) == encoder.toRow(fullBag)) +} diff --git a/swift/Tests/ForyTests/RowGoldenData.swift b/swift/Tests/ForyTests/RowGoldenData.swift new file mode 100644 index 0000000000..0f00911ea1 --- /dev/null +++ b/swift/Tests/ForyTests/RowGoldenData.swift @@ -0,0 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Bytes written by the reference row encoder in java/fory-format for the cross-language test models +// CrossLanguageTest.Foo, CrossLanguageTest.A (StringMap here), and GoCrossLanguageTest.Blob, with +// their create() values. Regenerate them from those models; never edit them by hand. + +// The Foo row without the schema hash. +let goldenFooRow: [UInt8] = [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, + 0x38, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0xa0, 0x00, 0x00, 0x00, + 0x70, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x10, 0x01, 0x00, 0x00, + 0x73, 0x74, 0x72, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, + 0x73, 0x74, 0x72, 0x31, 0x00, 0x00, 0x00, 0x00, 0x73, 0x74, 0x72, 0x32, + 0x00, 0x00, 0x00, 0x00, 0x70, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x48, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x50, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x58, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x60, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x68, 0x00, 0x00, 0x00, 0x6b, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x6b, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6b, 0x33, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x6b, 0x34, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x6b, 0x35, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6b, 0x36, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x05, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x03, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x73, 0x74, 0x72, 0x00, + 0x00, 0x00, 0x00, 0x00 +] + +// The Foo schema, one line per field. +let goldenFooSchema: [UInt8] = [ + 0x01, 0x05, + 0x46, 0x0b, 0xa8, 0x04, + 0x46, 0x0b, 0xb0, 0x15, + 0x46, 0x0b, 0xb8, 0x16, 0x49, 0x22, 0x64, 0x60, 0x15, + 0x46, 0x0b, 0xc0, 0x18, 0x05, 0x28, 0x98, 0x15, 0x4d, 0xd4, 0x0b, 0xa1, 0x00, 0x04, + 0x46, 0x0b, 0xc8, 0x1b, 0x02, 0x46, 0x0b, 0xa8, 0x04, 0x46, 0x0b, 0xb0, 0x15 +] + +// The StringMap schema hash followed by its row, as an encoder writes them. +let goldenStringMapEncoded: [UInt8] = [ + 0xc5, 0xbd, 0xf1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x88, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x28, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, + 0x03, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x6b, 0x31, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x69, 0x70, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x70, 0x69, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, + 0x30, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, + 0x76, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x2e, 0x30, 0x2e, + 0x30, 0x2e, 0x30, 0x00, 0x31, 0x32, 0x33, 0x34, 0x35, 0x00, 0x00, 0x00 +] + +// The Blob schema hash followed by its row, as an encoder writes them. +let goldenBlobEncoded: [UInt8] = [ + 0x38, 0x0d, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, + 0x05, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x01, 0xff, 0x7f, 0x80, 0x00, 0x00, 0x00, 0x62, 0x79, 0x74, 0x65, + 0x73, 0x00, 0x00, 0x00 +] diff --git a/swift/Tests/ForyTests/RowMacroEncodingTests.swift b/swift/Tests/ForyTests/RowMacroEncodingTests.swift new file mode 100644 index 0000000000..31f52b77f8 --- /dev/null +++ b/swift/Tests/ForyTests/RowMacroEncodingTests.swift @@ -0,0 +1,295 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation +import Fory +import Testing + +// The models every row format runtime shares in its cross-language tests. + +@ForyRow +private struct Bar: Equatable { + var f1: Int32? + var f2: String? +} + +@ForyRow +private struct Foo: Equatable { + var f1: Int32? + var f2: String? + var f3: [String?]? + var f4: [String: Int32?]? + var f5: Bar? +} + +@ForyRow +private struct StringMap: Equatable { + var f1: Int32? + var f2: [String: String]? +} + +@ForyRow +private struct Blob: Equatable { + var f1: Data? + var f2: String? +} + +private let fooValue = Foo( + f1: 1, f2: "str", f3: ["str1", nil, "str2"], f4: ["k1": 1, "k2": 2, "k3": 3, "k4": 4, "k5": 5, "k6": 6], + f5: Bar(f1: 1, f2: "str") +) +private let stringMapValue = StringMap(f1: 1, f2: ["pid": "12345", "ip": "0.0.0.0", "k1": "v1"]) +private let blobValue = Blob(f1: Data([0, 1, 0xff, 0x7f, 0x80]), f2: "bytes") + +// Field counts around the bitmap's 64-bit word boundary. + +@ForyRow +private struct Empty: Equatable {} + +@ForyRow +private struct Wide63: Equatable { + var f00: Int64 = 0, f01: Int64 = 0, f02: Int64 = 0, f03: Int64 = 0, f04: Int64 = 0, f05: Int64 = 0, f06: Int64 = 0, f07: Int64 = 0 + var f08: Int64 = 0, f09: Int64 = 0, f10: Int64 = 0, f11: Int64 = 0, f12: Int64 = 0, f13: Int64 = 0, f14: Int64 = 0, f15: Int64 = 0 + var f16: Int64 = 0, f17: Int64 = 0, f18: Int64 = 0, f19: Int64 = 0, f20: Int64 = 0, f21: Int64 = 0, f22: Int64 = 0, f23: Int64 = 0 + var f24: Int64 = 0, f25: Int64 = 0, f26: Int64 = 0, f27: Int64 = 0, f28: Int64 = 0, f29: Int64 = 0, f30: Int64 = 0, f31: Int64 = 0 + var f32: Int64 = 0, f33: Int64 = 0, f34: Int64 = 0, f35: Int64 = 0, f36: Int64 = 0, f37: Int64 = 0, f38: Int64 = 0, f39: Int64 = 0 + var f40: Int64 = 0, f41: Int64 = 0, f42: Int64 = 0, f43: Int64 = 0, f44: Int64 = 0, f45: Int64 = 0, f46: Int64 = 0, f47: Int64 = 0 + var f48: Int64 = 0, f49: Int64 = 0, f50: Int64 = 0, f51: Int64 = 0, f52: Int64 = 0, f53: Int64 = 0, f54: Int64 = 0, f55: Int64 = 0 + var f56: Int64 = 0, f57: Int64 = 0, f58: Int64 = 0, f59: Int64 = 0, f60: Int64 = 0, f61: Int64 = 0, f62: Int64 = 0 +} + +@ForyRow +private struct Wide64: Equatable { + var f00: Int64 = 0, f01: Int64 = 0, f02: Int64 = 0, f03: Int64 = 0, f04: Int64 = 0, f05: Int64 = 0, f06: Int64 = 0, f07: Int64 = 0 + var f08: Int64 = 0, f09: Int64 = 0, f10: Int64 = 0, f11: Int64 = 0, f12: Int64 = 0, f13: Int64 = 0, f14: Int64 = 0, f15: Int64 = 0 + var f16: Int64 = 0, f17: Int64 = 0, f18: Int64 = 0, f19: Int64 = 0, f20: Int64 = 0, f21: Int64 = 0, f22: Int64 = 0, f23: Int64 = 0 + var f24: Int64 = 0, f25: Int64 = 0, f26: Int64 = 0, f27: Int64 = 0, f28: Int64 = 0, f29: Int64 = 0, f30: Int64 = 0, f31: Int64 = 0 + var f32: Int64 = 0, f33: Int64 = 0, f34: Int64 = 0, f35: Int64 = 0, f36: Int64 = 0, f37: Int64 = 0, f38: Int64 = 0, f39: Int64 = 0 + var f40: Int64 = 0, f41: Int64 = 0, f42: Int64 = 0, f43: Int64 = 0, f44: Int64 = 0, f45: Int64 = 0, f46: Int64 = 0, f47: Int64 = 0 + var f48: Int64 = 0, f49: Int64 = 0, f50: Int64 = 0, f51: Int64 = 0, f52: Int64 = 0, f53: Int64 = 0, f54: Int64 = 0, f55: Int64 = 0 + var f56: Int64 = 0, f57: Int64 = 0, f58: Int64 = 0, f59: Int64 = 0, f60: Int64 = 0, f61: Int64 = 0, f62: Int64 = 0, f63: Int64 = 0 +} + +@ForyRow +private struct Wide65: Equatable { + var f00: Int64 = 0, f01: Int64 = 0, f02: Int64 = 0, f03: Int64 = 0, f04: Int64 = 0, f05: Int64 = 0, f06: Int64 = 0, f07: Int64 = 0 + var f08: Int64 = 0, f09: Int64 = 0, f10: Int64 = 0, f11: Int64 = 0, f12: Int64 = 0, f13: Int64 = 0, f14: Int64 = 0, f15: Int64 = 0 + var f16: Int64 = 0, f17: Int64 = 0, f18: Int64 = 0, f19: Int64 = 0, f20: Int64 = 0, f21: Int64 = 0, f22: Int64 = 0, f23: Int64 = 0 + var f24: Int64 = 0, f25: Int64 = 0, f26: Int64 = 0, f27: Int64 = 0, f28: Int64 = 0, f29: Int64 = 0, f30: Int64 = 0, f31: Int64 = 0 + var f32: Int64 = 0, f33: Int64 = 0, f34: Int64 = 0, f35: Int64 = 0, f36: Int64 = 0, f37: Int64 = 0, f38: Int64 = 0, f39: Int64 = 0 + var f40: Int64 = 0, f41: Int64 = 0, f42: Int64 = 0, f43: Int64 = 0, f44: Int64 = 0, f45: Int64 = 0, f46: Int64 = 0, f47: Int64 = 0 + var f48: Int64 = 0, f49: Int64 = 0, f50: Int64 = 0, f51: Int64 = 0, f52: Int64 = 0, f53: Int64 = 0, f54: Int64 = 0, f55: Int64 = 0 + var f56: Int64 = 0, f57: Int64 = 0, f58: Int64 = 0, f59: Int64 = 0, f60: Int64 = 0, f61: Int64 = 0, f62: Int64 = 0, f63: Int64 = 0 + var f64: Int64 = 0 +} + +private enum Color: String, RowCodable { + case red + case green +} + +@ForyRow +private struct AllNullable: Equatable { + var flag: Bool? + var tiny: Int8? + var short: Int16? + var int: Int32? + var long: Int64? + var word: Int? + var real: Float? + var double: Double? + var text: String? + var time: Date? + var day: LocalDate? + var span: Duration? + var blob: Data? + var list: [Int32]? + var tags: Set? + var counts: [String: Int32]? + var bar: Bar? + var color: Color? +} + +@ForyRow +private struct Nested: Equatable { + var groups: [String: [Bar?]]? +} + +@ForyRow +private struct Names: Equatable { + var userID: Int8 + var httpURL: Int8 + var f1: Int8 + var field2Name: Int8 + var URLValue: Int8 + // swiftlint:disable:next identifier_name + var x: Int8 + var already: Int8 +} + +@ForyRow +private struct Secret { + private var token: String + var id: Int32 +} + +// An initializer in the struct body would remove the memberwise initializer that decoding calls. +extension Secret { + init(id: Int32, secret: String) { + self.init(token: secret, id: id) + } + + var revealed: String { token } +} + +@ForyRow +private struct OptionalName { + var name: String? +} + +@ForyRow +private struct RequiredName { + var name: String +} + +private func checkRoundTrip(_ value: T, size: Int) throws { + let encoder = RowEncoder() + let row = try encoder.toRow(value) + #expect(row.count == size, "\(T.self)") + #expect(try encoder.fromRow(row) == value, "\(T.self)") +} + +@Test +func rowMacroSchemaGolden() throws { + #expect(try Foo.rowSchema.toBytes() == goldenFooSchema) + #expect(Foo.rowSchema.hash == 14_057_974_700_829_126) + #expect(StringMap.rowSchema.hash == 15_842_757) + #expect(Blob.rowSchema.hash == 527_672) +} + +@Test +func rowMacroDecodeGolden() throws { + let fooEncoder = RowEncoder() + #expect(try fooEncoder.fromRow(Data(goldenFooRow)) == fooValue) + #expect(try fooEncoder.decode(fooEncoder.encode(fooValue)) == fooValue) + let stringMapEncoder = RowEncoder() + #expect(try stringMapEncoder.decode(Data(goldenStringMapEncoded)) == stringMapValue) + #expect(try stringMapEncoder.decode(stringMapEncoder.encode(stringMapValue)) == stringMapValue) + #expect(try RowEncoder().decode(Data(goldenBlobEncoded)) == blobValue) +} + +// Blob holds no map, so its bytes do not depend on dictionary order and must match exactly. +@Test +func rowMacroBlobBytes() throws { + #expect(try RowEncoder().encode(blobValue) == Data(goldenBlobEncoded)) +} + +@Test +func rowMacroFieldCounts() throws { + try checkRoundTrip(Empty(), size: 0) + var wide63 = Wide63() + wide63.f00 = 1 + wide63.f62 = .min + try checkRoundTrip(wide63, size: 8 + 63 * 8) + var wide64 = Wide64() + wide64.f00 = 1 + wide64.f63 = .min + try checkRoundTrip(wide64, size: 8 + 64 * 8) + var wide65 = Wide65() + wide65.f00 = 1 + wide65.f64 = .min + try checkRoundTrip(wide65, size: 16 + 65 * 8) +} + +@Test +func rowMacroAllNull() throws { + let encoder = RowEncoder() + let row = try encoder.toRow(AllNullable()) + // Eighteen fields set the low 18 bitmap bits, and every slot stays zero. + #expect(Array(row) == [0xff, 0xff, 0x03] + [UInt8](repeating: 0, count: 5 + 18 * 8)) + #expect(try encoder.fromRow(row) == AllNullable()) +} + +@Test +func rowMacroAllValues() throws { + let full = AllNullable( + flag: true, tiny: -8, short: -16, int: 32, long: .min, word: .max, real: -0.0, double: .nan, text: "text", + time: Date(timeIntervalSince1970: 1_700_000_000), day: LocalDate(epochDay: 19_000), span: .seconds(-90), + blob: Data([1, 2]), list: [1, 2, 3], tags: ["a", "b"], counts: ["k": 1], bar: Bar(f1: nil, f2: "bar"), color: .green + ) + let encoder = RowEncoder() + var decoded = try encoder.fromRow(encoder.toRow(full)) + #expect(decoded.real?.bitPattern == full.real?.bitPattern) + #expect(decoded.double?.bitPattern == full.double?.bitPattern) + // NaN never equals itself, so the floats are compared by bit pattern above and cleared here. + var expected = full + (expected.real, expected.double, decoded.real, decoded.double) = (nil, nil, nil, nil) + #expect(decoded == expected) +} + +@Test +func rowMacroNestedInListInMap() throws { + let encoder = RowEncoder() + let values = [ + Nested(groups: ["a": [Bar(f1: 1, f2: "x"), nil, Bar(f1: nil, f2: nil)], "b": []]), + Nested(groups: [:]), + Nested(groups: nil) + ] + for value in values { + #expect(try encoder.decode(encoder.encode(value)) == value) + } +} + +@Test +func rowMacroPrivateProperty() throws { + let encoder = RowEncoder() + #expect(encoder.schema.fields.map(\.name) == ["id", "token"]) + let decoded = try encoder.decode(encoder.encode(Secret(id: 7, secret: "hidden"))) + #expect(decoded.id == 7) + #expect(decoded.revealed == "hidden") +} + +@Test +func rowMacroNames() throws { + let encoder = RowEncoder() + let expected = ["_u_r_l_value", "already", "f1", "field2_name", "http_u_r_l", "user_i_d", "x"] + #expect(encoder.schema.fields.map(\.name) == expected) + let names = Names(userID: 1, httpURL: 2, f1: 3, field2Name: 4, URLValue: 5, x: 6, already: 7) + let row = try encoder.toRow(names) + #expect(try encoder.fromRow(row) == names) + // Slots follow the sorted names, so URLValue comes first and x last. + let view = try Row(schema: encoder.schema, bytes: Array(row)) + #expect(try view.read(Int8.self, at: 0) == 5) + #expect(try view.read(Int8.self, at: 6) == 6) +} + +@Test +func rowMacroDeterministic() throws { + let encoder = RowEncoder() + #expect(try encoder.encode(fooValue) == encoder.encode(fooValue)) +} + +// A row schema marks String nullable whether or not the property is optional, so both structs share +// one schema, and a null written by one cannot be read into the other. +@Test +func rowMacroNullIntoValue() throws { + #expect(OptionalName.rowSchema == RequiredName.rowSchema) + let row = try RowEncoder().toRow(OptionalName(name: nil)) + #expect(throws: ForyError.invalidData("null row value cannot be read into non-optional String")) { + try RowEncoder().fromRow(row) + } +} diff --git a/swift/Tests/ForyTests/RowMacroTests.swift b/swift/Tests/ForyTests/RowMacroTests.swift new file mode 100644 index 0000000000..be518bae6c --- /dev/null +++ b/swift/Tests/ForyTests/RowMacroTests.swift @@ -0,0 +1,339 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import SwiftSyntaxMacroExpansion +import SwiftSyntaxMacros +import SwiftSyntaxMacrosTestSupport +import Testing +@testable import ForyMacro + +private let rowMacroSpecs = ["ForyRow": MacroSpec(type: ForyRowMacro.self, conformances: ["RowStruct"])] + +private func assertRowExpansion(_ source: String, _ expandedSource: String) { + assertMacroExpansion(source, expandedSource: expandedSource, macroSpecs: rowMacroSpecs) +} + +// A rejected declaration expands to itself, without the attribute and with nothing added. +private func assertRowRejects(_ source: String, _ expandedSource: String, _ message: String, line: Int, column: Int) { + assertMacroExpansion( + source, + expandedSource: expandedSource, + diagnostics: [DiagnosticSpec(message: message, line: line, column: column)], + macroSpecs: rowMacroSpecs + ) +} + +@Test +func rowMacroExpansion() { + assertRowExpansion( + """ + @ForyRow + struct Profile { + var userID: Int64 + var age: Int32? + } + """, + """ + struct Profile { + var userID: Int64 + var age: Int32? + + static let rowSchema = RowSchema([ + RowField.inferred("age", (Int32?).self), + RowField.inferred("user_i_d", (Int64).self) + ]) + + static func writeRowFields(_ value: Self, to writer: RowWriter) throws { + try writer.write(value.age, at: 0) + try writer.write(value.userID, at: 1) + } + + static func readRowFields(from row: Row) throws -> Self { + try Self( + userID: row.read(at: 1), + age: row.read(at: 0) + ) + } + } + + extension Profile: RowStruct { + } + """ + ) +} + +@Test +func rowMacroPublic() { + assertRowExpansion( + """ + @ForyRow + public struct Empty { + } + """, + """ + public struct Empty { + + public static let rowSchema = RowSchema([]) + + public static func writeRowFields(_ value: Self, to writer: RowWriter) throws { + + } + + public static func readRowFields(from row: Row) throws -> Self { + Self() + } + } + + extension Empty: RowStruct { + } + """ + ) +} + +@Test +func rowMacroSkips() { + assertRowExpansion( + """ + @ForyRow + struct Counter { + static var shared: Int32 = 0 + lazy var cache: Int32 = 0 + var doubled: Int32 { count * 2 } + var label: String { + get { "" } + set {} + } + var count: Int32 { + didSet {} + } + } + """, + """ + struct Counter { + static var shared: Int32 = 0 + lazy var cache: Int32 = 0 + var doubled: Int32 { count * 2 } + var label: String { + get { "" } + set {} + } + var count: Int32 { + didSet {} + } + + static let rowSchema = RowSchema([ + RowField.inferred("count", (Int32).self) + ]) + + static func writeRowFields(_ value: Self, to writer: RowWriter) throws { + try writer.write(value.count, at: 0) + } + + static func readRowFields(from row: Row) throws -> Self { + try Self( + count: row.read(at: 0) + ) + } + } + + extension Counter: RowStruct { + } + """ + ) +} + +@Test +func rowMacroFieldName() { + let cases = [ + ("userID", "user_i_d"), ("httpURL", "http_u_r_l"), ("f1", "f1"), ("field2Name", "field2_name"), + ("URLValue", "_u_r_l_value"), ("x", "x"), ("already", "already") + ] + for (name, expected) in cases { + #expect(rowFieldName(name) == expected, "\(name)") + } +} + +@Test +func rowMacroOrder() { + assertRowExpansion( + """ + @ForyRow + struct Names { + var x: Int8 + var userID: Int8 + var URLValue: Int8 + var already: Int8 + } + """, + """ + struct Names { + var x: Int8 + var userID: Int8 + var URLValue: Int8 + var already: Int8 + + static let rowSchema = RowSchema([ + RowField.inferred("_u_r_l_value", (Int8).self), + RowField.inferred("already", (Int8).self), + RowField.inferred("user_i_d", (Int8).self), + RowField.inferred("x", (Int8).self) + ]) + + static func writeRowFields(_ value: Self, to writer: RowWriter) throws { + try writer.write(value.URLValue, at: 0) + try writer.write(value.already, at: 1) + try writer.write(value.userID, at: 2) + try writer.write(value.x, at: 3) + } + + static func readRowFields(from row: Row) throws -> Self { + try Self( + x: row.read(at: 3), + userID: row.read(at: 2), + URLValue: row.read(at: 0), + already: row.read(at: 1) + ) + } + } + + extension Names: RowStruct { + } + """ + ) +} + +@Test +func rowMacroBacktick() { + assertRowExpansion( + """ + @ForyRow + struct Option { + var `default`: Int32 + } + """, + """ + struct Option { + var `default`: Int32 + + static let rowSchema = RowSchema([ + RowField.inferred("default", (Int32).self) + ]) + + static func writeRowFields(_ value: Self, to writer: RowWriter) throws { + try writer.write(value.`default`, at: 0) + } + + static func readRowFields(from row: Row) throws -> Self { + try Self( + `default`: row.read(at: 0) + ) + } + } + + extension Option: RowStruct { + } + """ + ) +} + +@Test +func rowMacroRejectsFieldTypes() { + let cases: [(declaration: String, message: String)] = [ + ("var count: UInt32", "@ForyRow field 'count' uses unsigned type 'UInt32'; use Data for bytes or a signed integer type"), + ("var ids: [UInt8]", "@ForyRow field 'ids' uses unsigned type 'UInt8'; use Data for bytes or a signed integer type"), + ("var value: Int??", "@ForyRow field 'value' uses the nested optional 'Int??'"), + ("var pair: (Int32, Int32)", "@ForyRow field 'pair' uses the tuple type '(Int32, Int32)', which has no row type"), + ("var times: [Date: Int32]", "@ForyRow field 'times' uses 'Date' as a map key, which is not supported"), + ("var names: [String?: Int32]", "@ForyRow field 'names' uses the optional map key 'String?'; map keys are never null"), + ("let version: Int32 = 1", "@ForyRow field 'version' is a let with a default value, so decoding cannot set it; use var"), + ("var count = 0", "@ForyRow field 'count' needs its own type annotation"), + ("var next: [Bad]", "@ForyRow field 'next' refers to 'Bad' itself, which a row cannot hold") + ] + for (declaration, message) in cases { + assertRowRejects( + "@ForyRow\nstruct Bad {\n \(declaration)\n}", + "struct Bad {\n \(declaration)\n}", + message, + line: 3, + column: 9 + ) + } +} + +@Test +func rowMacroRejectsDeclarations() { + assertRowRejects( + "@ForyRow\nstruct Bad {\n var (a, b): (Int32, Int32)\n}", + "struct Bad {\n var (a, b): (Int32, Int32)\n}", + "@ForyRow does not support the tuple pattern '(a, b)'; declare each property on its own", + line: 3, + column: 9 + ) + assertRowRejects( + "@ForyRow\nstruct Bad {\n @Wrapped var name: String\n}", + "struct Bad {\n @Wrapped var name: String\n}", + "@ForyRow field 'name' uses an attribute or property wrapper, which row encoding does not support", + line: 3, + column: 5 + ) + // In `var a, b: Int32` only `b` carries the annotation. + assertRowRejects( + "@ForyRow\nstruct Bad {\n var a, b: Int32\n}", + "struct Bad {\n var a, b: Int32\n}", + "@ForyRow field 'a' needs its own type annotation", + line: 3, + column: 9 + ) + assertRowRejects( + "@ForyRow\nenum Bad {\n case one\n}", + "enum Bad {\n case one\n}", + "@ForyRow can only be applied to a struct", + line: 2, + column: 1 + ) + assertRowRejects( + "@ForyRow\nclass Bad {\n}", + "class Bad {\n}", + "@ForyRow can only be applied to a struct", + line: 2, + column: 1 + ) + assertRowRejects( + "@ForyRow\nstruct Bad {\n}", + "struct Bad {\n}", + "@ForyRow does not support generic structs", + line: 2, + column: 11 + ) +} + +@Test +func rowMacroReportsEveryField() { + assertMacroExpansion( + "@ForyRow\nstruct Bad {\n var a: UInt8\n var b: Int??\n}", + expandedSource: "struct Bad {\n var a: UInt8\n var b: Int??\n}", + diagnostics: [ + DiagnosticSpec( + message: "@ForyRow field 'a' uses unsigned type 'UInt8'; use Data for bytes or a signed integer type", + line: 3, + column: 9 + ), + DiagnosticSpec(message: "@ForyRow field 'b' uses the nested optional 'Int??'", line: 4, column: 9) + ], + macroSpecs: rowMacroSpecs + ) +} diff --git a/swift/Tests/ForyTests/RowReaderTests.swift b/swift/Tests/ForyTests/RowReaderTests.swift new file mode 100644 index 0000000000..00170d0c7b --- /dev/null +++ b/swift/Tests/ForyTests/RowReaderTests.swift @@ -0,0 +1,188 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation +import Testing +@testable import Fory + +private let rowStringSchema = RowSchema([RowField(name: "s", dataType: .string, nullable: true)]) + +// A one-string row: 8-byte bitmap, then the given slot, then the given data. +private func rowWithString(slot: [UInt8], data: [UInt8] = [0x61, 0, 0, 0, 0, 0, 0, 0]) throws -> Row { + try Row(schema: rowStringSchema, bytes: [UInt8](repeating: 0, count: 8) + slot + data) +} + +@Test +func rowReaderLayout() throws { + let schema = RowSchema([ + RowField(name: "a", dataType: .int32, nullable: true), + RowField(name: "b", dataType: .int32, nullable: false), + RowField(name: "c", dataType: .string, nullable: true) + ]) + let bytes: [UInt8] = [ + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // bitmap: a is null + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // slot a + 0x2C, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // slot b = 300 + 0x02, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, // slot c: size 2, offset 0x20 + 0x68, 0x69, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // "hi" + ] + let row = try Row(schema: schema, bytes: bytes) + #expect(row.fieldCount == 3) + #expect(row.sizeInBytes == 40) + #expect(row.isNull(at: 0)) + #expect(row.int32(at: 0) == nil) + #expect(row.int32(at: 1) == 300) + #expect(try row.string(at: 2) == "hi") + #expect(row.toBytes() == bytes) +} + +@Test +func rowReaderGoLayout() throws { + let schema = RowSchema([ + RowField(name: "f1", dataType: .int64, nullable: false), + RowField(name: "f2", dataType: .string, nullable: true) + ]) + let bytes: [UInt8] = [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x08, 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, + 0x02, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, + 0x61, 0x62, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + ] + let row = try Row(schema: schema, bytes: bytes) + #expect(row.int64(at: 0) == 0x0102_0304_0506_0708) + #expect(try row.string(at: 1) == "ab") +} + +@Test +func rowReaderNested() throws { + let schema = RowSchema([ + RowField(name: "id", dataType: .int32, nullable: false), + RowField(name: "child", dataType: .struct([RowField(name: "x", dataType: .int32, nullable: false)]), nullable: true) + ]) + let bytes: [UInt8] = [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // parent bitmap + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // slot id = 1 + 0x10, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, // slot child: size 16, offset 0x18 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // child bitmap + 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // child slot x = 5 + ] + let row = try Row(schema: schema, bytes: bytes) + #expect(row.int32(at: 0) == 1) + let child = try #require(try row.row(at: 1)) + #expect(child.sizeInBytes == 16) + #expect(child.int32(at: 0) == 5) + #expect(child.toBytes() == Array(bytes[0x18..<0x28])) +} + +@Test +func rowReaderNullSkipsData() throws { + let schema = RowSchema([ + RowField(name: "s", dataType: .string, nullable: true), + RowField(name: "b", dataType: .binary, nullable: true), + RowField(name: "e", dataType: .string, nullable: true) + ]) + let bytes: [UInt8] = [ + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // s and b are null + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // slot s: garbage + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // slot b: garbage + 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00 // slot e: empty, not null + ] + let row = try Row(schema: schema, bytes: bytes) + #expect(try row.string(at: 0) == nil) + #expect(try row.binary(at: 1) == nil) + #expect(!row.isNull(at: 2)) + #expect(try row.string(at: 2) == "") +} + +@Test +func rowReaderFieldCounts() throws { + for (count, size) in [(0, 0), (63, 512), (64, 520), (65, 536)] { + let schema = RowSchema((0.. 0 { + bytes[(count - 1) / 8] |= 1 << ((count - 1) % 8) + // The first slot starts right after the bitmap, which grows to 16 bytes at 65 fields. + bytes[rowBitmapWidth(count)] = 7 + } + let row = try Row(schema: schema, bytes: bytes) + #expect(row.sizeInBytes == size) + if count > 0 { + #expect(row.isNull(at: count - 1)) + #expect(!row.isNull(at: count - 2)) + #expect(row.int64(at: 0) == 7) + } + } +} + +@Test +func rowReaderMultiByteUTF8() throws { + let row = try rowWithString( + slot: [0x06, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00], + data: [0x68, 0xC3, 0xA9, 0x6C, 0x6C, 0x6F, 0x00, 0x00] + ) + #expect(try row.string(at: 0) == "héllo") +} + +@Test +func rowReaderTemporal() throws { + let schema = RowSchema([ + RowField(name: "ts", dataType: .timestamp, nullable: false), + RowField(name: "d", dataType: .date32, nullable: false), + RowField(name: "dur", dataType: .duration, nullable: false) + ]) + let bytes: [UInt8] = [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // ts = -1 microsecond + 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, // d = -1 day + 0x24, 0xFA, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF // dur = -1500 microseconds + ] + let row = try Row(schema: schema, bytes: bytes) + // Counted back from 2001 in whole microseconds, so the check itself adds no rounding. + func micros(_ date: Date) -> Int64 { + Int64((date.timeIntervalSinceReferenceDate * 1_000_000).rounded()) + 978_307_200_000_000 + } + let timestamp = try #require(row.timestamp(at: 0)) + #expect(micros(timestamp) == -1) + #expect(timestamp < Date(timeIntervalSince1970: 0)) + #expect(micros(rowDate(micros: 1_234_567_890_123_456)) == 1_234_567_890_123_456) + #expect(row.localDate(at: 1) == LocalDate(epochDay: -1)) + #expect(row.duration(at: 2) == .microseconds(-1500)) +} + +@Test +func rowReaderCorrupt() throws { + #expect(throws: ForyError.invalidData("row of 15 bytes is shorter than its 16-byte fixed region")) { + try Row(schema: rowStringSchema, bytes: [UInt8](repeating: 0, count: 15)) + } + // Offset 0x18 is the end of the 24-byte row, so even one byte there is past it. + #expect(throws: ForyError.outOfBounds(cursor: 0x18, need: 1, length: 24)) { + try rowWithString(slot: [0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00]).string(at: 0) + } + #expect(throws: ForyError.outOfBounds(cursor: 0x10, need: 9, length: 24)) { + try rowWithString(slot: [0x09, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00]).string(at: 0) + } + let max = Int(clamping: UInt32.max) + #expect(throws: ForyError.outOfBounds(cursor: max, need: max, length: 24)) { + try rowWithString(slot: [UInt8](repeating: 0xFF, count: 8)).binary(at: 0) + } + #expect(throws: ForyError.invalidData("invalid UTF-8 sequence")) { + try rowWithString( + slot: [0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00], + data: [0xFF, 0xFE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ).string(at: 0) + } +} diff --git a/swift/Tests/ForyTests/RowSchemaBytesTests.swift b/swift/Tests/ForyTests/RowSchemaBytesTests.swift new file mode 100644 index 0000000000..7da47950f8 --- /dev/null +++ b/swift/Tests/ForyTests/RowSchemaBytesTests.swift @@ -0,0 +1,192 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation +import Testing +@testable import Fory + +private func field(_ name: String, _ dataType: RowDataType, nullable: Bool = true) -> RowField { + RowField(name: name, dataType: dataType, nullable: nullable) +} + +// Bytes and hashes that every row format runtime produces for these schemas. +private let simpleSchema = RowSchema([ + field("id", .int32), field("name", .string), field("score", .float64), field("active", .bool) +]) + +private let simpleBytes: [UInt8] = [ + 0x01, 0x04, 0x45, 0xa0, 0x60, 0x04, 0x49, 0x34, 0x0c, 0x20, 0x15, 0x4d, + 0xc8, 0x4e, 0x89, 0x00, 0x14, 0x4d, 0x00, 0x53, 0x45, 0x48, 0x01 +] + +private let nestedSchema = RowSchema([ + field("person", .struct([field("name", .string), field("age", .int32, nullable: false)])), + field("tags", .listOf(.string)), + field("attrs", .mapOf(.string, .int64)) +]) + +private let nestedBytes: [UInt8] = [ + 0x01, 0x03, 0x4d, 0x3c, 0x91, 0x93, 0x9a, 0x1b, 0x02, 0x49, 0x34, 0x0c, + 0x20, 0x15, 0x05, 0x00, 0xc4, 0x04, 0x49, 0x4c, 0x06, 0x90, 0x16, 0x49, + 0x22, 0x64, 0x60, 0x15, 0x4d, 0x82, 0x73, 0x8c, 0x80, 0x18, 0x05, 0x28, + 0x98, 0x15, 0x4d, 0xd4, 0x0b, 0xa1, 0x00, 0x06 +] + +private let specialSchema = RowSchema([ + field("a_very_long_field_name_exceeding_limit", .int64, nullable: false), + field("field2", .string), + field("$special_name", .date32), + field("CamelCase", .timestamp, nullable: false), + field("price", .decimal(precision: 10, scale: 2)) +]) + +private let specialBytes: [UInt8] = [ + 0x01, 0x05, 0x3d, 0x09, 0x03, 0x75, 0x24, 0x71, 0xb5, 0xb9, 0xa6, 0xd9, + 0x50, 0x45, 0x8f, 0x6d, 0x03, 0x09, 0xb2, 0x5c, 0x44, 0x20, 0xd0, 0xd3, + 0x6d, 0x68, 0x62, 0x26, 0x06, 0x52, 0x0a, 0x40, 0x85, 0x87, 0xb0, 0x15, + 0x61, 0xf2, 0x4f, 0x20, 0x90, 0x05, 0xed, 0xa0, 0x61, 0x00, 0x27, 0x1a, + 0x38, 0x01, 0x82, 0x16, 0xe0, 0x09, 0x08, 0x26, 0x4d, 0xbe, 0x28, 0x11, + 0x00, 0x28, 0x0a, 0x02 +] + +// The Foo schema from the cross-language row tests. +private let fooSchema = RowSchema([ + field("f1", .int32), + field("f2", .string), + field("f3", .listOf(.string)), + field("f4", .mapOf(.string, .int32)), + field("f5", .struct([field("f1", .int32), field("f2", .string)])) +]) + +private func nestedLists(_ levels: Int) -> RowDataType { + var dataType = RowDataType.int32 + for _ in 0.. [UInt8] { + var bytes: [UInt8] = [0x01, 0x01] + for _ in 0..") + #expect(RowDataType.mapOf(.string, .int32).description == "map") + let structType = RowDataType.struct([ + RowField(name: "a", dataType: .int32, nullable: false), + RowField(name: "b", dataType: .string, nullable: true) + ]) + #expect(structType.description == "struct") + #expect(RowSchema([RowField(name: "a", dataType: .int32, nullable: false)]).description == "schema") +} diff --git a/swift/Tests/ForyTests/RowSlotTests.swift b/swift/Tests/ForyTests/RowSlotTests.swift new file mode 100644 index 0000000000..188dee6e37 --- /dev/null +++ b/swift/Tests/ForyTests/RowSlotTests.swift @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation +import Testing +@testable import Fory + +@Test +func rowSlotEncode() throws { + let slot = try rowEncodeOffsetAndSize(offset: 0x20, size: 2) + #expect(slot == 0x0000_0020_0000_0002) + #expect(withUnsafeBytes(of: slot.littleEndian, Array.init) == [0x02, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00]) +} + +@Test +func rowSlotDecode() { + let ticket = rowDecodeOffsetAndSize(0x0000_0020_0000_0002) + #expect(ticket.offset == 0x20) + #expect(ticket.size == 2) + + let large = rowDecodeOffsetAndSize(0xFFFF_FFFF_0000_0001) + #expect(large.offset == UInt32.max) + #expect(large.size == 1) +} + +@Test +func rowSlotEncodeLimits() throws { + #expect(throws: ForyError.self) { + try rowEncodeOffsetAndSize(offset: Int(UInt32.max) + 1, size: 0) + } + #expect(throws: ForyError.self) { + try rowEncodeOffsetAndSize(offset: 0, size: -1) + } + let slot = try rowEncodeOffsetAndSize(offset: Int(UInt32.max), size: Int(UInt32.max)) + #expect(slot == UInt64.max) +} + +@Test +func rowSlotCheckedRange() throws { + #expect(try rowCheckedRange(offset: 8, size: 8, regionBase: 100, regionCount: 16) == 108..<116) + #expect(try rowCheckedRange(offset: 0, size: 0, regionBase: 0, regionCount: 0).isEmpty) + #expect(throws: ForyError.self) { + try rowCheckedRange(offset: 8, size: 9, regionBase: 0, regionCount: 16) + } + #expect(throws: ForyError.self) { + try rowCheckedRange(offset: UInt32.max, size: UInt32.max, regionBase: 0, regionCount: 16) + } +} diff --git a/swift/Tests/ForyTests/RowWriterTests.swift b/swift/Tests/ForyTests/RowWriterTests.swift new file mode 100644 index 0000000000..7154eaad74 --- /dev/null +++ b/swift/Tests/ForyTests/RowWriterTests.swift @@ -0,0 +1,257 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation +import Testing +@testable import Fory + +private let rowSection31Schema = RowSchema([ + RowField(name: "a", dataType: .int32, nullable: true), + RowField(name: "b", dataType: .int32, nullable: false), + RowField(name: "c", dataType: .string, nullable: true) +]) + +private let rowSection31Bytes: [UInt8] = [ + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // bitmap: a is null + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // slot a + 0x2C, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // slot b = 300 + 0x02, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, // slot c: size 2, offset 0x20 + 0x68, 0x69, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // "hi" +] + +private func writeSection31Row(into buffer: ByteBuffer) throws -> RowWriter { + var writer = RowWriter(schema: rowSection31Schema, buffer: buffer) + writer.reset() + writer.setNullAt(0) + writer.writeInt32(300, at: 1) + try writer.writeString("hi", at: 2) + return writer +} + +@Test +func rowWriterLayout() throws { + #expect(try writeSection31Row(into: ByteBuffer()).toBytes() == rowSection31Bytes) + + // A row that starts after other bytes still counts its offsets from its own start. + let buffer = ByteBuffer() + buffer.writeBytes([UInt8](repeating: 0xAA, count: 8)) + let writer = try writeSection31Row(into: buffer) + #expect(writer.base == 8) + #expect(writer.size == 40) + #expect(writer.toBytes() == rowSection31Bytes) +} + +@Test +func rowWriterGoLayout() throws { + let schema = RowSchema([ + RowField(name: "f1", dataType: .int64, nullable: false), + RowField(name: "f2", dataType: .string, nullable: true) + ]) + var writer = RowWriter(schema: schema) + writer.reset() + writer.writeInt64(0x0102_0304_0506_0708, at: 0) + try writer.writeString("ab", at: 1) + #expect( + writer.toBytes() == [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x08, 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, + 0x02, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, + 0x61, 0x62, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + ]) +} + +@Test +func rowWriterPrimitives() throws { + let schema = RowSchema([ + RowField(name: "b", dataType: .bool, nullable: false), + RowField(name: "i8", dataType: .int8, nullable: false), + RowField(name: "i16", dataType: .int16, nullable: false), + RowField(name: "i32", dataType: .int32, nullable: false), + RowField(name: "i64", dataType: .int64, nullable: false) + ]) + for useMin in [true, false] { + var writer = RowWriter(schema: schema) + writer.reset() + writer.writeBool(useMin, at: 0) + writer.writeInt8(useMin ? .min : .max, at: 1) + writer.writeInt16(useMin ? .min : .max, at: 2) + writer.writeInt32(useMin ? .min : .max, at: 3) + writer.writeInt64(useMin ? .min : .max, at: 4) + let row = try Row(schema: schema, bytes: writer.toBytes()) + #expect(row.bool(at: 0) == useMin) + #expect(row.int8(at: 1) == (useMin ? .min : .max)) + #expect(row.int16(at: 2) == (useMin ? .min : .max)) + #expect(row.int32(at: 3) == (useMin ? .min : .max)) + #expect(row.int64(at: 4) == (useMin ? .min : .max)) + } +} + +@Test +func rowWriterFloatBits() throws { + let floats: [Float] = [Float(bitPattern: 0x7FC0_0001), -0.0, .infinity, -.infinity] + let doubles: [Double] = [Double(bitPattern: 0x7FF8_0000_0000_0001), -0.0, .infinity, -.infinity] + let schema = RowSchema( + floats.indices.map { RowField(name: "f\($0)", dataType: .float32, nullable: false) } + + doubles.indices.map { RowField(name: "d\($0)", dataType: .float64, nullable: false) } + ) + var writer = RowWriter(schema: schema) + writer.reset() + for (index, value) in floats.enumerated() { + writer.writeFloat32(value, at: index) + } + for (index, value) in doubles.enumerated() { + writer.writeFloat64(value, at: floats.count + index) + } + let row = try Row(schema: schema, bytes: writer.toBytes()) + // NaN never equals itself, so the bits are compared instead of the values. + for (index, value) in floats.enumerated() { + #expect(row.float32(at: index)?.bitPattern == value.bitPattern) + } + for (index, value) in doubles.enumerated() { + #expect(row.float64(at: floats.count + index)?.bitPattern == value.bitPattern) + } +} + +@Test +func rowWriterNullEveryType() throws { + let types: [RowDataType] = [ + .bool, .int8, .int16, .int32, .int64, .float32, .float64, .date32, .timestamp, .duration, .string, .binary + ] + let schema = RowSchema(types.enumerated().map { RowField(name: "f\($0.offset)", dataType: $0.element, nullable: true) }) + var writer = RowWriter(schema: schema) + writer.reset() + // Fill the fixed-width slots first, so setNullAt has something to zero. + for index in 0..<10 { + writer.writeInt8(-1, at: index) + } + for index in types.indices { + writer.setNullAt(index) + } + let bytes = writer.toBytes() + #expect(writer.size == 8 + types.count * 8) + #expect(Array(bytes[0..<8]) == [0xFF, 0x0F, 0, 0, 0, 0, 0, 0]) + #expect(bytes[8...].allSatisfy { $0 == 0 }) +} + +@Test +func rowWriterEmptyString() throws { + var writer = RowWriter(schema: RowSchema([RowField(name: "s", dataType: .string, nullable: true)])) + writer.reset() + try writer.writeString("", at: 0) + #expect( + writer.toBytes() == [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00 // size 0, offset 0x10 (the end of the slots) + ]) +} + +@Test +func rowWriterMultiByteUTF8() throws { + var writer = RowWriter(schema: RowSchema([RowField(name: "s", dataType: .string, nullable: true)])) + writer.reset() + try writer.writeString("héllo", at: 0) + #expect( + Array(writer.toBytes()[8...]) == [ + 0x06, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x68, 0xC3, 0xA9, 0x6C, 0x6C, 0x6F, 0x00, 0x00 + ]) +} + +@Test +func rowWriterFieldCounts() throws { + for (count, size) in [(0, 0), (63, 512), (64, 520), (65, 536)] { + let schema = RowSchema((0.. 0 { + writer.setNullAt(count - 1) + let bytes = writer.toBytes() + #expect(bytes[(count - 1) / 8] == 1 << ((count - 1) % 8)) + let row = try Row(schema: schema, bytes: bytes) + #expect(row.isNull(at: count - 1)) + #expect(!row.isNull(at: count - 2)) + } + } +} + +@Test +func rowWriterDeterministic() throws { + let first = try writeSection31Row(into: ByteBuffer()).toBytes() + let second = try writeSection31Row(into: ByteBuffer()).toBytes() + let reused = ByteBuffer() + reused.writeBytes([UInt8](repeating: 0xFF, count: 128)) + reused.clear() + let third = try writeSection31Row(into: reused).toBytes() + #expect(first == second) + #expect(first == third) +} + +@Test +func rowWriterTimestamp() throws { + let window = Int64(8_589_934_591) + let samples: [Int64] = [ + 0, -1, 1, 1_700_000_000_123_457, -1_000_000_000_000_000, + (978_307_200 + window) * 1_000_000 + 999_999, + (978_307_200 - window) * 1_000_000 + 999_999 + ] + for micros in samples { + #expect(try rowTimestampMicros(rowDate(micros: micros)) == micros) + } + + let schema = RowSchema([RowField(name: "t", dataType: .timestamp, nullable: false)]) + var writer = RowWriter(schema: schema) + writer.reset() + try writer.writeTimestamp(rowDate(micros: -1), at: 0) + #expect(try Row(schema: schema, bytes: writer.toBytes()).int64(at: 0) == -1) + + #expect(throws: ForyError.encodingError("timestamp inf seconds from 2001 does not fit in 64-bit microseconds")) { + try writer.writeTimestamp(Date(timeIntervalSinceReferenceDate: .infinity), at: 0) + } +} + +@Test +func rowWriterDuration() throws { + #expect(try rowDurationMicros(.nanoseconds(-1)) == 0) + #expect(try rowDurationMicros(.nanoseconds(1_999)) == 1) + #expect(try rowDurationMicros(.nanoseconds(-1_999)) == -1) + #expect(try rowDurationMicros(.microseconds(Int64.min)) == Int64.min) + + let schema = RowSchema([RowField(name: "d", dataType: .duration, nullable: false)]) + var writer = RowWriter(schema: schema) + writer.reset() + try writer.writeDuration(.microseconds(-1_500), at: 0) + #expect(try Row(schema: schema, bytes: writer.toBytes()).duration(at: 0) == .microseconds(-1_500)) + + #expect(throws: ForyError.self) { + try writer.writeDuration(.seconds(Int64.max), at: 0) + } +} + +@Test +func rowWriterLocalDate() throws { + let schema = RowSchema([RowField(name: "d", dataType: .date32, nullable: false)]) + var writer = RowWriter(schema: schema) + writer.reset() + for epochDay in [-1, Int64(Int32.max)] { + try writer.writeLocalDate(LocalDate(epochDay: epochDay), at: 0) + #expect(try Row(schema: schema, bytes: writer.toBytes()).localDate(at: 0) == LocalDate(epochDay: epochDay)) + } + #expect(throws: ForyError.encodingError("date 2147483648 days from 1970 does not fit in 32 bits")) { + try writer.writeLocalDate(LocalDate(epochDay: Int64(Int32.max) + 1), at: 0) + } +}