diff --git a/benchmarks/java/src/main/java/org/apache/fory/benchmark/SchemaEvolutionSuite.java b/benchmarks/java/src/main/java/org/apache/fory/benchmark/SchemaEvolutionSuite.java new file mode 100644 index 0000000000..1e669908e7 --- /dev/null +++ b/benchmarks/java/src/main/java/org/apache/fory/benchmark/SchemaEvolutionSuite.java @@ -0,0 +1,264 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.benchmark; + +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.apache.fory.format.annotation.ForyRowVersion; +import org.apache.fory.format.encoder.ArrayEncoder; +import org.apache.fory.format.encoder.Encoders; +import org.apache.fory.format.encoder.MapEncoder; +import org.apache.fory.format.encoder.RowEncoder; +import org.apache.fory.logging.Logger; +import org.apache.fory.logging.LoggerFactory; +import org.apache.fory.reflect.TypeRef; +import org.openjdk.jmh.Main; +import org.openjdk.jmh.annotations.Benchmark; + +/** + * Row-codec schema-evolution throughput and allocation. Pair with the JMH gc profiler ({@code -prof + * gc}) to read {@code gc.alloc.rate.norm} (bytes per op). Two comparisons matter: {@code + * currentDecode} vs {@code olderDecode} shows that decoding an older payload through a projection + * codec allocates no more than decoding the current schema, because each projection holds its + * historical schema's row layout (no per-decode rebuild); and the {@code *NoEvolution} benchmarks + * vs their evolution-on counterparts show the steady-state cost of enabling {@code + * withSchemaEvolution()} when reading and writing current-version data. The {@code array*} and + * {@code map*} benchmarks repeat both comparisons for a top-level {@code List} and {@code + * Map}, whose evolution-on payloads carry an 8-byte hash prefix. + */ +public class SchemaEvolutionSuite { + private static final Logger LOG = LoggerFactory.getLogger(SchemaEvolutionSuite.class); + + public static class PersonV1 { + String name; + int age; + } + + public static class PersonV2 { + String name; + int age; + + @ForyRowVersion(since = 2) + String email; + } + + // Evolution-enabled codecs for the current (V2) schema; the V1 codec only produces a payload + // whose hash routes the V2 reader onto its projection path. Both standard and compact formats + // are measured: compact is where a per-projection cached row layout matters, so olderDecode vs + // currentDecode there is the parity check. + private static final RowEncoder v1Codec = + Encoders.buildBeanCodec(PersonV1.class).withSchemaEvolution().build().get(); + private static final RowEncoder v2Codec = + Encoders.buildBeanCodec(PersonV2.class).withSchemaEvolution().build().get(); + private static final RowEncoder v1CompactCodec = + Encoders.buildBeanCodec(PersonV1.class).compactEncoding().withSchemaEvolution().build().get(); + private static final RowEncoder v2CompactCodec = + Encoders.buildBeanCodec(PersonV2.class).compactEncoding().withSchemaEvolution().build().get(); + + // Evolution-disabled codecs for the same current (V2) schema. Comparing the *NoEvolution + // benchmarks against their evolution-on counterparts isolates the steady-state cost of the + // withSchemaEvolution() flag on the common path (reading and writing current-version data). + // Rows carry the 8-byte hash slot and decode compares it in both modes, so on current-version + // data the flag adds no work; only a hash mismatch takes the projection lookup. + private static final RowEncoder v2PlainCodec = + Encoders.buildBeanCodec(PersonV2.class).build().get(); + private static final RowEncoder v2PlainCompactCodec = + Encoders.buildBeanCodec(PersonV2.class).compactEncoding().build().get(); + + private static final PersonV2 person = newPerson(); + private static final byte[] currentBytes = v2Codec.encode(person); + private static final byte[] olderBytes = v1Codec.encode(newPersonV1()); + private static final byte[] currentCompactBytes = v2CompactCodec.encode(person); + private static final byte[] olderCompactBytes = v1CompactCodec.encode(newPersonV1()); + private static final byte[] plainBytes = v2PlainCodec.encode(person); + private static final byte[] plainCompactBytes = v2PlainCompactCodec.encode(person); + + private static final TypeRef> V1_LIST = new TypeRef>() {}; + private static final TypeRef> V2_LIST = new TypeRef>() {}; + private static final TypeRef> V1_MAP = + new TypeRef>() {}; + private static final TypeRef> V2_MAP = + new TypeRef>() {}; + + private static final ArrayEncoder> v2ArrayCodec = + Encoders.buildArrayCodec(V2_LIST).withSchemaEvolution().build().get(); + private static final ArrayEncoder> v2PlainArrayCodec = + Encoders.buildArrayCodec(V2_LIST).build().get(); + private static final MapEncoder> v2MapCodec = + Encoders.buildMapCodec(V2_MAP).withSchemaEvolution().build().get(); + private static final MapEncoder> v2PlainMapCodec = + Encoders.buildMapCodec(V2_MAP).build().get(); + + private static final List people = Arrays.asList(person, person, person, person); + private static final Map personMap = newPersonMap(person); + private static final byte[] currentArrayBytes = v2ArrayCodec.encode(people); + private static final byte[] olderArrayBytes = + Encoders.buildArrayCodec(V1_LIST) + .withSchemaEvolution() + .build() + .get() + .encode(Arrays.asList(newPersonV1(), newPersonV1(), newPersonV1(), newPersonV1())); + private static final byte[] plainArrayBytes = v2PlainArrayCodec.encode(people); + private static final byte[] currentMapBytes = v2MapCodec.encode(personMap); + private static final byte[] olderMapBytes = + Encoders.buildMapCodec(V1_MAP) + .withSchemaEvolution() + .build() + .get() + .encode(newPersonMap(newPersonV1())); + private static final byte[] plainMapBytes = v2PlainMapCodec.encode(personMap); + + private static Map newPersonMap(V value) { + Map map = new LinkedHashMap<>(); + for (int i = 0; i < 4; i++) { + map.put("person" + i, value); + } + return map; + } + + private static PersonV2 newPerson() { + PersonV2 p = new PersonV2(); + p.name = "Ada Lovelace"; + p.age = 36; + p.email = "ada@example.com"; + return p; + } + + private static PersonV1 newPersonV1() { + PersonV1 p = new PersonV1(); + p.name = "Ada Lovelace"; + p.age = 36; + return p; + } + + @Benchmark + public Object encode() { + return v2Codec.encode(person); + } + + @Benchmark + public Object currentDecode() { + return v2Codec.decode(currentBytes); + } + + @Benchmark + public Object olderDecode() { + return v2Codec.decode(olderBytes); + } + + @Benchmark + public Object compactEncode() { + return v2CompactCodec.encode(person); + } + + @Benchmark + public Object compactCurrentDecode() { + return v2CompactCodec.decode(currentCompactBytes); + } + + @Benchmark + public Object compactOlderDecode() { + return v2CompactCodec.decode(olderCompactBytes); + } + + // Evolution-off baselines for the current path. Pair each with its evolution-on counterpart + // (encode/currentDecode and the compact variants) to read the flag's overhead. + @Benchmark + public Object encodeNoEvolution() { + return v2PlainCodec.encode(person); + } + + @Benchmark + public Object currentDecodeNoEvolution() { + return v2PlainCodec.decode(plainBytes); + } + + @Benchmark + public Object compactEncodeNoEvolution() { + return v2PlainCompactCodec.encode(person); + } + + @Benchmark + public Object compactCurrentDecodeNoEvolution() { + return v2PlainCompactCodec.decode(plainCompactBytes); + } + + @Benchmark + public Object arrayEncode() { + return v2ArrayCodec.encode(people); + } + + @Benchmark + public Object arrayCurrentDecode() { + return v2ArrayCodec.decode(currentArrayBytes); + } + + @Benchmark + public Object arrayOlderDecode() { + return v2ArrayCodec.decode(olderArrayBytes); + } + + @Benchmark + public Object arrayEncodeNoEvolution() { + return v2PlainArrayCodec.encode(people); + } + + @Benchmark + public Object arrayCurrentDecodeNoEvolution() { + return v2PlainArrayCodec.decode(plainArrayBytes); + } + + @Benchmark + public Object mapEncode() { + return v2MapCodec.encode(personMap); + } + + @Benchmark + public Object mapCurrentDecode() { + return v2MapCodec.decode(currentMapBytes); + } + + @Benchmark + public Object mapOlderDecode() { + return v2MapCodec.decode(olderMapBytes); + } + + @Benchmark + public Object mapEncodeNoEvolution() { + return v2PlainMapCodec.encode(personMap); + } + + @Benchmark + public Object mapCurrentDecodeNoEvolution() { + return v2PlainMapCodec.decode(plainMapBytes); + } + + public static void main(String[] args) throws Exception { + if (args.length == 0) { + String commandLine = + "org.apache.fory.*SchemaEvolutionSuite.* -f 3 -wi 3 -i 3 -t 1 -w 2s -r 2s -prof gc -rf csv"; + args = commandLine.split(" "); + } + LOG.info("command line: {}", Arrays.toString(args)); + Main.main(args); + } +} diff --git a/docs/row-format/index.md b/docs/row-format/index.md index ac41122a90..52dd9429a5 100644 --- a/docs/row-format/index.md +++ b/docs/row-format/index.md @@ -96,5 +96,14 @@ Reuse the encoder within one thread. Create separate encoders for concurrent thr Choose Compact Row only when every reader is Java and the space reduction justifies the Java-specific layout. Use Standard Row for Java, Python, C++, Rust, and Go interchange. +## Schema Evolution + +Java row, array, and map codecs built with `withSchemaEvolution()` read payloads that older +versions of a bean wrote, in Standard Row and in Compact Row. Each payload then carries a schema +hash, and a reader rejects a payload whose hash it does not know with +`ClassNotCompatibleException`. Producers and consumers must agree on the setting. See the +[Java guide](java.md#schema-evolution) for the annotations, the rollout procedure, and the +limits. + See the [Row Format specification](../specification/row_format_spec.md) for the exact Standard and Compact binary layouts. diff --git a/docs/row-format/java.md b/docs/row-format/java.md index cb9f57560d..a8f68f259e 100644 --- a/docs/row-format/java.md +++ b/docs/row-format/java.md @@ -277,6 +277,204 @@ BinaryRow row = encoder.toRow(child); Parent decoded = encoder.fromRow(row); ``` +## Schema Evolution + +Enable `.withSchemaEvolution()` on a row, array, or map codec builder to read payloads written +by older versions of the same bean. Writing always uses the current version; reading detects +the payload's version from a strict hash at the head of the payload. Java is the only +implementation today. The protocol is language-neutral and defined in the +[row format specification](../specification/row_format_spec.md#schema-evolution). + +Annotate fields added after v1 with `@ForyRowVersion(since = N)`: + +```java +@Data +public class Person { + String name; + int age; + + @ForyRowVersion(since = 2) + String email; +} +``` + +Build the codec with schema evolution enabled. It writes the current version and reads the +current and older versions: + +```java +RowEncoder codec = + Encoders.buildBeanCodec(Person.class).withSchemaEvolution().build().get(); +byte[] bytes = codec.encode(person); +Person current = codec.decode(bytes); +// v1Bytes was written with schema evolution, before Person had the email field. +Person older = codec.decode(v1Bytes); +``` + +A v1 payload (with `name` and `age` only) decodes to a `Person` whose `email` is `null`. +Primitive fields added later default to `0`, `0.0`, or `false`, and `Optional` fields to +`Optional.empty()`. A field initializer does not apply to a field that the payload lacks. +Unannotated fields are treated as present from the first version, so a class can adopt versioning +by annotating only the fields added after v1. + +Put the annotation on the field of a class, on a record component, or on an accessor method of an +interface bean. The codec does not read it from other methods, so building the codec fails when +another method carries it, such as the getter of a class field. A class bean does not read the +annotations of an interface that it implements, because they belong to the codec of that +interface bean. Annotate the fields of the class. + +For a record, the absent component's default is passed to the canonical constructor, so a +constructor that rejects `null` for a reference component added in a later version throws when +decoding an older payload. Let the constructor tolerate the missing value, for example by +normalizing `null` to a default: + +```java +public record Person(String name, @ForyRowVersion(since = 2) String email) { + public Person { + if (email == null) { + email = ""; + } + } +} +``` + +Remove a field by deleting the Java member and declaring it on a nested history interface as a +method with a `@ForyRowVersion(until = N)`. The method's return type carries any parameterized +type information from the original field. `removedFields` must name an interface; building the +codec fails for a class. + +```java +@Data +@ForyRowSchema(removedFields = Person.History.class) +public class Person { + String name; + + @ForyRowVersion(since = 2) + String email; + + interface History { + @ForyRowVersion(until = 3) + int age(); + + @ForyRowVersion(until = 5) + List tags(); + } +} +``` + +The history method name matches the name of the original live member. For a class or a record, +including a Lombok `@Data` class, that is the field name (`age`, `tags`). For an interface bean, +it is the accessor method name (`getAge`). The codec cannot detect a history method with the wrong +name: a class that declares `getTags()` instead of `tags()` gets the wire name `get_tags`, so +every historical layout that holds that field hashes differently from the payloads that the old +version wrote, and those payloads fail to decode with `ClassNotCompatibleException`. Test that an +old payload still decodes before you retire a live field. + +A subclass inherits the removed fields that its superclasses declare with `@ForyRowSchema`, and +an interface bean inherits those of its superinterfaces, the same as it inherits their live fields. + +A field that is removed and later added again with the same name and the same Java type is one +field: older payloads that hold it decode with its value. If the Java type changes and the row +type or the nullability changes too (for example `String` to `int`, or `Integer` to `int`), older +values are discarded. If the Java type changes but the row type and the nullability stay the same +(for example `String` to an enum), or either type reaches a versioned bean (directly, or through a +list or map), building the codec fails, because an old layout could be identical to the new one +and old values would decode as the new type. To change the type of a field, give the new member a +new name. + +A field handled by a custom codec evolves as its encoded representation. When `getForyField` +returns null, the column follows `encodedType()`, and a versioned bean in that type evolves as +usual. When `getForyField` returns a field, that field is the whole column: the members of the +declared Java type do not change the layout, and their `@ForyRowVersion` annotations have no +effect. + +### Wire Format and Limitations + +Producers and consumers must agree on the `withSchemaEvolution()` flag — they are not +wire-compatible otherwise. Row payloads always carry an 8-byte hash slot; under evolution its +value is the strict hash (which includes field name and nullability), so a flag-mismatched +peer fails loudly with `ClassNotCompatibleException`. Arrays and maps prepend an 8-byte prefix +under evolution (the strict hash for an array, the combined key and value hash for a map) and no +prefix otherwise. An evolution-on consumer rejects evolution-off array and map bytes. An +evolution-off consumer cannot detect evolution-on array and map bytes, so the result of that read +is undefined. With evolution on, the hash also identifies the row format, so a compact consumer +rejects standard payloads with `ClassNotCompatibleException`, and the reverse. + +Only `encode` and `decode` write and check the hash. `toRow`, `fromRow`, `toArray`, `fromArray`, +`toMap`, and `fromMap` always use the current schema, and the bytes of a `BinaryRow` hold no hash. +To read older versions, store the output of `encode` and read it with `decode`. + +Turning the flag on changes the wire format of every payload, so do it as a cutover, not as a +rolling change. During a rolling deploy, flag-on and flag-off peers reject each other's rows, and +the result of a flag-off reader on flag-on arrays and maps is undefined. Rows written before the +cutover are not readable with the flag on. Stop or drain the producers, rewrite or stop reading +the stored payloads, and then start every peer with the flag on, in a release that changes no +schema. + +After that, deploy every reader of a schema change before any writer. A reader decodes payloads +whose layout it knows, from its own version and older versions, and it rejects a payload whose +layout it does not know with `ClassNotCompatibleException`. + +In a GraalVM native image, an evolution codec cannot decode a payload with an older layout: the +decode fails. This is a known limitation. + +The hash algorithm, wire names, field order, and framing are specified language-neutrally in +the [row format specification](../specification/row_format_spec.md#schema-evolution). Java is the +only implementation today, so Python, C++, and other runtimes cannot read evolution-enabled +payloads until they implement that protocol. + +Field identity on the wire is the snake_case form of the member name (`createdAt` becomes +`created_at`), and a versioned layout orders fields by that wire name. The `snake_case` conversion +inserts an underscore before each ASCII capital letter and lowercases it, and changes no other +character. + +With schema evolution, a member name or a history method name with an underscore, such as +`created_at` or `count_`, fails the codec build. The build also fails when the member-name order +and the wire-name order differ, which happens only for a character outside the Basic Multilingual +Plane. The error names the members to rename. Other Fory runtimes do not all convert acronyms, a +leading capital letter, or non-ASCII capital letters the same way, so use ordinary lowerCamelCase +member names (`createdAt`, not `fooURL` or `URLValue`) for portability. + +Because field names are wire identity, a rename is a removal plus an addition, not an in-place +mapping: declare the old name on the history interface with `until = N` and annotate the renamed +member with `since = N`. Older payloads then decode with the renamed field at its default, not +carrying over the old field's value; renaming without those declarations changes the payload hash +and older payloads stop decoding. The wire name derives from the field name for a class or a +record, and from the accessor method name for an interface bean, so converting a bean between a +class and an interface also changes every wire name and is a rename of every field. + +A reader selects the matching layout from the 8-byte strict hash on the payload. The hash includes +field names and nullability and is checked for collisions across a bean's own versions when the +codec is built, but it is still a 64-bit value: a payload whose hash coincides with one of the +reader's historical layouts is decoded against that layout. This is the same hash-based dispatch +the row format has always used, so feeding a codec bytes it was not built for has undefined results +whether or not evolution is enabled. Only hand a codec payloads produced for the same bean. + +Nested evolution works to arbitrary depth and places no restriction on shape: a versioned bean +may contain versioned beans that themselves contain versioned beans, the same versioned bean +class may back more than one field, and fields typed as a non-evolving bean, a list, or a map are +unrestricted. Each nesting level is routed to the correct historical layout. A versioned bean may +be used as a map key as well as a map value. When the key and value are different classes, they +evolve independently; when one class backs both, a payload holds one version of it. This +holds wherever the map appears: as the codec's top-level type, nested inside a bean field, or +reached through a top-level array or map (such as `List>`), and a single +map may evolve more than one distinct bean class across its key and value. A top-level map carries +its own hash identifying both layouts together; a map nested inside an array, another map, or a +bean field has its layouts folded into the enclosing payload's hash. + +When a versioned bean contains other versioned beans, the reader can read one layout per +combination of versions across the composition, with each bean class at one version. Building the +codec computes the layout and hash of every combination, so the build time and memory grow with +the product of the version counts of the distinct versioned bean classes, and the codec sets no +limit. A map whose key and value both evolve combines their versions the same way. The first +`decode` of a payload with each older layout takes longer than later ones, because the codec +prepares its reader for that layout inside that call. + +To bound the growth, stop reading the oldest layouts: remove an entry from a bean's `History` +interface, or remove or lower the `since` of a live field so that no layout without that field +remains. Either change removes every layout that differs only by that field, so the reader then +rejects every payload written with one of those layouts. This is a read-side decision only; the +writer always uses the current schema. + ## Related Topics - [Cross-Language Interoperability](../object-serialization/java/basic-serialization.md#cross-language-interoperability) - xlang mode diff --git a/docs/row-format/troubleshooting.md b/docs/row-format/troubleshooting.md index 34f51d1c6d..c8cd14c450 100644 --- a/docs/row-format/troubleshooting.md +++ b/docs/row-format/troubleshooting.md @@ -24,6 +24,20 @@ license: | Compact Row is a Java-only row family. Use Standard Row on Java, Python, C++, Rust, and Go for shared bytes. +## Decoding fails with `ClassNotCompatibleException` + +The schema hash at the start of the payload does not identify a schema that the reader knows. +Check that: + +- The writer and the reader use the same bean schema. With `withSchemaEvolution()`, the reader + also reads the older versions in the bean's history, but it rejects a version that it does not + know, such as a newer one, so deploy readers before writers. +- Both sides set `withSchemaEvolution()` the same way. +- With `withSchemaEvolution()`, both sides use the same layout: a Compact Row reader rejects + Standard Row payloads, and the reverse. + +See [Schema Evolution](java.md#schema-evolution) in the Java guide. + ## A field lookup fails Verify that the reader uses the same schema and field type as the encoded row. Cache typed field diff --git a/docs/specification/row_format_spec.md b/docs/specification/row_format_spec.md index a192e35f1e..2296ea765f 100644 --- a/docs/specification/row_format_spec.md +++ b/docs/specification/row_format_spec.md @@ -255,28 +255,29 @@ The compact format provides space-efficient encoding with additional optimizatio 1. **Field Slot Sizes**: Each field uses its natural width (Wi = type width or 8 for variable) 2. **Null Bitmap Position**: Placed after field slots, can borrow alignment padding -3. **Field Order**: Fields are sorted by alignment (8-byte → 4-byte → 2-byte → 1-byte → variable) +3. **Field Order**: Fields are sorted by slot width (see [Field Sorting Algorithm](#field-sorting-algorithm)) 4. **Conditional Bitmap**: Skipped entirely if all fields are non-nullable #### Null Bitmap (Compact) -- **Size**: `(num_nullable_fields + 7) / 8` bytes (byte-aligned, not 8-byte aligned) +- **Size**: `(num_fields + 7) / 8` bytes, one bit per field (byte-aligned, not 8-byte aligned) - **Skipped**: When all fields are primitive/non-nullable - **Position**: After all fixed-size field slots, can use alignment padding space #### Field Sorting Algorithm -Fields are sorted to minimize padding and optimize alignment: +Fields are sorted to reduce padding. The sort key of a field is its slot width `w`: its fixed +width (see [Fixed-Width Calculation](#fixed-width-calculation)), or 8 for a variable-width field, +whose slot holds an offset and a size. -``` -Priority order (highest to lowest): -1. Fields with 8-byte alignment (int64, float64, variable-width) -2. Fields with 4-byte alignment (int32, float32) -3. Fields with 2-byte alignment (int16) -4. Fields with 1-byte alignment (int8, bool) -``` +1. Group 0 holds the fields whose `w` is 2, 4, or a multiple of 8. Group 1 holds every other + width, such as 1, 3, 6, or 12. Group 0 comes first. +2. Within a group, a larger `w` comes first. +3. The sort is stable: fields with the same group and `w` keep their order in the schema. -Within each alignment group, larger fields come first. +A struct sorts its own fields by the same rule. For example, `bool`, `int16`, `int64`, `string`, +and a 12-byte inline struct `{int64, int32}` sort as `int64` and `string` (in schema order), +`int16`, the 12-byte struct, then `bool`. #### Fixed-Size Struct Inlining @@ -300,15 +301,22 @@ This eliminates the offset+size indirection for fixed-size nested structures. A field's fixed width is determined recursively: -- **Primitive types**: Natural byte width (1, 2, 4, or 8) -- **Struct types**: Sum of all child fixed widths (if all children are fixed-width) -- **Variable types** (string, array, map): Returns -1 (uses 8-byte offset+size slot) +- **Primitive types**: Natural byte width: 1 for `bool` and `int8`; 2 for `int16` and `float16`; + 4 for `int32`, `float32`, and `date32`; 8 for `int64`, `float64`, `timestamp`, and `duration` +- **Fixed-size binary**: Its byte width +- **Struct types**: The struct's null bitmap bytes plus the sum of all child fixed widths (if all + children are fixed-width). The bitmap bytes are 0 when every child is non-nullable, and + `(num_children + 7) / 8` otherwise +- **Variable types** (string, variable-size binary, decimal, list, map): Returns -1 (uses 8-byte + offset+size slot) ``` fixed_width(field) = - if primitive: type_width + if primitive or fixed-size binary: type_width if struct and all_children_fixed: header_bytes + sum(fixed_width(child) for each child) else: -1 (variable, uses 8-byte slot) + +header_bytes = 0 if no child is nullable, else (num_children + 7) / 8 ``` ### Compact Array Binary Layout @@ -345,6 +353,182 @@ if (fixed_width % 8 == 0): --- +## Schema Evolution + +Schema evolution lets a codec read payloads written by older versions of the same struct. It is +opt-in and currently implemented in Java only. It does not change the existing wire contract +above: a payload written without it is unchanged. When it is enabled, the row hash word holds the +strict hash below instead of the default schema hash, and every top-level array and map payload +starts with an 8-byte hash prefix. An array or map nested in a row, an array, or a map carries no +prefix. Producer and consumer must agree on whether it is enabled. The protocol below +is defined over language-neutral inputs so another runtime can implement it and interoperate with +Java payloads. + +See the [Java row format guide](../row-format/java.md#schema-evolution) for usage, annotations, +and limitations. + +### Version Model + +- Versions are per-struct integers starting at 1. Version numbers never appear on the wire; + payloads are identified by a strict schema hash. A reader decodes a payload when one of its + layouts has the payload's hash, so peers need no synchronized version counters. +- A schema history is a set of field entries `(wire name, type, nullability, since, until)`. + A live field has an unbounded `until`; a removed field declares the version it disappeared in. + How the entries are declared is language-specific (Java uses annotations and a removed-fields + interface); the protocol object is the entry tuple. +- A layout is materialized at version 1 and at every `since` and `until` version, even when no + field is active at that version (a version-1 payload of a struct whose fields all arrive later + has an empty layout). Layouts that come out identical collapse into one entry. +- Reading a payload that predates a field yields the reader type's absent value: an empty + optional for an optional type, null for another nullable field, and the type's zero value (`0`, + `0.0`, `false`, empty for languages without null) otherwise. A default that the reader's type + declares for the field (a field initializer, for example) does not apply. Fields removed in the + reader's schema are discarded. +- A removed entry and a live entry with the same wire name and the same declared type in the + implementation language are the same field: the reader reads the removed entry's value into the + live field in every layout that has it. A removed entry with a different declared type is + invalid, and building the history fails, when the two entries have the same row type and + nullability after the format puts every struct in wire order (in the compact format, the sort + can make two structs whose children differ only in order match: `{a: int32, b: int64}` and + `{b: int64, a: int32}` match, but `{x: int64, y: int64}` and `{y: int64, x: int64}` do not, + because children of equal width keep their order), or when either reaches a versioned struct: + an older layout could then be identical to a newer one, and the reader would decode old values + as the new type. A removed entry with a different declared type is otherwise discarded, because + its row type or nullability differs. + Layouts that come out identical are one entry, so a reader reads a field that any of the + collapsed versions reads. +- When a struct contains other versioned structs (directly, or through list elements, map keys, or + map values, to any depth), each distinct nested struct type is one version dimension and the + reader enumerates the cross-product of versions. A single payload carries one version per struct + type. + +### Wire Names and Canonical Field Order + +- A field's wire name is its declared member name converted to lower `snake_case` by the field-name + rule of the xlang object-graph format (a `camelCase` name converts; a name already in + `snake_case` is used as-is). That rule does not yet define the conversion of acronyms, a leading + capital letter, or non-ASCII capital letters, and the runtimes differ on them. Only ordinary + lowerCamelCase names, with no consecutive capital letters and only ASCII capital letters, are + portable across runtimes today. A runtime can restrict its member names further: the Java + runtime rejects a member name that contains an underscore. +- The canonical order of the fields of a versioned layout is ascending by wire name, compared by + Unicode code point (equivalently, byte-wise over the UTF-8 encodings). Duplicate wire names + within one version are invalid. +- The compact format then sorts the fields of every struct, at every depth (including structs in + list elements and in map keys and values), with a stable sort over the canonical order. The + sort key is the field's slot width `w`: the [fixed width](#fixed-width-calculation) of a + fixed-width field, and 8 for a variable-width field. A field whose `w` is 2, 4, or a multiple + of 8 sorts before a field with any other `w`. Within each of those two groups, a larger `w` + sorts first, and fields with equal `w` keep their canonical order. For example, a `string` and + an `int64` keep their canonical order, and a 12-byte inline struct `{int64, int32}` sorts after + an `int16` and before a `bool`. +- A custom codec's encoded shape (its struct children, their names, and their order) is fixed by + the codec's own definition, not by name sorting, and must be reproduced identically in every + runtime that registers an equivalent codec. The codec's order is the canonical order of its + struct. + +### Strict Schema Hash + +The strict hash distinguishes layouts that differ in row format, field name, or nullability, +unlike the default schema hash, which mixes only type IDs. All inputs are language-neutral. + +- State: an unsigned 64-bit accumulator `h`, seeded with the FNV-1a offset basis + `0xcbf29ce484222325`. Each input value `v` (one unsigned 64-bit integer) mixes as + `h = (h XOR v) * 0x100000001b3 mod 2^64` (the FNV-1a prime). A negative input (a negative + decimal scale) mixes as its 64-bit two's-complement value. +- The first value mixed after the seed is the format tag: `0` for the standard format and `1` for + the compact format. The two formats write a layout in the same field order when the compact sort + moves no field (for example, when every field is 8 bytes wide), so without the tag a reader of + one format would accept a payload of the other and decode wrong values. +- The hash covers fields in the order they are on the wire, at every depth: the canonical order + for the standard format, and the compact sort order above for the compact format, + including for structs in list elements and map entries. The hash of a layout mixes the format + tag, then folds its top-level fields in that order, each as a named field. +- A named field mixes: each byte of the wire name's UTF-8 encoding, then one `0` terminator, + then the field's shape. +- A field's shape mixes, in order: + 1. The Fory type ID (table below). + 2. Shape parameters the ID does not carry: for `binary`, the byte width (`0` for + variable-width); for `decimal`, the precision then the scale. + 3. Nullability: `1` if nullable, `0` otherwise. + 4. Children: a `list` mixes its element field's shape and a `map` mixes its key field's shape + then its value field's shape — child positions are fixed by the parent type, so no names are + mixed. At every depth, a map's key field mixes nullability `0` and its value field mixes + nullability `1`. A `struct` mixes its child count, then each child as a named field; the count + delimits the struct's extent so nesting structure stays unambiguous. +- Type IDs are the row format's subset of the Fory cross-language type table: `bool` 1, `int8` 2, + `int16` 3, `int32` 4, `int64` 6, `float16` 17, `float32` 19, `float64` 20, `string` 21, + `list` 22, `map` 24, `struct` 27, `duration` 37, `timestamp` 38, `date32` 39, `decimal` 40, + `binary` 41. +- The hash is 64 bits, so two distinct layouts in one history hash to the same value with odds of + about 2^-64. An implementation should reject a history with such a collision when it builds the + codec, but it need not detect every collision. +- Known-answer vectors for the strict hash in the standard format, except where a row names the + compact format (`n` is nullable, `nn` is not nullable): + + | Layout | Strict hash | + | -------------------------------------------------------- | -------------------- | + | `id: int32 nn` | `0x506a3242a9f93ae2` | + | `id: int32 nn`, compact format | `0xb7db30d6e6593b2f` | + | `name: string n` | `0x4bd0d984e7109ef6` | + | `größe: int64 nn` | `0xe5fcaf6a66d32a0e` | + | `amount: decimal(10, 2) nn` | `0x2ef05b2b90e987a9` | + | `amount: decimal(38, -3) nn` | `0x7e49dad4d1337a8a` | + | `blob: binary n, digest: binary(4) nn` | `0xca28ad1d8bf595e7` | + | `tags: list n` | `0x55f88fd672e92ea9` | + | `counts: map n` | `0x570b6dadf84868c8` | + | `point: struct n, z: int64 nn` | `0x793051ee66a15234` | + | `combine(0x506a3242a9f93ae2, 0x4bd0d984e7109ef6)` | `0x11ce9d3600e26f89` | + + The layout with the fields `day: date32 n`, `dur: duration n`, `f16: float16 nn`, + `f32: float32 nn`, `f64: float64 nn`, and `ts: timestamp n` has the strict hash + `0xf6320f76f21330bc` in the standard format. + + In the compact format, the layout with the fields `code: string n`, `flag: bool nn`, + `id: int64 nn`, `stats: struct n`, and `version: int16 nn` + hashes in the order `code`, `id`, `version`, `stats` (children `total`, `count`), `flag`. Its + strict hash is `0xa6ca2310859be6e6`. + +### Framing and Dispatch + +- A row payload's existing leading 8-byte hash word (little-endian) holds the strict hash of the + writer's current layout when evolution is enabled, and the default schema hash otherwise. +- Array and map payloads carry no hash word without evolution. With evolution, every top-level + array and map payload starts with an 8-byte prefix (little-endian). An array or map nested in a + row, an array, or a map carries no prefix. So every payload written with evolution enabled + starts, after any size prefix, with an 8-byte schema identifier: the strict hash of its layout + for a row or an array, and the combined key and value hash for a map. +- When the payload size is written before the payload (a little-endian int32), the size comes + first and the hash word or prefix follows it. The size counts the 8 bytes of the hash and the + body, but not its own 4 bytes. +- An array's prefix is the strict hash of a layout with one field: the empty name `""`, the element + type with each reachable struct at the payload's version, and the element's nullability. +- A map's prefix is one combined value, + `combine(keyHash, valueHash) = mix(mix(basis, keyHash), valueHash)` with the same basis and mix + step as above. `keyHash` and `valueHash` are each the strict hash of a one-field layout built as + for an array element: the key with nullability `0`, and the value with nullability `1`. A + position that reaches no struct still hashes its type. The combination is order-sensitive, so a + map key and value evolve independently while the payload carries one hash. Nested versioned + structs need no prefix of their own: their layouts substitute into the enclosing payload's + hash. +- The reader dispatches on the hash: the current layout's hash selects the current codec, a known + historical hash selects that layout's projection, and an unknown hash is an error. +- Known-answer vectors for the prefix in the standard format, except where a row names the + compact format (`n` is nullable, `nn` is not nullable): + + | Payload | Prefix | + | ---------------------------------------------------- | -------------------- | + | map, key `string nn`, value `int32 n` | `0xa8efad587d8a6b3b` | + | array of `struct n` | `0x66c342b5ad6942cb` | + | the same array, compact format | `0x905b89131d905a88` | + +- Whether evolution is enabled is agreed out of band. Array and map payloads written without + evolution have no header, so an evolution-off consumer cannot detect the prefix of an + evolution-enabled payload, and the result of that read is undefined; see the Java guide for the + rollout procedure. + +--- + ## Common Specifications The following specifications apply to both standard and compact formats. @@ -440,7 +624,7 @@ row_size = fixed_region_size + bitmap_size + variable_data_size where: fixed_region_size = sum of (fixed_width(field) or 8 for each field) - bitmap_size = all_non_nullable ? 0 : (num_nullable_fields + 7) / 8 + bitmap_size = all_non_nullable ? 0 : (num_fields + 7) / 8 // May be rounded to 8-byte boundary if has variable fields ``` diff --git a/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java b/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java index d87b86e0a9..ceedc77016 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java @@ -1013,7 +1013,11 @@ private static boolean isSupported(TypeRef typeRef, TypeResolutionContext ctx return false; } Tuple2, TypeRef> mapKeyValueType = getMapKeyValueType(typeRef); - return isSupported(mapKeyValueType.f0) && isSupported(mapKeyValueType.f1); + // Thread ctx through both key and value, matching the iterable branch above. The single-arg + // isSupported overload builds a fresh context with synthesizeInterfaces=false and the empty + // custom-type registry, which would reject an interface bean used as a map key or value even + // though the same type is supported as a direct field or list element. + return isSupported(mapKeyValueType.f0, ctx) && isSupported(mapKeyValueType.f1, ctx); } else if (cls.isEnum()) { return true; } else { diff --git a/java/fory-core/src/test/java/org/apache/fory/type/TypeUtilsTest.java b/java/fory-core/src/test/java/org/apache/fory/type/TypeUtilsTest.java index 3f970b3960..2821f431df 100644 --- a/java/fory-core/src/test/java/org/apache/fory/type/TypeUtilsTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/type/TypeUtilsTest.java @@ -179,6 +179,42 @@ public void isSupported() { TypeUtils.isSupported(TypeRef.of(AbstractList.class)); } + public interface Named { + String getName(); + } + + public static class NamedValues { + private Map byKey; + + public Map getByKey() { + return byKey; + } + + public void setByKey(Map byKey) { + this.byKey = byKey; + } + } + + public static class NamedKeys { + private Map byName; + + public Map getByName() { + return byName; + } + + public void setByName(Map byName) { + this.byName = byName; + } + } + + /** Map keys and values resolve with the caller's context, as list elements do. */ + @Test + public void mapTypesUseCallerContext() { + TypeResolutionContext ctx = new TypeResolutionContext(CustomTypeRegistry.EMPTY, true); + assertTrue(TypeUtils.isBean(NamedValues.class, ctx)); + assertTrue(TypeUtils.isBean(NamedKeys.class, ctx)); + } + public static class A { @Override public String toString() { diff --git a/java/fory-format/pom.xml b/java/fory-format/pom.xml index a35dc9fcb5..0d29a8cf1f 100644 --- a/java/fory-format/pom.xml +++ b/java/fory-format/pom.xml @@ -99,6 +99,22 @@ + + + org.apache.maven.plugins + maven-compiler-plugin + + 11 + + + + org.apache.maven.plugins maven-javadoc-plugin diff --git a/java/fory-format/src/main/java/org/apache/fory/format/annotation/ForyRowSchema.java b/java/fory-format/src/main/java/org/apache/fory/format/annotation/ForyRowSchema.java new file mode 100644 index 0000000000..5337c5112d --- /dev/null +++ b/java/fory-format/src/main/java/org/apache/fory/format/annotation/ForyRowSchema.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Class-level row-codec schema metadata used when the codec builder enables schema evolution. + * + *

Live fields without a {@link ForyRowVersion} annotation are treated as present from the first + * version, so a class can adopt versioning by annotating only the fields added later. + * + *

{@link #removedFields()} points at an interface (conventionally a nested one) whose accessor + * methods describe fields that have been removed from this bean but still appear on the wire in + * older payloads. Each method's return type is the original Java type of the removed field; each + * method must carry a {@link ForyRowVersion} annotation with {@code until} set, since removed + * fields have a known end-of-life version. + * + *

A class inherits the removed fields that its superclasses declare, and an interface bean + * inherits those of its superinterfaces, in the same way that it inherits their live fields. + * + *

Example: + * + * 

+ * @Data
+ * @ForyRowSchema(removedFields = MyBean.History.class)
+ * public class MyBean {
+ *   private String name;
+ *
+ *   interface History {
+ *     @ForyRowVersion(until = 3)
+ *     List<String> tags();
+ *
+ *     @ForyRowVersion(since = 2, until = 5)
+ *     Map<String, Long> counters();
+ *   }
+ * }
+ *
+ */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.TYPE) +public @interface ForyRowSchema { + /** + * An interface whose accessor methods describe historically-present-but-now-removed fields. + * Default {@code void.class} means there are no removed fields. The interface is never + * implemented; the codec reads its method signatures and annotations. Building the codec fails + * when this names a class. + */ + Class removedFields() default void.class; +} diff --git a/java/fory-format/src/main/java/org/apache/fory/format/annotation/ForyRowVersion.java b/java/fory-format/src/main/java/org/apache/fory/format/annotation/ForyRowVersion.java new file mode 100644 index 0000000000..0a80304cd6 --- /dev/null +++ b/java/fory-format/src/main/java/org/apache/fory/format/annotation/ForyRowVersion.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Declares the version window in which a row-codec field is logically present. The window is + * inclusive on the left and exclusive on the right, so {@code since=2, until=5} means versions 2, + * 3, and 4. + * + *

Only effective when the codec builder is configured with {@code withSchemaEvolution()}; + * otherwise the annotation is ignored and the field is treated as always present. + * + *

Place it on a field of a class, on a record component, or on an accessor method of an + * interface (a non-static method with no parameters and a return value). The codec reads it from no + * other method, so building a schema-evolution codec fails when another method of the bean or of a + * type that it inherits fields from carries it, such as the getter of a class field. A class bean + * does not read the annotations of an interface that it implements: they belong to the codec of + * that interface bean, so annotate the fields of the class. Record components are covered by {@code + * FIELD} and {@code METHOD} rather than {@code ElementType.RECORD_COMPONENT}: the compiler + * propagates a record-component annotation to the backing field and the accessor method (the + * targets it declares), and the codec reads the annotation from those elements. {@code + * RECORD_COMPONENT} is a JDK 16 enum constant and would break this Java 11 module at runtime, so it + * is intentionally omitted. + */ +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.FIELD, ElementType.METHOD}) +public @interface ForyRowVersion { + /** + * First version (inclusive) that contains this field. Defaults to 1, the first version. Must be + * at least 1 and less than {@link Integer#MAX_VALUE}. + */ + int since() default 1; + + /** + * First version (exclusive) that no longer contains this field. The default {@link + * Integer#MAX_VALUE} means the field has no upper bound and is present in every version from + * {@link #since()} onward. Only a method of the interface named by {@link + * ForyRowSchema#removedFields()} may set it; a live field or accessor that sets it fails the + * codec build. + */ + int until() default Integer.MAX_VALUE; +} diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/ArrayCodecBuilder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/ArrayCodecBuilder.java index 6e6c6d3645..12f1b51fab 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/encoder/ArrayCodecBuilder.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/ArrayCodecBuilder.java @@ -22,13 +22,14 @@ import static org.apache.fory.type.TypeUtils.getRawType; import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodHandles; -import java.lang.invoke.MethodType; import java.util.Collection; import java.util.HashSet; import java.util.Set; import java.util.function.Function; import java.util.function.Supplier; +import org.apache.fory.Fory; +import org.apache.fory.codegen.CodeGenerator; +import org.apache.fory.collection.LongMap; import org.apache.fory.format.row.binary.writer.BinaryArrayWriter; import org.apache.fory.format.type.DataTypes; import org.apache.fory.format.type.Field; @@ -62,18 +63,157 @@ public ArrayEncoder get() { } Function> buildWithWriter() { + final TypeRef elementType = TypeUtils.getElementType(collectionType); + final CodeGenerator codeGenerator = schemaEvolution ? Encoders.pinCodeGenerator() : null; loadArrayInnerCodecs(); - final Function generatedEncoderFactory = + if (!schemaEvolution) { + final Function generatedEncoderFactory = + generatedEncoderFactory(); + return new Function>() { + @Override + public ArrayEncoder apply(final BinaryArrayWriter writer) { + return new BinaryArrayEncoder<>( + writer, generatedEncoderFactory.apply(writer), sizeEmbedded); + } + }; + } + return buildVersionedWithWriter( + elementType, SchemaHistory.evolutionBean(elementType), codeGenerator); + } + + /** + * The evolution path, taken by every array codec with the flag on so that the payload always + * carries the element's strict hash. {@code elementClass} names generated projection classes; it + * is null only when no bean is reachable, and then the history has one version and no projection. + */ + private Function> buildVersionedWithWriter( + final TypeRef elementType, + final Class elementClass, + final CodeGenerator codeGenerator) { + // Enumerate the element field over every versioned bean reachable through its wrappers, so an + // element like Map evolves both the key bean and the value bean. The element + // schema's strict hash identifies the whole combination, and each combination's chosen versions + // are carried in vs.nestedBeanSchemas(), which ProjectionRouting uses to generate a projection + // row codec for every nested class (not just one). + SchemaHistory history = buildElementSchemaHistory(elementField.name(), elementType); + SchemaHistory.VersionedSchema current = history.current(); + Field element = DataTypes.arrayElementField(elementField); + checkCurrentLayout( + "the element of " + collectionType, + DataTypes.schema(DataTypes.field(elementField.name(), element.type(), element.nullable())), + current.schema(), + current.strictHash()); + + // Index of hash → deferred projection source per non-current combination. Building it compiles + // nothing: a combination's row and array codec classes are generated the first time a payload + // with that hash is decoded. The suffix encodes each chosen inner-bean version so distinct + // cross-product entries do not collide on a single generated class. + // + // Keyed by the raw strict hash straight from SchemaHistory, whose versions() hold one entry per + // strict hash, so the keys are unique and none is the current hash, and no builder-side + // collision check is needed here (unlike the map codec's combined (key, value) hash). Two + // distinct layouts with one strict hash would share an entry; SchemaHistory rejects most such + // collisions, which have odds of about 2^-64, but not all. + LongMap projectionSources = new LongMap<>(); + String elementName = elementField.name(); + for (SchemaHistory.VersionedSchema vs : history.versions()) { + if (vs == current) { + continue; + } + projectionSources.put( + vs.strictHash(), + new ArrayProjectionSource( + collectionType, codecFormat, elementClass, elementName, vs, codeGenerator)); + } + final Function currentFactory = generatedEncoderFactory(); + long currentHash = current.strictHash(); return new Function>() { @Override public ArrayEncoder apply(final BinaryArrayWriter writer) { - return new BinaryArrayEncoder<>( - writer, generatedEncoderFactory.apply(writer), sizeEmbedded); + return new VersionedBinaryArrayEncoder<>( + writer, + currentFactory.apply(writer), + sizeEmbedded, + currentHash, + new ProjectionCache<>(projectionSources, fory)); } }; } + /** + * Deferred projection codec for one historical element version, shared by every encoder of the + * codec. The row and array codec classes are compiled on the first {@link #compile} call (the + * first decode of this version's hash by any encoder), not at build time. Static and built from + * values captured at build time, so a later change to the builder cannot reach a lazy compile. + */ + private static final class ArrayProjectionSource + extends ProjectionSource< + ArrayProjectionSource.Classes, VersionedBinaryArrayEncoder.ProjectionArrayCodec> { + private final TypeRef> collectionType; + private final Encoding codecFormat; + private final Class elementClass; + private final String elementName; + private final SchemaHistory.VersionedSchema version; + private final CodeGenerator codeGenerator; + + ArrayProjectionSource( + TypeRef> collectionType, + Encoding codecFormat, + Class elementClass, + String elementName, + SchemaHistory.VersionedSchema version, + CodeGenerator codeGenerator) { + this.collectionType = collectionType; + this.codecFormat = codecFormat; + this.elementClass = elementClass; + this.elementName = elementName; + this.version = version; + this.codeGenerator = codeGenerator; + } + + @Override + VersionedBinaryArrayEncoder.ProjectionArrayCodec bind(Classes r, Fory fory) { + try { + BinaryArrayWriter projWriter = codecFormat.newArrayWriter(r.listField); + Object[] references = {r.listField, projWriter, fory}; + GeneratedArrayEncoder codec = (GeneratedArrayEncoder) r.ctor.invokeExact(references); + return new VersionedBinaryArrayEncoder.ProjectionArrayCodec(projWriter, codec); + } catch (Throwable e) { + throw ExceptionUtils.throwException(e); + } + } + + @Override + Classes generate() { + // Generates the projection row codec for every nested versioned bean class in this + // combination, both map key and value, so the array codec's references all resolve. + Class arrayClass = + Encoders.loadOrGenProjectionArrayCodecClass( + collectionType, + TypeRef.of(elementClass), + codecFormat, + ProjectionRouting.projectionSuffix(version), + ProjectionRouting.nestedSuffixesFor(version, codecFormat, codeGenerator), + codeGenerator); + // forElement substitutes each chosen historical struct into its leaf, so the element field + // at this combination is the single field of vs.schema(); wrap it back in the list field. + return new Classes( + Encoders.constructorHandleFor(arrayClass, GeneratedArrayEncoder.class), + DataTypes.arrayField(elementName, DataTypes.fieldOfSchema(version.schema(), 0))); + } + + static final class Classes { + final MethodHandle ctor; + final Field listField; + + Classes(MethodHandle ctor, Field listField) { + this.ctor = ctor; + this.listField = listField; + } + } + } + private void loadArrayInnerCodecs() { final Set> set = new HashSet<>(); Encoders.findBeanToken(collectionType, set); @@ -90,23 +230,8 @@ Function generatedEncoderFactory() { final TypeRef elementType = TypeUtils.getElementType(collectionType); final Class arrayCodecClass = Encoders.loadOrGenArrayCodecClass(collectionType, elementType, codecFormat); - - final MethodHandle constructorHandle; - try { - final var constructor = - arrayCodecClass.asSubclass(GeneratedArrayEncoder.class).getConstructor(Object[].class); - constructorHandle = - MethodHandles.lookup() - .unreflectConstructor(constructor) - .asType(MethodType.methodType(GeneratedArrayEncoder.class, Object[].class)); - } catch (final NoSuchMethodException | IllegalAccessException e) { - throw new EncoderException( - "Failed to construct array codec for " - + collectionType - + " with element class " - + elementType, - e); - } + final MethodHandle constructorHandle = + Encoders.constructorHandleFor(arrayCodecClass, GeneratedArrayEncoder.class); return new Function() { @Override public GeneratedArrayEncoder apply(final BinaryArrayWriter writer) { diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/ArrayEncoder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/ArrayEncoder.java index 5a365b7d35..1ce3888a08 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/encoder/ArrayEncoder.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/ArrayEncoder.java @@ -22,7 +22,10 @@ import org.apache.fory.format.row.binary.BinaryArray; import org.apache.fory.format.type.Field; -/** Encoder to encode/decode object in the list container by toArray/fromArray row. */ +/** + * Encoder to encode/decode object in the list container by toArray/fromArray row. An ArrayEncoder + * instance is reusable but not thread-safe. + */ public interface ArrayEncoder extends Encoder { Field field(); diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/ArrayEncoderBuilder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/ArrayEncoderBuilder.java index a5ec4715b2..b3bda2a4fd 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/encoder/ArrayEncoderBuilder.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/ArrayEncoderBuilder.java @@ -24,6 +24,7 @@ import java.lang.reflect.Array; import java.util.HashSet; +import java.util.Map; import org.apache.fory.Fory; import org.apache.fory.codegen.CodeGenerator; import org.apache.fory.codegen.CodegenContext; @@ -54,10 +55,26 @@ public ArrayEncoderBuilder(Class arrayCls, Class beanClass) { } public ArrayEncoderBuilder(TypeRef clsType, TypeRef beanType) { + this(clsType, beanType, null, null); + } + + /** + * Construct a projection array codec builder. {@code classSuffix} names this codec's own class + * (encoding the whole nested-bean version combination); {@code nestedSuffixes} routes each nested + * versioned bean class to its own projection row codec, so an element wrapping more than one bean + * (such as {@code Map}) embeds the right historical codec for each. + */ + ArrayEncoderBuilder( + TypeRef clsType, + TypeRef beanType, + String classSuffix, + Map, String> nestedSuffixes) { // A top-level collection has no enclosing bean, so scope element-codec resolution to Object to // match TypeInference's empty-path enclosing type; beanType still names the element type for // class naming and the empty-array template below. super(new CodegenContext(), beanType, Object.class); + this.classSuffix = classSuffix; + this.nestedClassSuffixes = nestedSuffixes; arrayToken = clsType; ctx.reserveName(ROOT_ARRAY_WRITER_NAME); ctx.reserveName(ROOT_ARRAY_NAME); @@ -86,7 +103,9 @@ public ArrayEncoderBuilder(TypeRef clsType, TypeRef beanType) { @Override public String genCode() { ctx.setPackage(CodeGenerator.getPackage(beanClass)); - String className = codecClassName(beanClass, TypeInference.inferTypeName(arrayToken)); + String className = + codecClassName(beanClass, TypeInference.inferTypeName(arrayToken)) + + (classSuffix == null ? "" : classSuffix); ctx.setClassName(className); // don't addImport(arrayClass), because user class may name collide. // janino don't support generics, so GeneratedCodec has no generics diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/BaseBinaryEncoderBuilder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/BaseBinaryEncoderBuilder.java index 530bf5a931..191a17e016 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/encoder/BaseBinaryEncoderBuilder.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/BaseBinaryEncoderBuilder.java @@ -97,6 +97,26 @@ public abstract class BaseBinaryEncoderBuilder extends CodecBuilder { protected final Map, Reference> arrayWriterMap = new HashMap<>(); protected final Map, Reference> beanEncoderMap = new HashMap<>(); + + /** + * Suffix of the generated row, array, or map codec class name, and of the implementation class + * that a row codec generates for an interface bean. A projection codec sets it so that each + * version combination gets its own classes. It does not route nested bean codecs; {@link + * #nestedClassSuffixes} does. + */ + String classSuffix; + + /** + * Per-class projection suffix for nested bean codecs, used when one generated codec embeds more + * than one distinct versioned bean class at independent versions — for example an element typed + * {@code Map}, whose key and value beans each project to their own historical row + * codec. Looked up by raw class in {@link #serializeForBean}; a class absent from the map, or a + * null map, reads at its current schema. Routing by raw class is exact because a writer writes + * one definition of a class, so every occurrence of a class in one payload is at the same + * version. + */ + Map, String> nestedClassSuffixes; + // We need to call beanEncoder's rowWriter.reset() before write a corresponding nested bean every // time. // Outermost beanEncoder's rowWriter.reset() should be called outside generated code before @@ -511,34 +531,9 @@ protected Expression serializeForBean( Field fieldIfKnown, TypeRef typeRef, Expression structField) { - Class rawType = getRawType(typeRef); - Reference rowWriter; + registerBeanCodec(writer, typeRef, structField); + Reference rowWriter = rowWriterMap.get(typeRef); Reference beanEncoder = beanEncoderMap.get(typeRef); - if (beanEncoder == null) { - // janino generics don't add cast, so this `<${type}>` is only for generated code readability - Expression schema = createSchemaFromStructField(structField); - String rowWriterName = - ctx.newName(StringUtils.uncapitalize(rawType.getSimpleName() + "RowWriter")); - NewInstance newRowWriter = new NewInstance(rowWriterType(), schema, writer); - ctx.addField(ctx.type(rowWriterType()), rowWriterName, newRowWriter); - - Preconditions.checkArgument(!codecClassName(rawType).contains(".")); - String encoderName = ctx.newName(StringUtils.uncapitalize(codecClassName(rawType))); - String encoderClass = codecQualifiedClassName(rawType); - TypeRef codecTypeRef = TypeRef.of(GeneratedRowEncoder.class); - NewInstance newEncoder = - new NewInstance( - codecTypeRef, - encoderClass, - ExpressionUtils.newObjectArray(schema, newRowWriter, foryRef)); - ctx.addField(encoderClass, encoderName, newEncoder); - - rowWriter = new Reference(rowWriterName, rowWriterType()); - rowWriterMap.put(typeRef, rowWriter); - beanEncoder = new Reference(encoderName, codecTypeRef); - beanEncoderMap.put(typeRef, beanEncoder); - } - rowWriter = rowWriterMap.get(typeRef); Expression expression = serializeForNotNullBean(ordinal, writer, inputObject, fieldIfKnown, rowWriter, beanEncoder); @@ -547,6 +542,49 @@ protected Expression serializeForBean( new Expression.IsNull(inputObject), new Invoke(writer, "setNullAt", ordinal), expression); } + /** + * Idempotently add the nested-bean row writer and row encoder as fields on the generated codec + * class and register them in {@link #beanEncoderMap} and {@link #rowWriterMap}. {@link + * #deserializeForBean} reads the encoder that this registers, so the encode pass must run before + * the decode pass, also for a decode-only projection codec. + */ + private void registerBeanCodec(Expression writer, TypeRef typeRef, Expression structField) { + if (beanEncoderMap.containsKey(typeRef)) { + return; + } + Class rawType = getRawType(typeRef); + Expression schema = createSchemaFromStructField(structField); + String rowWriterName = + ctx.newName(StringUtils.uncapitalize(rawType.getSimpleName() + "RowWriter")); + NewInstance newRowWriter = new NewInstance(rowWriterType(), schema, writer); + ctx.addField(ctx.type(rowWriterType()), rowWriterName, newRowWriter); + + Preconditions.checkArgument(!codecClassName(rawType).contains(".")); + String encoderName = ctx.newName(StringUtils.uncapitalize(codecClassName(rawType))); + String encoderClass = + codecQualifiedClassName(rawType) + suffixFor(nestedClassSuffixes, typeRef); + TypeRef codecTypeRef = TypeRef.of(GeneratedRowEncoder.class); + NewInstance newEncoder = + new NewInstance( + codecTypeRef, + encoderClass, + ExpressionUtils.newObjectArray(schema, newRowWriter, foryRef)); + ctx.addField(encoderClass, encoderName, newEncoder); + + rowWriterMap.put(typeRef, new Reference(rowWriterName, rowWriterType())); + beanEncoderMap.put(typeRef, new Reference(encoderName, codecTypeRef)); + } + + /** + * The suffix that {@code suffixes} routes {@code typeRef} to. The map is keyed from the same + * field walk as this descent, and SchemaHistory sites every bean with more than one version, so + * an absent class (or a null map) is single-version here and reads at its current schema. + */ + private static String suffixFor(Map, String> suffixes, TypeRef typeRef) { + String suffix = suffixes == null ? null : suffixes.get(getRawType(typeRef)); + return suffix == null ? "" : suffix; + } + protected Expression createSchemaFromStructField(Expression structField) { return new StaticInvoke( DataTypes.class, "schemaFromStructField", "schema", SCHEMA_TYPE, false, structField); diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/BaseCodecBuilder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/BaseCodecBuilder.java index 81f78ca247..f054e8b546 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/encoder/BaseCodecBuilder.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/BaseCodecBuilder.java @@ -23,13 +23,16 @@ import org.apache.fory.format.row.binary.CompactBinaryRow; import org.apache.fory.format.row.binary.writer.CompactBinaryRowWriter; import org.apache.fory.format.type.Schema; +import org.apache.fory.reflect.TypeRef; public class BaseCodecBuilder> { + protected Schema schema; protected int initialBufferSize = 16; protected boolean sizeEmbedded = true; protected Fory fory; protected Encoding codecFormat = DefaultCodecFormat.INSTANCE; + protected boolean schemaEvolution = false; BaseCodecBuilder(final Schema schema) { this.schema = schema; @@ -58,6 +61,30 @@ public B withSizeEmbedded(final boolean sizeEmbedded) { return castThis(); } + /** + * Enable schema evolution. The codec accepts payloads written by older versions of the same bean, + * using the {@link org.apache.fory.format.annotation.ForyRowVersion} and {@link + * org.apache.fory.format.annotation.ForyRowSchema} annotations to reconstruct historical schemas. + * Writing always uses the current version. Only {@code encode} and {@code decode} carry and check + * the schema hash; {@code toRow}, {@code fromRow}, and their array and map forms always use the + * current schema. + * + *

This flag is part of the wire contract: producers and consumers must agree on it, and it + * cannot be flipped on an existing dataset without rewriting. For array and map codecs it adds an + * 8-byte schema-hash prefix to the payload. Because those codecs have no header otherwise, an + * evolution-off reader has no way to tell that prefix from valid body bytes, so reading + * evolution-on bytes with evolution off is undefined. The reverse fails: an evolution-on reader + * reads the start of an evolution-off body as a hash that matches no known schema. Row payloads + * already carry an 8-byte hash slot, so a flag mismatch there is detected and rejected; but the + * slot is computed with a stricter hash under evolution (it also distinguishes field names and + * nullability), so flipping the flag rejects reads of previously written rows even when the field + * layout is byte-identical. + */ + public B withSchemaEvolution() { + this.schemaEvolution = true; + return castThis(); + } + /** * Configure compact encoding, which is more space efficient than the default encoding, but is not * yet stable. See {@link CompactBinaryRow} for details. @@ -68,6 +95,48 @@ public B compactEncoding() { return castThis(); } + /** + * Build the schema history for {@code targetClass} under the active codec format. The compact + * format sorts the fields of every struct, including structs in list elements and map entries, so + * historical schemas are sorted the same way: the strict hash then covers the fields in wire + * order. The default format passes schemas through unchanged. Each format tags its hashes, so a + * reader rejects a payload written in the other format. + */ + SchemaHistory buildSchemaHistory(final Class targetClass) { + return SchemaHistory.build(targetClass, codecFormat.historyFormat()); + } + + /** + * History of a top-level array element or map entry field, enumerated over the cross-product of + * every versioned bean reachable through the field's list/map/array wrappers (a directly-typed + * bean, a list element, or a map key or value). Used by the array/map codecs so the element + * schema's strict hash identifies all nested layouts jointly, letting a wrapper that reaches more + * than one distinct versioned bean class evolve every one of them. + */ + SchemaHistory buildElementSchemaHistory(final String fieldName, final TypeRef elementType) { + return SchemaHistory.forElement(fieldName, elementType, codecFormat.historyFormat()); + } + + /** + * Fail the build when the history's current layout of {@code owner}, with schema {@code history} + * and strict hash {@code historyHash}, is not the layout {@code inferred} that the codec writes. + * The history rebuilds the current layout from the annotations, and the writer tags its payloads + * with that layout's hash, so a divergence would tag payloads with the hash of another layout. + */ + void checkCurrentLayout( + final String owner, final Schema inferred, final Schema history, final long historyHash) { + final SchemaHistory.Format format = codecFormat.historyFormat(); + if (SchemaHistory.computeStrictSchemaHash(format, format.wireOrder(inferred)) != historyHash) { + throw new IllegalStateException( + "Schema history of " + + owner + + " resolved a current schema that differs from the inferred schema.\nhistory:\n" + + history + + "\ninferred:\n" + + inferred); + } + } + @SuppressWarnings("unchecked") protected B castThis() { return (B) this; diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/BinaryRowEncoder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/BinaryRowEncoder.java index 7cafa0ab2c..d62609e7fa 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/encoder/BinaryRowEncoder.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/BinaryRowEncoder.java @@ -33,18 +33,50 @@ class BinaryRowEncoder implements RowEncoder { private final BaseBinaryRowWriter writer; private final boolean sizeEmbedded; private final long schemaHash; + + /** + * Projection codecs for older versions. {@code null} when schema evolution is disabled or the + * bean has only one version; in that case a hash mismatch is a hard error. + */ + private final ProjectionCache projections; + private final MemoryBuffer buffer = MemoryUtils.buffer(16); + /** + * A projection codec and a row factory with the historical schema's layout precomputed so + * projection decodes match the current-schema path's per-call cost. + */ + static final class ProjectionCodec { + final RowFactory rowFactory; + final GeneratedRowEncoder codec; + + ProjectionCodec(RowFactory rowFactory, GeneratedRowEncoder codec) { + this.rowFactory = rowFactory; + this.codec = codec; + } + } + BinaryRowEncoder( final Schema schema, final GeneratedRowEncoder codec, final BaseBinaryRowWriter writer, final boolean sizeEmbedded) { + this(schema, codec, writer, sizeEmbedded, DataTypes.computeSchemaHash(schema), null); + } + + BinaryRowEncoder( + final Schema schema, + final GeneratedRowEncoder codec, + final BaseBinaryRowWriter writer, + final boolean sizeEmbedded, + final long schemaHash, + final ProjectionCache projections) { this.schema = schema; this.codec = codec; this.writer = writer; this.sizeEmbedded = sizeEmbedded; - this.schemaHash = DataTypes.computeSchemaHash(schema); + this.schemaHash = schemaHash; + this.projections = projections; } @Override @@ -69,28 +101,69 @@ public T decode(final MemoryBuffer buffer) { } T decode(final MemoryBuffer buffer, final int size) { - final long peerSchemaHash = buffer.readInt64(); - if (peerSchemaHash != schemaHash) { - throw new ClassNotCompatibleException( - String.format( - "Schema is not consistent, encoder schema is %s. " - + "self/peer schema hash are %s/%s. " - + "Please check writer schema.", - schema, schemaHash, peerSchemaHash)); + if (size < 8) { + throw sizeBelowHash(size); } + final long peerSchemaHash = buffer.readInt64(); + // The 8-byte hash has just been consumed; the row body occupies the remaining bytes. final int rowSize = size - 8; - final BinaryRow row = writer.newRow(); - row.pointTo(buffer, buffer.readerIndex(), rowSize); - buffer.increaseReaderIndex(rowSize); - return fromRow(row); + if (peerSchemaHash == schemaHash) { + // Hot path: writer.newRow() reuses the writer's cached row layout for the current schema. + final BinaryRow row = writer.newRow(); + row.pointTo(buffer, buffer.readerIndex(), rowSize); + buffer.increaseReaderIndex(rowSize); + return fromRow(row); + } + return decodeOtherSchema(buffer, peerSchemaHash, rowSize); } @Override public T decode(final byte[] bytes) { - // byte[] overloads ignore sizeEmbedded: encode writes no size prefix, decode uses bytes.length. + // byte[] overloads ignore sizeEmbedded: encode writes no length prefix (the 8-byte schema hash + // precedes the body and is not framing), so decode takes the size from bytes.length. return decode(MemoryUtils.wrap(bytes), bytes.length); } + /** + * Decodes a row whose hash is not the current schema's: an older version through its projection, + * or a schema that this codec does not know, which it rejects. Kept out of {@link #decode} so the + * current-schema path stays small. + */ + @SuppressWarnings("unchecked") + private T decodeOtherSchema( + final MemoryBuffer buffer, final long peerSchemaHash, final int rowSize) { + if (projections != null) { + ProjectionCodec projection = projections.get(peerSchemaHash); + if (projection != null) { + // The writer is bound to the current schema, so the historical row comes from the + // projection's own factory, which carries that schema's precomputed layout. + final BinaryRow row = projection.rowFactory.newRow(); + row.pointTo(buffer, buffer.readerIndex(), rowSize); + buffer.increaseReaderIndex(rowSize); + return (T) projection.codec.fromRow(row); + } + } + // Skip the body, so that a caller can go on to the next payload in the buffer. The skip moves + // the reader index past exactly the body, and loads the body from the stream if needed. + buffer.increaseReaderIndex(rowSize); + throw new ClassNotCompatibleException( + String.format( + "Schema is not consistent, encoder schema is %s. " + + "self/peer schema hash are %s/%s. " + + "Please check writer schema.", + schema, schemaHash, peerSchemaHash)); + } + + /** + * The failure of a decode whose payload {@code size} cannot hold the 8-byte schema hash that + * every row, and every array and map with schema evolution, starts with. A negative body size + * would move the reader index backward. + */ + static EncoderException sizeBelowHash(final int size) { + return new EncoderException( + "Payload size " + size + " is less than the 8-byte schema hash at its start"); + } + @Override public byte[] encode(final T obj) { buffer.writerIndex(0); diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/CompactArrayEncoderBuilder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/CompactArrayEncoderBuilder.java index 65f8508e35..e47d31c43a 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/encoder/CompactArrayEncoderBuilder.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/CompactArrayEncoderBuilder.java @@ -19,6 +19,7 @@ package org.apache.fory.format.encoder; +import java.util.Map; import org.apache.fory.codegen.Expression; import org.apache.fory.codegen.Expression.Invoke; import org.apache.fory.codegen.Expression.Reference; @@ -33,6 +34,14 @@ public CompactArrayEncoderBuilder(final TypeRef clsType, final TypeRef bea super(clsType, beanType); } + CompactArrayEncoderBuilder( + final TypeRef clsType, + final TypeRef beanType, + final String classSuffix, + final Map, String> nestedSuffixes) { + super(clsType, beanType, classSuffix, nestedSuffixes); + } + @Override protected Invoke beanWriterReset( final Expression writer, final Reference rowWriter, final Expression ordinal) { diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/CompactCodecFormat.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/CompactCodecFormat.java index c92be822b4..4e1751db3b 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/encoder/CompactCodecFormat.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/CompactCodecFormat.java @@ -21,10 +21,12 @@ import java.util.Collection; import java.util.Map; +import java.util.Set; import org.apache.fory.format.row.binary.BinaryArray; import org.apache.fory.format.row.binary.BinaryMap; import org.apache.fory.format.row.binary.CompactBinaryArray; import org.apache.fory.format.row.binary.CompactBinaryMap; +import org.apache.fory.format.row.binary.CompactRowLayout; import org.apache.fory.format.row.binary.writer.BaseBinaryRowWriter; import org.apache.fory.format.row.binary.writer.BinaryArrayWriter; import org.apache.fory.format.row.binary.writer.CompactBinaryArrayWriter; @@ -62,18 +64,60 @@ public RowEncoderBuilder newRowEncoder(final TypeRef beanType) { return new CompactRowEncoderBuilder(beanType); } + @Override + public RowEncoderBuilder newProjectionRowEncoder( + final TypeRef beanType, + final Schema historicalSchema, + final Set liveNames, + final String classSuffix, + final Map, String> nestedSuffixes) { + return new CompactRowEncoderBuilder( + beanType, historicalSchema, liveNames, classSuffix, nestedSuffixes); + } + @Override public ArrayEncoderBuilder newArrayEncoder( final TypeRef> collectionType, final TypeRef elementType) { return new CompactArrayEncoderBuilder(collectionType, elementType); } + @Override + public ArrayEncoderBuilder newProjectionArrayEncoder( + final TypeRef> collectionType, + final TypeRef elementType, + final String classSuffix, + final Map, String> nestedSuffixes) { + return new CompactArrayEncoderBuilder(collectionType, elementType, classSuffix, nestedSuffixes); + } + @Override public MapEncoderBuilder newMapEncoder( final TypeRef> mapType, final TypeRef beanToken) { return new CompactMapEncoderBuilder(mapType, beanToken); } + @Override + public MapEncoderBuilder newProjectionMapEncoder( + final TypeRef> mapType, + final TypeRef beanToken, + final String classSuffix, + final Map, String> nestedSuffixes) { + return new CompactMapEncoderBuilder(mapType, beanToken, classSuffix, nestedSuffixes); + } + + @Override + public RowFactory newRowFactory(final Schema schema) { + // Compute the compact layout once; every newRow() call reuses it (same model as the writer + // and the nested-slot read path). + final CompactRowLayout layout = new CompactRowLayout(schema); + return layout::newRow; + } + + @Override + public SchemaHistory.Format historyFormat() { + return SchemaHistory.Format.COMPACT; + } + @Override public BinaryArray newArray(final Field field) { return new CompactBinaryArray(field); diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/CompactMapEncoderBuilder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/CompactMapEncoderBuilder.java index be3d206d59..732737d5a5 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/encoder/CompactMapEncoderBuilder.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/CompactMapEncoderBuilder.java @@ -19,6 +19,7 @@ package org.apache.fory.format.encoder; +import java.util.Map; import org.apache.fory.codegen.Expression; import org.apache.fory.codegen.Expression.Invoke; import org.apache.fory.codegen.Expression.Reference; @@ -36,6 +37,14 @@ public CompactMapEncoderBuilder(final TypeRef clsType, final TypeRef beanT super(clsType, beanType); } + CompactMapEncoderBuilder( + final TypeRef clsType, + final TypeRef beanType, + final String classSuffix, + final Map, String> nestedSuffixes) { + super(clsType, beanType, classSuffix, nestedSuffixes); + } + @Override protected Invoke beanWriterReset( final Expression writer, final Reference rowWriter, final Expression ordinal) { diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/CompactRowEncoderBuilder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/CompactRowEncoderBuilder.java index 79ccc53391..b9d0012a4f 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/encoder/CompactRowEncoderBuilder.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/CompactRowEncoderBuilder.java @@ -19,6 +19,8 @@ package org.apache.fory.format.encoder; +import java.util.Map; +import java.util.Set; import org.apache.fory.codegen.Expression; import org.apache.fory.codegen.Expression.Invoke; import org.apache.fory.codegen.Expression.ListExpression; @@ -41,6 +43,15 @@ public CompactRowEncoderBuilder(final TypeRef beanType) { super(beanType); } + CompactRowEncoderBuilder( + final TypeRef beanType, + final Schema historicalSchema, + final Set liveNames, + final String classSuffix, + final Map, String> nestedSuffixes) { + super(beanType, historicalSchema, liveNames, classSuffix, nestedSuffixes); + } + @Override protected Schema inferSchema(final TypeRef beanType) { return CompactBinaryRowWriter.sortSchema(super.inferSchema(beanType)); diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/DefaultCodecFormat.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/DefaultCodecFormat.java index 8ee0f3a8f1..10b6d8ae92 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/encoder/DefaultCodecFormat.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/DefaultCodecFormat.java @@ -21,8 +21,10 @@ import java.util.Collection; import java.util.Map; +import java.util.Set; import org.apache.fory.format.row.binary.BinaryArray; import org.apache.fory.format.row.binary.BinaryMap; +import org.apache.fory.format.row.binary.BinaryRow; import org.apache.fory.format.row.binary.writer.BaseBinaryRowWriter; import org.apache.fory.format.row.binary.writer.BinaryArrayWriter; import org.apache.fory.format.row.binary.writer.BinaryRowWriter; @@ -59,18 +61,57 @@ public RowEncoderBuilder newRowEncoder(final TypeRef beanClass) { return new RowEncoderBuilder(beanClass); } + @Override + public RowEncoderBuilder newProjectionRowEncoder( + final TypeRef beanType, + final Schema historicalSchema, + final Set liveNames, + final String classSuffix, + final Map, String> nestedSuffixes) { + return new RowEncoderBuilder( + beanType, historicalSchema, liveNames, classSuffix, nestedSuffixes); + } + @Override public ArrayEncoderBuilder newArrayEncoder( final TypeRef> collectionType, final TypeRef elementType) { return new ArrayEncoderBuilder(collectionType, elementType); } + @Override + public ArrayEncoderBuilder newProjectionArrayEncoder( + final TypeRef> collectionType, + final TypeRef elementType, + final String classSuffix, + final Map, String> nestedSuffixes) { + return new ArrayEncoderBuilder(collectionType, elementType, classSuffix, nestedSuffixes); + } + @Override public MapEncoderBuilder newMapEncoder( final TypeRef> mapType, final TypeRef beanToken) { return new MapEncoderBuilder(mapType, beanToken); } + @Override + public MapEncoderBuilder newProjectionMapEncoder( + final TypeRef> mapType, + final TypeRef beanToken, + final String classSuffix, + final Map, String> nestedSuffixes) { + return new MapEncoderBuilder(mapType, beanToken, classSuffix, nestedSuffixes); + } + + @Override + public RowFactory newRowFactory(final Schema schema) { + return () -> new BinaryRow(schema); + } + + @Override + public SchemaHistory.Format historyFormat() { + return SchemaHistory.Format.STANDARD; + } + @Override public BinaryArray newArray(final Field field) { return new BinaryArray(field); diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/Encoders.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/Encoders.java index e8ab49cc15..410c8a7477 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/encoder/Encoders.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/Encoders.java @@ -22,6 +22,10 @@ import static org.apache.fory.type.TypeUtils.OBJECT_TYPE; import static org.apache.fory.type.TypeUtils.getRawType; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.lang.reflect.Constructor; import java.util.Collection; import java.util.HashSet; import java.util.LinkedHashSet; @@ -35,6 +39,7 @@ import org.apache.fory.format.row.binary.writer.BinaryRowWriter; import org.apache.fory.format.type.CustomTypeEncoderRegistry; import org.apache.fory.format.type.CustomTypeRegistration; +import org.apache.fory.format.type.Schema; import org.apache.fory.format.type.TypeInference; import org.apache.fory.logging.Logger; import org.apache.fory.logging.LoggerFactory; @@ -98,7 +103,7 @@ public static RowEncoder bean(Class beanClass, BinaryRowWriter writer) *

  • java.math.BigDecimal, java.math.BigInteger *
  • time related: java.sql.Date, java.sql.Timestamp, java.time.LocalDate, java.time.Instant *
  • Optional and friends: OptionalInt, OptionalLong, OptionalDouble - *
  • collection types: only array and java.util.List currently, map support is in progress + *
  • collection types: array, java.util.List, and java.util.Map *
  • record types *
  • nested java bean * @@ -230,8 +235,7 @@ public static , K, V> MapEncoder mapEncoder(TypeRef to *

    supported types for java bean field: - primitive types: boolean, int, double, etc. - boxed * types: Boolean, Integer, Double, etc. - String - java.math.BigDecimal, java.math.BigInteger - * time related: java.sql.Date, java.sql.Timestamp, java.time.LocalDate, java.time.Instant - - * collection types: only array and java.util.List currently, map support is in progress - nested - * java bean. + * collection types: array, java.util.List, and java.util.Map - nested java bean. */ public static , K, V> MapEncoder mapEncoder( TypeRef mapToken, TypeRef keyToken, TypeRef valToken, Fory fory) { @@ -331,7 +335,35 @@ static Class loadOrGenRowCodecClass(Class beanClass, Encoding codecFactory codecBuilder::genCode); }) .toArray(CompileUnit[]::new); - return loadCls(compileUnits); + return loadCls(Thread.currentThread().getContextClassLoader(), compileUnits); + } + + /** + * Compile and load a projection codec class for one historical version of {@code beanClass}. The + * current-version codec class is loaded separately by {@link #loadOrGenRowCodecClass}; this is + * used by schema-evolution code paths to materialize a decoder for each older version. The {@code + * nestedSuffixes} map directs codegen to the projection codec class to embed for each nested + * versioned bean type. {@code codeGenerator} is the generator pinned when the evolution encoder + * was built: it defined the current-version codecs that the projection embeds by name, so a + * first-decode compile on any thread resolves them. + */ + static Class loadOrGenProjectionRowCodecClass( + Class beanClass, + Encoding codecFactory, + Schema historicalSchema, + Set liveNames, + String classSuffix, + Map, String> nestedSuffixes, + CodeGenerator codeGenerator) { + final RowEncoderBuilder codecBuilder = + codecFactory.newProjectionRowEncoder( + TypeRef.of(beanClass), historicalSchema, liveNames, classSuffix, nestedSuffixes); + CompileUnit compileUnit = + new CompileUnit( + CodeGenerator.getPackage(beanClass), + codecBuilder.codecClassName(beanClass) + classSuffix, + codecBuilder::genCode); + return loadCls(codeGenerator, compileUnit); } static Class loadOrGenArrayCodecClass( @@ -348,7 +380,26 @@ static Class loadOrGenArrayCodecClass( codecBuilder.codecClassName(cls, prefix), codecBuilder::genCode); - return loadCls(compileUnit); + return loadCls(Thread.currentThread().getContextClassLoader(), compileUnit); + } + + static Class loadOrGenProjectionArrayCodecClass( + TypeRef> arrayCls, + TypeRef elementType, + Encoding codecFactory, + String classSuffix, + Map, String> nestedSuffixes, + CodeGenerator codeGenerator) { + Class cls = getRawType(elementType); + String prefix = TypeInference.inferTypeName(arrayCls); + ArrayEncoderBuilder codecBuilder = + codecFactory.newProjectionArrayEncoder(arrayCls, elementType, classSuffix, nestedSuffixes); + CompileUnit compileUnit = + new CompileUnit( + CodeGenerator.getPackage(cls), + codecBuilder.codecClassName(cls, prefix) + classSuffix, + codecBuilder::genCode); + return loadCls(codeGenerator, compileUnit); } static Class loadOrGenMapCodecClass( @@ -381,12 +432,45 @@ static Class loadOrGenMapCodecClass( codecBuilder.codecClassName(cls, prefix), codecBuilder::genCode); - return loadCls(compileUnit); + return loadCls(Thread.currentThread().getContextClassLoader(), compileUnit); } - private static Class loadCls(CompileUnit... compileUnit) { - CodeGenerator codeGenerator = - CodeGenerator.getSharedCodeGenerator(Thread.currentThread().getContextClassLoader()); + static Class loadOrGenProjectionMapCodecClass( + TypeRef> mapCls, + TypeRef beanToken, + Encoding codecFactory, + String classSuffix, + Map, String> nestedSuffixes, + CodeGenerator codeGenerator) { + Class cls = getRawType(beanToken); + String prefix = TypeInference.inferTypeName(mapCls); + MapEncoderBuilder codecBuilder = + codecFactory.newProjectionMapEncoder(mapCls, beanToken, classSuffix, nestedSuffixes); + CompileUnit compileUnit = + new CompileUnit( + CodeGenerator.getPackage(cls), + codecBuilder.codecClassName(cls, prefix) + classSuffix, + codecBuilder::genCode); + return loadCls(codeGenerator, compileUnit); + } + + /** + * The shared code generator for the calling thread's context classloader, for an evolution + * encoder to pin before it compiles its current-version codecs. A projection codec compiles + * lazily, on the first decode of its hash, and embeds those current-version codecs by class name, + * so it must compile in the generator that defined them. The shared generator is only softly + * reachable, and a replacement starts from the bare classloader, which cannot see them. Holding + * the generator from build through every projection compile keeps the shared lookup returning it. + */ + static CodeGenerator pinCodeGenerator() { + return CodeGenerator.getSharedCodeGenerator(Thread.currentThread().getContextClassLoader()); + } + + private static Class loadCls(ClassLoader classLoaderHint, CompileUnit... compileUnit) { + return loadCls(CodeGenerator.getSharedCodeGenerator(classLoaderHint), compileUnit); + } + + private static Class loadCls(CodeGenerator codeGenerator, CompileUnit... compileUnit) { ClassLoader classLoader = codeGenerator.compile(compileUnit); String className = compileUnit[0].getQualifiedClassName(); try { @@ -395,4 +479,21 @@ private static Class loadCls(CompileUnit... compileUnit) { throw new IllegalStateException("Impossible because we just compiled class", e); } } + + /** + * Build a {@link MethodHandle} bound to {@code generatedClass}'s {@code (Object[])} constructor, + * adapted so it returns {@code generatedType}. All generated row/array/map codec classes share + * this constructor shape; this helper centralises the reflection and exception wrapping. + */ + static MethodHandle constructorHandleFor(Class generatedClass, Class generatedType) { + try { + Constructor constructor = + generatedClass.asSubclass(generatedType).getConstructor(Object[].class); + return MethodHandles.lookup() + .unreflectConstructor(constructor) + .asType(MethodType.methodType(generatedType, Object[].class)); + } catch (NoSuchMethodException | IllegalAccessException e) { + throw new EncoderException("Failed to resolve constructor for " + generatedClass, e); + } + } } diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/Encoding.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/Encoding.java index c28b4d3b19..ccadb15392 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/encoder/Encoding.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/Encoding.java @@ -21,6 +21,7 @@ import java.util.Collection; import java.util.Map; +import java.util.Set; import org.apache.fory.format.row.binary.BinaryArray; import org.apache.fory.format.row.binary.BinaryMap; import org.apache.fory.format.row.binary.writer.BaseBinaryRowWriter; @@ -41,11 +42,65 @@ interface Encoding { RowEncoderBuilder newRowEncoder(TypeRef beanType); + /** + * Construct a projection codec builder for an older version of {@code beanType}, reading the + * supplied historical schema and producing instances of the current bean class. The {@code + * nestedSuffixes} map directs codegen to embed a specific projection codec class for each + * nested-bean type (used when a nested versioned bean was on the wire at an older version). An + * empty map means all nested beans use their current-version codecs. + */ + RowEncoderBuilder newProjectionRowEncoder( + TypeRef beanType, + Schema historicalSchema, + Set liveNames, + String classSuffix, + Map, String> nestedSuffixes); + ArrayEncoderBuilder newArrayEncoder( TypeRef> collectionType, TypeRef elementType); + /** + * Construct an array encoder builder for one historical combination of the element's nested + * versioned beans. {@code classSuffix} names the generated array codec class (encoding the whole + * combination); {@code nestedSuffixes} routes each nested bean class to its own projection row + * codec, so an element wrapping more than one distinct versioned bean (such as {@code Map}) embeds the right historical codec for each. + */ + ArrayEncoderBuilder newProjectionArrayEncoder( + TypeRef> collectionType, + TypeRef elementType, + String classSuffix, + Map, String> nestedSuffixes); + MapEncoderBuilder newMapEncoder(TypeRef> mapType, TypeRef beanToken); + /** + * Construct a map encoder builder whose generated code references the value and key bean row + * codec classes for one historical (key-version, value-version) combination. {@code classSuffix} + * names the generated map codec class for the whole combination; {@code nestedSuffixes} routes + * each nested bean class in either position to its own projection row codec. One map covers both + * positions because a payload holds one version of each bean class. An absent class means that + * bean is read at its current schema. + */ + MapEncoderBuilder newProjectionMapEncoder( + TypeRef> mapType, + TypeRef beanToken, + String classSuffix, + Map, String> nestedSuffixes); + + /** + * Build a {@link RowFactory} for {@code schema}, precomputing any schema-derived layout once. + * Used by the schema-evolution decode path to allocate rows for a historical schema without + * re-deriving the layout on every decode. + */ + RowFactory newRowFactory(Schema schema); + + /** + * The row format that schema evolution builds histories for: it orders the fields of historical + * schemas the way this format writes them, and tags their strict hashes. + */ + SchemaHistory.Format historyFormat(); + BinaryArray newArray(Field field); BinaryMap newMap(Field field); diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/MapCodecBuilder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/MapCodecBuilder.java index 44ad87e6de..1cb1265eea 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/encoder/MapCodecBuilder.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/MapCodecBuilder.java @@ -20,11 +20,12 @@ package org.apache.fory.format.encoder; import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodHandles; -import java.lang.invoke.MethodType; import java.util.Map; import java.util.function.BiFunction; import java.util.function.Supplier; +import org.apache.fory.Fory; +import org.apache.fory.codegen.CodeGenerator; +import org.apache.fory.collection.LongMap; import org.apache.fory.format.row.binary.writer.BinaryArrayWriter; import org.apache.fory.format.type.DataTypes; import org.apache.fory.format.type.Field; @@ -54,24 +55,310 @@ public class MapCodecBuilder> extends BaseCodecBuilder> build() { + final CodeGenerator codeGenerator = schemaEvolution ? Encoders.pinCodeGenerator() : null; loadMapInnerCodecs(); - final var mapEncoderFactory = generatedMapEncoder(); + if (!schemaEvolution) { + final var mapEncoderFactory = generatedMapEncoder(); + return new Supplier>() { + @Override + public MapEncoder get() { + final BinaryArrayWriter keyWriter = codecFormat.newArrayWriter(keyField); + final BinaryArrayWriter valWriter = + codecFormat.newArrayWriter(valField, keyWriter.getBuffer()); + final var codec = mapEncoderFactory.apply(keyWriter, valWriter); + return new BufferResettingMapEncoder<>( + initialBufferSize, + keyWriter, + valWriter, + new BinaryMapEncoder( + codecFormat, field, valWriter, keyWriter, codec, sizeEmbedded)); + } + }; + } + return buildVersioned( + SchemaHistory.evolutionBean(valType), SchemaHistory.evolutionBean(keyType), codeGenerator); + } + + /** + * Strict hash of one map position at {@code positionVs}: the position's one-field layout with the + * nullability the map writes it with (a key is non-nullable, a value nullable), which the element + * history's inferred field does not carry. A position without a bean still hashes its type, so a + * change to it changes the map hash. + */ + private long positionHash(SchemaHistory.VersionedSchema positionVs, Field wireField) { + Field position = DataTypes.fieldOfSchema(positionVs.schema(), 0); + return SchemaHistory.computeStrictSchemaHash( + codecFormat.historyFormat(), + DataTypes.schema(DataTypes.field(position.name(), position.type(), wireField.nullable()))); + } + + /** + * Fail the build when the history's current layout of one map position is not the position {@code + * wireField} that the codec writes. Only the type is compared: the history does not carry the + * nullability that the map writes the position with, which {@link #positionHash} supplies. + */ + private void checkCurrentPosition( + String position, SchemaHistory.VersionedSchema current, Field wireField) { + Field historyField = DataTypes.fieldOfSchema(current.schema(), 0); + checkCurrentLayout( + "the " + position + " of " + mapType, + DataTypes.schema( + DataTypes.field(historyField.name(), wireField.type(), wireField.nullable())), + current.schema(), + positionHash(current, wireField)); + } + + /** + * The evolution path, taken by every map codec with the flag on so that the payload always + * carries the combined key and value strict hash. {@code valClass} and {@code keyClass} name + * generated projection classes; a position without a reachable bean has one version and no + * projection. + */ + private Supplier> buildVersioned( + final Class valClass, final Class keyClass, final CodeGenerator codeGenerator) { + // The map's own key and value are independent positions on the wire (each its own array), and + // the map header's combined hash identifies the (key-layout, value-layout) pair. Each position + // is enumerated over every versioned bean reachable through its wrappers, so a position that + // itself wraps more than one bean (such as a value typed Map) evolves all of + // them. A position with no versioned bean has a single current-only layout, so the + // cross-product degenerates to the evolving position's versions (or to just the current layout + // when neither evolves, i.e. a non-versioned bean that still needs the hash prefix for + // flag-mismatch detection). + SchemaHistory valHistory = buildElementSchemaHistory(field.name(), valType); + SchemaHistory keyHistory = buildElementSchemaHistory(field.name(), keyType); + SchemaHistory.VersionedSchema valCurrent = valHistory.current(); + SchemaHistory.VersionedSchema keyCurrent = keyHistory.current(); + Field valWire = DataTypes.itemFieldForMap(field); + Field keyWire = DataTypes.keyFieldForMap(field); + checkCurrentPosition("key", keyCurrent, keyWire); + checkCurrentPosition("value", valCurrent, valWire); + + // Index one deferred projection source per (key-version, value-version) combination, keyed by + // the combined hash. The current/current combination is the hot path and is handled by the + // unsuffixed current codec, so it is skipped here. Building the index compiles nothing: a + // combination's codec classes are generated the first time its hash is decoded. The collision + // guards below still run eagerly over the full cross-product, so a hash clash fails fast at + // build rather than surfacing on an unlucky decode. + // + // A writer has one definition of each bean class, so a pair that puts a class reached from + // both the key and the value at two versions cannot occur on the wire; skip it, as the row + // enumeration does. The pair count is the product of the key and value version counts, with no + // cap, for the reason that SchemaHistory.enumerate gives. + LongMap projectionSources = new LongMap<>(); + for (SchemaHistory.VersionedSchema valVs : valHistory.versions()) { + for (SchemaHistory.VersionedSchema keyVs : keyHistory.versions()) { + if (valVs == valCurrent && keyVs == keyCurrent + || !SchemaHistory.sameClassVersions(keyVs, valVs)) { + continue; + } + // The map header carries a single hash, so combine the key and value strict hashes into + // one 64-bit map-layout hash that identifies the (key-version, value-version) combination + // jointly. The collision check below proves these combined hashes are unique at build time. + long hash = + SchemaHistory.combineHashes(positionHash(keyVs, keyWire), positionHash(valVs, valWire)); + if (projectionSources.containsKey(hash)) { + throw new IllegalStateException( + "Combined (key, value) schema-hash collision for map " + + mapType + + ": two distinct version combinations produced the same map-layout hash. " + + "Please file an issue with the key and value bean definitions."); + } + projectionSources.put( + hash, + new MapProjectionSource( + mapType, + field, + codecFormat, + valClass, + keyClass, + valVs, + keyVs, + valCurrent, + keyCurrent, + hash, + codeGenerator)); + } + } + final var currentFactory = generatedMapEncoder(); + long currentHash = + SchemaHistory.combineHashes( + positionHash(keyCurrent, keyWire), positionHash(valCurrent, valWire)); + // The decode hot path matches currentHash before consulting the projection map, so a projection + // colliding with it would be shadowed and never dispatched to. Prove that cannot happen. + if (projectionSources.containsKey(currentHash)) { + throw new IllegalStateException( + "Combined (key, value) schema-hash collision for map " + + mapType + + ": a historical version combination produced the same map-layout hash as the " + + "current schema. Please file an issue with the key and value bean definitions."); + } return new Supplier>() { @Override public MapEncoder get() { - final BinaryArrayWriter keyWriter = codecFormat.newArrayWriter(keyField); - final BinaryArrayWriter valWriter = - codecFormat.newArrayWriter(valField, keyWriter.getBuffer()); - final var codec = mapEncoderFactory.apply(keyWriter, valWriter); + BinaryArrayWriter keyWriter = codecFormat.newArrayWriter(keyField); + BinaryArrayWriter valWriter = codecFormat.newArrayWriter(valField, keyWriter.getBuffer()); + var codec = currentFactory.apply(keyWriter, valWriter); return new BufferResettingMapEncoder<>( initialBufferSize, keyWriter, valWriter, - new BinaryMapEncoder(codecFormat, field, valWriter, keyWriter, codec, sizeEmbedded)); + new VersionedBinaryMapEncoder( + codecFormat, + field, + valWriter, + keyWriter, + codec, + sizeEmbedded, + currentHash, + new ProjectionCache<>(projectionSources, fory))); } }; } + /** + * The map position field projected onto {@code positionVs}: the substituted type from the + * position's historical schema, but the current position field's name and nullability (a map key + * is non-nullable, a value nullable), which forElement's inferred field does not carry. + */ + private static Field projectedPositionField( + Field currentField, SchemaHistory.VersionedSchema positionVs) { + Field projected = DataTypes.fieldOfSchema(positionVs.schema(), 0); + return DataTypes.field(currentField.name(), projected.type(), currentField.nullable()); + } + + /** + * Deferred projection codec for one (key-version, value-version) combination, shared by every + * encoder of the codec. The per-position row codecs and the map codec class are generated on the + * first {@link #compile} call (the first decode of this combination's combined hash by any + * encoder), not at build time. The build-time collision guards already proved this combination's + * hash is unique. Static and built from values captured at build time, so a later change to the + * builder cannot reach a lazy compile. + */ + private static final class MapProjectionSource + extends ProjectionSource< + MapProjectionSource.Classes, VersionedBinaryMapEncoder.ProjectionMapCodec> { + private final TypeRef> mapType; + private final Field field; + private final Encoding codecFormat; + private final Class valClass; + private final Class keyClass; + private final SchemaHistory.VersionedSchema valVs; + private final SchemaHistory.VersionedSchema keyVs; + private final SchemaHistory.VersionedSchema valCurrent; + private final SchemaHistory.VersionedSchema keyCurrent; + private final long mapHash; + private final CodeGenerator codeGenerator; + + MapProjectionSource( + TypeRef> mapType, + Field field, + Encoding codecFormat, + Class valClass, + Class keyClass, + SchemaHistory.VersionedSchema valVs, + SchemaHistory.VersionedSchema keyVs, + SchemaHistory.VersionedSchema valCurrent, + SchemaHistory.VersionedSchema keyCurrent, + long mapHash, + CodeGenerator codeGenerator) { + this.mapType = mapType; + this.field = field; + this.codecFormat = codecFormat; + this.valClass = valClass; + this.keyClass = keyClass; + this.valVs = valVs; + this.keyVs = keyVs; + this.valCurrent = valCurrent; + this.keyCurrent = keyCurrent; + this.mapHash = mapHash; + this.codeGenerator = codeGenerator; + } + + /** + * Generate the projection row codec(s) and projection map codec for one (key-version, + * value-version) combination, and rebuild the map field with each position projected onto its + * historical schema. A position at its current schema gets an empty suffix and keeps its + * current field, so a map where only one side evolves pays no codegen for the unchanged side. + */ + @Override + VersionedBinaryMapEncoder.ProjectionMapCodec bind(Classes r, Fory fory) { + try { + BinaryArrayWriter projKey = codecFormat.newArrayWriter(r.keyArrayField); + BinaryArrayWriter projVal = + codecFormat.newArrayWriter(r.valueArrayField, projKey.getBuffer()); + Object[] references = { + r.keyArrayField, r.valueArrayField, projKey, projVal, fory, r.mapField + }; + GeneratedMapEncoder codec = (GeneratedMapEncoder) r.ctor.invokeExact(references); + return new VersionedBinaryMapEncoder.ProjectionMapCodec(codecFormat, r.mapField, codec); + } catch (Throwable e) { + throw ExceptionUtils.throwException(e); + } + } + + @Override + Classes generate() { + // Each position's history is forElement over the position type, so its schema's single field + // is the position field with every reachable bean projected onto this combination, and + // nestedSuffixesFor routes each bean class in the position to its own historical row codec. + // sameClassVersions holds for every indexed combination, so a class that both positions + // reach gets the same suffix from either side and one class-keyed map serves both. + // The projected field carries only the substituted type; the position's own nullability is + // taken from the current map field (map keys are non-nullable, values nullable). + Field currentVal = DataTypes.itemFieldForMap(field); + Field histVal = currentVal; + String valSuffix = ""; + Map, String> nested = null; + if (valVs != valCurrent) { + valSuffix = ProjectionRouting.projectionSuffix(valVs); + nested = ProjectionRouting.nestedSuffixesFor(valVs, codecFormat, codeGenerator); + histVal = projectedPositionField(currentVal, valVs); + } + Field currentKey = DataTypes.keyFieldForMap(field); + Field histKey = currentKey; + String keySuffix = ""; + if (keyVs != keyCurrent) { + keySuffix = ProjectionRouting.projectionSuffix(keyVs); + Map, String> keyNested = + ProjectionRouting.nestedSuffixesFor(keyVs, codecFormat, codeGenerator); + if (nested == null) { + nested = keyNested; + } else { + nested.putAll(keyNested); + } + histKey = projectedPositionField(currentKey, keyVs); + } + Class mapClass = + Encoders.loadOrGenProjectionMapCodecClass( + mapType, + TypeRef.of(valClass != null ? valClass : keyClass), + codecFormat, + ProjectionRouting.mapSuffix(valSuffix, keySuffix, mapHash), + nested, + codeGenerator); + Field histMapField = DataTypes.mapField(field.name(), histKey, histVal); + return new Classes( + Encoders.constructorHandleFor(mapClass, GeneratedMapEncoder.class), + histMapField, + DataTypes.keyArrayFieldForMap(histMapField), + DataTypes.itemArrayFieldForMap(histMapField)); + } + + static final class Classes { + final MethodHandle ctor; + final Field mapField; + final Field keyArrayField; + final Field valueArrayField; + + Classes(MethodHandle ctor, Field mapField, Field keyArrayField, Field valueArrayField) { + this.ctor = ctor; + this.mapField = mapField; + this.keyArrayField = keyArrayField; + this.valueArrayField = valueArrayField; + } + } + } + private void loadMapInnerCodecs() { Encoders.loadMapCodecs(keyType, codecFormat); Encoders.loadMapCodecs(valType, codecFormat); @@ -81,17 +368,8 @@ BiFunction generatedM final Class arrayCodecClass = Encoders.loadOrGenMapCodecClass(mapType, keyType, valType, codecFormat); - final MethodHandle constructorHandle; - try { - final var constructor = - arrayCodecClass.asSubclass(GeneratedMapEncoder.class).getConstructor(Object[].class); - constructorHandle = - MethodHandles.lookup() - .unreflectConstructor(constructor) - .asType(MethodType.methodType(GeneratedMapEncoder.class, Object[].class)); - } catch (final NoSuchMethodException | IllegalAccessException e) { - throw new EncoderException("Failed to construct array codec for " + mapType, e); - } + final MethodHandle constructorHandle = + Encoders.constructorHandleFor(arrayCodecClass, GeneratedMapEncoder.class); return new BiFunction() { @Override public GeneratedMapEncoder apply( diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/MapEncoder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/MapEncoder.java index dce6008b3a..92b24bf102 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/encoder/MapEncoder.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/MapEncoder.java @@ -23,7 +23,10 @@ import org.apache.fory.format.row.binary.BinaryMap; import org.apache.fory.format.type.Field; -/** Encoder to encode/decode object in the map container by toMap/fromMap row. */ +/** + * Encoder to encode/decode object in the map container by toMap/fromMap row. A MapEncoder instance + * is reusable but not thread-safe. + */ public interface MapEncoder extends Encoder { Field keyField(); diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/MapEncoderBuilder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/MapEncoderBuilder.java index 18abad605e..f6024b01a6 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/encoder/MapEncoderBuilder.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/MapEncoderBuilder.java @@ -58,10 +58,25 @@ public MapEncoderBuilder(Class mapCls, Class keyClass) { } public MapEncoderBuilder(TypeRef clsType, TypeRef beanType) { + this(clsType, beanType, null, null); + } + + /** + * Construct a projection map codec builder. {@code classSuffix} names this codec's own class + * (encoding the whole key and value version combination); {@code nestedSuffixes} routes each + * nested versioned bean class, in either position, to its own projection row codec. + */ + MapEncoderBuilder( + TypeRef clsType, + TypeRef beanType, + String classSuffix, + Map, String> nestedSuffixes) { // A top-level map has no enclosing bean, so scope key/value-codec resolution to Object to match // TypeInference's empty-path enclosing type; beanType still names the key/value bean for class // naming and schema generation. super(new CodegenContext(), beanType, Object.class); + this.classSuffix = classSuffix; + this.nestedClassSuffixes = nestedSuffixes; mapToken = clsType; ctx.reserveName(ROOT_KEY_WRITER_NAME); ctx.reserveName(ROOT_VALUE_WRITER_NAME); @@ -75,7 +90,9 @@ public MapEncoderBuilder(TypeRef clsType, TypeRef beanType) { @Override public String genCode() { ctx.setPackage(CodeGenerator.getPackage(beanClass)); - String className = codecClassName(beanClass, TypeInference.inferTypeName(mapToken)); + String className = + codecClassName(beanClass, TypeInference.inferTypeName(mapToken)) + + (classSuffix == null ? "" : classSuffix); ctx.setClassName(className); // don't addImport(arrayClass), because user class may name collide. // janino don't support generics, so GeneratedCodec has no generics diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/ProjectionCache.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/ProjectionCache.java new file mode 100644 index 0000000000..2192865d01 --- /dev/null +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/ProjectionCache.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format.encoder; + +import org.apache.fory.Fory; +import org.apache.fory.collection.LongMap; + +/** + * One encoder's projection codecs, keyed by the strict hash of their historical layout and compiled + * on the first decode of each hash. It needs no locking because an encoder is single-threaded; the + * shared {@link ProjectionSource}s generate each class once across encoders. + */ +final class ProjectionCache

    { + private final LongMap> sources; + private final LongMap

    compiled = new LongMap<>(); + private final Fory fory; + + /** + * {@code sources} is shared, immutable, and built without compiling anything. It can be empty + * when the codec has evolution on but no older layout. + */ + ProjectionCache(final LongMap> sources, final Fory fory) { + this.sources = sources; + this.fory = fory; + } + + /** The projection codec for {@code hash}, or {@code null} if no older layout has that hash. */ + P get(final long hash) { + P projection = compiled.get(hash); + if (projection != null) { + return projection; + } + ProjectionSource source = sources.get(hash); + if (source == null) { + return null; + } + projection = source.compile(fory); + compiled.put(hash, projection); + return projection; + } +} diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/ProjectionRouting.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/ProjectionRouting.java new file mode 100644 index 0000000000..e8ca56d343 --- /dev/null +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/ProjectionRouting.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format.encoder; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.fory.codegen.CodeGenerator; + +/** + * Suffix routing shared by row/array/map projection codec generation. Each cross-product entry gets + * a class-name suffix that uniquely identifies its full nested combination, and the per-nested-bean + * suffix map directs codegen to embed the right inner projection class for each nested-bean type at + * this combination's versions. + */ +final class ProjectionRouting { + /** + * Longest readable class-name suffix. Each nested versioned bean adds its simple name and a hash + * to a suffix, and a map codec joins a value and a key suffix, so a wide bean or map could push + * the source file that {@code FORY_CODE_DIR} writes for the class past the 255-byte file name + * limit of common file systems. A longer suffix is cut to this length and ends with a hash token + * instead, so a projection class name is at most 97 characters longer than the codec class name + * before its suffix. + */ + static final int MAX_READABLE_SUFFIX_LENGTH = 80; + + private ProjectionRouting() {} + + /** + * Build a class-name suffix that uniquely identifies {@code vs} across the whole cross-product, + * at any nesting depth. The suffix encodes the outer version and strict hash and, for each nested + * bean, that inner's simple name, version, and strict hash. Each hash is 16 hex digits, so each + * token has a fixed end, and the name follows the layout it decodes. + * + *

    The leading outer hash identifies the combination, and the nested tokens are for + * readability. + * + *

    A suffix longer than {@link #MAX_READABLE_SUFFIX_LENGTH} is cut, and the strict hash of + * {@code vs} identifies the combination instead; see {@link #bounded}. + */ + static String projectionSuffix(SchemaHistory.VersionedSchema vs) { + StringBuilder sb = + new StringBuilder("_V") + .append(vs.version()) + .append("h") + .append(String.format("%016x", vs.strictHash())); + if (!vs.nestedBeanSchemas().isEmpty()) { + List, SchemaHistory.VersionedSchema>> entries = + new ArrayList<>(vs.nestedBeanSchemas().entrySet()); + entries.sort((a, b) -> a.getKey().getName().compareTo(b.getKey().getName())); + for (Map.Entry, SchemaHistory.VersionedSchema> e : entries) { + SchemaHistory.VersionedSchema inner = e.getValue(); + sb.append("_") + .append(e.getKey().getSimpleName()) + .append(inner.version()) + .append("h") + .append(String.format("%016x", inner.strictHash())); + } + } + return bounded(sb.toString(), vs.strictHash()); + } + + /** + * Class-name suffix for the projection map codec of one (value-combination, key-combination) + * pair, from the {@link #projectionSuffix} of each position, empty for a position at its current + * schema. The codec class is cached by name, so distinct pairs must map to distinct suffixes or + * the second pair would reuse the first pair's codec class. Each half is written after its + * length, so no token on one side can imitate the boundary between the halves, whatever the bean + * simple names are. The current-schema codec, with both halves empty, gets no suffix. A suffix + * longer than {@link #MAX_READABLE_SUFFIX_LENGTH} is cut, and {@code mapHash}, the combined hash + * that the map payload carries for the pair, identifies it instead. + */ + static String mapSuffix(String valueSuffix, String keySuffix, long mapHash) { + if (valueSuffix.isEmpty() && keySuffix.isEmpty()) { + return ""; + } + return bounded( + "_" + valueSuffix.length() + valueSuffix + "_" + keySuffix.length() + keySuffix, mapHash); + } + + /** + * {@code readable} if it is at most {@link #MAX_READABLE_SUFFIX_LENGTH} characters long, else its + * first {@code MAX_READABLE_SUFFIX_LENGTH} characters (one fewer if the cut would split a + * surrogate pair), then {@code _} and the 16 hex digits of {@code hash}. A cut suffix is longer + * than every suffix that is not cut, so the two kinds never collide, and {@code hash} must be + * unique among the suffixes of one codec class. + */ + private static String bounded(String readable, long hash) { + if (readable.length() <= MAX_READABLE_SUFFIX_LENGTH) { + return readable; + } + int end = MAX_READABLE_SUFFIX_LENGTH; + if (Character.isHighSurrogate(readable.charAt(end - 1))) { + end--; + } + return readable.substring(0, end) + "_" + String.format("%016x", hash); + } + + /** + * Per-nested-bean-type suffix map for codegen, recursively materializing the inner projection + * class for each nested bean that a live field of {@code vs} reaches. Projection skips removed + * slots, so a bean reached only through removed fields gets no entry and no class. Empty string + * means the inner bean uses its current-version codec class. The chosen inner entry is taken + * directly from {@code vs}, so this resolves the correct combination to arbitrary depth without + * re-deriving it from a version number. + * + *

    Called when an outer combination is first compiled (its hash first decoded), so the inner + * classes are generated lazily alongside it rather than at builder time. {@code codeGenerator} is + * the generator pinned when the evolution encoder was built; every lazily generated class + * compiles in it, so an empty-suffix reference resolves to the current-version codec it already + * defined. + */ + static Map, String> nestedSuffixesFor( + SchemaHistory.VersionedSchema vs, Encoding codecFormat, CodeGenerator codeGenerator) { + Map, String> out = new HashMap<>(); + for (Map.Entry, SchemaHistory.VersionedSchema> e : vs.nestedBeanSchemas().entrySet()) { + Class innerClass = e.getKey(); + SchemaHistory.VersionedSchema innerVs = e.getValue(); + if (!vs.liveNestedClasses().contains(innerClass)) { + continue; + } + if (innerVs.isCurrent()) { + out.put(innerClass, ""); + } else { + String innerSuffix = projectionSuffix(innerVs); + out.put(innerClass, innerSuffix); + // Generate the inner's projection class so the outer's `new InnerCodec` resolves at + // class load. Recurses through the inner's own nested combination. + Encoders.loadOrGenProjectionRowCodecClass( + innerClass, + codecFormat, + innerVs.schema(), + innerVs.liveFieldNames(), + innerSuffix, + nestedSuffixesFor(innerVs, codecFormat, codeGenerator), + codeGenerator); + } + } + return out; + } +} diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/ProjectionSource.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/ProjectionSource.java new file mode 100644 index 0000000000..0e4acc68de --- /dev/null +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/ProjectionSource.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format.encoder; + +import org.apache.fory.Fory; +import org.apache.fory.util.ExceptionUtils; + +/** + * The projection codec for one historical layout of a schema-evolution codec. The generated classes + * {@code C} are shared by every encoder of the codec and are generated once, on the first decode of + * the layout by any encoder. Each encoder then binds its own codec {@code P} with {@link #compile}. + */ +abstract class ProjectionSource { + private volatile C classes; + + /** + * The failure of {@link #generate}, the cause of the exception of every later compile. Guarded by + * {@code this}. + */ + private Throwable failure; + + /** + * Generates the projection classes for this layout. Runs at most once, unless it fails with a + * {@link VirtualMachineError}. + */ + abstract C generate(); + + /** Creates one encoder's projection codec from the generated classes. */ + abstract P bind(C classes, Fory fory); + + final P compile(final Fory fory) { + C c = classes; + if (c == null) { + synchronized (this) { + c = classes; + if (c == null) { + if (failure != null) { + // A new exception for each throw: decodes on several threads must not share one + // instance, to which each can add suppressed exceptions, and whose stack trace shows + // only the first decode. + throw new EncoderException("Projection code generation failed earlier", failure); + } + try { + c = generate(); + } catch (Throwable t) { + // Generation is deterministic, so a retry would rerun the code generator only to fail + // again, on every decode of this layout. A VirtualMachineError such as an + // OutOfMemoryError can be transient, so a later decode retries it. + if (!(t instanceof VirtualMachineError)) { + failure = t; + } + throw ExceptionUtils.throwException(t); + } + classes = c; + } + } + } + return bind(c, fory); + } +} diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowCodecBuilder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowCodecBuilder.java index eeb624d706..623449f6ca 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowCodecBuilder.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowCodecBuilder.java @@ -20,11 +20,13 @@ package org.apache.fory.format.encoder; import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodHandles; -import java.lang.invoke.MethodType; import java.util.function.Function; import java.util.function.Supplier; +import org.apache.fory.Fory; +import org.apache.fory.codegen.CodeGenerator; +import org.apache.fory.collection.LongMap; import org.apache.fory.format.row.binary.writer.BaseBinaryRowWriter; +import org.apache.fory.format.type.Schema; import org.apache.fory.format.type.TypeInference; import org.apache.fory.util.ExceptionUtils; @@ -45,51 +47,186 @@ public class RowCodecBuilder extends BaseCodecBuilder> { * virtual thread. */ public Supplier> build() { - final Function> rowEncoderFactory = buildForWriter(); + final RowEncoderFactory factory = buildForWriter(); return new Supplier>() { @Override public RowEncoder get() { - final BaseBinaryRowWriter writer = codecFormat.newWriter(schema); - return new BufferResettingRowEncoder( - initialBufferSize, writer, rowEncoderFactory.apply(writer)); + final BaseBinaryRowWriter writer = codecFormat.newWriter(factory.schema); + return new BufferResettingRowEncoder(initialBufferSize, writer, factory.apply(writer)); } }; } - Function> buildForWriter() { + /** + * Resolve the schema and the per-writer encoder factory together. The evolution path rotates the + * schema to the history-derived current version; returning it alongside the factory keeps that + * resolution out of the mutable builder state, so a reused builder is unaffected. + */ + RowEncoderFactory buildForWriter() { + return schemaEvolution ? evolvingBuildForWriter() : defaultBuildForWriter(); + } + + private RowEncoderFactory defaultBuildForWriter() { + final Schema currentSchema = schema; final Function rowEncoderFactory = - rowEncoderFactory(); - return new Function>() { + rowEncoderFactory(currentSchema); + return new RowEncoderFactory(currentSchema) { @Override public RowEncoder apply(final BaseBinaryRowWriter writer) { return new BinaryRowEncoder( - schema, rowEncoderFactory.apply(writer), writer, sizeEmbedded); + currentSchema, rowEncoderFactory.apply(writer), writer, sizeEmbedded); } }; } - Function rowEncoderFactory() { - final Class rowCodecClass = Encoders.loadOrGenRowCodecClass(beanClass, codecFormat); - MethodHandle constructorHandle; - try { - final var constructor = - rowCodecClass.asSubclass(GeneratedRowEncoder.class).getConstructor(Object[].class); - constructorHandle = - MethodHandles.lookup() - .unreflectConstructor(constructor) - .asType(MethodType.methodType(GeneratedRowEncoder.class, Object[].class)); - } catch (final NoSuchMethodException | IllegalAccessException e) { - throw new EncoderException("Failed to construct codec for " + beanClass, e); + private RowEncoderFactory evolvingBuildForWriter() { + SchemaHistory history = buildSchemaHistory(beanClass); + SchemaHistory.VersionedSchema currentVersion = history.current(); + // The history-derived schema is what writers, generated codec, and decode dispatch must agree + // on. It travels back to build() through the returned factory rather than the mutable schema + // field, so building does not rotate builder state that a later build()/buildForWriter() reads. + final Schema currentSchema = currentVersion.schema(); + checkCurrentLayout(beanClass.getName(), schema, currentSchema, currentVersion.strictHash()); + + final CodeGenerator codeGenerator = Encoders.pinCodeGenerator(); + final Function currentFactory = + rowEncoderFactory(currentSchema); + // Index of hash → deferred projection source for each non-current combination of + // (outer-version, inner-versions). Building it compiles nothing: a combination's codec class is + // generated only the first time a payload with that hash is decoded. The suffix encodes the + // combination so distinct cross-product entries get distinct generated classes; the nested-bean + // version map directs the projection codec to embed the right inner projection class. + // + // Keyed by the raw strict hash straight from SchemaHistory, whose versions() hold one entry per + // strict hash, so the keys are unique and none is the current hash. No builder-side collision + // check is needed here, unlike the map codec, whose key is a combined (key, value) hash that + // is computed outside SchemaHistory. Two distinct layouts with one strict hash would share an + // entry. SchemaHistory rejects most, not all, such collisions, which have odds of about 2^-64. + final LongMap projectionSources = new LongMap<>(); + for (SchemaHistory.VersionedSchema vs : history.versions()) { + if (vs == currentVersion) { + continue; + } + projectionSources.put( + vs.strictHash(), new RowProjectionSource(beanClass, codecFormat, vs, codeGenerator)); + } + + final long currentHash = currentVersion.strictHash(); + // The writer and codec need the history schema, which the compact format sorts at every depth + // so the hash covers the wire order. schema() reports the builder's schema instead, the same + // value an evolution-off codec reports. The two hold the same fields in the same top-level + // order and differ only in the field order inside nested structs. + final Schema reportedSchema = schema; + return new RowEncoderFactory(currentSchema) { + @Override + public RowEncoder apply(final BaseBinaryRowWriter writer) { + return new BinaryRowEncoder( + reportedSchema, + currentFactory.apply(writer), + writer, + sizeEmbedded, + currentHash, + projectionSources.size == 0 ? null : new ProjectionCache<>(projectionSources, fory)); + } + }; + } + + /** + * A per-writer encoder factory that also carries the schema the writer must be created with. The + * schema travels with the factory instead of through the mutable builder, so {@link #build()} can + * create the writer without reading builder state that the evolution path would otherwise rotate. + */ + abstract static class RowEncoderFactory + implements Function> { + final Schema schema; + + RowEncoderFactory(final Schema schema) { + this.schema = schema; } + } + + /** + * Deferred projection codec for one historical version, shared by every encoder of the codec. The + * class is compiled on the first {@link #compile} call (the first decode of this version's hash + * by any encoder), not at build time. + */ + private static final class RowProjectionSource + extends ProjectionSource { + private final Class beanClass; + private final Encoding codecFormat; + private final SchemaHistory.VersionedSchema version; + private final CodeGenerator codeGenerator; + + RowProjectionSource( + Class beanClass, + Encoding codecFormat, + SchemaHistory.VersionedSchema version, + CodeGenerator codeGenerator) { + this.beanClass = beanClass; + this.codecFormat = codecFormat; + this.version = version; + this.codeGenerator = codeGenerator; + } + + @Override + BinaryRowEncoder.ProjectionCodec bind(Classes r, Fory fory) { + try { + // A decode-only codec still builds child writers under its root writer, so give it a + // detached one: the encoder's own writer would collect those children and walk them on + // every encode. + Object[] references = {version.schema(), codecFormat.newWriter(version.schema()), fory}; + GeneratedRowEncoder codec = (GeneratedRowEncoder) r.ctor.invokeExact(references); + return new BinaryRowEncoder.ProjectionCodec(r.rowFactory, codec); + } catch (final ReflectiveOperationException e) { + throw new EncoderException( + "Failed to construct projection codec for schema " + version.schema(), e); + } catch (Throwable e) { + throw ExceptionUtils.throwException(e); + } + } + + /** Generates the projection class and the row factory for its historical schema. */ + @Override + Classes generate() { + Schema historicalSchema = version.schema(); + Class projectionClass = + Encoders.loadOrGenProjectionRowCodecClass( + beanClass, + codecFormat, + historicalSchema, + version.liveFieldNames(), + ProjectionRouting.projectionSuffix(version), + ProjectionRouting.nestedSuffixesFor(version, codecFormat, codeGenerator), + codeGenerator); + return new Classes( + Encoders.constructorHandleFor(projectionClass, GeneratedRowEncoder.class), + codecFormat.newRowFactory(historicalSchema)); + } + + static final class Classes { + final MethodHandle ctor; + final RowFactory rowFactory; + + Classes(MethodHandle ctor, RowFactory rowFactory) { + this.ctor = ctor; + this.rowFactory = rowFactory; + } + } + } + + Function rowEncoderFactory(final Schema codecSchema) { + final Class rowCodecClass = Encoders.loadOrGenRowCodecClass(beanClass, codecFormat); + final MethodHandle constructorHandle = + Encoders.constructorHandleFor(rowCodecClass, GeneratedRowEncoder.class); return new Function() { @Override public GeneratedRowEncoder apply(final BaseBinaryRowWriter writer) { try { - final Object[] references = {schema, writer, fory}; + final Object[] references = {codecSchema, writer, fory}; return (GeneratedRowEncoder) constructorHandle.invokeExact(references); } catch (final ReflectiveOperationException e) { throw new EncoderException("Failed to construct codec for " + beanClass, e); - } catch (final Throwable e) { + } catch (Throwable e) { throw ExceptionUtils.throwException(e); } } diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowEncoderBuilder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowEncoderBuilder.java index fe1fab9196..ae246745b8 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowEncoderBuilder.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowEncoderBuilder.java @@ -31,6 +31,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.SortedMap; import org.apache.fory.Fory; import org.apache.fory.builder.CodecBuilder; @@ -59,6 +60,7 @@ import org.apache.fory.type.TypeUtils; import org.apache.fory.util.Preconditions; import org.apache.fory.util.StringUtils; +import org.apache.fory.util.record.RecordComponent; import org.apache.fory.util.record.RecordUtils; /** Expression builder for building jit row encoder class. */ @@ -77,15 +79,44 @@ class RowEncoderBuilder extends BaseBinaryEncoderBuilder { private final CodegenContext generatedBeanImpl; private final String generatedBeanImplName; + /** + * When non-null, this builder produces a decode-only projection codec: schema fields whose name + * is in {@code projectionLiveNames} are assigned to the bean as usual; others are skipped + * entirely, which is safe because the row layout addresses every field by its fixed slot rather + * than by sequential reads. {@code toRow} on a projection codec throws. + */ + private final Set projectionLiveNames; + public RowEncoderBuilder(Class beanClass) { this(TypeRef.of(beanClass)); } public RowEncoderBuilder(TypeRef beanType) { + this(beanType, null, null, null, null); + } + + /** + * Construct a decode-only projection builder for an older version of {@code beanType}. The + * supplied {@code historicalSchema} is used as the layout to decode; only fields whose name is in + * {@code liveNames} are written into the resulting bean. {@code classSuffix} distinguishes this + * codec from the current-version codec and from other historical projections. {@code + * nestedSuffixes} routes each nested-bean type to a specific projection codec class (used when an + * inner versioned bean was on the wire at an older version). + */ + RowEncoderBuilder( + TypeRef beanType, + Schema historicalSchema, + Set liveNames, + String classSuffix, + Map, String> nestedSuffixes) { super(new CodegenContext(), beanType); Preconditions.checkArgument(beanClass.isInterface() || TypeUtils.isBean(beanType, typeCtx)); - className = codecClassName(beanClass); - this.schema = inferSchema(beanType); + this.projectionLiveNames = liveNames; + this.classSuffix = classSuffix; + this.nestedClassSuffixes = nestedSuffixes; + className = + classSuffix == null ? codecClassName(beanClass) : codecClassName(beanClass) + classSuffix; + this.schema = historicalSchema != null ? historicalSchema : inferSchema(beanType); this.descriptorsMap = Descriptor.getDescriptorsMap(beanClass); ctx.reserveName(ROOT_ROW_WRITER_NAME); ctx.reserveName(SCHEMA_NAME); @@ -105,7 +136,11 @@ public RowEncoderBuilder(TypeRef beanType) { ctx.addImports(Row.class, ArrayData.class, MapData.class); ctx.addImports(BinaryRow.class, BinaryArray.class, BinaryMap.class); if (beanClass.isInterface()) { - generatedBeanImplName = beanClass.getSimpleName() + "GeneratedImpl"; + // The impl class is a member class of this codec, whose class name is unique, so the impl + // names cannot collide. The projection suffix only makes the impl class of a historical + // version easy to identify, for example in a stack trace. + generatedBeanImplName = + beanClass.getSimpleName() + "GeneratedImpl" + (classSuffix == null ? "" : classSuffix); generatedBeanImpl = buildImplClass(); } else { generatedBeanImplName = null; @@ -158,6 +193,10 @@ public String genCode() { ctx.addField(rowWriterType, ROOT_ROW_WRITER_NAME); ctx.addField(ctx.type(Fory.class), FORY_NAME); + // Order matters: the encode pass registers nested-bean encoder fields on ctx as a side effect + // (see serializeForBean), and the decode pass reads them. Building decode first would fail in + // deserializeForBean, which requires the registered encoder. A projection codec builds the + // encode pass for this reason only and then replaces it. Keep encode genCode before decode. Expression encodeExpr = buildEncodeExpression(); String encodeCode = encodeExpr.genCode(ctx).code(); Expression decodeExpr = buildDecodeExpression(); @@ -203,8 +242,13 @@ public Expression buildEncodeExpression() { // schema field's name must correspond to descriptor's name. for (int i = 0; i < numFields; i++) { Field field = schema.field(i); + if (projectionLiveNames != null && !projectionLiveNames.contains(field.name())) { + // A slot that the projection does not read: a removed field without a live field of the + // same name and declared type. A projection codec never encodes. + continue; + } Descriptor d = getDescriptorByFieldName(field.name()); - Preconditions.checkNotNull(d); + Preconditions.checkNotNull(d, "missing descriptor for schema field " + field.name()); TypeRef fieldType = d.getTypeRef(); Expression fieldValue = getFieldValue(bean, d); Literal ordinal = Literal.ofInt(i); @@ -215,6 +259,12 @@ public Expression buildEncodeExpression() { serializeFor(ordinal, fieldValue, writer, fieldType, field, foryField, new HashSet<>()); expressions.add(fieldExpr); } + if (projectionLiveNames != null) { + // Decode-only: never run the writer logic. The expressions above were generated only for + // their side effects on the codegen context (registering nested-bean encoder fields). + return new Expression.Block( + "throw new UnsupportedOperationException(\"projection codec is decode-only\");\n"); + } expressions.add( new Expression.Return( new Expression.Invoke(writer, "getRow", TypeRef.of(BinaryRow.class)))); @@ -237,19 +287,24 @@ public Expression buildDecodeExpression() { bean = new Expression.Reference("new " + generatedBeanImplName + "(row)"); } else { int numFields = schema.numFields(); - List fieldNames = new ArrayList<>(numFields); - Expression[] values = new Expression[numFields]; - Descriptor[] descriptors = new Descriptor[numFields]; - // schema field's name must correspond to descriptor's name. + // Build, in schema order, the per-slot bean-side info for the fields that the projection + // reads; the other slots stay in the row layout and are skipped. Offsets are keyed on slot + // index, so skipping them is safe. + List liveDescriptors = new ArrayList<>(); + List liveValues = new ArrayList<>(); for (int i = 0; i < numFields; i++) { Literal ordinal = Literal.ofInt(i); - Descriptor d = getDescriptorByFieldName(schema.field(i).name()); - fieldNames.add(d.getName()); - descriptors[i] = d; + String wireName = schema.field(i).name(); + if (projectionLiveNames != null && !projectionLiveNames.contains(wireName)) { + continue; + } + Descriptor d = getDescriptorByFieldName(wireName); + Preconditions.checkNotNull(d, "missing descriptor for wire field " + wireName); TypeRef fieldType = d.getTypeRef(); Expression.Variable value = new Expression.Variable("value_" + d.getName(), nullValue(fieldType)); - values[i] = value; + liveDescriptors.add(d); + liveValues.add(value); expressions.add(value); Expression.Invoke isNullAt = new Expression.Invoke( @@ -267,17 +322,25 @@ public Expression buildDecodeExpression() { expressions.add(decode); } if (RecordUtils.isRecord(beanClass)) { - int[] map = RecordUtils.buildRecordComponentMapping(beanClass, fieldNames); - Expression[] args = new Expression[numFields]; - for (int i = 0; i < numFields; i++) { - args[i] = values[map[i]]; - } - bean = new Expression.NewInstance(beanType, beanType.getRawType().getName(), args); + bean = buildRecordInstance(liveDescriptors, liveValues); } else { bean = newBean(); expressions.add(bean); - for (int i = 0; i < values.length; i++) { - expressions.add(setFieldValue(bean, descriptors[i], values[i])); + for (int i = 0; i < liveDescriptors.size(); i++) { + expressions.add(setFieldValue(bean, liveDescriptors.get(i), liveValues.get(i))); + } + if (projectionLiveNames != null) { + // A field the older payload lacks gets the default a record component gets, not + // whatever the constructor or a field initializer left there. + Set assigned = new HashSet<>(); + for (Descriptor d : liveDescriptors) { + assigned.add(d.getName()); + } + for (Descriptor d : descriptorsMap.values()) { + if (!assigned.contains(d.getName())) { + expressions.add(setFieldValue(bean, d, nullValue(d.getTypeRef()))); + } + } } } } @@ -290,6 +353,30 @@ public Expression buildDecodeExpression() { return expressions; } + /** + * Build a record instance, supplying defaults for components not contributed by the wire. The + * non-projection path always supplies every component; the projection path may supply a subset. + */ + private Expression buildRecordInstance( + List liveDescriptors, List liveValues) { + Map byName = new HashMap<>(liveDescriptors.size() * 2); + for (int i = 0; i < liveDescriptors.size(); i++) { + byName.put(liveDescriptors.get(i).getName(), liveValues.get(i)); + } + RecordComponent[] components = RecordUtils.getRecordComponents(beanClass); + Expression[] args = new Expression[components.length]; + for (int i = 0; i < components.length; i++) { + String compName = components[i].getName(); + Expression value = byName.get(compName); + if (value == null) { + TypeRef compType = TypeRef.of(components[i].getGenericType()); + value = nullValue(compType); + } + args[i] = value; + } + return new Expression.NewInstance(beanType, beanType.getRawType().getName(), args); + } + private static Expression nullValue(TypeRef fieldType) { Class rawType = fieldType.getRawType(); if (TypeUtils.isOptionalType(rawType)) { @@ -303,7 +390,11 @@ private void addDecoderMethods() { int numFields = schema.numFields(); for (int i = 0; i < numFields; i++) { Literal ordinal = Literal.ofInt(i); - Descriptor d = getDescriptorByFieldName(schema.field(i).name()); + String wireName = schema.field(i).name(); + if (projectionLiveNames != null && !projectionLiveNames.contains(wireName)) { + continue; + } + Descriptor d = getDescriptorByFieldName(wireName); TypeRef fieldType = d.getTypeRef(); Class rawFieldType = fieldType.getRawType(); // Resolve a codec on the raw field type before any Optional unwrap; keep in lockstep with the @@ -358,7 +449,12 @@ private CodegenContext buildImplClass() { int numFields = schema.numFields(); for (int i = 0; i < numFields; i++) { Literal ordinal = Literal.ofInt(i); - Descriptor d = getDescriptorByFieldName(schema.field(i).name()); + String wireName = schema.field(i).name(); + if (projectionLiveNames != null && !projectionLiveNames.contains(wireName)) { + // A slot that the projection skips: it does not read the field. + continue; + } + Descriptor d = getDescriptorByFieldName(wireName); TypeRef fieldType = d.getTypeRef(); Class rawFieldType = fieldType.getRawType(); @@ -410,11 +506,15 @@ private CodegenContext buildImplClass() { // Note: adding constructor captures init code, so must happen after all fields are collected implClass.addConstructor("this.row = row;", BinaryRow.class, "row"); + final boolean projecting = projectionLiveNames != null; methodsNeedingImpl.forEach( (methodName, signatures) -> signatures.forEach( (methodType, method) -> { - if (method.isDefault()) { + // An absent field's default accessor returns the absent value, not its body. + boolean absentAccessor = + projecting && isAccessorOfAbsentField(methodName, methodType); + if (!absentAccessor && method.isDefault()) { return; } Object[] params = new Object[methodType.parameterCount() * 2]; @@ -422,16 +522,58 @@ private CodegenContext buildImplClass() { params[i * 2] = methodType.parameterType(i); params[i * 2 + 1] = "unused" + i; } - implClass.addMethod( - methodName, - "throw new UnsupportedOperationException();", - methodType.returnType(), - params); + String body; + if (absentAccessor) { + body = + "return " + + defaultValueExpression(methodType.returnType(), implClass) + + ";"; + } else { + body = "throw new UnsupportedOperationException();"; + } + implClass.addMethod(methodName, body, methodType.returnType(), params); })); return implClass; } + /** + * True when {@code methodName(returnType)} on the current bean class names a property whose field + * is not in the historical schema this projection codec is generating. Such a method gets a + * default-value body instead of {@code throw} or its interface {@code default} body, so the + * interface proxy can serve callers that don't know the field is missing in this version. + */ + private boolean isAccessorOfAbsentField(String methodName, MethodType methodType) { + // An accessor takes no arguments; the live-member pass above only removes the no-arg signature. + // A parameterized method sharing a name and return type with a descriptor is not that field's + // accessor, so it must still throw rather than be silenced into a default value. + if (methodType.parameterCount() != 0) { + return false; + } + // Look up by raw method name, not the wire-name conversion: this path runs only for interface + // beans (see buildImplClass), whose descriptors are keyed by the method names themselves. + Descriptor d = descriptorsMap.get(methodName); + if (d == null) { + return false; + } + // Match the raw return type, the same identity the live-field pass uses to remove an accessor + // from methodsNeedingImpl above. A method whose return type differs is a different overload, + // not this field's accessor, and must still throw. + if (d.getTypeRef().getRawType() != methodType.returnType()) { + return false; + } + // Name and return type match a descriptor, but the live-member loop did not emit it: the field + // is absent in this version. + return true; + } + + private static String defaultValueExpression(Class returnType, CodegenContext ctx) { + if (TypeUtils.isOptionalType(returnType)) { + return ctx.type(returnType) + ".empty()"; + } + return TypeUtils.defaultValue(returnType); + } + private Descriptor getDescriptorByFieldName(String fieldName) { String name = StringUtils.lowerUnderscoreToLowerCamelCase(fieldName); return descriptorsMap.get(name); diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowFactory.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowFactory.java new file mode 100644 index 0000000000..4e8e94f0cc --- /dev/null +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowFactory.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format.encoder; + +import org.apache.fory.format.row.binary.BinaryRow; + +/** + * Allocates fresh {@link BinaryRow} instances for a fixed schema. Obtained once per schema from + * {@link Encoding#newRowFactory}. The compact format captures its schema-derived layout (offsets, + * widths, nullability) in the factory so every {@link #newRow} call reuses it; the default format + * builds a {@link BinaryRow} directly per call, matching {@code BinaryRowWriter#newRow}. Either way + * the schema-evolution decode path holds one factory per historical schema, giving it the same + * per-decode cost as the current-schema path that reads through the writer's cached layout. + */ +@FunctionalInterface +interface RowFactory { + BinaryRow newRow(); +} diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/SchemaHistory.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/SchemaHistory.java new file mode 100644 index 0000000000..027d7386fc --- /dev/null +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/SchemaHistory.java @@ -0,0 +1,1186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format.encoder; + +import java.lang.reflect.AnnotatedElement; +import java.lang.reflect.Method; +import java.nio.charset.StandardCharsets; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import org.apache.fory.format.annotation.ForyRowSchema; +import org.apache.fory.format.annotation.ForyRowVersion; +import org.apache.fory.format.row.binary.writer.CompactBinaryRowWriter; +import org.apache.fory.format.type.CustomTypeEncoderRegistry; +import org.apache.fory.format.type.DataType; +import org.apache.fory.format.type.DataTypes; +import org.apache.fory.format.type.Field; +import org.apache.fory.format.type.Schema; +import org.apache.fory.format.type.TypeInference; +import org.apache.fory.reflect.TypeRef; +import org.apache.fory.type.Descriptor; +import org.apache.fory.type.TypeResolutionContext; +import org.apache.fory.type.TypeUtils; +import org.apache.fory.util.StringUtils; + +/** + * Resolves the version history of a row-codec bean. Each entry exposes the schema as it appeared at + * a particular version, along with a strict hash that uniquely identifies the historical layout. + * Only used when {@code withSchemaEvolution()} is configured on the codec builder. + * + *

    The hash mixes the row format, field names, and nullability in addition to types, so that two + * schemas that differ only in field order or naming are distinguishable, and a reader of one format + * rejects a payload of the other even when both formats write the same field order. This is + * intentionally a different hash from {@link DataTypes#computeSchemaHash} and is used only by + * versioning code paths. + * + *

    Every hash input is language-neutral so another runtime can reproduce the same value from the + * same declared schema: snake_case wire names, Fory type IDs, and the shape parameters a type ID + * does not carry (decimal precision/scale, fixed-size binary width). The algorithm is specified in + * {@code docs/specification/row_format_spec.md}. Versioned schemas additionally require the + * canonical field order — ascending by wire name — which for Java coincides with the member-name + * order the generated codecs bake in. Member names with an underscore are rejected at build time, + * and so are beans whose member names sort differently by wire name (a character outside the Basic + * Multilingual Plane against one above the surrogate range). + */ +final class SchemaHistory { + + /** Implicit version of a live field that carries no {@link ForyRowVersion}. */ + private static final int FIRST_VERSION = 1; + + /** FNV-1a 64-bit offset basis, the seed for every strict-hash mix in this class. */ + private static final long FNV_OFFSET_BASIS = 0xcbf29ce484222325L; + + /** + * A row format as schema evolution sees it: the tag that every strict hash mixes first, and the + * order in which the format writes the fields of each struct. The standard and compact formats + * write one schema in the same field order when the compact sort moves no field (every field 8 + * bytes wide, for example), so only the tag keeps their hashes apart. + */ + enum Format { + STANDARD(0), + COMPACT(1); + + private final long tag; + + Format(long tag) { + this.tag = tag; + } + + /** + * Return {@code schema} with the fields of every struct, at any depth, in the order this format + * writes them: unchanged for the standard format, and sorted at every depth, including structs + * in list elements and map entries, for the compact format. + */ + Schema wireOrder(Schema schema) { + return this == COMPACT ? CompactBinaryRowWriter.sortSchemaDeep(schema) : schema; + } + } + + /** One entry in a {@link SchemaHistory}. */ + static final class VersionedSchema { + private final int version; + private final Schema schema; + private final long strictHash; + private final boolean current; + private final Set liveFieldNames; + private final Map, VersionedSchema> nestedBeanSchemas; + private final Set> liveNestedClasses; + + VersionedSchema( + int version, + Schema schema, + long strictHash, + boolean current, + Set liveFieldNames, + Map, VersionedSchema> nestedBeanSchemas, + Set> liveNestedClasses) { + this.version = version; + this.schema = schema; + this.strictHash = strictHash; + this.current = current; + this.liveFieldNames = liveFieldNames; + this.nestedBeanSchemas = nestedBeanSchemas; + this.liveNestedClasses = liveNestedClasses; + } + + int version() { + return version; + } + + Schema schema() { + return schema; + } + + long strictHash() { + return strictHash; + } + + /** + * True when this entry is its bean's current (writer-side) schema. Routing uses this to decide + * whether a nested-bean slot embeds the current-version codec class (no suffix) or a historical + * projection class. + */ + boolean isCurrent() { + return current; + } + + /** + * Wire names of the fields in this version that a projection reads into the current bean: every + * live field, and every removed field that has a live field with the same name and declared + * type. A projection skips the slots of the other fields and does not read them. + */ + Set liveFieldNames() { + return liveFieldNames; + } + + /** + * For each nested versioned bean class referenced by this schema, the exact inner entry chosen + * for this combination. Empty when the schema has no nested versioned beans. Each value carries + * its own {@code strictHash} and {@code nestedBeanSchemas}, so routing can identify and recurse + * into the inner subtree to arbitrary depth without re-deriving it from a version number. + * + *

    Keyed by class, not by field. A writer writes one definition of a given bean class, so + * every field of that class in a single payload is at the same version; the enumeration carries + * one entry per class, and a class may back more than one field. + */ + Map, VersionedSchema> nestedBeanSchemas() { + return nestedBeanSchemas; + } + + /** + * The keys of {@link #nestedBeanSchemas()} that a live field reaches. A class reached only + * through removed fields shapes the hash, but projection skips removed slots and never decodes + * it, so it needs no codec. + */ + Set> liveNestedClasses() { + return liveNestedClasses; + } + } + + private final List versions; + private final VersionedSchema current; + + private SchemaHistory(List versions, VersionedSchema current) { + this.versions = versions; + this.current = current; + } + + /** + * The writer-side schema (latest version, all nested beans current). This is the same {@link + * VersionedSchema} instance that also appears in {@link #versions()}, so callers building + * per-version projection codecs may skip it with a reference identity check ({@code vs == + * current()}) rather than comparing hashes. + */ + VersionedSchema current() { + return current; + } + + /** + * All distinct historical schemas, in build order: outer version ascending, and within an outer + * version, one entry per nested cross-product combination. This is not a strict sort by {@link + * VersionedSchema#version()} — collapsing field-set duplicates can place a later combination in + * an earlier slot. Dispatch keys on the strict hash, so callers must not rely on positional + * order. + */ + List versions() { + return versions; + } + + /** + * Build a history from the bean's annotations. The schema for each version is put in the wire + * order of {@code format} and hashed with its tag. + */ + static SchemaHistory build(Class beanClass, Format format) { + return build(beanClass, format, new HashMap<>()); + } + + /** + * Build a history, reusing {@code built} for any bean class already enumerated in this build, so + * a bean reached through several field paths (a diamond in the type graph, or two fields of the + * same type) is enumerated once rather than once per path. The memo is per top-level build, not + * static: the format (standard vs compact) and registry state must not leak across builds. + */ + private static SchemaHistory build( + Class beanClass, Format format, Map, SchemaHistory> built) { + if (built.containsKey(beanClass)) { + SchemaHistory memoized = built.get(beanClass); + // A null memo entry marks a build in progress: this bean's history reaches back to itself, + // which only removed-field declarations can do (live-field cycles are rejected by + // inferSchema's cycle check before build() runs). The historical schema would be cyclic. + if (memoized == null) { + throw new IllegalStateException( + "Circular schema history for " + + beanClass.getName() + + ": a removed field reaches back to a bean whose history is still being built. " + + "Row format does not support circular references."); + } + return memoized; + } + built.put(beanClass, null); + List all = collectLiveFields(beanClass); + // A subclass's History interface can extend its parent's, so the same declaration can arrive + // from both classes. Keep one copy of each, so an inherited removal is not an overlapping one. + List removed = new ArrayList<>(); + for (Class removedFieldsClass : removedFieldsClasses(beanClass)) { + for (FieldEntry fe : collectRemovedFields(removedFieldsClass)) { + if (!containsSameDeclaration(removed, fe)) { + removed.add(fe); + } + } + } + all.addAll(removed); + + // Infer each field once and recursively expand the history of every versioned bean it reaches. + // A versioned bean can be the field type directly, the element of a list, or the key or value + // of a map; we locate it at any of those sites so the outer's enumeration can cross-product + // over the inner's versions. A map contributes a key site and a value site independently. The + // inner schema substitutes back into the same site at materialization time. + // + // Live-field cycles are rejected before build() runs: TypeInference.inferField calls + // ctx.checkNoCycle on every bean it descends into, and RowCodecBuilder runs inferSchema in its + // constructor. Removed fields never pass through inferSchema, so a cycle reachable only through + // a removed-field declaration is caught by the in-progress memo entry above instead. + for (FieldEntry fe : all) { + inferEntry(beanClass, fe, format, built); + } + + validateNoNameCollision(beanClass, all, format); + SchemaHistory history = enumerate(beanClass.getName(), all, format); + built.put(beanClass, history); + return history; + } + + /** + * History of a top-level array element or map entry field, which has no version annotations of + * its own but may reach versioned beans through its list/map/array wrappers (a directly-typed + * bean, a list element, or a map key or value, to any depth). The single element field is + * enumerated over the cross-product of every reachable bean's versions, exactly as a bean field + * is on the row path, so the element schema's strict hash identifies all nested layouts jointly + * and an older payload restores every nested bean — on either map side — at its historical + * layout. + */ + static SchemaHistory forElement(String fieldName, TypeRef elementType, Format format) { + FieldEntry element = + new FieldEntry(fieldName, fieldName, elementType, FIRST_VERSION, Integer.MAX_VALUE, true); + // A top-level element has no enclosing bean; codec lookups behave as inferField's do at the + // top level, where only globally-registered (enclosing-agnostic) codecs can match. + inferEntry(void.class, element, format, new HashMap<>()); + List all = new ArrayList<>(1); + all.add(element); + return enumerate("element " + elementType, all, format); + } + + /** + * Enumerate every distinct historical schema for {@code all} over its version boundaries and the + * cross-product of nested bean versions, returning a {@link SchemaHistory}. {@code label} names + * the owner for collision diagnostics. Each entry's field is already inferred. + */ + private static SchemaHistory enumerate(String label, List all, Format format) { + // Materialize a schema at every version V where the field set changes — both "since" and + // "until" boundaries qualify, because either adds or removes a field from the active set. + // FIRST_VERSION is always materialized even when every field declares since >= 2: a payload + // written before those fields existed carries the v1 layout, so that schema must be decodable. + TreeSet schemaVersions = new TreeSet<>(); + schemaVersions.add(FIRST_VERSION); + for (FieldEntry fe : all) { + schemaVersions.add(fe.since); + if (fe.until != Integer.MAX_VALUE) { + schemaVersions.add(fe.until); + } + } + + // Sort by Java member name so the per-version schema matches the order + // TypeInference.inferSchema produces (which iterates Descriptor.getDescriptors, alphabetical + // by Java member name). + // The canonical cross-language order is ascending by wire name; checkCanonicalOrder below + // verifies per version that the member-name order coincides with it, so the emitted schemas + // comply with the spec while staying byte-identical to the generated codec's baked order. + all.sort((a, b) -> a.javaName.compareTo(b.javaName)); + // A field with finite [since, until) can leave two boundaries with identical field sets + // (e.g. v1 and v4 both lack a field that lived in [v2, v4)). Collapse boundaries that + // produce the same layout into one VersionedSchema, since they round-trip identically. + // The strict hash is the layout identity, so it is the dedup key. Schema.equals is weaker: it + // ignores list element nullability and fixed-size binary width, so it would merge a + // List layout with an int[] layout and drop one of their hashes. The guard on + // hashToSchema below rejects most real strict-hash collisions (two distinct schemas with one + // hash), but not all of them. + int latestVersion = schemaVersions.last(); + Map byHash = new LinkedHashMap<>(); + // A removed field that a live field adds again with the same wire name and declared type holds + // the same data, so the projection reads it in every older layout. Reading it only where its + // layout happens to collapse with a version that has the field live would make the result + // depend on unrelated fields. A removed field with another declared type is discarded; + // validateNoTypeChange already rejected the case where both types encode to the same row field. + Map> liveTypes = new HashMap<>(); + for (FieldEntry fe : all) { + if (fe.live) { + liveTypes.put(fe.name, fe.typeRef); + } + } + Set readEntries = new HashSet<>(); + for (FieldEntry fe : all) { + if (fe.live || fe.typeRef.equals(liveTypes.get(fe.name))) { + readEntries.add(fe); + } + } + // Versions that collapse into one entry read the same fields: a removed entry is read only with + // a live twin of the same declared type, and validateNoTypeChange rejects every other type + // change whose layout could coincide with a newer one. + Map hashToSchema = new HashMap<>(); + Long currentHash = null; + for (int v : schemaVersions) { + List activeEntries = new ArrayList<>(); + for (FieldEntry fe : all) { + if (fe.since <= v && v < fe.until) { + activeEntries.add(fe); + } + } + checkCanonicalOrder(label, v, activeEntries); + // Cross-product over each nested versioned bean *class*, not each field. A writer always + // writes one definition of a given bean class, so every field of that class in a single + // payload is at the same version; the off-diagonal combinations (the same class at two + // versions in one record) are unreachable on the wire. Enumerating one dimension per class + // keeps the count a product over distinct nested classes rather than over fields, and lets + // a class appear in more than one field. A class also reached through another nested bean's + // own history is held to that same version (see cartesian). If no entries have nested + // histories, this yields a single combination. + // + // The combination count is the product of the per-class version counts. There is no cap on + // purpose: what a host can afford ranges from a microcontroller to a large server, so no one + // limit fits. If the growth becomes a concern, drop entries from each bean's History + // interface once you no longer need to read payloads from that range — that removes the + // corresponding VersionedSchema from this enumeration. Retiring history entries is purely a + // read-side concern; the writer always uses the current schema. + LinkedHashMap, List> innerChoices = new LinkedHashMap<>(); + Set> liveNestedClasses = new HashSet<>(); + for (FieldEntry fe : activeEntries) { + for (NestedSite site : fe.nestedSites) { + innerChoices.putIfAbsent(site.beanClass, site.history.versions()); + if (readEntries.contains(fe)) { + liveNestedClasses.add(site.beanClass); + } + } + } + Set> liveNested = Collections.unmodifiableSet(liveNestedClasses); + for (Map, VersionedSchema> combination : cartesian(innerChoices)) { + List fields = new ArrayList<>(activeEntries.size()); + Set liveNames = new HashSet<>(); + for (FieldEntry fe : activeEntries) { + // Substitute each nested versioned site (a direct field, list element, map value, or map + // key) with the version this combination chose for its bean class, keeping every + // collection/map wrapper intact. A map can have both a key site and a value site, which + // are substituted independently. + Field field = fe.field; + for (NestedSite site : fe.nestedSites) { + VersionedSchema innerVs = combination.get(site.beanClass); + field = site.substitute(field, new DataTypes.StructType(innerVs.schema().fields())); + } + fields.add(field); + if (readEntries.contains(fe)) { + liveNames.add(fe.name); + } + } + Schema schema = format.wireOrder(new Schema(fields)); + long hash = computeStrictSchemaHash(format, schema); + // Schema.equals ignores list element nullability and fixed-size binary width, so this guard + // misses a collision between two schemas that differ only there. A collision has odds of + // about 2^-64, so the gap is accepted. + Schema previousSchema = hashToSchema.putIfAbsent(hash, schema); + if (previousSchema != null && !previousSchema.equals(schema)) { + throw new IllegalStateException( + "Strict hash collision for " + + label + + " at version " + + v + + ": two distinct historical schemas hashed to the same value. Please file an " + + "issue with the bean definition."); + } + // This combination represents the writer-side configuration at outer version v only when + // every chosen inner is itself that inner's current schema. The bean's own current schema + // is the writer-side configuration at the latest version. + boolean innerAllCurrent = true; + for (VersionedSchema inner : combination.values()) { + if (!inner.isCurrent()) { + innerAllCurrent = false; + break; + } + } + boolean isCurrent = v == latestVersion && innerAllCurrent; + VersionedSchema vs = + new VersionedSchema( + v, + schema, + hash, + isCurrent, + Collections.unmodifiableSet(liveNames), + Collections.unmodifiableMap(new HashMap<>(combination)), + liveNested); + // Versions collapse when their layouts are identical: v1 and v4 of a field that lived in + // [v2, v4), or a field removed and later added again. The latest version must win, so the + // stored entry is the one flagged current. Enclosing histories test inner.isCurrent() to + // find their own current combination, and fail to build if no inner entry is current. + if (innerAllCurrent) { + byHash.put(hash, vs); + } else { + byHash.putIfAbsent(hash, vs); + } + if (isCurrent) { + currentHash = hash; + } + } + } + // The all-current combination at the latest version is always one of the cartesian entries, + // so currentHash is always set and present here. + VersionedSchema current = byHash.get(currentHash); + if (current == null) { + throw new IllegalStateException("No current schema resolved for " + label); + } + return new SchemaHistory( + Collections.unmodifiableList(new ArrayList<>(byHash.values())), current); + } + + /** + * Cartesian product over (nested bean class, list-of-inner-VersionedSchema), keeping only the + * combinations that put every class at one version. A class reached both directly and through + * another chosen inner schema is at the same version in both places on the wire, because a writer + * has one definition of each class. Each class has one history per build (the build memo), so its + * entries compare by identity. + */ + private static List, VersionedSchema>> cartesian( + LinkedHashMap, List> choices) { + List, VersionedSchema>> out = new ArrayList<>(); + // Parallel to out: every class each partial combination fixes, at any depth. + List, VersionedSchema>> pinned = new ArrayList<>(); + out.add(new HashMap<>()); + pinned.add(new HashMap<>()); + for (Map.Entry, List> choice : choices.entrySet()) { + Class cls = choice.getKey(); + List options = choice.getValue(); + List, VersionedSchema>> next = new ArrayList<>(out.size() * options.size()); + List, VersionedSchema>> nextPinned = new ArrayList<>(next.size()); + for (int i = 0; i < out.size(); i++) { + for (VersionedSchema opt : options) { + Map, VersionedSchema> fixed = new HashMap<>(pinned.get(i)); + if (!pin(fixed, cls, opt)) { + continue; + } + Map, VersionedSchema> extended = new HashMap<>(out.get(i)); + extended.put(cls, opt); + next.add(extended); + nextPinned.add(fixed); + } + } + out = next; + pinned = nextPinned; + } + return out; + } + + /** + * Whether {@code first} and {@code second}, which can come from separate builds, put every bean + * class that both reach, at any depth, at the same version. Separate builds hold separate entries + * for one version, so this compares versions by strict hash, which is unique within a history. + */ + static boolean sameClassVersions(VersionedSchema first, VersionedSchema second) { + Map, Long> versions = new HashMap<>(); + return collectVersions(first, versions) && collectVersions(second, versions); + } + + private static boolean collectVersions(VersionedSchema vs, Map, Long> versions) { + for (Map.Entry, VersionedSchema> nested : vs.nestedBeanSchemas().entrySet()) { + Long prior = versions.putIfAbsent(nested.getKey(), nested.getValue().strictHash()); + if (prior != null && prior != nested.getValue().strictHash()) { + return false; + } + if (!collectVersions(nested.getValue(), versions)) { + return false; + } + } + return true; + } + + /** + * Fix {@code cls} at {@code vs} in {@code fixed}, together with every class that {@code vs} fixes + * at any depth. Returns false when one of them is already fixed at another version. + */ + private static boolean pin( + Map, VersionedSchema> fixed, Class cls, VersionedSchema vs) { + VersionedSchema prior = fixed.putIfAbsent(cls, vs); + if (prior != null) { + return prior == vs; + } + for (Map.Entry, VersionedSchema> nested : vs.nestedBeanSchemas().entrySet()) { + if (!pin(fixed, nested.getKey(), nested.getValue())) { + return false; + } + } + return true; + } + + /** + * A representative bean reachable from a top-level array element or map position, which names and + * places the generated projection codecs: the first bean that the element type reaches when the + * value side of every map comes first, or null when no bean is reachable; then the position has + * one version and needs no projection. The per-version enumeration is done by {@link + * #forElement}, which sites every reachable bean, so the choice here only names the codec. Do not + * derive codec structure from it. + */ + static Class evolutionBean(TypeRef elementType) { + // The descent reaches a map's key before its value, so its last bean is the value-first one. + List> beans = new ArrayList<>(); + TypeInference.inferNamedField( + "", + elementType, + void.class, + (leafType, path) -> { + if (isBean(leafType)) { + beans.add(leafType); + } + }); + return beans.isEmpty() ? null : beans.get(beans.size() - 1); + } + + /** + * A versioned bean reachable from a field, together with the branch taken at each map on the way + * down so the historical struct substitutes back into exactly that leaf. The path lists one entry + * per map crossed, in order from the field root; list/array wrappers add no entry because they + * have a single element leaf. A map field contributes a key site and a value site independently, + * each with its own path; every other field shape has at most one site. + */ + private static final class NestedSite { + final Class beanClass; + final SchemaHistory history; + final List path; + + NestedSite(Class beanClass, SchemaHistory history, List path) { + this.beanClass = beanClass; + this.history = history; + this.path = path; + } + + /** + * Replace this site's bean struct in {@code current} with {@code historical}, descending list + * elements and taking {@code path[depth]} at each map, while leaving every other leaf intact so + * independent sites (a map's key and value, or beans under different map sides at any depth) + * substitute without disturbing one another. {@code current} is the inferred field, whose shape + * follows the descent that found the site: a custom codec or an Optional adds no level, an + * array or iterable is a list, a map is a map, and the bean is the struct at the end. + */ + Field substitute(Field current, DataTypes.StructType historical) { + return substitute(current, historical, 0); + } + + private Field substitute(Field current, DataTypes.StructType historical, int depth) { + DataType type = current.type(); + if (type instanceof DataTypes.ListType) { + Field element = substitute(DataTypes.arrayElementField(current), historical, depth); + return DataTypes.arrayField(current.name(), element); + } + if (type instanceof DataTypes.MapType) { + Field keyField = DataTypes.keyFieldForMap(current); + Field itemField = DataTypes.itemFieldForMap(current); + if (path.get(depth) == TypeInference.MapBranch.KEY) { + keyField = substitute(keyField, historical, depth + 1); + } else { + itemField = substitute(itemField, historical, depth + 1); + } + // Rebuild through MapType, which keeps the key non-nullable: a substituted key that is a + // list or map comes back nullable, and DataTypes.mapField rejects a nullable key. + DataTypes.MapType mapType = (DataTypes.MapType) type; + return new Field( + current.name(), + new DataTypes.MapType(keyField.type(), itemField.type(), mapType.keysSorted()), + current.nullable()); + } + return DataTypes.field(current.name(), historical, current.nullable()); + } + } + + /** + * Infer {@code fe}'s row field and collect the versioned beans that its type reaches: the field + * type itself, a list/array element, a map value, and a map key, to arbitrary nesting depth. Each + * site records the map-branch path needed to substitute its historical struct back. A map + * contributes a key site and a value site independently, so an evolving key and an evolving value + * each become their own cross-product dimension. {@code enclosing} is the class that declares the + * field ({@code void.class} for a top-level element), so custom-codec lookups match the live + * inference path. + */ + private static void inferEntry( + Class enclosing, FieldEntry fe, Format format, Map, SchemaHistory> built) { + fe.field = + TypeInference.inferNamedField( + fe.name, + fe.typeRef, + enclosing, + (leafType, path) -> { + if (!isBean(leafType)) { + return; + } + // A bean is an evolution site when it has more than one version, whether the + // variation comes from its own @ForyRowVersion fields or only from a nested versioned + // bean it reaches through its fields (e.g. a wrapper struct used as a map key, whose + // own fields are stable but whose detail field evolves). build() already expands that + // nested cross-product, so its version count is the exact evolution test; a + // single-version history means nothing here evolves and the site needs no projection. + SchemaHistory history = build(leafType, format, built); + if (history.versions().size() > 1) { + fe.nestedSites.add(new NestedSite(leafType, history, path)); + } + }); + } + + /** + * True if the row format treats {@code cls} as a bean, so its history is built for evolution-site + * discovery. Whether the bean actually evolves is decided by its version count in {@link + * #inferEntry}, not here. + * + *

    The inference descent reports every leaf that is not a scalar, so this skips a type that the + * row format does not support; inference then fails on it as it does on the encode path. Use the + * same synthesize-interfaces context as inferField, so an interface bean nested as a field type, + * list element, or map key/value is discovered as a bean rather than rejected; otherwise its + * older versions are never enumerated and an older payload decodes at the interface's current + * layout. The context is a fresh one, not the descent's own: a bean on the descent's walked path + * is not a bean there, and build() must see it to report a cycle through a removed field. + */ + private static boolean isBean(Class cls) { + return TypeUtils.isBean( + cls, new TypeResolutionContext(CustomTypeEncoderRegistry.customTypeHandler(), true)); + } + + /** + * The {@link ForyRowSchema#removedFields} classes declared on {@code beanClass} and every type it + * inherits live fields from: its superclasses, or for an interface bean its superinterfaces. Live + * fields come from the whole hierarchy, so the removed fields must too, or a subclass codec would + * omit the slots a parent removed and reject the parent's older payloads. + */ + private static Set> removedFieldsClasses(Class beanClass) { + Set> out = new LinkedHashSet<>(); + Set> visited = new HashSet<>(); + ArrayDeque> pending = new ArrayDeque<>(); + pending.add(beanClass); + while (!pending.isEmpty()) { + Class cls = pending.poll(); + if (!visited.add(cls)) { + continue; + } + ForyRowSchema schemaAnn = cls.getAnnotation(ForyRowSchema.class); + if (schemaAnn != null && schemaAnn.removedFields() != void.class) { + Class removedFields = schemaAnn.removedFields(); + // The removed fields are the descriptors of this type, and only an interface has its + // methods as descriptors. A class would yield its fields, so its methods would be ignored. + if (!removedFields.isInterface()) { + throw new IllegalStateException( + "@ForyRowSchema(removedFields = " + + removedFields.getName() + + ") on " + + cls.getName() + + " must name an interface: removed fields are declared as its accessor methods."); + } + out.add(removedFields); + } + if (cls.isInterface()) { + pending.addAll(Arrays.asList(cls.getInterfaces())); + } else if (cls.getSuperclass() != null && cls.getSuperclass() != Object.class) { + pending.add(cls.getSuperclass()); + } + } + return out; + } + + private static boolean containsSameDeclaration(List entries, FieldEntry fe) { + for (FieldEntry other : entries) { + if (other.name.equals(fe.name) + && other.since == fe.since + && other.until == fe.until + && other.typeRef.equals(fe.typeRef)) { + return true; + } + } + return false; + } + + private static List collectRemovedFields(Class historyClass) { + List descriptors = Descriptor.getDescriptors(historyClass); + checkVersionedMethodsRead(historyClass, descriptors); + List out = new ArrayList<>(descriptors.size()); + for (Descriptor d : descriptors) { + ForyRowVersion ann = lookupForyVersion(d); + if (ann == null) { + throw new IllegalStateException( + "Removed-field declaration " + + historyClass.getName() + + "." + + d.getName() + + " requires a @ForyRowVersion(until = ...) annotation"); + } + if (ann.until() == Integer.MAX_VALUE) { + throw new IllegalStateException( + "Removed-field declaration " + + historyClass.getName() + + "." + + d.getName() + + " must specify @ForyRowVersion.until (no upper bound makes no sense for a field " + + "that has been removed)"); + } + if (ann.since() < FIRST_VERSION) { + throw new IllegalStateException( + "Invalid @ForyRowVersion on " + + historyClass.getName() + + "." + + d.getName() + + ": since (" + + ann.since() + + ") must be >= " + + FIRST_VERSION + + " (the first schema version). A since below that adds a version no writer can " + + "emit."); + } + if (ann.since() >= ann.until()) { + throw new IllegalStateException( + "Invalid @ForyRowVersion on " + + historyClass.getName() + + "." + + d.getName() + + ": since (" + + ann.since() + + ") must be strictly less than until (" + + ann.until() + + ")"); + } + // The history method's name must mirror the live field/method name. Wire names are + // derived the same way the live path derives them: descriptor name -> lower_underscore. + // For a class bean or a record the descriptor name is the field name ("tags"); for an + // interface bean it is the accessor method name ("getTags"). The user writes the history + // method to match. + String wireName = wireName(historyClass, d); + out.add( + new FieldEntry( + wireName, d.getName(), d.getTypeRef(), ann.since(), ann.until(), /*live*/ false)); + } + return out; + } + + private static List collectLiveFields(Class beanClass) { + List descriptors = Descriptor.getDescriptors(beanClass); + checkVersionedMethodsRead(beanClass, descriptors); + List out = new ArrayList<>(descriptors.size()); + for (Descriptor d : descriptors) { + ForyRowVersion ann = lookupForyVersion(d); + int since = ann == null ? FIRST_VERSION : ann.since(); + int until = ann == null ? Integer.MAX_VALUE : ann.until(); + if (since < FIRST_VERSION) { + throw new IllegalStateException( + "Invalid @ForyRowVersion on " + + beanClass.getName() + + "." + + d.getName() + + ": since (" + + since + + ") must be >= " + + FIRST_VERSION + + " (the first schema version). A since below that adds a version no writer can " + + "emit."); + } + // until is exclusive and Integer.MAX_VALUE is its open bound, so a window that starts there + // is empty. Every live field would leave the latest version, and the current schema would be + // empty. + if (since == Integer.MAX_VALUE) { + throw new IllegalStateException( + "Invalid @ForyRowVersion on " + + beanClass.getName() + + "." + + d.getName() + + ": since must be less than Integer.MAX_VALUE, the open upper bound of every " + + "version window."); + } + // A live field still exists as a Java member, so it has no end-of-life version. A finite + // until would silently drop it from the current schema (until extends the version set, so + // latestVersion >= until excludes the field), and the writer would stop serializing a field + // the bean still has. Removals are declared on the history class via + // @ForyRowSchema.removedFields. + if (until != Integer.MAX_VALUE) { + throw new IllegalStateException( + "Invalid @ForyRowVersion on " + + beanClass.getName() + + "." + + d.getName() + + ": a live field must not set until (" + + until + + "). Declare removed fields on the @ForyRowSchema.removedFields history class instead."); + } + // No since/until ordering check here: a live field always has until == MAX_VALUE (enforced + // above), so the ordering check lives only on the removed-field path in collectRemovedFields. + String wireName = wireName(beanClass, d); + out.add(new FieldEntry(wireName, d.getName(), d.getTypeRef(), since, until, /*live*/ true)); + } + return out; + } + + /** + * The wire name of member {@code d} of {@code type}. The generated codec finds the member of a + * wire name by removing its underscores and capitalizing a lowercase letter after each one + * ({@code a_b} to {@code aB}), so a member name with an underscore would bind to another member + * ({@code a_} to {@code a}) or to none. Reject it. + */ + private static String wireName(Class type, Descriptor d) { + String name = d.getName(); + if (name.indexOf('_') >= 0) { + throw new IllegalStateException( + "Member '" + + name + + "' of " + + type.getName() + + " has an underscore in its name, which schema evolution rejects. Rename it, for " + + "example to lowerCamelCase."); + } + return StringUtils.lowerCamelToLowerUnderscore(name); + } + + private static ForyRowVersion lookupForyVersion(Descriptor d) { + ForyRowVersion ann = readAnnotation(d.getField()); + if (ann != null) { + return ann; + } + return readAnnotation(d.getReadMethod()); + } + + private static ForyRowVersion readAnnotation(AnnotatedElement element) { + return element == null ? null : element.getAnnotation(ForyRowVersion.class); + } + + /** + * Reject a {@link ForyRowVersion} on a method of {@code type} or the supertypes that it inherits + * descriptors from (its superclasses, or for an interface its superinterfaces) that is not the + * read method of one of its descriptors, such as a getter of a field-backed bean or an interface + * method that takes parameters. {@link #lookupForyVersion} reads only the descriptor field and + * read method, so such an annotation would have no effect. The interfaces of a class are not + * checked: their annotations belong to the codecs of those interface beans. + */ + private static void checkVersionedMethodsRead(Class type, List descriptors) { + Set readMethods = new HashSet<>(); + for (Descriptor d : descriptors) { + if (d.getReadMethod() != null) { + readMethods.add(d.getReadMethod()); + } + } + Set> visited = new HashSet<>(); + ArrayDeque> pending = new ArrayDeque<>(); + pending.add(type); + while (!pending.isEmpty()) { + Class cls = pending.poll(); + if (cls == Object.class || !visited.add(cls)) { + continue; + } + for (Method method : cls.getDeclaredMethods()) { + // A bridge method carries the annotations of the method that it forwards to. + if (!method.isBridge() + && method.isAnnotationPresent(ForyRowVersion.class) + && !readMethods.contains(method)) { + throw new IllegalStateException( + "@ForyRowVersion on " + + cls.getName() + + "." + + method.getName() + + "() has no effect on " + + type.getName() + + ": schema evolution reads it only from fields, record components, and the " + + "accessor methods of an interface. On a class, annotate the field, not its " + + "getter."); + } + } + if (cls.isInterface()) { + pending.addAll(Arrays.asList(cls.getInterfaces())); + } else if (cls.getSuperclass() != null) { + pending.add(cls.getSuperclass()); + } + } + } + + private static void validateNoNameCollision( + Class beanClass, List entries, Format format) { + // For each pair with the same name, their [since, until) windows must not overlap. + Map> byName = new HashMap<>(); + for (FieldEntry fe : entries) { + byName.computeIfAbsent(fe.name, k -> new ArrayList<>()).add(fe); + } + for (Map.Entry> e : byName.entrySet()) { + List group = e.getValue(); + if (group.size() < 2) { + continue; + } + group.sort((a, b) -> Integer.compare(a.since, b.since)); + for (int i = 1; i < group.size(); i++) { + FieldEntry prev = group.get(i - 1); + FieldEntry curr = group.get(i); + if (curr.since < prev.until) { + throw new IllegalStateException( + "Field name '" + + e.getKey() + + "' is declared with overlapping version windows [" + + prev.since + + "," + + prev.until + + ") and [" + + curr.since + + "," + + curr.until + + "); each version must have one definition per name. Adjust the @ForyRowVersion " + + "annotations on the live field or in the removed-fields class to make the " + + "windows disjoint."); + } + } + validateNoTypeChange(beanClass, e.getKey(), group, format); + } + } + + /** + * Reject a removed entry whose declared type differs from the live entry's when an older layout + * could be byte-identical to a newer one, so the reader would decode old values with the new type + * (a String re-added as an enum fails in valueOf). That holds when both encode to the same row + * field in the wire order of {@code format} (the same strict hash; the compact sort can put the + * children of two codecs that write one struct in different orders into one order, but children + * of equal width keep the codec's order), and also when either reaches a versioned bean, whose + * older versions can make two different current shapes coincide. Otherwise the layouts stay + * distinct and the reader discards the old value. + */ + private static void validateNoTypeChange( + Class beanClass, String name, List group, Format format) { + FieldEntry live = null; + for (FieldEntry fe : group) { + if (fe.live) { + live = fe; + } + } + if (live == null) { + return; + } + long liveHash = computeStrictSchemaHash(format, format.wireOrder(DataTypes.schema(live.field))); + for (FieldEntry fe : group) { + if (fe.live || fe.typeRef.equals(live.typeRef)) { + continue; + } + boolean reachesVersionedBean = !fe.nestedSites.isEmpty() || !live.nestedSites.isEmpty(); + if (reachesVersionedBean + || computeStrictSchemaHash(format, format.wireOrder(DataTypes.schema(fe.field))) + == liveHash) { + throw new IllegalStateException( + "Field '" + + name + + "' of " + + beanClass.getName() + + " is declared as " + + fe.typeRef + + " in the removed-fields class and as " + + live.typeRef + + " on the live member. An older layout can then be identical to a newer one, so " + + "old values would decode as the new type. Give the live member a new name."); + } + } + } + + /** + * Strict schema hash of {@code schema}, whose fields are already in the wire order of {@code + * format}; used only by versioning code paths. Distinguishes schemas that differ in row format, + * field name, or nullability, unlike {@link DataTypes#computeSchemaHash}. Every input is + * language-neutral so another runtime reproduces the value from the same declared schema: the + * format tag, wire names as UTF-8 bytes, Fory type IDs, and the shape parameters a type ID does + * not carry. List and map children have positions fixed by the parent type, so only struct-owned + * fields mix a name. The algorithm is specified in {@code docs/specification/row_format_spec.md}; + * do not change it without updating the spec. + */ + static long computeStrictSchemaHash(Format format, Schema schema) { + long hash = mix(FNV_OFFSET_BASIS, format.tag); + for (Field field : schema.fields()) { + hash = hashNamedField(hash, field); + } + return hash; + } + + private static long hashNamedField(long hash, Field field) { + hash = mix(hash, field.name()); + return hashFieldShape(hash, field); + } + + private static long hashFieldShape(long hash, Field field) { + DataType type = field.type(); + // The Fory type ID identifies every type except the parameterized ones: TYPE_BINARY covers + // both variable and fixed-size binary, so the byte width follows the ID (0 = variable), and a + // decimal's precision and scale are not part of its ID either. + hash = mix(hash, type.typeId()); + if (type instanceof DataTypes.BinaryType) { + hash = mix(hash, 0); + } else if (type instanceof DataTypes.FixedWidthBinaryType) { + hash = mix(hash, type.byteWidth()); + } else if (type instanceof DataTypes.DecimalType) { + hash = mix(hash, ((DataTypes.DecimalType) type).precision()); + hash = mix(hash, ((DataTypes.DecimalType) type).scale()); + } + hash = mix(hash, field.nullable() ? 1 : 0); + if (type instanceof DataTypes.ListType) { + hash = hashFieldShape(hash, DataTypes.arrayElementField(field)); + } else if (type instanceof DataTypes.MapType) { + hash = hashFieldShape(hash, DataTypes.keyFieldForMap(field)); + hash = hashFieldShape(hash, DataTypes.itemFieldForMap(field)); + } else if (type instanceof DataTypes.StructType) { + // Mix the child count before recursing. Unlike list and map, whose arity is fixed by the + // type kind, a struct has a variable number of children with no boundary marker between a + // nested struct's last child and the parent's next field. Without the count, {a:struct,b} + // and {a:struct} mix an identical byte sequence and collide. The count delimits the + // struct's extent so the hash stays injective over nesting structure. + List children = type.fields(); + hash = mix(hash, children.size()); + for (Field child : children) { + hash = hashNamedField(hash, child); + } + } + return hash; + } + + /** + * Verify that {@code active} — one version's field entries, already sorted by Java member name — + * is also strictly ascending by wire name, the canonical cross-language order versioned schemas + * are defined in. Member names have no underscore ({@link #wireName}), so the two orders diverge + * only where {@link String#compareTo}, which compares UTF-16 code units, disagrees with code + * point order: a character outside the Basic Multilingual Plane against one above the surrogate + * range. Java cannot emit the canonical order then, because the generated codec bakes member-name + * order, so reject the bean with rename guidance. Duplicate wire names within one version are + * unreachable here ({@link #validateNoNameCollision} forces their windows apart), but the strict + * comparison covers them all the same. + */ + private static void checkCanonicalOrder(String label, int version, List active) { + for (int i = 1; i < active.size(); i++) { + FieldEntry prev = active.get(i - 1); + FieldEntry curr = active.get(i); + if (compareWireNames(prev.name, curr.name) >= 0) { + throw new IllegalStateException( + "Version " + + version + + " of " + + label + + " orders member '" + + prev.javaName + + "' (wire name '" + + prev.name + + "') before '" + + curr.javaName + + "' (wire name '" + + curr.name + + "'), but schema evolution orders fields by wire name, which requires '" + + curr.name + + "' to sort strictly after '" + + prev.name + + "'. Rename one of the members so the member-name order and the wire-name order " + + "agree. The orders differ for a character outside the Basic Multilingual Plane."); + } + } + } + + /** + * Compare wire names by Unicode code point, equivalent to byte-wise comparison of their UTF-8 + * encodings — the order the spec defines. {@link String#compareTo} would misorder supplementary + * characters against BMP characters above the surrogate range. + */ + private static int compareWireNames(String a, String b) { + int i = 0; + int limit = Math.min(a.length(), b.length()); + while (i < limit) { + int cpA = a.codePointAt(i); + int cpB = b.codePointAt(i); + if (cpA != cpB) { + return Integer.compare(cpA, cpB); + } + // Equal code points consume equal char counts, so one index serves both strings. + i += Character.charCount(cpA); + } + return Integer.compare(a.length(), b.length()); + } + + /** + * Combine two strict hashes into one 64-bit value with the same FNV-1a mix used for schema + * hashing, so a map header carrying a single hash can identify a (key, value) layout combination + * jointly. Order-sensitive: {@code combineHashes(a, b) != combineHashes(b, a)}. + */ + static long combineHashes(long first, long second) { + return mix(mix(FNV_OFFSET_BASIS, first), second); + } + + private static long mix(long hash, long value) { + hash ^= value; + hash *= 1099511628211L; // FNV prime + return hash; + } + + private static long mix(long hash, String value) { + // A wire name mixes as its UTF-8 bytes followed by a zero terminator, so a runtime that + // stores names as raw bytes reproduces the hash without a UTF-16 round trip. + byte[] utf8 = value.getBytes(StandardCharsets.UTF_8); + for (byte b : utf8) { + hash = mix(hash, b & 0xFFL); + } + return mix(hash, 0); + } + + private static final class FieldEntry { + final String name; + + /** + * Java member name used for canonical ordering. Matches {@link Descriptor#getName} so live + * fields and removed fields (declared on the history class) sort into the same order as {@link + * TypeInference#inferSchema} produces. + */ + final String javaName; + + final TypeRef typeRef; + final int since; + final int until; + final boolean live; + + /** The row field inferred from {@link #typeRef}, set by {@link #inferEntry}. */ + Field field; + + /** + * The versioned beans reachable from this field (the field type, a list element, a map value, + * or a map key), empty when none. Each site keys the outer cross-product by its bean class, so + * every field backed by the same class shares one version dimension; a map field with an + * evolving key and value contributes two sites. + */ + final List nestedSites = new ArrayList<>(2); + + FieldEntry( + String name, String javaName, TypeRef typeRef, int since, int until, boolean live) { + this.name = name; + this.javaName = javaName; + this.typeRef = typeRef; + this.since = since; + this.until = until; + this.live = live; + } + } +} diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/VersionedBinaryArrayEncoder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/VersionedBinaryArrayEncoder.java new file mode 100644 index 0000000000..4a880ec461 --- /dev/null +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/VersionedBinaryArrayEncoder.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format.encoder; + +import org.apache.fory.exception.ClassNotCompatibleException; +import org.apache.fory.format.row.binary.BinaryArray; +import org.apache.fory.format.row.binary.writer.BinaryArrayWriter; +import org.apache.fory.format.type.Field; +import org.apache.fory.memory.LittleEndian; +import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.memory.MemoryUtils; + +/** + * The array encoder of a codec built with schema evolution. Every payload starts, after any size + * prefix, with the 8-byte strict hash of its element layout, and the hash precedes the body. Decode + * reads an older layout through its projection codec. Codecs without evolution use {@link + * BinaryArrayEncoder}, which has no hash on any path. + */ +class VersionedBinaryArrayEncoder implements ArrayEncoder { + private final BinaryArrayWriter writer; + private final GeneratedArrayEncoder codec; + private final boolean sizeEmbedded; + + /** Strict hash of the current one-field element layout (the element need not be a bean). */ + private final long currentHash; + + /** Projection codecs for older element layouts; empty when the element has no older layout. */ + private final ProjectionCache projections; + + /** + * A projection variant of the array codec along with the writer used to materialize an array + * instance of the right physical type (standard vs. compact) for the historical element field. + */ + static final class ProjectionArrayCodec { + final BinaryArrayWriter writer; + final GeneratedArrayEncoder codec; + + ProjectionArrayCodec(BinaryArrayWriter writer, GeneratedArrayEncoder codec) { + this.writer = writer; + this.codec = codec; + } + } + + VersionedBinaryArrayEncoder( + final BinaryArrayWriter writer, + final GeneratedArrayEncoder codec, + final boolean sizeEmbedded, + final long currentHash, + final ProjectionCache projections) { + this.writer = writer; + this.codec = codec; + this.sizeEmbedded = sizeEmbedded; + this.currentHash = currentHash; + this.projections = projections; + } + + @Override + public Field field() { + return writer.getField(); + } + + @SuppressWarnings("unchecked") + @Override + public T fromArray(final BinaryArray array) { + return (T) codec.fromArray(array); + } + + @Override + public BinaryArray toArray(final T obj) { + return codec.toArray(obj); + } + + @Override + public T decode(final MemoryBuffer buffer) { + return decode(buffer, sizeEmbedded ? buffer.readInt32() : buffer.remaining()); + } + + @Override + public T decode(final byte[] bytes) { + // byte[] overloads ignore sizeEmbedded: encode writes no length prefix (the 8-byte hash + // precedes the body, but that is data, not framing), so decode takes the size from + // bytes.length. + return decode(MemoryUtils.wrap(bytes), bytes.length); + } + + T decode(final MemoryBuffer buffer, final int size) { + if (size < 8) { + throw BinaryRowEncoder.sizeBelowHash(size); + } + final long peerHash = buffer.readInt64(); + final int bodySize = size - 8; + if (peerHash == currentHash) { + final BinaryArray array = writer.newArray(); + final int readerIndex = buffer.readerIndex(); + // Advance first: on a stream-backed buffer this loads the body that pointTo reads. + buffer.readerIndex(readerIndex + bodySize); + array.pointTo(buffer, readerIndex, bodySize); + return fromArray(array); + } + return decodeOtherSchema(buffer, peerHash, bodySize); + } + + /** + * Decodes an array whose hash is not the current schema's: an older version through its + * projection, or a schema that this codec does not know, which it rejects. Kept out of {@link + * #decode(MemoryBuffer, int)} so the current-schema path stays small. + */ + @SuppressWarnings("unchecked") + private T decodeOtherSchema(final MemoryBuffer buffer, final long peerHash, final int bodySize) { + ProjectionArrayCodec projection = projections.get(peerHash); + if (projection == null) { + // Skip the body, so that a caller can go on to the next payload in the buffer. + buffer.increaseReaderIndex(bodySize); + throw new ClassNotCompatibleException( + String.format( + "Array element schema is not consistent. self/peer hash are %s/%s.", + currentHash, peerHash)); + } + BinaryArray array = projection.writer.newArray(); + final int readerIndex = buffer.readerIndex(); + // Advance first: on a stream-backed buffer this loads the body that pointTo reads. + buffer.readerIndex(readerIndex + bodySize); + array.pointTo(buffer, readerIndex, bodySize); + return (T) projection.codec.fromArray(array); + } + + @Override + public byte[] encode(final T obj) { + final BinaryArray array = toArray(obj); + // Build the result with a single allocation: the result byte[]. The hash header is poked + // in via LittleEndian (no buffer wrapper) and the body is bulk-copied out of the writer's + // backing buffer into the result. + final int n = array.getSizeInBytes(); + if (n > Integer.MAX_VALUE - 8) { + throw new EncoderException("Array body too large to prepend schema hash header: " + n); + } + final byte[] result = new byte[8 + n]; + LittleEndian.putInt64(result, 0, currentHash); + writer.getBuffer().get(0, result, 8, n); + return result; + } + + @Override + public int encode(final MemoryBuffer buffer, final T obj) { + final MemoryBuffer prevBuffer = writer.getBuffer(); + final int writerIndex = buffer.writerIndex(); + if (sizeEmbedded) { + buffer.writeInt32(-1); + } + buffer.writeInt64(currentHash); + try { + writer.setBuffer(buffer); + toArray(obj); + final int size = buffer.writerIndex() - writerIndex; + if (sizeEmbedded) { + buffer.putInt32(writerIndex, size - 4); + } + return size; + } finally { + writer.setBuffer(prevBuffer); + } + } +} diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/VersionedBinaryMapEncoder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/VersionedBinaryMapEncoder.java new file mode 100644 index 0000000000..a19e93934a --- /dev/null +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/VersionedBinaryMapEncoder.java @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format.encoder; + +import org.apache.fory.exception.ClassNotCompatibleException; +import org.apache.fory.format.row.binary.BinaryArray; +import org.apache.fory.format.row.binary.BinaryMap; +import org.apache.fory.format.row.binary.writer.BinaryArrayWriter; +import org.apache.fory.format.type.Field; +import org.apache.fory.memory.LittleEndian; +import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.memory.MemoryUtils; + +/** + * The map encoder of a codec built with schema evolution. Every payload starts, after any size + * prefix, with the 8-byte combined key and value strict hash, and the hash precedes the body. + * Decode reads an older (key, value) layout through its projection codec. Codecs without evolution + * use {@link BinaryMapEncoder}, which has no hash on any path. + */ +class VersionedBinaryMapEncoder implements MapEncoder { + private final Encoding format; + private final Field mapField; + private final BinaryArrayWriter valWriter; + private final BinaryArrayWriter keyWriter; + private final GeneratedMapEncoder codec; + private final boolean sizeEmbedded; + private final long currentHash; + + /** + * Projection codecs for older (key, value) layouts, keyed by the combined hash; empty when no + * older combination exists. + */ + private final ProjectionCache projections; + + /** + * Per-version projection codec; the {@code Encoding} and historical {@code mapField} together + * materialize an empty map shaped for the historical layout (standard vs. compact). + */ + static final class ProjectionMapCodec { + final Encoding format; + final Field mapField; + final GeneratedMapEncoder codec; + + ProjectionMapCodec(Encoding format, Field mapField, GeneratedMapEncoder codec) { + this.format = format; + this.mapField = mapField; + this.codec = codec; + } + } + + VersionedBinaryMapEncoder( + final Encoding format, + final Field mapField, + final BinaryArrayWriter valWriter, + final BinaryArrayWriter keyWriter, + final GeneratedMapEncoder codec, + final boolean sizeEmbedded, + final long currentHash, + final ProjectionCache projections) { + this.format = format; + this.mapField = mapField; + this.valWriter = valWriter; + this.keyWriter = keyWriter; + this.codec = codec; + this.sizeEmbedded = sizeEmbedded; + this.currentHash = currentHash; + this.projections = projections; + } + + @Override + public Field keyField() { + return keyWriter.getField(); + } + + @Override + public Field valueField() { + return valWriter.getField(); + } + + @SuppressWarnings("unchecked") + @Override + public M fromMap(final BinaryArray key, final BinaryArray value) { + return (M) codec.fromMap(key, value); + } + + @Override + public BinaryMap toMap(final M obj) { + return codec.toMap(obj); + } + + @Override + public M decode(final MemoryBuffer buffer) { + return decode(buffer, sizeEmbedded ? buffer.readInt32() : buffer.remaining()); + } + + M decode(final MemoryBuffer buffer, final int size) { + if (size < 8) { + throw BinaryRowEncoder.sizeBelowHash(size); + } + long peerHash = buffer.readInt64(); + int bodySize = size - 8; + if (peerHash == currentHash) { + final BinaryMap map = format.newMap(mapField); + int readerIndex = buffer.readerIndex(); + // Advance first: on a stream-backed buffer this loads the body that pointTo reads. + buffer.readerIndex(readerIndex + bodySize); + map.pointTo(buffer, readerIndex, bodySize); + return fromMap(map); + } + return decodeOtherSchema(buffer, peerHash, bodySize); + } + + @Override + public M decode(final byte[] bytes) { + // byte[] overloads ignore sizeEmbedded: encode writes no length prefix (the 8-byte hash + // precedes the body, but that is data, not framing), so decode takes the size from + // bytes.length. + return decode(MemoryUtils.wrap(bytes), bytes.length); + } + + /** + * Decodes a map whose hash is not the current schema's: an older version through its projection, + * or a schema that this codec does not know, which it rejects. Kept out of {@link + * #decode(MemoryBuffer, int)} so the current-schema path stays small. + */ + @SuppressWarnings("unchecked") + private M decodeOtherSchema(final MemoryBuffer buffer, final long peerHash, final int bodySize) { + ProjectionMapCodec projection = projections.get(peerHash); + if (projection == null) { + // Skip the body, so that a caller can go on to the next payload in the buffer. + buffer.increaseReaderIndex(bodySize); + throw new ClassNotCompatibleException( + String.format( + "Map (key,value) schema is not consistent. self/peer hash are %s/%s.", + currentHash, peerHash)); + } + BinaryMap map = projection.format.newMap(projection.mapField); + int readerIndex = buffer.readerIndex(); + // Advance first: on a stream-backed buffer this loads the body that pointTo reads. + buffer.readerIndex(readerIndex + bodySize); + map.pointTo(buffer, readerIndex, bodySize); + return (M) projection.codec.fromMap(map); + } + + @Override + public byte[] encode(final M obj) { + final BinaryMap map = toMap(obj); + // Build the result with a single allocation: the result byte[]. The hash header is poked + // in via LittleEndian (no buffer wrapper) and the body is bulk-copied out of the map's + // backing buffer into the result. + final int n = map.getSizeInBytes(); + if (n > Integer.MAX_VALUE - 8) { + throw new EncoderException("Map body too large to prepend schema hash header: " + n); + } + final byte[] result = new byte[8 + n]; + LittleEndian.putInt64(result, 0, currentHash); + map.getBuf().get(map.getBaseOffset(), result, 8, n); + return result; + } + + @Override + public int encode(final MemoryBuffer buffer, final M obj) { + final MemoryBuffer prevBuffer = keyWriter.getBuffer(); + final int writerIndex = buffer.writerIndex(); + if (sizeEmbedded) { + buffer.writeInt32(-1); + } + buffer.writeInt64(currentHash); + try { + keyWriter.setBuffer(buffer); + valWriter.setBuffer(buffer); + toMap(obj); + final int size = buffer.writerIndex() - writerIndex; + if (sizeEmbedded) { + buffer.putInt32(writerIndex, size - 4); + } + return size; + } finally { + keyWriter.setBuffer(prevBuffer); + valWriter.setBuffer(prevBuffer); + } + } +} diff --git a/java/fory-format/src/main/java/org/apache/fory/format/row/binary/writer/CompactBinaryRowWriter.java b/java/fory-format/src/main/java/org/apache/fory/format/row/binary/writer/CompactBinaryRowWriter.java index c915866757..af01d91fca 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/row/binary/writer/CompactBinaryRowWriter.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/row/binary/writer/CompactBinaryRowWriter.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import org.apache.fory.annotation.Internal; import org.apache.fory.format.row.binary.BinaryRow; import org.apache.fory.format.row.binary.CompactBinaryRow; import org.apache.fory.format.row.binary.CompactRowLayout; @@ -91,6 +92,17 @@ public static Schema sortSchema(final Schema schema) { return new Schema(sortFields(schema.fields()), schema.metadata()); } + /** + * Sorts every struct in {@code schema} into compact write order, including structs in list + * elements and in map keys and values. {@link #sortSchema} stops at list and map boundaries + * because each nested row writer and reader sorts its own struct. This form gives the order of + * every struct on the wire in one schema, which the schema-evolution strict hash needs. + */ + @Internal + public static Schema sortSchemaDeep(final Schema schema) { + return new Schema(sortFields(schema.fields(), true), schema.metadata()); + } + @Override protected int fixedSize() { return fixedSize; @@ -167,26 +179,57 @@ public static int[] fixedOffsets(final Schema schema) { } static List sortFields(final List fields) { + return sortFields(fields, false); + } + + /** + * Sorts {@code fields} and the fields of every struct they hold. With {@code deep}, the sort also + * descends into list elements and map keys and values; see {@link #sortSchemaDeep}. + */ + private static List sortFields(final List fields, final boolean deep) { final ArrayList sortedFields = new ArrayList<>(fields); for (int i = 0; i < sortedFields.size(); i++) { - sortedFields.set(i, sortField(sortedFields.get(i))); + sortedFields.set(i, sortField(sortedFields.get(i), deep)); } Collections.sort(sortedFields, new FieldAlignmentComparator()); return sortedFields; } static Field sortField(final Field field) { - DataType type = field.type(); - if (!(type instanceof DataTypes.StructType)) { - return field; - } - DataTypes.StructType structType = (DataTypes.StructType) type; - final List children = structType.fields(); - final List sortedChildren = sortFields(children); - if (children.equals(sortedChildren)) { + return sortField(field, false); + } + + private static Field sortField(final Field field, final boolean deep) { + final DataType type = field.type(); + final DataType sortedType; + if (type instanceof DataTypes.StructType) { + final List children = ((DataTypes.StructType) type).fields(); + final List sortedChildren = sortFields(children, deep); + if (children.equals(sortedChildren)) { + return field; + } + sortedType = new DataTypes.StructType(sortedChildren); + } else if (deep && type instanceof DataTypes.ListType) { + final Field value = ((DataTypes.ListType) type).valueField(); + final Field sortedValue = sortField(value, true); + if (sortedValue == value) { + return field; + } + sortedType = new DataTypes.ListType(sortedValue); + } else if (deep && type instanceof DataTypes.MapType) { + final DataTypes.MapType mapType = (DataTypes.MapType) type; + final Field key = mapType.keyField(); + final Field item = mapType.itemField(); + final Field sortedKey = sortField(key, true); + final Field sortedItem = sortField(item, true); + if (sortedKey == key && sortedItem == item) { + return field; + } + sortedType = new DataTypes.MapType(sortedKey.type(), sortedItem.type(), mapType.keysSorted()); + } else { return field; } - return new Field(field.name(), new DataTypes.StructType(sortedChildren), field.nullable()); + return new Field(field.name(), sortedType, field.nullable()); } public void resetFor(final CompactBinaryRowWriter nestedWriter, final int ordinal) { diff --git a/java/fory-format/src/main/java/org/apache/fory/format/type/TypeInference.java b/java/fory-format/src/main/java/org/apache/fory/format/type/TypeInference.java index 1f3843f438..083c4c6d33 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/type/TypeInference.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/type/TypeInference.java @@ -26,12 +26,14 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.OptionalDouble; import java.util.OptionalInt; import java.util.OptionalLong; +import org.apache.fory.annotation.Internal; import org.apache.fory.collection.Tuple2; import org.apache.fory.format.encoder.CustomCodec; import org.apache.fory.format.encoder.CustomCollectionFactory; @@ -129,20 +131,33 @@ private static Field inferField(TypeRef arrayTypeRef, TypeRef typeRef) { } } + private static Field inferField(String name, TypeRef typeRef, TypeResolutionContext ctx) { + return inferField(name, typeRef, ctx, null, null); + } + /** * When type is both iterable and bean, we take it as iterable in row-format. Note circular * references in bean class is not allowed. * + *

    {@code leaves} and {@code path} are null except under {@link #inferNamedField}, which + * describes what the descent reports. Schema evolution finds the nested beans of a field through + * them, so this is the only descent through custom codecs, Optional, arrays, iterables, and maps. + * * @return DataType of a typeToken */ - private static Field inferField(String name, TypeRef typeRef, TypeResolutionContext ctx) { + private static Field inferField( + String name, + TypeRef typeRef, + TypeResolutionContext ctx, + LeafVisitor leaves, + List path) { // Handle TypeVariable (e.g., K, V from Map) by resolving to its bound. // This can happen with Scala 3 LTS where generic type information may not be fully resolved. Type type = typeRef.getType(); if (type instanceof TypeVariable) { TypeVariable typeVariable = (TypeVariable) type; Type bound = typeVariable.getBounds()[0]; // First bound is a class, others are interfaces - return inferField(name, TypeRef.of(bound), ctx); + return inferField(name, TypeRef.of(bound), ctx, leaves, path); } Class rawType = getRawType(typeRef); Class enclosingType = ctx.getEnclosingType().getRawType(); @@ -164,11 +179,11 @@ private static Field inferField(String name, TypeRef typeRef, TypeResolutionC } TypeRef replacementType = customEncoder.encodedType(); if (replacementType != null && !typeRef.equals(replacementType)) { - return inferField(name, replacementType, ctx); + return inferField(name, replacementType, ctx, leaves, path); } } else if (rawType == Optional.class) { TypeRef elemType = TypeUtils.getTypeArguments(typeRef).get(0); - Field result = inferField(name, elemType, ctx); + Field result = inferField(name, elemType, ctx, leaves, path); if (result.nullable()) { return result; } @@ -225,22 +240,37 @@ private static Field inferField(String name, TypeRef typeRef, TypeResolutionC } else if (rawType.isArray()) { // array Field f = inferField( - DataTypes.ARRAY_ITEM_NAME, Objects.requireNonNull(typeRef.getComponentType()), ctx); + DataTypes.ARRAY_ITEM_NAME, + Objects.requireNonNull(typeRef.getComponentType()), + ctx, + leaves, + path); return DataTypes.arrayField(name, f); } else if (TypeUtils.ITERABLE_TYPE.isSupertypeOf(typeRef)) { // iterable // when type is both iterable and bean, we take it as iterable in row-format - Field f = inferField(DataTypes.ARRAY_ITEM_NAME, TypeUtils.getElementType(typeRef), ctx); + Field f = + inferField( + DataTypes.ARRAY_ITEM_NAME, TypeUtils.getElementType(typeRef), ctx, leaves, path); return DataTypes.arrayField(name, f); } else if (TypeUtils.MAP_TYPE.isSupertypeOf(typeRef)) { Tuple2, TypeRef> kvType = TypeUtils.getMapKeyValueType(typeRef); - Field keyField = inferField(DataTypes.MAP_KEY_NAME, kvType.f0, ctx); + Field keyField = + inferField(DataTypes.MAP_KEY_NAME, kvType.f0, ctx, leaves, append(path, MapBranch.KEY)); // Map's keys must be non-nullable if (keyField.nullable()) { keyField = keyField.withNullable(false); } - Field valueField = inferField(DataTypes.MAP_VALUE_NAME, kvType.f1, ctx); + Field valueField = + inferField( + DataTypes.MAP_VALUE_NAME, kvType.f1, ctx, leaves, append(path, MapBranch.VALUE)); return DataTypes.mapField(name, keyField, valueField); - } else if (TypeUtils.isBean(rawType, ctx)) { // bean field + } + // Report the leaf before the bean check: that check fails for a type already on the walked + // path, and schema evolution reports a cycle through a removed field itself. + if (leaves != null) { + leaves.visit(rawType, path); + } + if (TypeUtils.isBean(rawType, ctx)) { // bean field ctx.checkNoCycle(rawType); List descriptors = Descriptor.getDescriptors(rawType); List fields = new ArrayList<>(descriptors.size()); @@ -250,12 +280,58 @@ private static Field inferField(String name, TypeRef typeRef, TypeResolutionC fields.add(inferField(n, fieldType, ctx.appendTypePath(rawType))); } return DataTypes.structField(name, true, fields); - } else { - throw new UnsupportedOperationException( - String.format( - "Unsupported type %s for field %s, seen type set is %s", - typeRef, name, ctx.getWalkedTypePath())); } + throw new UnsupportedOperationException( + String.format( + "Unsupported type %s for field %s, seen type set is %s", + typeRef, name, ctx.getWalkedTypePath())); + } + + private static List append(List path, MapBranch branch) { + if (path == null) { + return null; + } + List next = new ArrayList<>(path.size() + 1); + next.addAll(path); + next.add(branch); + return next; + } + + /** The branch that a descent takes at a map: its key or its value. */ + @Internal + public enum MapBranch { + KEY, + VALUE + } + + /** Receives the leaves of a field-type descent; see {@link #inferNamedField}. */ + @Internal + @FunctionalInterface + public interface LeafVisitor { + void visit(Class leafType, List path); + } + + /** + * Infer a single named field from its Java type, used by schema-evolution code paths that need to + * reconstruct historical fields by name and type without going through a Java member. {@code + * enclosingType} is the bean class declaring the field, so bean-scoped custom codecs resolve + * exactly as they do when {@link #inferSchema} walks the live bean; pass {@code void.class} for a + * top-level array/map element, which has no enclosing bean and matches only enclosing-agnostic + * registrations. + * + *

    {@code leaves} receives, in descent order, each type at which the descent stops unwrapping + * custom codecs, Optional, arrays, iterables, and map keys and values, unless that type is a + * scalar: a bean, or a type the row format does not support. A map's key comes before its value. + * The path lists the branch taken at each map from the field root; the descent reports nothing + * inside a reported bean. + */ + @Internal + public static Field inferNamedField( + String name, TypeRef typeRef, Class enclosingType, LeafVisitor leaves) { + TypeResolutionContext ctx = + new TypeResolutionContext(CustomTypeEncoderRegistry.customTypeHandler(), true); + return inferField( + name, typeRef, ctx.appendTypePath(enclosingType), leaves, Collections.emptyList()); } public static String inferTypeName(TypeRef token) { diff --git a/java/fory-format/src/main/java11/module-info.java b/java/fory-format/src/main/java11/module-info.java index 0f6064b8e9..82f6432366 100644 --- a/java/fory-format/src/main/java11/module-info.java +++ b/java/fory-format/src/main/java11/module-info.java @@ -24,6 +24,7 @@ requires static transitive org.apache.arrow.memory.core; requires static transitive org.apache.arrow.vector; + exports org.apache.fory.format.annotation; exports org.apache.fory.format.encoder; exports org.apache.fory.format.row; exports org.apache.fory.format.row.binary; diff --git a/java/fory-format/src/test/java/org/apache/fory/format/encoder/ImplementInterfaceTest.java b/java/fory-format/src/test/java/org/apache/fory/format/encoder/ImplementInterfaceTest.java index 448ea689d3..a1afc69880 100644 --- a/java/fory-format/src/test/java/org/apache/fory/format/encoder/ImplementInterfaceTest.java +++ b/java/fory-format/src/test/java/org/apache/fory/format/encoder/ImplementInterfaceTest.java @@ -20,7 +20,9 @@ package org.apache.fory.format.encoder; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.OptionalDouble; import java.util.OptionalInt; @@ -329,6 +331,77 @@ public void testListTooLazy() { Assert.assertEquals(deserializedBean.f1().get(0).f1(), 42); } + public interface MapOuter { + Map f1(); + } + + static class MapOuterImpl implements MapOuter { + private final Map f1; + + MapOuterImpl(final Map f1) { + this.f1 = f1; + } + + @Override + public Map f1() { + return f1; + } + } + + /** + * Interface bean as a map value. Type inference reaches the map value type through {@code + * isSupported}, which must recognize the interface as a synthesizable bean the same way it does + * for a direct field or list element. + */ + @Test + public void testMapValueInterface() { + final Map map = new HashMap<>(); + map.put("k", new ListInnerImpl(42)); + final MapOuter bean1 = new MapOuterImpl(map); + final RowEncoder encoder = Encoders.bean(MapOuter.class); + final BinaryRow row = encoder.toRow(bean1); + final MemoryBuffer buffer = MemoryUtils.wrap(row.toBytes()); + row.pointTo(buffer, 0, buffer.size()); + final MapOuter deserializedBean = encoder.fromRow(row); + Assert.assertEquals(deserializedBean.f1().get("k").f1(), 42); + } + + public interface KeyMapOuter { + Map f1(); + } + + static class KeyMapOuterImpl implements KeyMapOuter { + private final Map f1; + + KeyMapOuterImpl(final Map f1) { + this.f1 = f1; + } + + @Override + public Map f1() { + return f1; + } + } + + /** + * Interface bean as a map key. The key side of {@code isSupported} threads the resolution context + * the same way as the value side, so the interface must synthesize as a bean here too. + */ + @Test + public void testMapKeyInterface() { + final Map map = new HashMap<>(); + map.put(new ListInnerImpl(42), "v"); + final KeyMapOuter bean1 = new KeyMapOuterImpl(map); + final RowEncoder encoder = Encoders.bean(KeyMapOuter.class); + final BinaryRow row = encoder.toRow(bean1); + final MemoryBuffer buffer = MemoryUtils.wrap(row.toBytes()); + row.pointTo(buffer, 0, buffer.size()); + final KeyMapOuter deserializedBean = encoder.fromRow(row); + final Map.Entry entry = deserializedBean.f1().entrySet().iterator().next(); + Assert.assertEquals(entry.getKey().f1(), 42); + Assert.assertEquals(entry.getValue(), "v"); + } + public interface Value extends Comparable { int v(); @@ -433,12 +506,19 @@ public void testListElementsLazy() { new ListLazyElemInner(4))); final RowEncoder encoder = Encoders.bean(ListLazyElemOuter.class); final BinaryRow row = encoder.toRow(bean1); + // Only the accessed element (index 2, value 42) should be constructed; the constructor's check + // guard asserts that. Reset it in finally so the flag never leaks into another test decoding a + // ListLazyElemInner through its globally-registered codec. ListLazyElemInner.check = true; - final MemoryBuffer buffer = MemoryUtils.wrap(row.toBytes()); - row.pointTo(buffer, 0, buffer.size()); - final ListLazyElemOuter deserializedBean = encoder.fromRow(row); - Assert.assertEquals(deserializedBean.f1().get(2).f1(), 42); - Assert.assertEquals(deserializedBean.f1().get(3), null); + try { + final MemoryBuffer buffer = MemoryUtils.wrap(row.toBytes()); + row.pointTo(buffer, 0, buffer.size()); + final ListLazyElemOuter deserializedBean = encoder.fromRow(row); + Assert.assertEquals(deserializedBean.f1().get(2).f1(), 42); + Assert.assertEquals(deserializedBean.f1().get(3), null); + } finally { + ListLazyElemInner.check = false; + } } public interface IgnoredMethods { diff --git a/java/fory-format/src/test/java/org/apache/fory/format/encoder/SchemaEvolutionEdgeCaseTest.java b/java/fory-format/src/test/java/org/apache/fory/format/encoder/SchemaEvolutionEdgeCaseTest.java new file mode 100644 index 0000000000..49d386521e --- /dev/null +++ b/java/fory-format/src/test/java/org/apache/fory/format/encoder/SchemaEvolutionEdgeCaseTest.java @@ -0,0 +1,2758 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format.encoder; + +import java.io.ByteArrayInputStream; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import lombok.Data; +import org.apache.fory.Fory; +import org.apache.fory.exception.ClassNotCompatibleException; +import org.apache.fory.format.annotation.ForyRowSchema; +import org.apache.fory.format.annotation.ForyRowVersion; +import org.apache.fory.io.ForyInputStream; +import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.reflect.TypeRef; +import org.testng.Assert; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +/** + * Edge cases of row-codec schema evolution: long version chains, nested and collection-typed beans, + * maps, and the build-time checks. Each test probes one case. + */ +public class SchemaEvolutionEdgeCaseTest { + + // --------------------------------------------------------------------------- + // Long version chain: a field added at each version 1..5, plus the removal of 'a' at v5. + // Verifies projection codecs are built and dispatched for every historical version. + // --------------------------------------------------------------------------- + + @Data + public static class ChainV1 { + private int a; // since 1 + } + + @Data + public static class ChainV2 { + private int a; + + @ForyRowVersion(since = 2) + private String b; + } + + @Data + public static class ChainV3 { + private int a; + + @ForyRowVersion(since = 2) + private String b; + + @ForyRowVersion(since = 3) + private long c; + } + + @Data + public static class ChainV4 { + private int a; + + @ForyRowVersion(since = 2) + private String b; + + @ForyRowVersion(since = 3) + private long c; + + @ForyRowVersion(since = 4) + private double d; + } + + /** + * v5 also removes the v1 'a' field ({@code until = 5}). Each version adds a field, so the reader + * must know four older layouts, v1 to v4, besides the current v5 layout. + */ + @Data + @ForyRowSchema(removedFields = ChainV5.History.class) + public static class ChainV5 { + @ForyRowVersion(since = 2) + private String b; + + @ForyRowVersion(since = 3) + private long c; + + @ForyRowVersion(since = 4) + private double d; + + @ForyRowVersion(since = 5) + private boolean e; + + interface History { + @ForyRowVersion(until = 5) + int a(); + } + } + + @Test + public void longChainAllVersionsReadable() { + RowEncoder w1 = evolvingCodec(ChainV1.class); + RowEncoder w2 = evolvingCodec(ChainV2.class); + RowEncoder w3 = evolvingCodec(ChainV3.class); + RowEncoder w4 = evolvingCodec(ChainV4.class); + RowEncoder reader = evolvingCodec(ChainV5.class); + + ChainV1 v1 = new ChainV1(); + v1.setA(11); + ChainV2 v2 = new ChainV2(); + v2.setA(21); + v2.setB("two"); + ChainV3 v3 = new ChainV3(); + v3.setA(31); + v3.setB("three"); + v3.setC(333L); + ChainV4 v4 = new ChainV4(); + v4.setA(41); + v4.setB("four"); + v4.setC(444L); + v4.setD(4.4); + + ChainV5 out1 = reader.decode(w1.encode(v1)); + Assert.assertNull(out1.getB()); + Assert.assertEquals(out1.getC(), 0L); + Assert.assertEquals(out1.getD(), 0.0); + Assert.assertFalse(out1.isE()); + + ChainV5 out2 = reader.decode(w2.encode(v2)); + Assert.assertEquals(out2.getB(), "two"); + Assert.assertEquals(out2.getC(), 0L); + + ChainV5 out3 = reader.decode(w3.encode(v3)); + Assert.assertEquals(out3.getC(), 333L); + Assert.assertEquals(out3.getD(), 0.0); + + ChainV5 out4 = reader.decode(w4.encode(v4)); + Assert.assertEquals(out4.getB(), "four"); + Assert.assertEquals(out4.getC(), 444L); + Assert.assertEquals(out4.getD(), 4.4); + Assert.assertFalse(out4.isE()); + } + + // --------------------------------------------------------------------------- + // Compact format with alignment shuffle: v1 has only longs; v2 adds a byte. + // Compact sorts fields by alignment width so the v1 and v2 schemas have + // different physical orders, even though their logical field sets differ by + // only the added byte. + // --------------------------------------------------------------------------- + + @Data + public static class AlignV1 { + private long x; + private long y; + } + + @Data + public static class AlignV2 { + private long x; + private long y; + + @ForyRowVersion(since = 2) + private byte flag; + } + + @Test + public void compactAlignmentReshuffleAcrossVersions() { + RowEncoder writer = + Encoders.buildBeanCodec(AlignV1.class) + .compactEncoding() + .withSchemaEvolution() + .build() + .get(); + RowEncoder reader = + Encoders.buildBeanCodec(AlignV2.class) + .compactEncoding() + .withSchemaEvolution() + .build() + .get(); + AlignV1 in = new AlignV1(); + in.setX(11); + in.setY(22); + AlignV2 out = reader.decode(writer.encode(in)); + Assert.assertEquals(out.getX(), 11); + Assert.assertEquals(out.getY(), 22); + Assert.assertEquals(out.getFlag(), (byte) 0); // primitive default + } + + // --------------------------------------------------------------------------- + // Boxed vs primitive default for an absent field. + // --------------------------------------------------------------------------- + + @Data + public static class DefaultsV1 { + private String name; + } + + @Data + public static class DefaultsV2 { + private String name; + + @ForyRowVersion(since = 2) + private int primitiveCount; // default 0 + + @ForyRowVersion(since = 2) + private Integer boxedCount; // default null + } + + @Test + public void primitiveAndBoxedDefaults() { + RowEncoder writer = evolvingCodec(DefaultsV1.class); + RowEncoder reader = evolvingCodec(DefaultsV2.class); + DefaultsV1 in = new DefaultsV1(); + in.setName("n"); + DefaultsV2 out = reader.decode(writer.encode(in)); + Assert.assertEquals(out.getName(), "n"); + Assert.assertEquals(out.getPrimitiveCount(), 0); + Assert.assertNull(out.getBoxedCount()); + } + + // --------------------------------------------------------------------------- + // Disjoint-window false collision. A field whose [since, until) + // window starts above the base version and ends below infinity leaves the + // pre-since and post-until boundaries with identical field sets. SchemaHistory + // must collapse those into one entry rather than flagging a false collision. + // --------------------------------------------------------------------------- + + @Data + @ForyRowSchema(removedFields = GappedWindow.History.class) + public static class GappedWindow { + private String name; + + interface History { + @ForyRowVersion(since = 2, until = 4) + int oldField(); + } + } + + /** A bean field keyed by a list of versioned beans: the substituted key stays non-nullable. */ + @Data + public static class ListKeyHolder { + private Map, String> byList; + } + + @Test + public void listOfVersionedBeansAsMapKey() { + DefaultsV2 d = new DefaultsV2(); + d.setName("k"); + ListKeyHolder in = new ListKeyHolder(); + in.setByList(Collections.singletonMap(Arrays.asList(d), "v")); + RowEncoder codec = evolvingCodec(ListKeyHolder.class); + Map.Entry, String> e = + codec.decode(codec.encode(in)).getByList().entrySet().iterator().next(); + Assert.assertEquals(e.getKey().get(0).getName(), "k"); + Assert.assertEquals(e.getValue(), "v"); + } + + @Data + public static class OldDetail { + private String name; + } + + /** {@code detail} is removed as a plain bean and added again as the versioned DefaultsV2. */ + @Data + @ForyRowSchema(removedFields = RetypedDetail.History.class) + public static class RetypedDetail { + @ForyRowVersion(since = 3) + private DefaultsV2 detail; + + @ForyRowVersion(since = 2) + private String label; + + interface History { + @ForyRowVersion(until = 2) + OldDetail detail(); + } + } + + /** DefaultsV2 at v1 has OldDetail's layout, so the two versions could collapse; reject it. */ + @Test + public void retypedVersionedBeanRejected() { + try { + evolvingCodec(RetypedDetail.class); + Assert.fail("expected the codec build to reject the type change"); + } catch (IllegalStateException e) { + Assert.assertTrue(e.getMessage().contains("'detail'"), e.getMessage()); + } + } + + /** Holds GappedWindow, whose v1 and v4 layouts collapse, in every nested position. */ + @Data + public static class GappedHolder { + private GappedWindow window; + private List windows; + private Map byName; + } + + @Test + public void collapsedNestedBean() { + GappedWindow w = new GappedWindow(); + w.setName("w"); + GappedHolder in = new GappedHolder(); + in.setWindow(w); + in.setWindows(Arrays.asList(w)); + in.setByName(Collections.singletonMap("k", w)); + RowEncoder holder = + Encoders.buildBeanCodec(GappedHolder.class).withSchemaEvolution().build().get(); + GappedHolder out = holder.decode(holder.encode(in)); + Assert.assertEquals(out.getWindow().getName(), "w"); + Assert.assertEquals(out.getWindows().get(0).getName(), "w"); + Assert.assertEquals(out.getByName().get("k").getName(), "w"); + + ArrayEncoder> list = + Encoders.buildArrayCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + Assert.assertEquals(list.decode(list.encode(Arrays.asList(w))).get(0).getName(), "w"); + MapEncoder> map = + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + Assert.assertEquals( + map.decode(map.encode(Collections.singletonMap("k", w))).get("k").getName(), "w"); + } + + @Data + public static class InitializedV1 { + private String name; + } + + @Data + public static class InitializedV2 { + private String name; + + @ForyRowVersion(since = 2) + private Optional nick; + + @ForyRowVersion(since = 2) + private String label = "unset"; + } + + @Test + public void absentFieldsGetDefaultsOnPlainBeans() { + RowEncoder writer = evolvingCodec(InitializedV1.class); + RowEncoder reader = evolvingCodec(InitializedV2.class); + InitializedV1 in = new InitializedV1(); + in.setName("a"); + InitializedV2 out = reader.decode(writer.encode(in)); + Assert.assertEquals(out.getName(), "a"); + Assert.assertEquals(out.getNick(), Optional.empty()); + Assert.assertNull(out.getLabel()); + } + + @Test + public void disjointWindowDoesNotFalseCollide() { + // Build alone is the assertion: a false collision fails the build. + RowEncoder codec = evolvingCodec(GappedWindow.class); + GappedWindow in = new GappedWindow(); + in.setName("hi"); + Assert.assertEquals(codec.decode(codec.encode(in)).getName(), "hi"); + } + + // --------------------------------------------------------------------------- + // Removed field whose original type was a nested struct. The projection + // codec must skip the slot without trying to read or decode it. + // --------------------------------------------------------------------------- + + @Data + public static class StructRefV1 { + private String id; + private DefaultsV1 detail; // removed at v2 + private long tail; // live in both versions, positioned after the removed slot + } + + @Data + @ForyRowSchema(removedFields = StructRefV2.History.class) + public static class StructRefV2 { + private String id; + private long tail; + + interface History { + @ForyRowVersion(until = 2) + DefaultsV1 detail(); + } + } + + // A struct that itself contains a live, evolving nested bean. Used as a map KEY so the nested + // bean, not only the top-level key bean, must route to its projection codec. + @Data + public static class KeyHolderV1 { + private String id; + private DefaultsV1 detail; + } + + @Data + public static class KeyHolderV2 { + private String id; + private DefaultsV2 detail; + } + + @Test + public void removedNestedStructField() { + RowEncoder writer = evolvingCodec(StructRefV1.class); + RowEncoder reader = evolvingCodec(StructRefV2.class); + StructRefV1 in = new StructRefV1(); + in.setId("x"); + DefaultsV1 d = new DefaultsV1(); + d.setName("inner"); + in.setDetail(d); + in.setTail(42L); + StructRefV2 out = reader.decode(writer.encode(in)); + Assert.assertEquals(out.getId(), "x"); + // The field after the removed slot must read correctly, proving projection skipped exactly the + // removed struct's width. + Assert.assertEquals(out.getTail(), 42L); + } + + // A removed field whose type is a versioned bean that in turn holds an unversioned bean. Nothing + // live on the reader reaches either class, so the reader never compiles their current-version + // codecs. The writer uses look-alike classes so that it does not compile them either: the strict + // hash keys on field names, not class names. + @Data + public static class RemovedGeo { + private double lat; + } + + @Data + public static class RemovedGeoWriter { + private double lat; + } + + @Data + public static class RemovedAddrWriter { + private RemovedGeoWriter geo; + private String street; + } + + @Data + public static class RemovedAddr { + private RemovedGeo geo; + private String street; + + @ForyRowVersion(since = 2) + private String zip; + } + + @Data + public static class RemovedAddrHolderV1 { + private RemovedAddrWriter addr; + private String id; + private long tail; + } + + @Data + @ForyRowSchema(removedFields = RemovedAddrHolderV2.History.class) + public static class RemovedAddrHolderV2 { + private String id; + private long tail; + + interface History { + @ForyRowVersion(until = 2) + RemovedAddr addr(); + } + } + + @Test + public void removedVersionedBeanFieldAtHistoricalVersion() { + RowEncoder writer = evolvingCodec(RemovedAddrHolderV1.class); + RowEncoder reader = evolvingCodec(RemovedAddrHolderV2.class); + RemovedAddrHolderV1 in = new RemovedAddrHolderV1(); + RemovedAddrWriter addr = new RemovedAddrWriter(); + RemovedGeoWriter geo = new RemovedGeoWriter(); + geo.setLat(1.5); + addr.setGeo(geo); + addr.setStreet("main"); + in.setAddr(addr); + in.setId("x"); + in.setTail(42L); + RemovedAddrHolderV2 out = reader.decode(writer.encode(in)); + Assert.assertEquals(out.getId(), "x"); + Assert.assertEquals(out.getTail(), 42L); + } + + // --------------------------------------------------------------------------- + // Removed collection-typed field. The history interface preserves the full + // parameterized type, so List and Map round-trip + // through the projection without losing element-type information. + // --------------------------------------------------------------------------- + + @Data + public static class CollectionsV1 { + private String id; + private List tags; // removed at v2 + private Map counters; // removed at v2 + private long tail; // live in both versions, positioned after the removed slots + } + + @Data + @ForyRowSchema(removedFields = CollectionsV2.History.class) + public static class CollectionsV2 { + private String id; + private long tail; + + interface History { + @ForyRowVersion(until = 2) + List tags(); + + @ForyRowVersion(until = 2) + Map counters(); + } + } + + @Test + public void removedParameterizedCollectionFields() { + RowEncoder writer = evolvingCodec(CollectionsV1.class); + RowEncoder reader = evolvingCodec(CollectionsV2.class); + CollectionsV1 in = new CollectionsV1(); + in.setId("c"); + in.setTags(Arrays.asList("alpha", "beta")); + Map counters = new HashMap<>(); + counters.put("k1", 1L); + counters.put("k2", 2L); + in.setCounters(counters); + in.setTail(42L); + CollectionsV2 out = reader.decode(writer.encode(in)); + Assert.assertEquals(out.getId(), "c"); + // The field after the removed slots must read correctly, proving projection skipped exactly the + // two removed collection slots' width. + Assert.assertEquals(out.getTail(), 42L); + } + + // --------------------------------------------------------------------------- + // Same wire-name retyped across versions: 'tag' was int [1,3), then String [3,inf). + // --------------------------------------------------------------------------- + + @Data + public static class RetypeV1 { + private int tag; // present in v1, v2 + } + + @Data + @ForyRowSchema(removedFields = RetypeV3.History.class) + public static class RetypeV3 { + @ForyRowVersion(since = 3) + private String tag; + + interface History { + @ForyRowVersion(until = 3) + int tag(); + } + } + + @Test + public void retypedSameNameAcrossVersions() { + RowEncoder writer = evolvingCodec(RetypeV1.class); + RowEncoder reader = evolvingCodec(RetypeV3.class); + RetypeV1 in = new RetypeV1(); + in.setTag(7); + RetypeV3 out = reader.decode(writer.encode(in)); + // The 'tag' on the wire was int and is dropped during projection; the v3 String 'tag' has + // no source in this payload so defaults to null. + Assert.assertNull(out.getTag()); + } + + // --------------------------------------------------------------------------- + // List and int[] infer list fields that differ only in element + // nullability, so their layouts and strict hashes differ. Schema.equals ignores + // element nullability; the history must not treat the two layouts as one. + // --------------------------------------------------------------------------- + + @Data + public static class ScoresListV1 { + private String name; + private List scores; + } + + @Data + @ForyRowSchema(removedFields = ScoresArrayV3.History.class) + public static class ScoresArrayV3 { + private String name; + + @ForyRowVersion(since = 3) + private int[] scores; + + interface History { + @ForyRowVersion(until = 3) + List scores(); + } + } + + @Data + @ForyRowSchema(removedFields = ScoresBase.History.class) + public static class ScoresBase { + private String name; + + interface History { + @ForyRowVersion(until = 3) + List scores(); + } + } + + /** + * Removes the {@code int[]} scores of v3 and v4; the base removes the {@code List} + * scores of v1 and v2. + */ + @Data + @lombok.EqualsAndHashCode(callSuper = true) + @ForyRowSchema(removedFields = ScoresRemovedV5.History.class) + public static class ScoresRemovedV5 extends ScoresBase { + interface History { + @ForyRowVersion(since = 3, until = 5) + int[] scores(); + } + } + + @Test + public void listRetypedToArray() { + ScoresListV1 in = new ScoresListV1(); + in.setName("alice"); + in.setScores(Arrays.asList(1, 2)); + RowEncoder reader = evolvingCodec(ScoresArrayV3.class); + ScoresArrayV3 out = reader.decode(evolvingCodec(ScoresListV1.class).encode(in)); + Assert.assertEquals(out.getName(), "alice"); + Assert.assertNull(out.getScores()); + + ScoresArrayV3 current = new ScoresArrayV3(); + current.setName("bob"); + current.setScores(new int[] {3}); + Assert.assertEquals(reader.decode(reader.encode(current)), current); + } + + @Test + public void removedListAndArrayLayouts() { + RowEncoder reader = evolvingCodec(ScoresRemovedV5.class); + ScoresListV1 v1 = new ScoresListV1(); + v1.setName("alice"); + v1.setScores(Arrays.asList(1, 2)); + Assert.assertEquals( + reader.decode(evolvingCodec(ScoresListV1.class).encode(v1)).getName(), "alice"); + + ScoresArrayV3 v3 = new ScoresArrayV3(); + v3.setName("bob"); + v3.setScores(new int[] {3}); + Assert.assertEquals( + reader.decode(evolvingCodec(ScoresArrayV3.class).encode(v3)).getName(), "bob"); + } + + // --------------------------------------------------------------------------- + // Wide schema (more than 64 fields) crossing the null-bitmap word boundary. + // --------------------------------------------------------------------------- + + @Data + public static class WideV1 { + private int f00, f01, f02, f03, f04, f05, f06, f07, f08, f09; + private int f10, f11, f12, f13, f14, f15, f16, f17, f18, f19; + private int f20, f21, f22, f23, f24, f25, f26, f27, f28, f29; + private int f30, f31, f32, f33, f34, f35, f36, f37, f38, f39; + private int f40, f41, f42, f43, f44, f45, f46, f47, f48, f49; + private int f50, f51, f52, f53, f54, f55, f56, f57, f58, f59; + private int f60, f61, f62, f63, f64, f65, f66, f67; + } + + @Data + public static class WideV2 { + private int f00, f01, f02, f03, f04, f05, f06, f07, f08, f09; + private int f10, f11, f12, f13, f14, f15, f16, f17, f18, f19; + private int f20, f21, f22, f23, f24, f25, f26, f27, f28, f29; + private int f30, f31, f32, f33, f34, f35, f36, f37, f38, f39; + private int f40, f41, f42, f43, f44, f45, f46, f47, f48, f49; + private int f50, f51, f52, f53, f54, f55, f56, f57, f58, f59; + private int f60, f61, f62, f63, f64, f65, f66, f67; + + @ForyRowVersion(since = 2) + private String extra; + } + + @Test + public void wideSchemaAcrossBitmapWord() { + RowEncoder writer = evolvingCodec(WideV1.class); + RowEncoder reader = evolvingCodec(WideV2.class); + WideV1 in = new WideV1(); + in.setF00(100); + in.setF63(163); + in.setF67(167); // past the first 64-bit bitmap word + WideV2 out = reader.decode(writer.encode(in)); + Assert.assertEquals(out.getF00(), 100); + Assert.assertEquals(out.getF63(), 163); + Assert.assertEquals(out.getF67(), 167); + Assert.assertNull(out.getExtra()); + } + + // --------------------------------------------------------------------------- + // Many elements through a single projection codec: 100 elements written by the + // same older version must all decode correctly via the same projection codec, + // with each element's data preserved and no carry-over of state across slots. + // --------------------------------------------------------------------------- + + @Test + public void arrayManyElementsThroughOneProjection() { + ArrayEncoder> writer = + Encoders.buildArrayCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + ArrayEncoder> reader = + Encoders.buildArrayCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + List in = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + ChainV2 e = new ChainV2(); + e.setA(i); + e.setB("elem-" + i); + in.add(e); + } + List out = reader.decode(writer.encode(in)); + Assert.assertEquals(out.size(), 100); + for (int i = 0; i < 100; i++) { + Assert.assertEquals(out.get(i).getB(), "elem-" + i); + Assert.assertEquals(out.get(i).getC(), 0L); + Assert.assertFalse(out.get(i).isE()); + } + } + + // --------------------------------------------------------------------------- + // Sanity: two readers for the same (class, history) co-exist without + // interfering. The two readers share the cached generated codec class (by + // design of the codec cache), so the test exercises whether + // BinaryRowEncoder's per-instance projection map and current-codec instance + // are correctly per-reader rather than accidentally shared. + // --------------------------------------------------------------------------- + + @Test + public void twoIndependentReadersForSameClass() { + RowEncoder writer = evolvingCodec(DefaultsV1.class); + RowEncoder r1 = evolvingCodec(DefaultsV2.class); + RowEncoder r2 = evolvingCodec(DefaultsV2.class); + DefaultsV1 in1 = new DefaultsV1(); + in1.setName("first"); + DefaultsV1 in2 = new DefaultsV1(); + in2.setName("second"); + byte[] b1 = writer.encode(in1); + byte[] b2 = writer.encode(in2); + Assert.assertEquals(r1.decode(b1).getName(), "first"); + Assert.assertEquals(r2.decode(b2).getName(), "second"); + Assert.assertEquals(r1.decode(b2).getName(), "second"); + Assert.assertEquals(r2.decode(b1).getName(), "first"); + } + + // --------------------------------------------------------------------------- + // Schema-history misconfiguration: overlapping windows for the same name + // must fail builder construction, not at first bad payload. + // --------------------------------------------------------------------------- + + @Data + @ForyRowSchema(removedFields = OverlapMisconfig.History.class) + public static class OverlapMisconfig { + // Live field 'x' since 1 (default) collides with the removed window [1, 5). + private int x; + + interface History { + @ForyRowVersion(since = 1, until = 5) + int x(); + } + } + + @Test(expectedExceptions = IllegalStateException.class) + public void overlappingWindowFailsAtBuild() { + evolvingCodec(OverlapMisconfig.class); + } + + // --------------------------------------------------------------------------- + // A removed-field history declaration must carry a well-formed @ForyRowVersion. + // Each misconfiguration fails at build with a message that names the offending + // declaration, so the user can fix the annotation rather than chase a decode error. + // --------------------------------------------------------------------------- + + @Data + @ForyRowSchema(removedFields = MissingAnnotation.History.class) + public static class MissingAnnotation { + private int x; + + interface History { + // No @ForyRowVersion: a removed field has no [since, until) window without it. + String legacy(); + } + } + + @Data + @ForyRowSchema(removedFields = MissingUntil.History.class) + public static class MissingUntil { + private int x; + + interface History { + @ForyRowVersion(since = 2) + String legacy(); + } + } + + @Data + @ForyRowSchema(removedFields = EmptyWindow.History.class) + public static class EmptyWindow { + private int x; + + interface History { + @ForyRowVersion(since = 5, until = 5) + String legacy(); + } + } + + @Test + public void removedFieldWithoutForyVersionFailsAtBuild() { + IllegalStateException e = + Assert.expectThrows( + IllegalStateException.class, + () -> + Encoders.buildBeanCodec(MissingAnnotation.class) + .withSchemaEvolution() + .build() + .get()); + Assert.assertTrue(e.getMessage().contains("requires a @ForyRowVersion"), e.getMessage()); + } + + @Test + public void removedFieldWithoutUntilFailsAtBuild() { + IllegalStateException e = + Assert.expectThrows(IllegalStateException.class, () -> evolvingCodec(MissingUntil.class)); + Assert.assertTrue( + e.getMessage().contains("must specify @ForyRowVersion.until"), e.getMessage()); + } + + @Test + public void removedFieldEmptyWindowFailsAtBuild() { + IllegalStateException e = + Assert.expectThrows(IllegalStateException.class, () -> evolvingCodec(EmptyWindow.class)); + Assert.assertTrue(e.getMessage().contains("must be strictly less than until"), e.getMessage()); + } + + @Data + @ForyRowSchema(removedFields = RemovedFieldCycle.History.class) + public static class RemovedFieldCycle { + private int x; + + interface History { + // A removed field typed as the bean itself: the historical schema would be cyclic. Live + // fields cannot do this (inferSchema rejects them), so only the history path reaches it. + @ForyRowVersion(until = 2) + RemovedFieldCycle previous(); + } + } + + @Test + public void removedFieldCycleFailsAtBuild() { + IllegalStateException e = + Assert.expectThrows( + IllegalStateException.class, () -> evolvingCodec(RemovedFieldCycle.class)); + Assert.assertTrue(e.getMessage().contains("Circular schema history"), e.getMessage()); + } + + /** A still-present field carrying a finite until; removals belong on the history class. */ + @Data + public static class LiveFieldWithUntil { + private int x; + + @ForyRowVersion(until = 3) + private String stillHere; + } + + @Test + public void liveFieldWithUntilFailsAtBuild() { + IllegalStateException e = + Assert.expectThrows( + IllegalStateException.class, () -> evolvingCodec(LiveFieldWithUntil.class)); + Assert.assertTrue(e.getMessage().contains("live field must not set until"), e.getMessage()); + } + + /** A since below the first version adds a schema version no writer can emit. */ + @Data + public static class LiveFieldSinceBelowFirst { + private int x; + + @ForyRowVersion(since = 0) + private String added; + } + + @Test + public void liveFieldSinceBelowFirstFailsAtBuild() { + IllegalStateException e = + Assert.expectThrows( + IllegalStateException.class, () -> evolvingCodec(LiveFieldSinceBelowFirst.class)); + Assert.assertTrue(e.getMessage().contains("must be >= 1"), e.getMessage()); + } + + /** A window that starts at the open bound of until is empty: no version would hold a field. */ + @Data + public static class LiveFieldSinceMax { + private int x; + + @ForyRowVersion(since = Integer.MAX_VALUE) + private String added; + } + + @Test + public void liveFieldSinceMaxFailsAtBuild() { + IllegalStateException e = + Assert.expectThrows( + IllegalStateException.class, () -> evolvingCodec(LiveFieldSinceMax.class)); + Assert.assertTrue(e.getMessage().contains("less than Integer.MAX_VALUE"), e.getMessage()); + } + + @Data + @ForyRowSchema(removedFields = RemovedFieldSinceBelowFirst.History.class) + public static class RemovedFieldSinceBelowFirst { + private int x; + + interface History { + @ForyRowVersion(since = 0, until = 3) + String legacy(); + } + } + + @Test + public void removedFieldSinceBelowFirstFailsAtBuild() { + IllegalStateException e = + Assert.expectThrows( + IllegalStateException.class, () -> evolvingCodec(RemovedFieldSinceBelowFirst.class)); + Assert.assertTrue(e.getMessage().contains("must be >= 1"), e.getMessage()); + } + + /** The codec reads the version of a class field from the field, never from its getter. */ + @Data + public static class VersionedGetter { + private String name; + private String email; + + @ForyRowVersion(since = 2) + public String getEmail() { + return email; + } + } + + @Test + public void versionedGetterFailsAtBuild() { + IllegalStateException e = + Assert.expectThrows( + IllegalStateException.class, () -> evolvingCodec(VersionedGetter.class)); + Assert.assertTrue(e.getMessage().contains("getEmail() has no effect"), e.getMessage()); + } + + public interface VersionedContact { + String getName(); + + @ForyRowVersion(since = 2) + String getNickname(); + } + + /** + * The descriptors of a class bean are its fields, which carry their own versions. The annotations + * of an interface that it implements belong to the codec of that interface bean. + */ + @Data + public static class VersionedContactImpl implements VersionedContact { + private String name; + + @ForyRowVersion(since = 2) + private String nickname; + } + + @Data + public static class VersionedContactV1 { + private String name; + } + + @Test + public void classImplementingVersionedInterfaceBeanBuilds() { + VersionedContactV1 in = new VersionedContactV1(); + in.setName("ann"); + VersionedContactImpl out = + evolvingCodec(VersionedContactImpl.class) + .decode(evolvingCodec(VersionedContactV1.class).encode(in)); + Assert.assertEquals(out.getName(), "ann"); + Assert.assertNull(out.getNickname()); + evolvingCodec(VersionedContact.class); + } + + /** A class has its fields, not its methods, as descriptors, so it cannot declare removals. */ + @Data + @ForyRowSchema(removedFields = RemovedFieldsClass.History.class) + public static class RemovedFieldsClass { + private String name; + + abstract static class History { + @ForyRowVersion(until = 2) + abstract int age(); + } + } + + @Test + public void removedFieldsClassFailsAtBuild() { + IllegalStateException e = + Assert.expectThrows( + IllegalStateException.class, () -> evolvingCodec(RemovedFieldsClass.class)); + Assert.assertTrue(e.getMessage().contains("must name an interface"), e.getMessage()); + } + + // --------------------------------------------------------------------------- + // A field whose type is a Collection subclass that shadows a field name across + // its own hierarchy. The row format encodes it through the iterable branch and + // never introspects it as a bean, so it round-trips fine. SchemaHistory must + // apply the same iterable/map/bean classification before introspecting a nested + // field type; otherwise it calls Descriptor.getDescriptors on the shadowed + // collection class and fails the whole history build on a bean that works. + // --------------------------------------------------------------------------- + + public static class TaggedListBase extends ArrayList { + protected String marker; + } + + // Shadows TaggedListBase.marker, which makes Descriptor.getDescriptors reject + // this class even though the codec treats it purely as a List. + public static class TaggedList extends TaggedListBase { + protected String marker; + } + + @Data + public static class ShadowedCollectionV2 { + private TaggedList labels; + + @ForyRowVersion(since = 2) + private String tag; + } + + @Test + public void versionedBeanWithShadowedCollectionFieldBuilds() { + RowEncoder codec = evolvingCodec(ShadowedCollectionV2.class); + ShadowedCollectionV2 in = new ShadowedCollectionV2(); + TaggedList labels = new TaggedList<>(); + labels.add("a"); + labels.add("b"); + in.setLabels(labels); + in.setTag("t"); + ShadowedCollectionV2 out = codec.decode(codec.encode(in)); + Assert.assertEquals(out.getLabels(), Arrays.asList("a", "b")); + Assert.assertEquals(out.getTag(), "t"); + } + + // --------------------------------------------------------------------------- + // Roundtrip a List field nested inside a versioned outer record. + // Verifies the projection codec generated for the outer correctly handles + // an inline list of plain beans whose layout is fixed. + // --------------------------------------------------------------------------- + + @Data + public static class NestedListV1 { + private List items; + } + + @Data + public static class NestedListV2 { + private List items; + + @ForyRowVersion(since = 2) + private String tag; + } + + // --------------------------------------------------------------------------- + // Evolution flag asymmetry: same class, one side opt-in, the other not. + // Documented as wire-incompatible across the row, array, and map codecs. + // An evolution-on reader rejects evolution-off bytes on every codec: it reads + // the start of the body as a hash that matches no known layout. A row has a + // hash slot in both modes, so an evolution-off row reader also rejects + // evolution-on bytes. Array and map payloads have no hash without evolution, + // so an evolution-off reader cannot detect evolution-on bytes, and the spec + // leaves that read undefined; no test pins its result. + // --------------------------------------------------------------------------- + + @DataProvider + public Object[][] flagAsymmetryCases() { + DefaultsV1 v = new DefaultsV1(); + v.setName("hi"); + + RowEncoder rowOn = evolvingCodec(DefaultsV1.class); + RowEncoder rowOff = Encoders.buildBeanCodec(DefaultsV1.class).build().get(); + + ArrayEncoder> arrayOn = + Encoders.buildArrayCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + ArrayEncoder> arrayOff = + Encoders.buildArrayCodec(new TypeRef>() {}).build().get(); + List list = Arrays.asList(v); + + MapEncoder> mapOn = + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + MapEncoder> mapOff = + Encoders.buildMapCodec(new TypeRef>() {}).build().get(); + LinkedHashMap map = new LinkedHashMap<>(); + map.put("k", v); + + return new Object[][] { + {"row", rowOff.encode(v), (Consumer) rowOn::decode}, + {"array", arrayOff.encode(list), (Consumer) arrayOn::decode}, + {"map", mapOff.encode(map), (Consumer) mapOn::decode}, + }; + } + + @Test(dataProvider = "flagAsymmetryCases") + public void evolutionOnReaderRejectsEvolutionOffBytes( + String label, byte[] offBytes, Consumer onDecoder) { + Assert.expectThrows(ClassNotCompatibleException.class, () -> onDecoder.accept(offBytes)); + } + + @Test + public void evolutionOffRowReaderRejectsEvolutionOnBytes() { + DefaultsV1 v = new DefaultsV1(); + v.setName("hi"); + RowEncoder rowOff = Encoders.buildBeanCodec(DefaultsV1.class).build().get(); + byte[] onBytes = evolvingCodec(DefaultsV1.class).encode(v); + Assert.expectThrows(ClassNotCompatibleException.class, () -> rowOff.decode(onBytes)); + } + + // --------------------------------------------------------------------------- + // Framing: payloads with a size prefix, several in one buffer. A decode that + // rejects a payload must leave the buffer at the start of the next payload, + // and a size too small to hold the hash must not move the buffer backward. + // --------------------------------------------------------------------------- + + @DataProvider + public Object[][] framedCodecs() { + DefaultsV1 v = new DefaultsV1(); + v.setName("hi"); + List list = Arrays.asList(v); + LinkedHashMap map = new LinkedHashMap<>(); + map.put("k", v); + + RowEncoder row = + Encoders.buildBeanCodec(DefaultsV1.class) + .withSchemaEvolution() + .withSizeEmbedded(true) + .build() + .get(); + RowEncoder rowOff = + Encoders.buildBeanCodec(DefaultsV1.class).withSizeEmbedded(true).build().get(); + ArrayEncoder> array = + Encoders.buildArrayCodec(new TypeRef>() {}) + .withSchemaEvolution() + .withSizeEmbedded(true) + .build() + .get(); + MapEncoder> mapCodec = + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .withSizeEmbedded(true) + .build() + .get(); + + return new Object[][] { + {"row", new FramedCodec(b -> row.encode(b, v), row::decode, v)}, + {"row without evolution", new FramedCodec(b -> rowOff.encode(b, v), rowOff::decode, v)}, + {"array", new FramedCodec(b -> array.encode(b, list), array::decode, list)}, + {"map", new FramedCodec(b -> mapCodec.encode(b, map), mapCodec::decode, map)}, + }; + } + + @Test(dataProvider = "framedCodecs") + public void unknownHashSkipsPayload(String label, FramedCodec c) { + MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(64); + c.encode.accept(buffer); + c.encode.accept(buffer); + // The hash of the first payload follows its 4-byte size prefix. + buffer.putInt64(4, buffer.getInt64(4) ^ 1); + Assert.expectThrows(ClassNotCompatibleException.class, () -> c.decode.apply(buffer)); + Assert.assertEquals(c.decode.apply(buffer), c.value); + } + + /** + * On a stream-backed buffer, the skip of an unknown payload moves the reader index past exactly + * its body, so the next payload still decodes. The stream returns one byte per read, so the + * buffer holds no byte that a decode did not ask for. + */ + @Test(dataProvider = "framedCodecs") + public void unknownHashSkipsStreamedPayload(String label, FramedCodec c) { + MemoryBuffer encoded = MemoryBuffer.newHeapBuffer(64); + c.encode.accept(encoded); + int next = encoded.writerIndex(); + c.encode.accept(encoded); + encoded.putInt64(4, encoded.getInt64(4) ^ 1); + MemoryBuffer buffer = oneByteStream(encoded); + Assert.expectThrows(ClassNotCompatibleException.class, () -> c.decode.apply(buffer)); + Assert.assertEquals(buffer.readerIndex(), next); + Assert.assertEquals(c.decode.apply(buffer), c.value); + } + + /** + * A row, array, or map decode loads the body from the stream for the current and an older layout. + */ + @Test + public void streamedDecode() { + DefaultsV1 old = new DefaultsV1(); + old.setName("old"); + DefaultsV2 current = new DefaultsV2(); + current.setName("current"); + current.setPrimitiveCount(3); + current.setBoxedCount(4); + DefaultsV2 oldAsCurrent = new DefaultsV2(); + oldAsCurrent.setName("old"); + + MemoryBuffer rows = MemoryBuffer.newHeapBuffer(64); + framedRowCodec(DefaultsV1.class).encode(rows, old); + RowEncoder rowReader = framedRowCodec(DefaultsV2.class); + rowReader.encode(rows, current); + MemoryBuffer rowStream = oneByteStream(rows); + Assert.assertEquals(rowReader.decode(rowStream), oldAsCurrent); + Assert.assertEquals(rowReader.decode(rowStream), current); + + MemoryBuffer arrays = MemoryBuffer.newHeapBuffer(64); + framedArrayCodec(new TypeRef>() {}).encode(arrays, Arrays.asList(old)); + ArrayEncoder> arrayReader = + framedArrayCodec(new TypeRef>() {}); + arrayReader.encode(arrays, Arrays.asList(current)); + MemoryBuffer arrayStream = oneByteStream(arrays); + Assert.assertEquals(arrayReader.decode(arrayStream), Arrays.asList(oldAsCurrent)); + Assert.assertEquals(arrayReader.decode(arrayStream), Arrays.asList(current)); + + MemoryBuffer maps = MemoryBuffer.newHeapBuffer(64); + framedMapCodec(new TypeRef>() {}) + .encode(maps, Collections.singletonMap("k", old)); + MapEncoder> mapReader = + framedMapCodec(new TypeRef>() {}); + mapReader.encode(maps, Collections.singletonMap("k", current)); + MemoryBuffer mapStream = oneByteStream(maps); + Assert.assertEquals(mapReader.decode(mapStream), Collections.singletonMap("k", oldAsCurrent)); + Assert.assertEquals(mapReader.decode(mapStream), Collections.singletonMap("k", current)); + } + + private static RowEncoder framedRowCodec(Class type) { + return Encoders.buildBeanCodec(type).withSchemaEvolution().withSizeEmbedded(true).build().get(); + } + + private static > ArrayEncoder framedArrayCodec(TypeRef type) { + return Encoders.buildArrayCodec(type) + .withSchemaEvolution() + .withSizeEmbedded(true) + .build() + .get(); + } + + private static > MapEncoder framedMapCodec(TypeRef type) { + return Encoders.buildMapCodec(type).withSchemaEvolution().withSizeEmbedded(true).build().get(); + } + + /** Streams the written bytes of {@code encoded} one byte per read. */ + private static MemoryBuffer oneByteStream(MemoryBuffer encoded) { + InputStream stream = + new FilterInputStream( + new ByteArrayInputStream(encoded.getBytes(0, encoded.writerIndex()))) { + @Override + public int read(byte[] b, int off, int len) throws IOException { + return super.read(b, off, Math.min(len, 1)); + } + }; + return new ForyInputStream(stream).getBuffer(); + } + + /** + * A size prefix below the 8-byte hash fails the decode, and the decode does not move the reader + * index backward. The stream returns one byte per read, so the buffer holds only the bytes that + * the decode asked for, and a decode that moves back leaves the reader index before their end. + */ + @Test(dataProvider = "framedCodecs") + public void sizeBelowHashRejected(String label, FramedCodec c) { + MemoryBuffer encoded = MemoryBuffer.newHeapBuffer(64); + c.encode.accept(encoded); + // The payload keeps its valid hash, but its size prefix says 4 bytes, less than the hash. + encoded.putInt32(0, 4); + MemoryBuffer buffer = oneByteStream(encoded); + Assert.expectThrows(RuntimeException.class, () -> c.decode.apply(buffer)); + Assert.assertEquals(buffer.readerIndex(), buffer.size()); + } + + private static final class FramedCodec { + final Consumer encode; + final Function decode; + final Object value; + + FramedCodec( + Consumer encode, Function decode, Object value) { + this.encode = encode; + this.decode = decode; + this.value = value; + } + } + + // --------------------------------------------------------------------------- + // mapSuffix injectivity: the generated map codec class name joins the value and key suffixes, + // and the class is cached by name, so two distinct (value-combo, key-combo) pairs must never + // produce the same suffix or the second pair would reuse the first's codec class. The boundary + // between the halves is the only place value and key strings meet, so the adversarial case is a + // value-side nested bean whose own suffix token contains a boundary-like string such as "_K_V". + // A bean named K_V yields exactly that token. + // --------------------------------------------------------------------------- + + @Data + public static class KVTokenV2 { + private int a; + + @ForyRowVersion(since = 2) + private String b; + } + + /** + * Outer value bean carrying a nested bean whose simple name is K_V, so its suffix token is _K_V*. + */ + @Data + public static class ValueWithKVNested { + private K_V inner; + + @ForyRowVersion(since = 2) + private String tag; + } + + /** Simple name is literally "K_V": its projection token is "_K_Vh". */ + @Data + public static class K_V { + private int x; + + @ForyRowVersion(since = 2) + private String y; + } + + @Test + public void mapSuffixIsInjective() { + // Build histories for a value position that contains the adversarial _K_V token and a key + // position that also evolves, then join every (value, key) combination the way the map codec + // does and require the joined suffixes to be distinct. + SchemaHistory valHistory = + SchemaHistory.build(ValueWithKVNested.class, SchemaHistory.Format.STANDARD); + SchemaHistory keyHistory = SchemaHistory.build(KVTokenV2.class, SchemaHistory.Format.STANDARD); + + Set joined = new HashSet<>(); + Set pairs = new HashSet<>(); + for (SchemaHistory.VersionedSchema valVs : valHistory.versions()) { + String val = valVs.isCurrent() ? "" : ProjectionRouting.projectionSuffix(valVs); + for (SchemaHistory.VersionedSchema keyVs : keyHistory.versions()) { + String key = keyVs.isCurrent() ? "" : ProjectionRouting.projectionSuffix(keyVs); + String suffix = + ProjectionRouting.mapSuffix( + val, key, SchemaHistory.combineHashes(keyVs.strictHash(), valVs.strictHash())); + pairs.add(val + "\0" + key); + Assert.assertTrue( + joined.add(suffix), + "mapSuffix collision: distinct (value, key) combinations produced the same class " + + "suffix \"" + + suffix + + "\""); + } + } + // Sanity: we actually exercised more than one distinct (value, key) pair, including a value + // suffix that contains the _K_V boundary string. + Assert.assertTrue(pairs.size() > 1, "expected multiple combinations"); + Assert.assertTrue( + valHistory.versions().stream() + .anyMatch( + vs -> !vs.isCurrent() && ProjectionRouting.projectionSuffix(vs).contains("_K_V")), + "test did not exercise the _K_V boundary token it was designed to probe"); + // A value-side token that repeats the start of a key-side suffix must not join to the same + // name as that key suffix. + String h = "h0123456789abcdef"; + Assert.assertNotEquals( + ProjectionRouting.mapSuffix("_V1_K_V1_X2" + h, "", 0), + ProjectionRouting.mapSuffix("_V1", "_V1_X2" + h, 0)); + } + + // --------------------------------------------------------------------------- + // Projection class names: a projection suffix holds a token per nested versioned class, so a + // bean with several of them, several levels down, gets a long class name. FORY_CODE_DIR writes + // the source of every generated class to a file named after it, which fails past the 255-byte + // file name limit unless the suffix is bounded. + // --------------------------------------------------------------------------- + + @Data + public static class LongNameLeafV1 { + private int a; + } + + @Data + public static class VersionedLeafWithASimpleNameLongEnoughToWidenA { + private int a; + + @ForyRowVersion(since = 2) + private int b; + } + + @Data + public static class VersionedLeafWithASimpleNameLongEnoughToWidenB { + private int a; + + @ForyRowVersion(since = 2) + private int b; + } + + @Data + public static class VersionedLeafWithASimpleNameLongEnoughToWidenC { + private int a; + + @ForyRowVersion(since = 2) + private int b; + } + + @Data + public static class WideMiddleV1 { + private LongNameLeafV1 alpha; + private LongNameLeafV1 beta; + private LongNameLeafV1 gamma; + } + + /** Its projection suffix has a token for each of three leaf classes with long simple names. */ + @Data + public static class WideMiddle { + private VersionedLeafWithASimpleNameLongEnoughToWidenA alpha; + private VersionedLeafWithASimpleNameLongEnoughToWidenB beta; + private VersionedLeafWithASimpleNameLongEnoughToWidenC gamma; + } + + @Data + public static class DeepTopV1 { + private WideMiddleV1 middle; + private int x; + } + + @Data + public static class DeepTop { + private WideMiddle middle; + private int x; + } + + private static DeepTopV1 deepTopV1(int x) { + LongNameLeafV1 leaf = new LongNameLeafV1(); + leaf.setA(x * 10); + WideMiddleV1 middle = new WideMiddleV1(); + middle.setAlpha(leaf); + middle.setBeta(leaf); + middle.setGamma(leaf); + DeepTopV1 top = new DeepTopV1(); + top.setMiddle(middle); + top.setX(x); + return top; + } + + private static void assertDeepTop(DeepTop top, int x) { + Assert.assertEquals(top.getX(), x); + Assert.assertEquals(top.getMiddle().getAlpha().getA(), x * 10); + Assert.assertEquals(top.getMiddle().getBeta().getA(), x * 10); + Assert.assertEquals(top.getMiddle().getGamma().getA(), x * 10); + Assert.assertEquals(top.getMiddle().getGamma().getB(), 0); + } + + /** + * Older payloads of a deeply nested bean decode, as a row, an array element, and a map value, + * while {@code FORY_CODE_DIR} writes the source of every generated class, and every file name + * stays under the 255-byte limit. + */ + @Test + public void projectionClassNamesAreBounded() throws IOException { + Path codeDir = Files.createTempDirectory("fory-projection-code"); + String previous = System.getProperty("FORY_CODE_DIR"); + System.setProperty("FORY_CODE_DIR", codeDir.toString()); + List files; + try { + DeepTop row = + Encoders.buildBeanCodec(DeepTop.class) + .withSchemaEvolution() + .build() + .get() + .decode( + Encoders.buildBeanCodec(DeepTopV1.class) + .withSchemaEvolution() + .build() + .get() + .encode(deepTopV1(1))); + assertDeepTop(row, 1); + List list = + Encoders.buildArrayCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get() + .decode( + Encoders.buildArrayCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get() + .encode(Collections.singletonList(deepTopV1(2)))); + assertDeepTop(list.get(0), 2); + Map map = + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get() + .decode( + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get() + .encode(Collections.singletonMap("k", deepTopV1(3)))); + assertDeepTop(map.get("k"), 3); + } finally { + if (previous == null) { + System.clearProperty("FORY_CODE_DIR"); + } else { + System.setProperty("FORY_CODE_DIR", previous); + } + try (Stream walk = Files.walk(codeDir)) { + files = walk.sorted(Comparator.reverseOrder()).collect(Collectors.toList()); + } + for (Path file : files) { + Files.delete(file); + } + } + Assert.assertTrue( + files.stream() + .map(f -> f.getFileName().toString()) + .anyMatch(name -> name.contains("WideMiddleRowCodec") && name.contains("_V1h")), + files.toString()); + for (Path file : files) { + String name = file.getFileName().toString(); + Assert.assertTrue(name.getBytes(StandardCharsets.UTF_8).length < 255, name); + } + } + + // --------------------------------------------------------------------------- + // Map with a versioned bean as the KEY: current-version round-trip. + // Cross-version key evolution is covered by evolveMapKey* below; this pins the + // same-schema baseline so a regression there is distinguishable from a build + // or encode/decode fault. + // --------------------------------------------------------------------------- + + @Test + public void mapNonBeanKeyTypeChangeFailsLoud() { + MapEncoder> writer = + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + MapEncoder> reader = + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + Map in = new HashMap<>(); + DefaultsV1 value = new DefaultsV1(); + value.setName("v"); + in.put("k", value); + byte[] bytes = writer.encode(in); + Assert.expectThrows(ClassNotCompatibleException.class, () -> reader.decode(bytes)); + } + + @Test + public void mapWithoutBeanValueTypeChangeFailsLoud() { + MapEncoder> writer = + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + MapEncoder> reader = + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + Map in = new HashMap<>(); + in.put("k", 7L); + byte[] bytes = writer.encode(in); + Assert.expectThrows(ClassNotCompatibleException.class, () -> reader.decode(bytes)); + } + + @Data + public static class FramingElement { + private byte narrow; + private long wide; + } + + /** + * The array and map hash prefixes, built from the one-field layouts that the spec defines. The + * expected values are the spec's framing vectors, computed independently of this implementation. + */ + @Test + public void hashPrefixMatchesSpecFraming() { + Map map = new HashMap<>(); + map.put("k", 1); + Assert.assertEquals( + prefix( + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get() + .encode(map)), + 0xa8efad587d8a6b3bL); + List list = new ArrayList<>(); + list.add(new FramingElement()); + Assert.assertEquals( + prefix( + Encoders.buildArrayCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get() + .encode(list)), + 0x66c342b5ad6942cbL); + Assert.assertEquals( + prefix( + Encoders.buildArrayCodec(new TypeRef>() {}) + .compactEncoding() + .withSchemaEvolution() + .build() + .get() + .encode(list)), + 0x905b89131d905a88L); + } + + private static long prefix(byte[] bytes) { + return MemoryBuffer.fromByteArray(bytes).readInt64(); + } + + /** + * A projection whose code generation fails must fail on every later decode of its layout, without + * generating again. A later decode throws a new exception, caused by the first failure, so that + * threads do not share one exception instance and its stack trace. + */ + @Test + public void projectionGenerationFailureIsCached() { + int[] generated = {0}; + IllegalStateException failure = new IllegalStateException("generation failed"); + ProjectionSource source = + new ProjectionSource() { + @Override + Object generate() { + generated[0]++; + throw failure; + } + + @Override + Object bind(Object classes, Fory fory) { + return classes; + } + }; + Assert.assertSame( + Assert.expectThrows(IllegalStateException.class, () -> source.compile(null)), failure); + EncoderException second = + Assert.expectThrows(EncoderException.class, () -> source.compile(null)); + EncoderException third = + Assert.expectThrows(EncoderException.class, () -> source.compile(null)); + Assert.assertSame(second.getCause(), failure); + Assert.assertSame(third.getCause(), failure); + Assert.assertNotSame(second, third); + Assert.assertEquals(generated[0], 1); + } + + /** Every field is 8 bytes wide, so the compact sort keeps the standard field order. */ + @Data + public static class EightByteFields { + private long count; + private String name; + } + + /** + * The standard and compact formats write {@link EightByteFields} in the same field order, so only + * the format tag in the strict hash tells their layouts apart. A reader of either format must + * reject the other format's row, array, and map payloads instead of decoding wrong values. + */ + @Test + public void formatMismatchIsRejected() { + EightByteFields bean = new EightByteFields(); + bean.setCount(7); + bean.setName("n"); + List list = Collections.singletonList(bean); + Map map = Collections.singletonMap("k", bean); + TypeRef> listType = new TypeRef>() {}; + TypeRef> mapType = new TypeRef>() {}; + for (boolean compactWriter : new boolean[] {false, true}) { + RowCodecBuilder rowWriter = Encoders.buildBeanCodec(EightByteFields.class); + RowCodecBuilder rowReader = Encoders.buildBeanCodec(EightByteFields.class); + ArrayCodecBuilder> arrayWriter = Encoders.buildArrayCodec(listType); + ArrayCodecBuilder> arrayReader = Encoders.buildArrayCodec(listType); + MapCodecBuilder> mapWriter = Encoders.buildMapCodec(mapType); + MapCodecBuilder> mapReader = Encoders.buildMapCodec(mapType); + if (compactWriter) { + rowWriter.compactEncoding(); + arrayWriter.compactEncoding(); + mapWriter.compactEncoding(); + } else { + rowReader.compactEncoding(); + arrayReader.compactEncoding(); + mapReader.compactEncoding(); + } + byte[] row = rowWriter.withSchemaEvolution().build().get().encode(bean); + byte[] array = arrayWriter.withSchemaEvolution().build().get().encode(list); + byte[] mapBytes = mapWriter.withSchemaEvolution().build().get().encode(map); + RowEncoder rowCodec = rowReader.withSchemaEvolution().build().get(); + ArrayEncoder> arrayCodec = + arrayReader.withSchemaEvolution().build().get(); + MapEncoder> mapCodec = + mapReader.withSchemaEvolution().build().get(); + Assert.expectThrows(ClassNotCompatibleException.class, () -> rowCodec.decode(row)); + Assert.expectThrows(ClassNotCompatibleException.class, () -> arrayCodec.decode(array)); + Assert.expectThrows(ClassNotCompatibleException.class, () -> mapCodec.decode(mapBytes)); + } + } + + @Data + public static class CompactOrderStats { + private int count; + private long total; + } + + @Data + public static class CompactOrderRow { + private String code; + private boolean flag; + private long id; + private CompactOrderStats stats; + private short version; + } + + /** + * The compact row hash covers the fields in the spec's compact sort order: the string keeps its + * wire-name order among the 8-byte fields, and the 12-byte inline struct sorts after the int16 + * and before the bool. The expected value is the spec's compact vector, computed independently of + * this implementation. + */ + @Test + public void compactRowHashMatchesSpecVector() { + byte[] bytes = + Encoders.buildBeanCodec(CompactOrderRow.class) + .compactEncoding() + .withSchemaEvolution() + .build() + .get() + .encode(new CompactOrderRow()); + Assert.assertEquals(prefix(bytes), 0xa6ca2310859be6e6L); + } + + @Test + public void mapWithVersionedKey() { + MapEncoder> codec = + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + DefaultsV2 k = new DefaultsV2(); + k.setName("k"); + k.setPrimitiveCount(1); + k.setBoxedCount(2); + Map in = new HashMap<>(); + in.put(k, "v"); + Map out = codec.decode(codec.encode(in)); + Assert.assertEquals(out.size(), 1); + DefaultsV2 outKey = out.keySet().iterator().next(); + Assert.assertEquals(outKey.getName(), "k"); + Assert.assertEquals(outKey.getPrimitiveCount(), 1); + Assert.assertEquals(outKey.getBoxedCount(), Integer.valueOf(2)); + } + + // A top-level map whose value evolves while the key is an unversioned struct bean with the same + // shape on both sides. The key must round-trip unchanged at its current schema. + @Test + public void mapStructKeyValueEvolution() { + MapEncoder> writer = + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + MapEncoder> reader = + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + RemovedGeo key = new RemovedGeo(); + key.setLat(7.5); + DefaultsV1 val = new DefaultsV1(); + val.setName("val"); + Map in = new HashMap<>(); + in.put(key, val); + Map out = reader.decode(writer.encode(in)); + Assert.assertEquals(out.size(), 1); + Map.Entry entry = out.entrySet().iterator().next(); + Assert.assertEquals(entry.getKey().getLat(), 7.5); + Assert.assertEquals(entry.getValue().getName(), "val"); + Assert.assertEquals(entry.getValue().getPrimitiveCount(), 0); + Assert.assertNull(entry.getValue().getBoxedCount()); + } + + // A row field typed as Map with the key bean at its current version on both + // sides. A map key is a version site of the outer history, like a map value (see + // evolvingMapKeyInRowField). Here the writer and reader differ only in the outer bean, so the + // keyed map round-trips with the key at its current layout. + @Data + public static class KeyMapHolderV1 { + private Map byKey; + } + + @Data + public static class KeyMapHolderV2 { + private Map byKey; + + @ForyRowVersion(since = 2) + private String note; + } + + @Test + public void versionedBeanAsMapKeyInRowField() { + RowEncoder writer = evolvingCodec(KeyMapHolderV1.class); + RowEncoder reader = evolvingCodec(KeyMapHolderV2.class); + DefaultsV2 key = new DefaultsV2(); + key.setName("k"); + key.setPrimitiveCount(7); + key.setBoxedCount(8); + KeyMapHolderV1 in = new KeyMapHolderV1(); + in.setByKey(new HashMap<>()); + in.getByKey().put(key, "v"); + KeyMapHolderV2 out = reader.decode(writer.encode(in)); + Assert.assertEquals(out.getByKey().size(), 1); + DefaultsV2 outKey = out.getByKey().keySet().iterator().next(); + Assert.assertEquals(outKey.getName(), "k"); + Assert.assertEquals(outKey.getPrimitiveCount(), 7); + Assert.assertEquals(out.getByKey().get(outKey), "v"); + Assert.assertNull(out.getNote()); // note added at v2; v1 payload defaults it + } + + // A row field whose map KEY is a versioned bean that actually skews across writer and reader: + // writer's key is DefaultsV1 (name only), reader's key is DefaultsV2 (adds two since=2 fields). + // Unlike versionedBeanAsMapKeyInRowField, the key version differs on the two sides, so this is + // the case that must not corrupt: the reader must materialize the since=2 fields at their + // defaults rather than reinterpreting the writer's bytes against the current key layout. + @Data + public static class SkewKeyMapHolderV1 { + private Map byKey; + } + + @Data + public static class SkewKeyMapHolderV2 { + private Map byKey; + } + + @Test + public void evolvingMapKeyInRowField() { + RowEncoder writer = evolvingCodec(SkewKeyMapHolderV1.class); + RowEncoder reader = evolvingCodec(SkewKeyMapHolderV2.class); + DefaultsV1 key = new DefaultsV1(); + key.setName("k"); + SkewKeyMapHolderV1 in = new SkewKeyMapHolderV1(); + in.setByKey(new HashMap<>()); + in.getByKey().put(key, "v"); + SkewKeyMapHolderV2 out = reader.decode(writer.encode(in)); + Assert.assertEquals(out.getByKey().size(), 1); + DefaultsV2 outKey = out.getByKey().keySet().iterator().next(); + Assert.assertEquals(outKey.getName(), "k"); + Assert.assertEquals(outKey.getPrimitiveCount(), 0); // since=2 field absent in writer; defaults + Assert.assertNull(outKey.getBoxedCount()); + Assert.assertEquals(out.getByKey().get(outKey), "v"); + } + + // A versioned bean as the KEY of a map nested inside another map's value: the substitution path + // must take value at the outer map and key at the inner map. A single key/value flag cannot + // express that two-step descent, so this exercises the multi-step branch path. + @Data + public static class NestedKeyMapHolderV1 { + private Map> outer; + } + + @Data + public static class NestedKeyMapHolderV2 { + private Map> outer; + } + + @Test + public void evolvingKeyOfNestedMap() { + RowEncoder writer = evolvingCodec(NestedKeyMapHolderV1.class); + RowEncoder reader = evolvingCodec(NestedKeyMapHolderV2.class); + DefaultsV1 key = new DefaultsV1(); + key.setName("k"); + Map inner = new HashMap<>(); + inner.put(key, "v"); + NestedKeyMapHolderV1 in = new NestedKeyMapHolderV1(); + in.setOuter(new HashMap<>()); + in.getOuter().put("o", inner); + NestedKeyMapHolderV2 out = reader.decode(writer.encode(in)); + Map outInner = out.getOuter().get("o"); + Assert.assertEquals(outInner.size(), 1); + DefaultsV2 outKey = outInner.keySet().iterator().next(); + Assert.assertEquals(outKey.getName(), "k"); + Assert.assertEquals(outKey.getPrimitiveCount(), 0); // since=2 field absent in writer; defaults + Assert.assertNull(outKey.getBoxedCount()); + Assert.assertEquals(outInner.get(outKey), "v"); + } + + // --------------------------------------------------------------------------- + // Evolving map key: the map header's combined (key,value) hash selects the + // historical key layout, so an older key decodes correctly instead of being + // read against the current layout and corrupting silently. + // --------------------------------------------------------------------------- + + @Test + public void evolveMapKeyValuePrimitive() { + // Writer key is DefaultsV1 (name only); reader key is DefaultsV2 (adds two since=2 fields). + // Value is a plain String on both sides, so only the key dimension evolves. + MapEncoder> writer = + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + MapEncoder> reader = + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + DefaultsV1 k = new DefaultsV1(); + k.setName("k"); + Map in = new HashMap<>(); + in.put(k, "v"); + Map out = reader.decode(writer.encode(in)); + Assert.assertEquals(out.size(), 1); + Map.Entry e = out.entrySet().iterator().next(); + Assert.assertEquals(e.getKey().getName(), "k"); + Assert.assertEquals(e.getKey().getPrimitiveCount(), 0); // absent since=2 field defaults + Assert.assertNull(e.getKey().getBoxedCount()); + Assert.assertEquals(e.getValue(), "v"); + } + + @Test + public void evolveBothMapKeyAndValue() { + // Both key and value evolve independently: V1 writer, V2 reader on each side. + MapEncoder> writer = + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + MapEncoder> reader = + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + DefaultsV1 k = new DefaultsV1(); + k.setName("k"); + DefaultsV1 v = new DefaultsV1(); + v.setName("val"); + Map in = new HashMap<>(); + in.put(k, v); + Map out = reader.decode(writer.encode(in)); + Assert.assertEquals(out.size(), 1); + Map.Entry e = out.entrySet().iterator().next(); + Assert.assertEquals(e.getKey().getName(), "k"); + Assert.assertEquals(e.getKey().getPrimitiveCount(), 0); + Assert.assertNull(e.getKey().getBoxedCount()); + Assert.assertEquals(e.getValue().getName(), "val"); + Assert.assertEquals(e.getValue().getPrimitiveCount(), 0); + Assert.assertNull(e.getValue().getBoxedCount()); + } + + @Test + public void evolveMapKeyCurrentValueOlder() { + // The key is a versioned bean already at its current version; the value is a different + // versioned bean at an older version. Exercises the (key-current, value-older) projection. + MapEncoder> writer = + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + MapEncoder> reader = + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + InitializedV2 k = new InitializedV2(); + k.setName("k"); + k.setNick(Optional.of("n")); + k.setLabel("l"); + DefaultsV1 v = new DefaultsV1(); + v.setName("val"); + Map in = new HashMap<>(); + in.put(k, v); + Map out = reader.decode(writer.encode(in)); + Assert.assertEquals(out.size(), 1); + Map.Entry e = out.entrySet().iterator().next(); + Assert.assertEquals(e.getKey().getName(), "k"); + Assert.assertEquals(e.getKey().getNick(), Optional.of("n")); // key fully present + Assert.assertEquals(e.getKey().getLabel(), "l"); + Assert.assertEquals(e.getValue().getName(), "val"); + Assert.assertEquals(e.getValue().getPrimitiveCount(), 0); + Assert.assertNull(e.getValue().getBoxedCount()); + } + + @Test + public void evolveMapSameBeanKeyAndValue() { + // Key and value are the same versioned bean class. A writer has one definition of the class, + // so both positions are at one version: a payload with both at V1 decodes, and the reader does + // not accept a payload that puts the class at two versions. + MapEncoder> olderWriter = + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + MapEncoder> mixedWriter = + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + MapEncoder> reader = + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + + DefaultsV1 oldKey = new DefaultsV1(); + oldKey.setName("ok"); + DefaultsV1 oldVal = new DefaultsV1(); + oldVal.setName("ov"); + Map older = new HashMap<>(); + older.put(oldKey, oldVal); + Map.Entry e = + reader.decode(olderWriter.encode(older)).entrySet().iterator().next(); + Assert.assertEquals(e.getKey().getName(), "ok"); + Assert.assertEquals(e.getKey().getPrimitiveCount(), 0); + Assert.assertEquals(e.getValue().getName(), "ov"); + Assert.assertNull(e.getValue().getBoxedCount()); + + DefaultsV2 newVal = new DefaultsV2(); + newVal.setName("nv"); + Map mixed = new HashMap<>(); + mixed.put(oldKey, newVal); + byte[] mixedBytes = mixedWriter.encode(mixed); + Assert.expectThrows(ClassNotCompatibleException.class, () -> reader.decode(mixedBytes)); + } + + @Test + public void evolveMapKeyCompact() { + // The key dimension must also project under the compact format, which sorts schema fields by + // alignment; the historical key schema is sorted the same way as the writer's, so the layouts + // line up. + MapEncoder> writer = + Encoders.buildMapCodec(new TypeRef>() {}) + .compactEncoding() + .withSchemaEvolution() + .build() + .get(); + MapEncoder> reader = + Encoders.buildMapCodec(new TypeRef>() {}) + .compactEncoding() + .withSchemaEvolution() + .build() + .get(); + DefaultsV1 k = new DefaultsV1(); + k.setName("k"); + Map in = new HashMap<>(); + in.put(k, "v"); + Map out = reader.decode(writer.encode(in)); + Assert.assertEquals(out.size(), 1); + Map.Entry e = out.entrySet().iterator().next(); + Assert.assertEquals(e.getKey().getName(), "k"); + Assert.assertEquals(e.getKey().getPrimitiveCount(), 0); + Assert.assertEquals(e.getValue(), "v"); + } + + @Test + public void evolveMapStructKeyWithNestedBean() { + // The key is a struct that itself wraps an evolving nested bean. The nested detail is written + // at V1 and read by a V2 reader, so the nested key bean must project to its historical layout. + MapEncoder> writer = + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + MapEncoder> reader = + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + KeyHolderV1 k = new KeyHolderV1(); + k.setId("h"); + DefaultsV1 d = new DefaultsV1(); + d.setName("inner"); + k.setDetail(d); + Map in = new HashMap<>(); + in.put(k, "v"); + Map out = reader.decode(writer.encode(in)); + Assert.assertEquals(out.size(), 1); + Map.Entry e = out.entrySet().iterator().next(); + Assert.assertEquals(e.getKey().getId(), "h"); + Assert.assertEquals(e.getKey().getDetail().getName(), "inner"); + Assert.assertEquals(e.getKey().getDetail().getPrimitiveCount(), 0); // since=2, absent in writer + Assert.assertNull(e.getKey().getDetail().getBoxedCount()); + Assert.assertEquals(e.getValue(), "v"); + } + + @Test + public void evolveMapCollectionKey() { + // The key is a List of versioned beans, so the key subtree is a collection rather than a bare + // bean. The element bean must project to its historical layout through the collection decode. + MapEncoder, String>> writer = + Encoders.buildMapCodec(new TypeRef, String>>() {}) + .withSchemaEvolution() + .build() + .get(); + MapEncoder, String>> reader = + Encoders.buildMapCodec(new TypeRef, String>>() {}) + .withSchemaEvolution() + .build() + .get(); + DefaultsV1 d = new DefaultsV1(); + d.setName("k"); + Map, String> in = new HashMap<>(); + in.put(Arrays.asList(d), "v"); + Map, String> out = reader.decode(writer.encode(in)); + Assert.assertEquals(out.size(), 1); + Map.Entry, String> e = out.entrySet().iterator().next(); + Assert.assertEquals(e.getKey().size(), 1); + Assert.assertEquals(e.getKey().get(0).getName(), "k"); + Assert.assertEquals(e.getKey().get(0).getPrimitiveCount(), 0); // since=2, absent in writer + Assert.assertNull(e.getKey().get(0).getBoxedCount()); + Assert.assertEquals(e.getValue(), "v"); + } + + // A versioned bean as the KEY of an inner map reached through a top-level array or map wrapper. + // The top-level codec's own key/value hash only covers a top-level map's own key, and the + // row-field path (the nested sites of a bean field) only applies inside a bean, so this + // wrapper-reached inner key is the shape that must still evolve through the top-level array/map + // entry point. + + @Test + public void arrayOfMapKeyOlderPayload() { + ArrayEncoder>> writer = + Encoders.buildArrayCodec(new TypeRef>>() {}) + .withSchemaEvolution() + .build() + .get(); + ArrayEncoder>> reader = + Encoders.buildArrayCodec(new TypeRef>>() {}) + .withSchemaEvolution() + .build() + .get(); + DefaultsV1 k = new DefaultsV1(); + k.setName("k"); + Map inner = new HashMap<>(); + inner.put(k, "v"); + List> out = reader.decode(writer.encode(Arrays.asList(inner))); + Assert.assertEquals(out.size(), 1); + Map.Entry e = out.get(0).entrySet().iterator().next(); + Assert.assertEquals(e.getKey().getName(), "k"); + Assert.assertEquals(e.getKey().getPrimitiveCount(), 0); // since=2 field absent in writer + Assert.assertNull(e.getKey().getBoxedCount()); + Assert.assertEquals(e.getValue(), "v"); + } + + // A top-level map whose VALUE is itself a Map with two DIFFERENT versioned bean + // classes. The value position must evolve both inner beans and generate a codec for each. + @Test + public void mapValueWrappingTwoBeansOlderPayload() { + MapEncoder>> writer = + Encoders.buildMapCodec(new TypeRef>>() {}) + .withSchemaEvolution() + .build() + .get(); + MapEncoder>> reader = + Encoders.buildMapCodec(new TypeRef>>() {}) + .withSchemaEvolution() + .build() + .get(); + AlignV1 k = new AlignV1(); + k.setX(11); + k.setY(22); + DefaultsV1 v = new DefaultsV1(); + v.setName("val"); + Map inner = new HashMap<>(); + inner.put(k, v); + Map> in = new HashMap<>(); + in.put("o", inner); + Map> out = reader.decode(writer.encode(in)); + Map.Entry e = out.get("o").entrySet().iterator().next(); + Assert.assertEquals(e.getKey().getX(), 11); + Assert.assertEquals(e.getKey().getY(), 22); + Assert.assertEquals(e.getKey().getFlag(), (byte) 0); + Assert.assertEquals(e.getValue().getName(), "val"); + Assert.assertEquals(e.getValue().getPrimitiveCount(), 0); + Assert.assertNull(e.getValue().getBoxedCount()); + } + + @Test + public void mapOfMapKeyOlderPayload() { + MapEncoder>> writer = + Encoders.buildMapCodec(new TypeRef>>() {}) + .withSchemaEvolution() + .build() + .get(); + MapEncoder>> reader = + Encoders.buildMapCodec(new TypeRef>>() {}) + .withSchemaEvolution() + .build() + .get(); + DefaultsV1 k = new DefaultsV1(); + k.setName("k"); + Map inner = new HashMap<>(); + inner.put(k, "v"); + Map> in = new HashMap<>(); + in.put("o", inner); + Map> out = reader.decode(writer.encode(in)); + Map.Entry e = out.get("o").entrySet().iterator().next(); + Assert.assertEquals(e.getKey().getName(), "k"); + Assert.assertEquals(e.getKey().getPrimitiveCount(), 0); // since=2 field absent in writer + Assert.assertNull(e.getKey().getBoxedCount()); + Assert.assertEquals(e.getValue(), "v"); + } + + // A nested map whose KEY and VALUE are DIFFERENT versioned beans, reached through a top-level + // array wrapper. Key is AlignV1/V2 (V2 adds a byte flag), value is DefaultsV1/V2 (V2 adds two + // since=2 fields). The element schema's strict hash must cover both sides at their chosen + // versions, just as the enclosing-bean hash does on the row-field path, so an older payload + // restores both beans at their historical layout. Old field values are non-default so reading + // either side at the wrong (current) layout would misread the bytes rather than coincide with a + // default. The two beans evolve independently, so dispatch must distinguish their combination. + @Test + public void arrayOfMapBothSidesOlderPayload() { + ArrayEncoder>> writer = + Encoders.buildArrayCodec(new TypeRef>>() {}) + .withSchemaEvolution() + .build() + .get(); + ArrayEncoder>> reader = + Encoders.buildArrayCodec(new TypeRef>>() {}) + .withSchemaEvolution() + .build() + .get(); + AlignV1 k = new AlignV1(); + k.setX(11); + k.setY(22); + DefaultsV1 v = new DefaultsV1(); + v.setName("val"); + Map inner = new HashMap<>(); + inner.put(k, v); + List> out = reader.decode(writer.encode(Arrays.asList(inner))); + Assert.assertEquals(out.size(), 1); + Map.Entry e = out.get(0).entrySet().iterator().next(); + Assert.assertEquals(e.getKey().getX(), 11); + Assert.assertEquals(e.getKey().getY(), 22); + Assert.assertEquals(e.getKey().getFlag(), (byte) 0); // key since=2 field absent in writer + Assert.assertEquals(e.getValue().getName(), "val"); + Assert.assertEquals(e.getValue().getPrimitiveCount(), 0); // value since=2 fields absent + Assert.assertNull(e.getValue().getBoxedCount()); + } + + // --------------------------------------------------------------------------- + // Removed nullable struct that was null on the wire: the v1 writer leaves + // the slot's null bit set; the v2 reader skips the slot during projection. + // --------------------------------------------------------------------------- + + @Data + public static class NullableStructV1 { + private String id; + private DefaultsV1 detail; // nullable, removed at v2 + private long tail; // live in both versions, positioned after the removed slot + } + + @Data + @ForyRowSchema(removedFields = NullableStructV2.History.class) + public static class NullableStructV2 { + private String id; + private long tail; + + interface History { + @ForyRowVersion(until = 2) + DefaultsV1 detail(); + } + } + + @Test + public void removedNullableStructWasNullOnWire() { + RowEncoder writer = evolvingCodec(NullableStructV1.class); + RowEncoder reader = evolvingCodec(NullableStructV2.class); + NullableStructV1 in = new NullableStructV1(); + in.setId("only-id"); + // detail intentionally left null + in.setTail(42L); + NullableStructV2 out = reader.decode(writer.encode(in)); + Assert.assertEquals(out.getId(), "only-id"); + // The field after the removed (null-on-wire) slot must read correctly, proving projection + // skipped the slot rather than reading its null bit as part of a later field. + Assert.assertEquals(out.getTail(), 42L); + } + + // --------------------------------------------------------------------------- + // Builder method ordering: compactEncoding() before vs after withSchemaEvolution() + // must produce equivalent codecs. + // --------------------------------------------------------------------------- + + @Test + public void builderMethodOrderingIsCommutative() { + RowEncoder w = + Encoders.buildBeanCodec(DefaultsV1.class) + .compactEncoding() + .withSchemaEvolution() + .build() + .get(); + RowEncoder rOrderA = + Encoders.buildBeanCodec(DefaultsV2.class) + .compactEncoding() + .withSchemaEvolution() + .build() + .get(); + RowEncoder rOrderB = + Encoders.buildBeanCodec(DefaultsV2.class) + .withSchemaEvolution() + .compactEncoding() + .build() + .get(); + DefaultsV1 in = new DefaultsV1(); + in.setName("commute"); + byte[] bytes = w.encode(in); + Assert.assertEquals(rOrderA.decode(bytes).getName(), "commute"); + Assert.assertEquals(rOrderB.decode(bytes).getName(), "commute"); + } + + @Test + public void nestedListSurvivesOuterProjection() { + RowEncoder writer = evolvingCodec(NestedListV1.class); + RowEncoder reader = evolvingCodec(NestedListV2.class); + DefaultsV1 a = new DefaultsV1(); + a.setName("a"); + DefaultsV1 b = new DefaultsV1(); + b.setName("b"); + NestedListV1 in = new NestedListV1(); + in.setItems(Arrays.asList(a, b)); + NestedListV2 out = reader.decode(writer.encode(in)); + Assert.assertEquals(out.getItems().size(), 2); + Assert.assertEquals(out.getItems().get(0).getName(), "a"); + Assert.assertEquals(out.getItems().get(1).getName(), "b"); + Assert.assertNull(out.getTag()); + } + + // --------------------------------------------------------------------------- + // Nested versioned bean: a parent bean with a struct field whose own type is + // versioned independently. The wire layout for the inner struct is inline in + // the parent's bytes with no per-inner hash. The reader, dispatching on the + // parent's strict hash, needs to choose an inner schema consistent with what + // the writer used. + // --------------------------------------------------------------------------- + + /** Stand-in for "older code that wrote the inner struct without field x". */ + @Data + public static class NestedInnerWriter { + private String name; + } + + /** Stand-in for "older code that wrote the outer containing NestedInnerWriter". */ + @Data + public static class NestedOuterWriter { + private long id; + private NestedInnerWriter inner; + } + + /** Newer inner with an added field at v2. */ + @Data + public static class NestedInnerV2 { + private String name; + + @ForyRowVersion(since = 2) + private String addedField; + } + + /** Newer outer that still has just (id, inner) but its inner type evolved. */ + @Data + public static class NestedOuterV2 { + private long id; + private NestedInnerV2 inner; + } + + @Test + public void readerInnerNewerThanWriter() { + // Writer uses the "older shape" inner. Both writer and reader are evolution-on so they + // agree on strict-hash framing. + RowEncoder writer = evolvingCodec(NestedOuterWriter.class); + RowEncoder reader = evolvingCodec(NestedOuterV2.class); + + NestedOuterWriter in = new NestedOuterWriter(); + in.setId(42); + NestedInnerWriter inn = new NestedInnerWriter(); + inn.setName("hello"); + in.setInner(inn); + + byte[] bytes = writer.encode(in); + NestedOuterV2 out = reader.decode(bytes); + Assert.assertEquals(out.getId(), 42); + Assert.assertNotNull(out.getInner()); + Assert.assertEquals(out.getInner().getName(), "hello"); + Assert.assertNull(out.getInner().getAddedField()); + } + + // --------------------------------------------------------------------------- + // Outer + inner versioned independently. The cross-product enumeration must + // generate a projection codec for each (outer-version, inner-version) pair + // that isn't the current combination. + // --------------------------------------------------------------------------- + + /** Outer with its own added field at v2; inner stays at v1. */ + @Data + public static class CrossOuterV2_InnerV1 { + private long id; + private NestedInnerWriter inner; + + @ForyRowVersion(since = 2) + private String label; + } + + /** Outer v2 reader with inner evolved to v2. Both dimensions evolve independently. */ + @Data + public static class CrossOuterV2_InnerV2 { + private long id; + private NestedInnerV2 inner; + + @ForyRowVersion(since = 2) + private String label; + } + + @Test + public void crossOuterAndInnerEvolution() { + // Writer writes outer V1 + inner V1 (no label, no addedField). + RowEncoder writer = evolvingCodec(NestedOuterWriter.class); + RowEncoder reader = evolvingCodec(CrossOuterV2_InnerV2.class); + + NestedOuterWriter in = new NestedOuterWriter(); + in.setId(100); + NestedInnerWriter inn = new NestedInnerWriter(); + inn.setName("legacy-inner"); + in.setInner(inn); + + byte[] bytes = writer.encode(in); + CrossOuterV2_InnerV2 out = reader.decode(bytes); + Assert.assertEquals(out.getId(), 100); + Assert.assertEquals(out.getInner().getName(), "legacy-inner"); + Assert.assertNull(out.getInner().getAddedField()); + Assert.assertNull(out.getLabel()); + } + + /** + * Contract: {@code SchemaHistory.current().nestedBeanSchemas()} must report each nested bean at + * its current entry. Two cross-product combinations canonicalizing to the same signature is rare + * today (the inner's own strict-hash collapse, {@code byHash}, merges wire-equal schemas before + * the outer sees them) but the contract is documented and future callers may rely on it. + */ + @Test + public void schemaHistoryCurrentReflectsCurrentInnerVersions() { + SchemaHistory history = + SchemaHistory.build(CrossOuterV2_InnerV2.class, SchemaHistory.Format.STANDARD); + SchemaHistory.VersionedSchema current = history.current(); + Assert.assertTrue(current.isCurrent(), "history.current() must be marked current"); + for (Map.Entry, SchemaHistory.VersionedSchema> e : + current.nestedBeanSchemas().entrySet()) { + SchemaHistory innerHistory = SchemaHistory.build(e.getKey(), SchemaHistory.Format.STANDARD); + Assert.assertTrue( + e.getValue().isCurrent(), + "current().nestedBeanSchemas() must report inner " + e.getKey() + " at its current"); + Assert.assertEquals( + e.getValue().version(), + innerHistory.current().version(), + "inner current version mismatch for " + e.getKey()); + } + } + + // --------------------------------------------------------------------------- + // Cross-product enumeration must route inner-bean versions through array and + // map projection codecs, not just through the row codec. The reader's outer + // type has N outer versions x M inner versions; multiple cross-product entries + // share an outer version number, so the per-class suffix must encode the + // inner version to keep them from colliding on the codegen cache. + // --------------------------------------------------------------------------- + + @Test + public void crossOuterAndInnerEvolutionArray() { + ArrayEncoder> writer = + Encoders.buildArrayCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + ArrayEncoder> reader = + Encoders.buildArrayCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + + List in = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + NestedOuterWriter e = new NestedOuterWriter(); + e.setId(i); + NestedInnerWriter inn = new NestedInnerWriter(); + inn.setName("legacy-" + i); + e.setInner(inn); + in.add(e); + } + + List out = reader.decode(writer.encode(in)); + Assert.assertEquals(out.size(), 3); + for (int i = 0; i < 3; i++) { + Assert.assertEquals(out.get(i).getId(), i); + Assert.assertEquals(out.get(i).getInner().getName(), "legacy-" + i); + Assert.assertNull(out.get(i).getInner().getAddedField()); + Assert.assertNull(out.get(i).getLabel()); + } + } + + @Test + public void crossOuterAndInnerEvolutionMap() { + MapEncoder> writer = + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + MapEncoder> reader = + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + + LinkedHashMap in = new LinkedHashMap<>(); + for (int i = 0; i < 3; i++) { + NestedOuterWriter e = new NestedOuterWriter(); + e.setId(i); + NestedInnerWriter inn = new NestedInnerWriter(); + inn.setName("legacy-" + i); + e.setInner(inn); + in.put("k" + i, e); + } + + Map out = reader.decode(writer.encode(in)); + Assert.assertEquals(out.size(), 3); + for (int i = 0; i < 3; i++) { + CrossOuterV2_InnerV2 v = out.get("k" + i); + Assert.assertNotNull(v, "missing key k" + i); + Assert.assertEquals(v.getId(), i); + Assert.assertEquals(v.getInner().getName(), "legacy-" + i); + Assert.assertNull(v.getInner().getAddedField()); + Assert.assertNull(v.getLabel()); + } + } + + // --------------------------------------------------------------------------- + // A full-length payload whose strict schema hash matches neither the reader's + // current schema nor any entry in its projection history must fail loudly. + // This is the projection-map miss. Encoding one bean and decoding it with an + // evolving codec for a structurally different bean reaches that branch + // deterministically. + // --------------------------------------------------------------------------- + + @Test + public void rowUnknownSchemaHashFailsLoudly() { + ChainV2 writer = new ChainV2(); + writer.setA(7); + writer.setB("foreign"); + byte[] foreign = evolvingCodec(ChainV2.class).encode(writer); + Assert.assertTrue(foreign.length > 8, "payload must clear the 8-byte hash prefix"); + RowEncoder reader = evolvingCodec(DefaultsV1.class); + Assert.expectThrows(ClassNotCompatibleException.class, () -> reader.decode(foreign)); + } + + @Test + public void mapUnknownSchemaHashFailsLoudly() { + MapEncoder> writer = + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + ChainV2 v = new ChainV2(); + v.setA(7); + v.setB("foreign"); + LinkedHashMap in = new LinkedHashMap<>(); + in.put("k", v); + byte[] foreign = writer.encode(in); + Assert.assertTrue(foreign.length > 8, "payload must clear the 8-byte hash prefix"); + MapEncoder> reader = + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + Assert.expectThrows(ClassNotCompatibleException.class, () -> reader.decode(foreign)); + } + + @Test + public void arrayUnknownSchemaHashFailsLoudly() { + ArrayEncoder> writer = + Encoders.buildArrayCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + ChainV2 v = new ChainV2(); + v.setA(7); + v.setB("foreign"); + List in = new ArrayList<>(); + in.add(v); + byte[] foreign = writer.encode(in); + Assert.assertTrue(foreign.length > 8, "payload must clear the 8-byte hash prefix"); + ArrayEncoder> reader = + Encoders.buildArrayCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + Assert.expectThrows(ClassNotCompatibleException.class, () -> reader.decode(foreign)); + } + + // --------------------------------------------------------------------------- + // Three-level nesting: L1 -> L2 -> L3, each independently versioned. Because + // L2's own history cross-products over L3's versions, L2's history holds two + // entries that share a version number but differ in their L3 layout. Routing + // must pick the L2 entry whose L3 matches the writer, not the first one with a + // matching version number. Identifies the inner combination by strict hash, so + // it resolves the correct subtree to arbitrary depth. + // --------------------------------------------------------------------------- + + @Data + public static class L3Writer { + private String name; + } + + @Data + public static class L2Writer { + private long tag; + private L3Writer leaf; + } + + @Data + public static class L1Writer { + private long id; + private L2Writer mid; + } + + @Data + public static class L3V2 { + private String name; + + @ForyRowVersion(since = 2) + private String note; + } + + @Data + public static class L2V2 { + private long tag; + private L3V2 leaf; + + @ForyRowVersion(since = 2) + private String midLabel; + } + + @Data + public static class L1V2 { + private long id; + private L2V2 mid; + + @ForyRowVersion(since = 2) + private String outerLabel; + } + + @Test + public void threeLevelNestedEvolution() { + RowEncoder writer = evolvingCodec(L1Writer.class); + RowEncoder reader = evolvingCodec(L1V2.class); + + L1Writer in = new L1Writer(); + in.setId(7); + L2Writer mid = new L2Writer(); + mid.setTag(11); + L3Writer leaf = new L3Writer(); + leaf.setName("deep"); + mid.setLeaf(leaf); + in.setMid(mid); + + L1V2 out = reader.decode(writer.encode(in)); + Assert.assertEquals(out.getId(), 7); + Assert.assertNull(out.getOuterLabel()); + Assert.assertEquals(out.getMid().getTag(), 11); + Assert.assertNull(out.getMid().getMidLabel()); + Assert.assertEquals(out.getMid().getLeaf().getName(), "deep"); + Assert.assertNull(out.getMid().getLeaf().getNote()); + } + + // --------------------------------------------------------------------------- + // The same versioned bean class in two fields. A writer writes one definition + // of that class, so both fields are always at the same version on the wire; + // the enumeration carries one version dimension per class, not per field, so a + // class may back more than one slot. + // --------------------------------------------------------------------------- + + @Data + public static class TwoLeafWriter { + private L3Writer first; + private L3Writer second; + } + + @Data + public static class TwoLeafV2 { + private L3V2 first; + private L3V2 second; + } + + @Test + public void sameClassInTwoFields() { + RowEncoder writer = evolvingCodec(TwoLeafWriter.class); + RowEncoder reader = evolvingCodec(TwoLeafV2.class); + + TwoLeafWriter in = new TwoLeafWriter(); + L3Writer a = new L3Writer(); + a.setName("alpha"); + L3Writer b = new L3Writer(); + b.setName("beta"); + in.setFirst(a); + in.setSecond(b); + + TwoLeafV2 out = reader.decode(writer.encode(in)); + Assert.assertEquals(out.getFirst().getName(), "alpha"); + Assert.assertNull(out.getFirst().getNote()); + Assert.assertEquals(out.getSecond().getName(), "beta"); + Assert.assertNull(out.getSecond().getNote()); + } + + private static RowEncoder evolvingCodec(Class beanClass) { + return Encoders.buildBeanCodec(beanClass).withSchemaEvolution().build().get(); + } +} diff --git a/java/fory-format/src/test/java/org/apache/fory/format/encoder/SchemaEvolutionTest.java b/java/fory-format/src/test/java/org/apache/fory/format/encoder/SchemaEvolutionTest.java new file mode 100644 index 0000000000..2fa7f1d8d7 --- /dev/null +++ b/java/fory-format/src/test/java/org/apache/fory/format/encoder/SchemaEvolutionTest.java @@ -0,0 +1,2048 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format.encoder; + +import java.net.URL; +import java.net.URLClassLoader; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; +import lombok.Data; +import org.apache.fory.codegen.CodeGenerator; +import org.apache.fory.exception.ClassNotCompatibleException; +import org.apache.fory.format.annotation.ForyRowSchema; +import org.apache.fory.format.annotation.ForyRowVersion; +import org.apache.fory.format.row.binary.writer.BinaryRowWriter; +import org.apache.fory.format.row.binary.writer.BinaryWriter; +import org.apache.fory.format.type.DataTypes; +import org.apache.fory.format.type.Field; +import org.apache.fory.format.type.TypeInference; +import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.reflect.TypeRef; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class SchemaEvolutionTest { + + /** Original v1 bean: just a name and an age. */ + @Data + public static class PersonV1 { + private String name; + private int age; + } + + /** + * v2: added an email. The codec built against this class must still be able to read v1 payloads + * (email will default to null). + */ + @Data + public static class PersonV2 { + private String name; + private int age; + + @ForyRowVersion(since = 2) + private String email; + } + + /** + * v3: same as v2 with the age field removed. The codec built against this class must read v1 + * payloads (with age) and v2 payloads (with age + email). + */ + @Data + @ForyRowSchema(removedFields = PersonV3.History.class) + public static class PersonV3 { + private String name; + + @ForyRowVersion(since = 2) + private String email; + + interface History { + @ForyRowVersion(until = 3) + int age(); + } + } + + /** Round-trip at the current version: writing PersonV2, reading PersonV2 with evolution on. */ + @Test + public void currentVersionRoundTrip() { + RowEncoder codec = evolvingCodec(PersonV2.class); + PersonV2 in = new PersonV2(); + in.setName("alice"); + in.setAge(30); + in.setEmail("alice@example.com"); + byte[] bytes = codec.encode(in); + PersonV2 out = codec.decode(bytes); + Assert.assertEquals(out.getName(), "alice"); + Assert.assertEquals(out.getAge(), 30); + Assert.assertEquals(out.getEmail(), "alice@example.com"); + } + + /** + * The crux: a payload produced by PersonV1 (literally a different Java class with the v1-shaped + * schema) decoded by PersonV2's evolution-enabled codec. We use PersonV1 as a stand-in for "what + * older code wrote." Both classes are encoded with schema evolution on so they share the + * strict-hash format; PersonV1's history is a single entry, and PersonV2's history contains both + * v1 (without email) and v2 (with email) entries that match PersonV1's single entry by hash. + */ + @Test + public void olderPayload() { + RowEncoder oldWriter = evolvingCodec(PersonV1.class); + RowEncoder newReader = evolvingCodec(PersonV2.class); + + PersonV1 in = new PersonV1(); + in.setName("alice"); + in.setAge(30); + byte[] bytes = oldWriter.encode(in); + + PersonV2 out = newReader.decode(bytes); + Assert.assertEquals(out.getName(), "alice"); + Assert.assertEquals(out.getAge(), 30); + Assert.assertNull(out.getEmail()); + } + + // --- Compact row format --- + + @Test + public void compactRowOlderPayload() { + RowEncoder oldWriter = + Encoders.buildBeanCodec(PersonV1.class) + .compactEncoding() + .withSchemaEvolution() + .build() + .get(); + RowEncoder newReader = + Encoders.buildBeanCodec(PersonV2.class) + .compactEncoding() + .withSchemaEvolution() + .build() + .get(); + PersonV1 in = new PersonV1(); + in.setName("bob"); + in.setAge(42); + byte[] bytes = oldWriter.encode(in); + PersonV2 out = newReader.decode(bytes); + Assert.assertEquals(out.getName(), "bob"); + Assert.assertEquals(out.getAge(), 42); + Assert.assertNull(out.getEmail()); + } + + /** The compact sort puts the 8-byte String slot before the int, against the name order. */ + @Data + public static class Pair { + private int a; + private String b; + } + + @Data + public static class PairListV2 { + private List pairs; + + @ForyRowVersion(since = 2) + private String label; + } + + /** + * A compact codec reports the same schema with evolution on as with it off, although the + * evolution writer sorts the struct inside the list element too. + */ + @Test + public void compactSchemaMatchesEvolutionOff() { + Assert.assertEquals( + Encoders.buildBeanCodec(PairListV2.class) + .compactEncoding() + .withSchemaEvolution() + .build() + .get() + .schema(), + Encoders.buildBeanCodec(PairListV2.class).compactEncoding().build().get().schema()); + } + + /** One 8-byte field: the compact format inlines it as an 8-byte struct, with the 8-byte slots. */ + @Data + public static class SlotV1 { + private long a; + } + + /** + * Version 2 adds an int, so the inline struct is 12 bytes wide. The compact sort then moves it + * from the group of 2-, 4-, and 8-multiple widths to the group of other widths, after them. + */ + @Data + public static class SlotV2 { + private long a; + + @ForyRowVersion(since = 2) + private int b; + } + + /** The compact order is slot, tag, x, flag at version 1 and tag, x, slot, flag at version 2. */ + @Data + public static class SlotFieldV1 { + private boolean flag; + private SlotV1 slot; + private String tag; + private int x; + } + + @Data + public static class SlotFieldV2 { + private boolean flag; + private SlotV2 slot; + private String tag; + private int x; + } + + @Data + public static class SlotListV1 { + private List slots; + private int x; + } + + @Data + public static class SlotListV2 { + private List slots; + private int x; + } + + @Data + public static class SlotMapV1 { + private Map slots; + private int x; + } + + @Data + public static class SlotMapV2 { + private Map slots; + private int x; + } + + private static SlotV1 slotV1(long a) { + SlotV1 slot = new SlotV1(); + slot.setA(a); + return slot; + } + + private static RowEncoder compactEvolvingCodec(Class beanClass) { + return Encoders.buildBeanCodec(beanClass).compactEncoding().withSchemaEvolution().build().get(); + } + + /** + * An older compact payload whose inline nested bean is in another sort group than the current + * one: the projection must read the bean, and the fields that the sort moved around it, at their + * version-1 offsets. + */ + @Test + public void compactNestedBeanChangesSortGroup() { + SlotFieldV1 in = new SlotFieldV1(); + in.setFlag(true); + in.setSlot(slotV1(0x0102030405060708L)); + in.setTag("t"); + in.setX(-7); + SlotFieldV2 out = + compactEvolvingCodec(SlotFieldV2.class) + .decode(compactEvolvingCodec(SlotFieldV1.class).encode(in)); + Assert.assertTrue(out.isFlag()); + Assert.assertEquals(out.getSlot().getA(), 0x0102030405060708L); + Assert.assertEquals(out.getSlot().getB(), 0); + Assert.assertEquals(out.getTag(), "t"); + Assert.assertEquals(out.getX(), -7); + } + + /** The same change for list elements, which the compact format stores inline at their width. */ + @Test + public void compactListElementChangesSortGroup() { + SlotListV1 in = new SlotListV1(); + in.setSlots(Arrays.asList(slotV1(11), slotV1(-12), slotV1(Long.MAX_VALUE))); + in.setX(5); + SlotListV2 out = + compactEvolvingCodec(SlotListV2.class) + .decode(compactEvolvingCodec(SlotListV1.class).encode(in)); + Assert.assertEquals(out.getSlots().size(), 3); + Assert.assertEquals(out.getSlots().get(0).getA(), 11); + Assert.assertEquals(out.getSlots().get(1).getA(), -12); + Assert.assertEquals(out.getSlots().get(2).getA(), Long.MAX_VALUE); + for (SlotV2 slot : out.getSlots()) { + Assert.assertEquals(slot.getB(), 0); + } + Assert.assertEquals(out.getX(), 5); + } + + /** The same change for the values of a map in a row field. */ + @Test + public void compactMapValueChangesSortGroup() { + Map slots = new LinkedHashMap<>(); + slots.put("p", slotV1(21)); + slots.put("q", slotV1(-22)); + SlotMapV1 in = new SlotMapV1(); + in.setSlots(slots); + in.setX(6); + SlotMapV2 out = + compactEvolvingCodec(SlotMapV2.class) + .decode(compactEvolvingCodec(SlotMapV1.class).encode(in)); + Assert.assertEquals(out.getSlots().size(), 2); + Assert.assertEquals(out.getSlots().get("p").getA(), 21); + Assert.assertEquals(out.getSlots().get("q").getA(), -22); + Assert.assertEquals(out.getSlots().get("p").getB(), 0); + Assert.assertEquals(out.getSlots().get("q").getB(), 0); + Assert.assertEquals(out.getX(), 6); + } + + /** + * The byte[] overloads use bytes.length for the body size; the MemoryBuffer overloads write and + * read an embedded int32 size prefix ahead of the 8-byte hash. That framing is a distinct code + * path, so exercise a projection hit (older payload, newer reader) through it. Two records are + * written into one buffer and read back in order to confirm the reader advances past each + * record's embedded size. + */ + @Test + public void sequentialOlderPayloads() { + RowEncoder oldWriter = evolvingCodec(PersonV1.class); + RowEncoder newReader = evolvingCodec(PersonV2.class); + + PersonV1 alice = new PersonV1(); + alice.setName("alice"); + alice.setAge(30); + PersonV1 bob = new PersonV1(); + bob.setName("bob"); + bob.setAge(42); + + MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(32); + oldWriter.encode(buffer, alice); + oldWriter.encode(buffer, bob); + + PersonV2 outAlice = newReader.decode(buffer); + PersonV2 outBob = newReader.decode(buffer); + Assert.assertEquals(outAlice.getName(), "alice"); + Assert.assertEquals(outAlice.getAge(), 30); + Assert.assertNull(outAlice.getEmail()); + Assert.assertEquals(outBob.getName(), "bob"); + Assert.assertEquals(outBob.getAge(), 42); + Assert.assertNull(outBob.getEmail()); + } + + // --- Array of versioned beans --- + + @Test + public void arrayStandardOlderPayload() { + ArrayEncoder> oldWriter = + Encoders.buildArrayCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + ArrayEncoder> newReader = + Encoders.buildArrayCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + PersonV1 a = new PersonV1(); + a.setName("alice"); + a.setAge(30); + PersonV1 b = new PersonV1(); + b.setName("bob"); + b.setAge(42); + byte[] bytes = oldWriter.encode(Arrays.asList(a, b)); + List out = newReader.decode(bytes); + Assert.assertEquals(out.size(), 2); + Assert.assertEquals(out.get(0).getName(), "alice"); + Assert.assertEquals(out.get(0).getAge(), 30); + Assert.assertNull(out.get(0).getEmail()); + Assert.assertEquals(out.get(1).getName(), "bob"); + } + + @Test + public void arrayCompactOlderPayload() { + ArrayEncoder> oldWriter = + Encoders.buildArrayCodec(new TypeRef>() {}) + .compactEncoding() + .withSchemaEvolution() + .build() + .get(); + ArrayEncoder> newReader = + Encoders.buildArrayCodec(new TypeRef>() {}) + .compactEncoding() + .withSchemaEvolution() + .build() + .get(); + PersonV1 p = new PersonV1(); + p.setName("carol"); + p.setAge(25); + byte[] bytes = oldWriter.encode(Arrays.asList(p)); + List out = newReader.decode(bytes); + Assert.assertEquals(out.size(), 1); + Assert.assertEquals(out.get(0).getName(), "carol"); + Assert.assertEquals(out.get(0).getAge(), 25); + Assert.assertNull(out.get(0).getEmail()); + } + + // --- Map with versioned bean values --- + + @Test + public void mapStandardOlderPayload() { + MapEncoder> oldWriter = + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + MapEncoder> newReader = + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + Map in = new HashMap<>(); + PersonV1 p = new PersonV1(); + p.setName("dave"); + p.setAge(40); + in.put("k1", p); + byte[] bytes = oldWriter.encode(in); + Map out = newReader.decode(bytes); + Assert.assertEquals(out.size(), 1); + Assert.assertEquals(out.get("k1").getName(), "dave"); + Assert.assertEquals(out.get("k1").getAge(), 40); + Assert.assertNull(out.get("k1").getEmail()); + } + + @Test + public void mapCompactOlderPayload() { + MapEncoder> oldWriter = + Encoders.buildMapCodec(new TypeRef>() {}) + .compactEncoding() + .withSchemaEvolution() + .build() + .get(); + MapEncoder> newReader = + Encoders.buildMapCodec(new TypeRef>() {}) + .compactEncoding() + .withSchemaEvolution() + .build() + .get(); + Map in = new HashMap<>(); + PersonV1 p = new PersonV1(); + p.setName("eve"); + p.setAge(28); + in.put("k1", p); + byte[] bytes = oldWriter.encode(in); + Map out = newReader.decode(bytes); + Assert.assertEquals(out.get("k1").getName(), "eve"); + Assert.assertEquals(out.get("k1").getAge(), 28); + Assert.assertNull(out.get("k1").getEmail()); + } + + // An evolution-on array/map prepends the 8-byte schema hash even when the collection is empty, so + // the framed payload is exactly the prefix with a zero-length body (size == 8, bodySize == 0). + // The reader must dispatch on the hash and return an empty collection rather than tripping the + // size guard or reading past the prefix. + @Test + public void emptyArrayRoundTrip() { + ArrayEncoder> oldWriter = + Encoders.buildArrayCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + ArrayEncoder> newReader = + Encoders.buildArrayCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + byte[] bytes = oldWriter.encode(Arrays.asList()); + List out = newReader.decode(bytes); + Assert.assertTrue(out.isEmpty()); + } + + @Test + public void emptyMapRoundTrip() { + MapEncoder> oldWriter = + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + MapEncoder> newReader = + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + byte[] bytes = oldWriter.encode(new HashMap<>()); + Map out = newReader.decode(bytes); + Assert.assertTrue(out.isEmpty()); + } + + /** + * Streaming counterpart of {@link #arrayStandardOlderPayload}: the encode(MemoryBuffer) overload + * frames each record with an embedded int32 size. Two older-schema lists in one buffer confirm + * the reader advances past each record's size on a projection hit. + */ + @Test + public void arrayStreamingOlderPayload() { + ArrayEncoder> oldWriter = + Encoders.buildArrayCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + ArrayEncoder> newReader = + Encoders.buildArrayCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + PersonV1 a = new PersonV1(); + a.setName("alice"); + a.setAge(30); + PersonV1 b = new PersonV1(); + b.setName("bob"); + b.setAge(42); + + MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(32); + oldWriter.encode(buffer, Arrays.asList(a)); + oldWriter.encode(buffer, Arrays.asList(b)); + + List outA = newReader.decode(buffer); + List outB = newReader.decode(buffer); + Assert.assertEquals(outA.size(), 1); + Assert.assertEquals(outA.get(0).getName(), "alice"); + Assert.assertEquals(outA.get(0).getAge(), 30); + Assert.assertNull(outA.get(0).getEmail()); + Assert.assertEquals(outB.get(0).getName(), "bob"); + Assert.assertEquals(outB.get(0).getAge(), 42); + } + + /** + * Streaming counterpart of {@link #mapStandardOlderPayload}. The map encode(MemoryBuffer) path + * restores both the key and value writer buffers in a finally block; two records in one buffer + * exercise that restore across records. + */ + @Test + public void mapStreamingOlderPayload() { + MapEncoder> oldWriter = + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + MapEncoder> newReader = + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + Map first = new HashMap<>(); + PersonV1 dave = new PersonV1(); + dave.setName("dave"); + dave.setAge(40); + first.put("k1", dave); + Map second = new HashMap<>(); + PersonV1 erin = new PersonV1(); + erin.setName("erin"); + erin.setAge(35); + second.put("k2", erin); + + MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(32); + oldWriter.encode(buffer, first); + oldWriter.encode(buffer, second); + + Map outFirst = newReader.decode(buffer); + Map outSecond = newReader.decode(buffer); + Assert.assertEquals(outFirst.get("k1").getName(), "dave"); + Assert.assertEquals(outFirst.get("k1").getAge(), 40); + Assert.assertNull(outFirst.get("k1").getEmail()); + Assert.assertEquals(outSecond.get("k2").getName(), "erin"); + Assert.assertEquals(outSecond.get("k2").getAge(), 35); + } + + // --- Unversioned element bean still carries the strict-hash prefix --- + // + // A top-level array/map built with withSchemaEvolution() must emit the 8-byte prefix even when + // its element/value bean has no @ForyRowVersion fields, so two independently-built evolution-on + // codecs stay wire-compatible (see SchemaHistory#evolutionBean). Each test compares the evolving + // payload against a non-evolving one: the lengths must differ by exactly the prefix, which a + // regression skipping the evolution path for an unversioned element would not produce. + + @Test + public void arrayOfUnversionedBeanCarriesHashPrefix() { + ArrayEncoder> evolving = + Encoders.buildArrayCodec(new TypeRef>() {}).withSchemaEvolution().build().get(); + ArrayEncoder> plain = + Encoders.buildArrayCodec(new TypeRef>() {}).build().get(); + Item item = new Item(); + item.setName("widget"); + item.setQuantity(7); + List in = Arrays.asList(item); + + byte[] evolvingBytes = evolving.encode(in); + byte[] plainBytes = plain.encode(in); + Assert.assertEquals(evolvingBytes.length, plainBytes.length + 8); + + // A separately-built evolution-on codec reads the prefixed payload back. + ArrayEncoder> reader = + Encoders.buildArrayCodec(new TypeRef>() {}).withSchemaEvolution().build().get(); + List out = reader.decode(evolvingBytes); + Assert.assertEquals(out.size(), 1); + Assert.assertEquals(out.get(0).getName(), "widget"); + Assert.assertEquals(out.get(0).getQuantity(), 7); + } + + @Test + public void mapOfUnversionedBeanCarriesHashPrefix() { + MapEncoder> evolving = + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + MapEncoder> plain = + Encoders.buildMapCodec(new TypeRef>() {}).build().get(); + Item item = new Item(); + item.setName("gadget"); + item.setQuantity(3); + Map in = new HashMap<>(); + in.put("k1", item); + + byte[] evolvingBytes = evolving.encode(in); + byte[] plainBytes = plain.encode(in); + Assert.assertEquals(evolvingBytes.length, plainBytes.length + 8); + + MapEncoder> reader = + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + Map out = reader.decode(evolvingBytes); + Assert.assertEquals(out.size(), 1); + Assert.assertEquals(out.get("k1").getName(), "gadget"); + Assert.assertEquals(out.get("k1").getQuantity(), 3); + } + + // --- Interface-typed beans --- + // + // The wire field name is derived from each interface's accessor method name (via + // lowerCamelToLowerUnderscore), so two interfaces that share the same accessor names produce + // the same wire layout. Use accessor-style getters consistently across versions. + + /** v1 interface: just name and age. */ + public interface PersonIfaceV1 { + String getName(); + + int getAge(); + } + + /** v2 interface: adds email. Same accessor naming so the wire field names match. */ + public interface PersonIfaceV2 { + String getName(); + + int getAge(); + + @ForyRowVersion(since = 2) + String getEmail(); + } + + @Test + public void interfaceOlderPayload() { + RowEncoder oldWriter = evolvingCodec(PersonIfaceV1.class); + RowEncoder newReader = evolvingCodec(PersonIfaceV2.class); + PersonIfaceV1 in = + new PersonIfaceV1() { + public String getName() { + return "alice"; + } + + public int getAge() { + return 30; + } + }; + byte[] bytes = oldWriter.encode(in); + PersonIfaceV2 out = newReader.decode(bytes); + Assert.assertEquals(out.getName(), "alice"); + Assert.assertEquals(out.getAge(), 30); + // email was added in v2; v1 payload has none. The interface proxy returns the default. + Assert.assertNull(out.getEmail()); + } + + /** v2 interface whose added accessor returns an Optional. */ + public interface PersonIfaceOptionalV2 { + String getName(); + + int getAge(); + + @ForyRowVersion(since = 2) + Optional getNickname(); + } + + @Test + public void interfaceAddedOptionalFieldIsEmpty() { + PersonIfaceV1 in = + new PersonIfaceV1() { + public String getName() { + return "alice"; + } + + public int getAge() { + return 30; + } + }; + byte[] bytes = evolvingCodec(PersonIfaceV1.class).encode(in); + PersonIfaceOptionalV2 out = evolvingCodec(PersonIfaceOptionalV2.class).decode(bytes); + Assert.assertEquals(out.getName(), "alice"); + Assert.assertEquals(out.getNickname(), Optional.empty()); + } + + /** v2 interface whose added accessor is a {@code default} method. */ + public interface PersonIfaceDefaultV2 { + String getName(); + + int getAge(); + + @ForyRowVersion(since = 2) + default String getEmail() { + return "none"; + } + } + + @Test + public void interfaceAddedDefaultAccessorIsAbsent() { + PersonIfaceV1 in = + new PersonIfaceV1() { + public String getName() { + return "alice"; + } + + public int getAge() { + return 30; + } + }; + byte[] bytes = evolvingCodec(PersonIfaceV1.class).encode(in); + PersonIfaceDefaultV2 out = evolvingCodec(PersonIfaceDefaultV2.class).decode(bytes); + Assert.assertEquals(out.getName(), "alice"); + Assert.assertNull(out.getEmail()); + } + + /** + * v3 interface: name and email; age removed (only present in v1 and v2). The history interface + * declares the removed field's original signature; its method name is the live accessor's name, + * so {@code getAge()} maps to the live accessor's wire name, {@code get_age}. + */ + @ForyRowSchema(removedFields = PersonIfaceV3.History.class) + public interface PersonIfaceV3 { + String getName(); + + @ForyRowVersion(since = 2) + String getEmail(); + + interface History { + @ForyRowVersion(until = 3) + int getAge(); + } + } + + @ForyRowSchema(removedFields = PersonIfaceBaseV3.History.class) + public interface PersonIfaceBaseV3 { + String getName(); + + interface History { + @ForyRowVersion(until = 3) + int getAge(); + } + } + + /** Inherits name and the removed age from its superinterface. */ + public interface PersonIfaceChildV3 extends PersonIfaceBaseV3 { + @ForyRowVersion(since = 2) + String getEmail(); + } + + @Test + public void superinterfaceRemovedField() { + PersonIfaceV2 in = + new PersonIfaceV2() { + public String getName() { + return "alice"; + } + + public int getAge() { + return 30; + } + + public String getEmail() { + return "alice@example.com"; + } + }; + byte[] bytes = evolvingCodec(PersonIfaceV2.class).encode(in); + PersonIfaceChildV3 out = evolvingCodec(PersonIfaceChildV3.class).decode(bytes); + Assert.assertEquals(out.getName(), "alice"); + Assert.assertEquals(out.getEmail(), "alice@example.com"); + } + + @Test + public void interfaceRemovedField() { + RowEncoder v2Writer = evolvingCodec(PersonIfaceV2.class); + RowEncoder v3Reader = evolvingCodec(PersonIfaceV3.class); + PersonIfaceV2 in = + new PersonIfaceV2() { + public String getName() { + return "alice"; + } + + public int getAge() { + return 30; + } + + public String getEmail() { + return "alice@example.com"; + } + }; + byte[] bytes = v2Writer.encode(in); + PersonIfaceV3 out = v3Reader.decode(bytes); + Assert.assertEquals(out.getName(), "alice"); + Assert.assertEquals(out.getEmail(), "alice@example.com"); + } + + /** + * v1 writer interface: just a name. Used to produce a payload that the reader below projects to + * its v1 schema, where {@code score} is absent. + */ + public interface ScoredV1 { + String getName(); + } + + /** + * Current reader interface. {@code getScore()} is a live {@code since=2} accessor, so when a v1 + * payload is projected it is absent and gets a default-value body. {@code getScore(int)} is a + * parameterized overload sharing that name and return type. It is not an accessor — accessors + * take no arguments — so the projection proxy must throw for it rather than silence it into a + * default. Without the {@code parameterCount() != 0} guard in {@code isAccessorOfAbsentField}, it + * would match the absent {@code score} descriptor by name and return type and return {@code 0}. + */ + public interface ScoredV2 { + String getName(); + + @ForyRowVersion(since = 2) + int getScore(); + + int getScore(int seed); + } + + @Test + public void projectionNonAccessorOverloadStillThrows() { + RowEncoder v1Writer = evolvingCodec(ScoredV1.class); + RowEncoder reader = evolvingCodec(ScoredV2.class); + ScoredV1 in = () -> "alice"; + ScoredV2 out = reader.decode(v1Writer.encode(in)); + Assert.assertEquals(out.getName(), "alice"); + // score was added in v2; the v1 payload has none, so the no-arg accessor defaults to 0. + Assert.assertEquals(out.getScore(), 0); + try { + out.getScore(7); + Assert.fail( + "parameterized getScore is not an accessor and must not be silenced to a default"); + } catch (UnsupportedOperationException expected) { + // The projection proxy does not implement non-accessor methods. + } + } + + /** Removed-field test: v3 codec reads v2 payload, dropping the no-longer-present 'age'. */ + @Test + public void removedField() { + RowEncoder v2Writer = evolvingCodec(PersonV2.class); + RowEncoder v3Reader = evolvingCodec(PersonV3.class); + + PersonV2 in = new PersonV2(); + in.setName("alice"); + in.setAge(30); + in.setEmail("alice@example.com"); + byte[] bytes = v2Writer.encode(in); + + PersonV3 out = v3Reader.decode(bytes); + Assert.assertEquals(out.getName(), "alice"); + Assert.assertEquals(out.getEmail(), "alice@example.com"); + } + + @Data + public static class AccountBaseV1 { + private String name; + private int age; + } + + @Data + @lombok.EqualsAndHashCode(callSuper = true) + public static class AccountV1 extends AccountBaseV1 { + private String email; + } + + @Data + @ForyRowSchema(removedFields = AccountBaseV2.History.class) + public static class AccountBaseV2 { + private String name; + + interface History { + @ForyRowVersion(until = 2) + int age(); + } + } + + @Data + @lombok.EqualsAndHashCode(callSuper = true) + public static class AccountV2 extends AccountBaseV2 { + private String email; + } + + /** Encoders from one codec share its projection; each still decodes with its own writer. */ + @Test + public void encodersShareProjection() { + PersonV2 in = new PersonV2(); + in.setName("alice"); + in.setAge(30); + in.setEmail("alice@example.com"); + byte[] bytes = evolvingCodec(PersonV2.class).encode(in); + Supplier> codec = + Encoders.buildBeanCodec(PersonV3.class).withSchemaEvolution().build(); + RowEncoder first = codec.get(); + RowEncoder second = codec.get(); + Assert.assertEquals(first.decode(bytes).getEmail(), "alice@example.com"); + Assert.assertEquals(second.decode(bytes).getEmail(), "alice@example.com"); + } + + /** Removes email at v3; its History extends the parent's, which removes age at v2. */ + @Data + @lombok.EqualsAndHashCode(callSuper = true) + @ForyRowSchema(removedFields = AccountV3.History.class) + public static class AccountV3 extends AccountBaseV2 { + interface History extends AccountBaseV2.History { + @ForyRowVersion(until = 3) + String email(); + } + } + + @Test + public void extendedHistoryInterface() { + AccountV2 in = new AccountV2(); + in.setName("alice"); + in.setEmail("alice@example.com"); + AccountV3 out = + evolvingCodec(AccountV3.class).decode(evolvingCodec(AccountV2.class).encode(in)); + Assert.assertEquals(out.getName(), "alice"); + } + + /** A field that a superclass removed is still read from older payloads of the subclass. */ + @Test + public void superclassRemovedField() { + AccountV1 in = new AccountV1(); + in.setName("alice"); + in.setAge(30); + in.setEmail("alice@example.com"); + byte[] bytes = evolvingCodec(AccountV1.class).encode(in); + + AccountV2 out = evolvingCodec(AccountV2.class).decode(bytes); + Assert.assertEquals(out.getName(), "alice"); + Assert.assertEquals(out.getEmail(), "alice@example.com"); + } + + @Data + public static class CounterV1 { + private int count; + } + + /** + * {@code count} is removed at v2 and added again with the same type at v3. {@code label} arrives + * at v2, so the v1 layout does not collapse with any version where {@code count} is live. + */ + @Data + @ForyRowSchema(removedFields = CounterV3.History.class) + public static class CounterV3 { + @ForyRowVersion(since = 3) + private int count; + + @ForyRowVersion(since = 2) + private String label; + + interface History { + @ForyRowVersion(until = 2) + int count(); + } + } + + @Test + public void readdedFieldReadFromOlderPayload() { + CounterV1 in = new CounterV1(); + in.setCount(42); + CounterV3 out = + evolvingCodec(CounterV3.class).decode(evolvingCodec(CounterV1.class).encode(in)); + Assert.assertEquals(out.getCount(), 42); + Assert.assertNull(out.getLabel()); + } + + public enum Status { + ACTIVE, + } + + @Data + public static class TicketV1 { + private String status; + } + + /** + * {@code status} is removed as a String at v2 and added again as an enum at v3. Both encode as + * utf8, so the v1 and v3 layouts could be identical; the codec build rejects the change. + */ + @Data + @ForyRowSchema(removedFields = TicketV3.History.class) + public static class TicketV3 { + @ForyRowVersion(since = 2) + private String label; + + @ForyRowVersion(since = 3) + private Status status; + + interface History { + @ForyRowVersion(until = 2) + String status(); + } + } + + @Test + public void typeChangeWithSameRowTypeRejected() { + try { + evolvingCodec(TicketV3.class); + Assert.fail("expected the codec build to reject the type change"); + } catch (IllegalStateException e) { + Assert.assertTrue(e.getMessage().contains("'status'"), e.getMessage()); + } + } + + @Data + public static class CounterTextV1 { + private String count; + } + + /** {@code count} is removed as a String at v2 and added again as an int at v3. */ + @Data + @ForyRowSchema(removedFields = CounterIntV3.History.class) + public static class CounterIntV3 { + @ForyRowVersion(since = 3) + private int count; + + @ForyRowVersion(since = 2) + private String label; + + interface History { + @ForyRowVersion(until = 2) + String count(); + } + } + + @Test + public void readdedFieldWithNewTypeDiscarded() { + CounterTextV1 in = new CounterTextV1(); + in.setCount("forty-two"); + CounterIntV3 out = + evolvingCodec(CounterIntV3.class).decode(evolvingCodec(CounterTextV1.class).encode(in)); + Assert.assertEquals(out.getCount(), 0); + } + + // --------------------------------------------------------------------------- + // Compositional test + // + // Outer mutable bean evolves v1 -> v2 (adds displayName, removes legacyName). + // The bean carries diverse nested data shapes that themselves do not evolve: + // a concrete struct, an interface-typed struct (lazy proxy), an inline list + // of structs, and an inline map. The test exercises one + // dispatch boundary (the outer codec, or the outer list codec) and verifies + // that the projected outer correctly carries every nested shape through. + // --------------------------------------------------------------------------- + + @Data + public static class Profile { + private String bio; + private int rating; + } + + /** Address is interface-typed; the row codec generates a lazy proxy for reads. */ + public interface Address { + String getStreet(); + + String getCity(); + } + + @Data + public static class Item { + private String name; + private long quantity; + } + + @Data + public static class OuterV1 { + private long id; + private String legacyName; + private Profile profile; + private Address address; + private List items; + private Map properties; + } + + /** + * OuterV2 adds {@code displayName} at version 2 and removes {@code legacyName} at version 2. + * Everything else carries forward unchanged. The compositional test writes an OuterV1 and reads + * as OuterV2. + */ + @Data + @ForyRowSchema(removedFields = OuterV2.History.class) + public static class OuterV2 { + private long id; + + @ForyRowVersion(since = 2) + private String displayName; + + private Profile profile; + private Address address; + private List items; + private Map properties; + + interface History { + @ForyRowVersion(until = 2) + String legacyName(); + } + } + + private static OuterV1 sampleV1() { + OuterV1 in = new OuterV1(); + in.setId(7); + in.setLegacyName("retired"); + Profile p = new Profile(); + p.setBio("hello"); + p.setRating(5); + in.setProfile(p); + in.setAddress( + new Address() { + public String getStreet() { + return "1 Main"; + } + + public String getCity() { + return "Springfield"; + } + }); + Item a = new Item(); + a.setName("a"); + a.setQuantity(1); + Item b = new Item(); + b.setName("b"); + b.setQuantity(2); + in.setItems(Arrays.asList(a, b)); + Map props = new HashMap<>(); + props.put("k1", a); + props.put("k2", b); + in.setProperties(props); + return in; + } + + private static void assertProjectedToV2(OuterV2 out) { + Assert.assertEquals(out.getId(), 7); + Assert.assertNull(out.getDisplayName()); // added in v2, absent in v1 wire + Assert.assertEquals(out.getProfile().getBio(), "hello"); + Assert.assertEquals(out.getProfile().getRating(), 5); + Assert.assertEquals(out.getAddress().getStreet(), "1 Main"); + Assert.assertEquals(out.getAddress().getCity(), "Springfield"); + Assert.assertEquals(out.getItems().size(), 2); + Assert.assertEquals(out.getItems().get(0).getName(), "a"); + Assert.assertEquals(out.getItems().get(1).getQuantity(), 2); + Assert.assertEquals(out.getProperties().get("k1").getName(), "a"); + Assert.assertEquals(out.getProperties().get("k2").getQuantity(), 2); + } + + @Test + public void compositionalRowEvolution() { + RowEncoder writer = evolvingCodec(OuterV1.class); + RowEncoder reader = evolvingCodec(OuterV2.class); + byte[] bytes = writer.encode(sampleV1()); + assertProjectedToV2(reader.decode(bytes)); + } + + @Test + public void compositionalArrayEvolution() { + ArrayEncoder> writer = + Encoders.buildArrayCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + ArrayEncoder> reader = + Encoders.buildArrayCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + byte[] bytes = writer.encode(Arrays.asList(sampleV1(), sampleV1())); + List out = reader.decode(bytes); + Assert.assertEquals(out.size(), 2); + assertProjectedToV2(out.get(0)); + assertProjectedToV2(out.get(1)); + } + + // --------------------------------------------------------------------------- + // A versioned bean nested inside a collection field of the outer bean. The + // outer's SchemaHistory must look through the list/map wrapper to discover the + // inner bean and enumerate its versions, so an older payload (inner at v1) is + // projected into the newer reader (inner at v2). Without that, the outer has no + // projection matching the older inner layout and decode throws. + // --------------------------------------------------------------------------- + + @Data + public static class TagV1 { + private String key; + } + + @Data + public static class TagV2 { + private String key; + + @ForyRowVersion(since = 2) + private long weight; + } + + @Data + public static class CatalogV1 { + private String id; + private List tags; + private Map labels; + } + + @Data + public static class CatalogV2 { + private String id; + private List tags; + private Map labels; + } + + private static CatalogV1 sampleCatalog() { + CatalogV1 in = new CatalogV1(); + in.setId("c1"); + TagV1 a = new TagV1(); + a.setKey("alpha"); + TagV1 b = new TagV1(); + b.setKey("beta"); + in.setTags(Arrays.asList(a, b)); + Map labels = new HashMap<>(); + labels.put("k1", a); + in.setLabels(labels); + return in; + } + + @Test + public void evolvingBeanInCollectionField() { + RowEncoder writer = evolvingCodec(CatalogV1.class); + RowEncoder reader = evolvingCodec(CatalogV2.class); + CatalogV2 out = reader.decode(writer.encode(sampleCatalog())); + Assert.assertEquals(out.getId(), "c1"); + Assert.assertEquals(out.getTags().size(), 2); + Assert.assertEquals(out.getTags().get(0).getKey(), "alpha"); + Assert.assertEquals(out.getTags().get(1).getKey(), "beta"); + // weight was added at v2; the v1 payload has no source for it. + Assert.assertEquals(out.getTags().get(0).getWeight(), 0L); + Assert.assertEquals(out.getLabels().get("k1").getKey(), "alpha"); + } + + // --------------------------------------------------------------------------- + // A versioned *interface* bean nested inside an evolving outer bean. Interface + // beans are valid versioned row beans at the top level (see PersonIfaceV1/V2), + // so they must also be discovered when nested as a field type, a list element, + // or a map value. SchemaHistory's evolution-site discovery has to recognize an + // interface the same way the top-level container path does (synthesizing the interface as + // a bean); otherwise the outer's cross-product never enumerates the inner's + // older versions, an older inner payload has no matching projection, and decode + // fails with a schema-hash mismatch (ClassNotCompatibleException). + // --------------------------------------------------------------------------- + + /** v1 interface bean: a single key accessor. */ + public interface SlugV1 { + String getKey(); + } + + /** v2 interface bean: adds a weight at version 2. Same accessor naming as v1. */ + public interface SlugV2 { + String getKey(); + + @ForyRowVersion(since = 2) + long getWeight(); + } + + @Data + public static class BoxV1 { + private String id; + private SlugV1 slug; + private List slugs; + private Map labels; + } + + @Data + public static class BoxV2 { + private String id; + private SlugV2 slug; + private List slugs; + private Map labels; + } + + private static SlugV1 slugV1(String key) { + return () -> key; + } + + @Test + public void evolvingInterfaceBeanNestedInOuterBean() { + RowEncoder writer = evolvingCodec(BoxV1.class); + RowEncoder reader = evolvingCodec(BoxV2.class); + + BoxV1 in = new BoxV1(); + in.setId("b1"); + in.setSlug(slugV1("direct")); + in.setSlugs(Arrays.asList(slugV1("alpha"), slugV1("beta"))); + Map labels = new HashMap<>(); + labels.put("k1", slugV1("gamma")); + in.setLabels(labels); + + BoxV2 out = reader.decode(writer.encode(in)); + + Assert.assertEquals(out.getId(), "b1"); + Assert.assertEquals(out.getSlug().getKey(), "direct"); + Assert.assertEquals(out.getSlugs().size(), 2); + Assert.assertEquals(out.getSlugs().get(0).getKey(), "alpha"); + Assert.assertEquals(out.getSlugs().get(1).getKey(), "beta"); + Assert.assertEquals(out.getLabels().get("k1").getKey(), "gamma"); + // weight was added at v2; the v1 payload has no source, so it defaults. + Assert.assertEquals(out.getSlug().getWeight(), 0L); + Assert.assertEquals(out.getSlugs().get(0).getWeight(), 0L); + Assert.assertEquals(out.getLabels().get("k1").getWeight(), 0L); + } + + // --- Versioned bean nested inside a top-level container's element/value --- + // + // A top-level array or map whose element/value is itself a collection of a versioned bean + // (List, Map<.., Person>) must still evolve. The versioned bean is reachable through + // the container element/value the same way SchemaHistory's site discovery descends, so an + // older payload must decode under the newer codec rather than being read at a stale layout. + + @Test + public void mapOfListValueOlderPayload() { + MapEncoder>> oldWriter = + Encoders.buildMapCodec(new TypeRef>>() {}) + .withSchemaEvolution() + .build() + .get(); + MapEncoder>> newReader = + Encoders.buildMapCodec(new TypeRef>>() {}) + .withSchemaEvolution() + .build() + .get(); + Map> in = new HashMap<>(); + PersonV1 p = new PersonV1(); + p.setName("dave"); + p.setAge(40); + in.put("k1", Arrays.asList(p)); + byte[] bytes = oldWriter.encode(in); + Map> out = newReader.decode(bytes); + Assert.assertEquals(out.size(), 1); + PersonV2 read = out.get("k1").get(0); + Assert.assertEquals(read.getName(), "dave"); + Assert.assertEquals(read.getAge(), 40); + Assert.assertNull(read.getEmail()); + } + + @Test + public void arrayOfListElementOlderPayload() { + ArrayEncoder>> oldWriter = + Encoders.buildArrayCodec(new TypeRef>>() {}) + .withSchemaEvolution() + .build() + .get(); + ArrayEncoder>> newReader = + Encoders.buildArrayCodec(new TypeRef>>() {}) + .withSchemaEvolution() + .build() + .get(); + PersonV1 p = new PersonV1(); + p.setName("dave"); + p.setAge(40); + byte[] bytes = oldWriter.encode(Arrays.asList(Arrays.asList(p))); + List> out = newReader.decode(bytes); + Assert.assertEquals(out.size(), 1); + PersonV2 read = out.get(0).get(0); + Assert.assertEquals(read.getName(), "dave"); + Assert.assertEquals(read.getAge(), 40); + Assert.assertNull(read.getEmail()); + } + + /** Map value is itself a map of the versioned bean, exercising the map-wrapper projection. */ + @Test + public void mapOfMapValueOlderPayload() { + MapEncoder>> oldWriter = + Encoders.buildMapCodec(new TypeRef>>() {}) + .withSchemaEvolution() + .build() + .get(); + MapEncoder>> newReader = + Encoders.buildMapCodec(new TypeRef>>() {}) + .withSchemaEvolution() + .build() + .get(); + PersonV1 p = new PersonV1(); + p.setName("dave"); + p.setAge(40); + Map inner = new HashMap<>(); + inner.put("inner", p); + Map> in = new HashMap<>(); + in.put("k1", inner); + Map> out = newReader.decode(oldWriter.encode(in)); + PersonV2 read = out.get("k1").get("inner"); + Assert.assertEquals(read.getName(), "dave"); + Assert.assertEquals(read.getAge(), 40); + Assert.assertNull(read.getEmail()); + } + + // --------------------------------------------------------------------------- + // Added reference-typed fields. Every other added-field test defaults a scalar + // (String/int/...); defaulting an added struct or collection slot is a distinct + // projection path. v2 adds a nested struct and a list of structs that are absent + // from the v1 wire, so both must read back as null. + // --------------------------------------------------------------------------- + + @Data + public static class HolderV1 { + private long id; + } + + @Data + public static class HolderV2 { + private long id; + + @ForyRowVersion(since = 2) + private Profile profile; + + @ForyRowVersion(since = 2) + private List items; + } + + @Test + public void addedReferenceFieldsDefaultToNull() { + RowEncoder writer = evolvingCodec(HolderV1.class); + RowEncoder reader = evolvingCodec(HolderV2.class); + + HolderV1 in = new HolderV1(); + in.setId(7); + HolderV2 out = reader.decode(writer.encode(in)); + + Assert.assertEquals(out.getId(), 7); + Assert.assertNull(out.getProfile()); + Assert.assertNull(out.getItems()); + } + + // --------------------------------------------------------------------------- + // A versioned bean reached only through an Optional field. TypeInference.inferField + // unwraps Optional to a nullable T and encodes the inner bean as a struct, so the + // evolution walk must look through Optional the same way. Without that, the outer's + // cross-product never enumerates the inner's older versions and an older payload + // (inner at v1) has no matching projection under the newer reader. + // --------------------------------------------------------------------------- + + @Data + public static class OptionalHolderV1 { + private String id; + private Optional tag; + } + + @Data + public static class OptionalHolderV2 { + private String id; + private Optional tag; + } + + @Test + public void evolvingBeanInOptionalField() { + RowEncoder writer = evolvingCodec(OptionalHolderV1.class); + RowEncoder reader = evolvingCodec(OptionalHolderV2.class); + + OptionalHolderV1 in = new OptionalHolderV1(); + in.setId("o1"); + TagV1 tag = new TagV1(); + tag.setKey("alpha"); + in.setTag(Optional.of(tag)); + + OptionalHolderV2 out = reader.decode(writer.encode(in)); + Assert.assertEquals(out.getId(), "o1"); + Assert.assertEquals(out.getTag().get().getKey(), "alpha"); + // weight was added at v2; the v1 payload has no source for it. + Assert.assertEquals(out.getTag().get().getWeight(), 0L); + } + + // --------------------------------------------------------------------------- + // A versioned bean reached through a bare Iterable field. TypeInference encodes + // any Iterable (ITERABLE_TYPE.isSupertypeOf), not just Collection, as an array, + // so the evolution walk must descend an Iterable element too. A field typed + // Iterable (not Collection) otherwise slips past the collection + // branch and its element bean is never enumerated. + // --------------------------------------------------------------------------- + + @Data + public static class IterableHolderV1 { + private String id; + private Iterable tags; + } + + @Data + public static class IterableHolderV2 { + private String id; + private Iterable tags; + } + + @Test + public void evolvingBeanInIterableField() { + RowEncoder writer = evolvingCodec(IterableHolderV1.class); + RowEncoder reader = evolvingCodec(IterableHolderV2.class); + + IterableHolderV1 in = new IterableHolderV1(); + in.setId("i1"); + TagV1 a = new TagV1(); + a.setKey("alpha"); + TagV1 b = new TagV1(); + b.setKey("beta"); + in.setTags(Arrays.asList(a, b)); + + IterableHolderV2 out = reader.decode(writer.encode(in)); + Assert.assertEquals(out.getId(), "i1"); + List tags = new ArrayList<>(); + out.getTags().forEach(tags::add); + Assert.assertEquals(tags.size(), 2); + Assert.assertEquals(tags.get(0).getKey(), "alpha"); + Assert.assertEquals(tags.get(1).getKey(), "beta"); + Assert.assertEquals(tags.get(0).getWeight(), 0L); + } + + // --------------------------------------------------------------------------- + // A versioned bean reached only through a custom codec's encodedType(). inferField + // resolves the codec and recurses into encodedType(), encoding the versioned struct, + // so the evolution walk must resolve the codec the same way. The declared field type + // (Money) is not itself a bean, so without codec resolution the inner versioned bean + // is never enumerated and an older payload decodes at the current layout, reading a + // field that does not exist in the older row. + // --------------------------------------------------------------------------- + + /** A domain type encoded through a custom codec into a versioned struct. */ + public static final class Money { + final long cents; + + Money(long cents) { + this.cents = cents; + } + } + + @Data + public static class AmountV1 { + private long cents; + } + + @Data + public static class AmountV2 { + private long cents; + + @ForyRowVersion(since = 2) + private String currency; + } + + static final class MoneyCodecV1 implements CustomCodec { + @Override + public Field getForyField(String fieldName) { + return null; // default inference: recurse into encodedType() + } + + @Override + public AmountV1 encode(Money value) { + AmountV1 a = new AmountV1(); + a.setCents(value.cents); + return a; + } + + @Override + public Money decode(AmountV1 a) { + return new Money(a.getCents()); + } + + @Override + public TypeRef encodedType() { + return TypeRef.of(AmountV1.class); + } + } + + static final class MoneyCodecV2 implements CustomCodec { + @Override + public Field getForyField(String fieldName) { + return null; + } + + @Override + public AmountV2 encode(Money value) { + AmountV2 a = new AmountV2(); + a.setCents(value.cents); + return a; + } + + @Override + public Money decode(AmountV2 a) { + return new Money(a.getCents()); + } + + @Override + public TypeRef encodedType() { + return TypeRef.of(AmountV2.class); + } + } + + @Data + public static class WalletV1 { + private String id; + private Money balance; + } + + @Data + public static class WalletV2 { + private String id; + private Money balance; + } + + @Test + public void evolvingBeanThroughCustomCodec() { + Encoders.registerCustomCodec(WalletV1.class, Money.class, new MoneyCodecV1()); + Encoders.registerCustomCodec(WalletV2.class, Money.class, new MoneyCodecV2()); + + RowEncoder writer = evolvingCodec(WalletV1.class); + RowEncoder reader = evolvingCodec(WalletV2.class); + + WalletV1 in = new WalletV1(); + in.setId("w1"); + in.setBalance(new Money(500)); + + WalletV2 out = reader.decode(writer.encode(in)); + Assert.assertEquals(out.getId(), "w1"); + // The v1 payload encoded balance as AmountV1 (cents only); the v2 codec projects it. + Assert.assertEquals(out.getBalance().cents, 500L); + } + + static final class MoneyCentsCodec implements CustomCodec { + @Override + public Field getForyField(String fieldName) { + return DataTypes.notNullField(fieldName, DataTypes.int64()); + } + + @Override + public Long encode(Money value) { + return value.cents; + } + + @Override + public Money decode(Long cents) { + return new Money(cents); + } + + @Override + public TypeRef encodedType() { + return TypeRef.of(Long.class); + } + } + + @Data + public static class LedgerV1 { + private String id; + private Money balance; + } + + @Data + public static class LedgerV2 { + private String id; + private Money balance; + + @ForyRowVersion(since = 2) + private String note; + } + + /** + * A bean-scoped custom codec whose encoded type is not a versioned bean, so no nested-site + * substitution rewrites the field. Historical field reconstruction must resolve the codec with + * the declaring bean as the enclosing type, exactly as live inference does; a miss reconstructs + * the field from the raw domain type instead of the codec's declaration. The compact format + * exposes the divergence: the codec declares a non-nullable fixed int64 slot, while the missed + * reconstruction yields a nullable variable-width struct, so the projection reader's layout no + * longer matches the bytes the writer produced. + */ + @Test + public void evolvingBeanWithScalarCodec() { + Encoders.registerCustomCodec(LedgerV1.class, Money.class, new MoneyCentsCodec()); + Encoders.registerCustomCodec(LedgerV2.class, Money.class, new MoneyCentsCodec()); + + RowEncoder writer = + Encoders.buildBeanCodec(LedgerV1.class) + .compactEncoding() + .withSchemaEvolution() + .build() + .get(); + RowEncoder reader = + Encoders.buildBeanCodec(LedgerV2.class) + .compactEncoding() + .withSchemaEvolution() + .build() + .get(); + + LedgerV1 in = new LedgerV1(); + in.setId("l1"); + in.setBalance(new Money(250)); + + LedgerV2 out = reader.decode(writer.encode(in)); + Assert.assertEquals(out.getId(), "l1"); + Assert.assertEquals(out.getBalance().cents, 250L); + Assert.assertNull(out.getNote()); + + // The evolution-enabled encoder's public schema must carry the codec's declared column type, + // not a reconstruction from the raw domain type; int64Field validates the declared type. + Assert.assertEquals(writer.schema().int64Field("balance").get(writer.toRow(in)), 250L); + } + + /** + * A class the codec exists to hide, with member names that schema evolution rejects: they have + * underscores, the libphonenumber PhoneNumber shape. + */ + @Data + public static class Handle { + private long code_; + private long codeSource_; + } + + static final class HandleCodec implements CustomCodec { + @Override + public Field getForyField(String fieldName) { + return new Field(fieldName, DataTypes.utf8(), true); + } + + @Override + public String encode(Handle value) { + return value.getCode_() + ":" + value.getCodeSource_(); + } + + @Override + public Handle decode(String value) { + int sep = value.indexOf(':'); + Handle h = new Handle(); + h.setCode_(Long.parseLong(value.substring(0, sep))); + h.setCodeSource_(Long.parseLong(value.substring(sep + 1))); + return h; + } + + @Override + public TypeRef encodedType() { + return TypeRef.of(String.class); + } + } + + @Data + public static class ContactV1 { + private String id; + private Handle handle; + } + + @Data + public static class ContactV2 { + private String id; + private Handle handle; + + @ForyRowVersion(since = 2) + private String note; + } + + /** + * A codec supplying its own terminal foryField replaces the whole column, so the evolution walk + * must not introspect the declared type as a bean: Handle's member names would fail the build if + * it did. + */ + @Test + public void terminalCodecFieldIsNotIntrospected() { + Encoders.registerCustomCodec(ContactV1.class, Handle.class, new HandleCodec()); + Encoders.registerCustomCodec(ContactV2.class, Handle.class, new HandleCodec()); + + RowEncoder writer = evolvingCodec(ContactV1.class); + RowEncoder reader = evolvingCodec(ContactV2.class); + + ContactV1 in = new ContactV1(); + in.setId("c1"); + Handle handle = new Handle(); + handle.setCode_(7); + handle.setCodeSource_(9); + in.setHandle(handle); + + ContactV2 out = reader.decode(writer.encode(in)); + Assert.assertEquals(out.getId(), "c1"); + Assert.assertEquals(out.getHandle().getCode_(), 7L); + Assert.assertEquals(out.getHandle().getCodeSource_(), 9L); + Assert.assertNull(out.getNote()); + } + + /** + * Projection codecs compile on the first decode of a historical hash, on whatever thread that + * happens. The compile must resolve classes with the loader captured when the encoder was built, + * not the decoding thread's context classloader, so a decode from an executor with an unrelated + * or null context classloader behaves exactly like a build-thread decode. The decode thread here + * carries a loader that can see neither Fory nor the bean classes. + */ + @Test + public void projectionCompileUsesBuildTimeClassLoader() throws Exception { + RowEncoder oldWriter = evolvingCodec(PersonV1.class); + RowEncoder newReader = evolvingCodec(PersonV2.class); + PersonV1 in = new PersonV1(); + in.setName("alice"); + in.setAge(30); + byte[] bytes = oldWriter.encode(in); + + CompletableFuture result = new CompletableFuture<>(); + Thread decoder = + new Thread( + () -> { + try { + result.complete(newReader.decode(bytes)); + } catch (Throwable t) { + result.completeExceptionally(t); + } + }); + decoder.setContextClassLoader(new URLClassLoader(new URL[0], null)); + decoder.start(); + decoder.join(); + PersonV2 out = result.get(); + Assert.assertEquals(out.getName(), "alice"); + Assert.assertEquals(out.getAge(), 30); + Assert.assertNull(out.getEmail()); + } + + /** + * The shared code generator for a classloader is only softly reachable, so the GC can drop it + * between build and the first decode of a historical hash. A projection that embeds a nested + * bean's current-version codec must still compile after that: the replacement generator starts + * from the bare classloader, which cannot see the codec classes the dropped generator defined. + * Each test below drops the generator after build, as the GC would, then decodes an old payload. + * The helper restores the dropped generators afterward, so later tests in the JVM see the state + * they would have seen without this one. + */ + @SuppressWarnings("unchecked") + private static R withSharedCodeGeneratorsDropped(Supplier decode) + throws ReflectiveOperationException { + java.lang.reflect.Field field = CodeGenerator.class.getDeclaredField("sharedCodeGenerator"); + field.setAccessible(true); + Map shared = (Map) field.get(null); + Map saved = new HashMap<>(shared); + shared.clear(); + try { + return decode.get(); + } finally { + shared.clear(); + shared.putAll(saved); + } + } + + @Test + public void rowProjectionAfterGeneratorDropped() throws Exception { + RowEncoder writer = evolvingCodec(OuterV1.class); + RowEncoder reader = evolvingCodec(OuterV2.class); + byte[] bytes = writer.encode(sampleV1()); + assertProjectedToV2(withSharedCodeGeneratorsDropped(() -> reader.decode(bytes))); + } + + @Test + public void arrayProjectionAfterGeneratorDropped() throws Exception { + ArrayEncoder> writer = + Encoders.buildArrayCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + ArrayEncoder> reader = + Encoders.buildArrayCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + byte[] bytes = writer.encode(Arrays.asList(sampleV1())); + List out = withSharedCodeGeneratorsDropped(() -> reader.decode(bytes)); + Assert.assertEquals(out.size(), 1); + assertProjectedToV2(out.get(0)); + } + + @Test + public void mapProjectionAfterGeneratorDropped() throws Exception { + MapEncoder> writer = + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + MapEncoder> reader = + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get(); + Map in = new HashMap<>(); + in.put("k", sampleV1()); + byte[] bytes = writer.encode(in); + Map out = withSharedCodeGeneratorsDropped(() -> reader.decode(bytes)); + Assert.assertEquals(out.size(), 1); + assertProjectedToV2(out.get("k")); + } + + // Used only by concurrentFirstProjectedDecodes, so the race compiles their projection classes, + // including the nested inner projection, for the first time. + @Data + public static class RaceInnerV1 { + private String a; + } + + @Data + public static class RaceInnerV2 { + private String a; + + @ForyRowVersion(since = 2) + private String b; + } + + @Data + public static class RaceV1 { + private String id; + private RaceInnerV1 inner; + } + + @Data + public static class RaceV2 { + @ForyRowVersion(since = 2) + private String extra; + + private String id; + private RaceInnerV2 inner; + } + + private static RaceV1 raceSample() { + RaceInnerV1 inner = new RaceInnerV1(); + inner.setA("a"); + RaceV1 v1 = new RaceV1(); + v1.setId("id"); + v1.setInner(inner); + return v1; + } + + private static void assertRaceProjected(RaceV2 out) { + Assert.assertEquals(out.getId(), "id"); + Assert.assertNull(out.getExtra()); + Assert.assertEquals(out.getInner().getA(), "a"); + Assert.assertNull(out.getInner().getB()); + } + + /** + * Encoders from one codec share each projection's lazily resolved state. Several encoders that + * decode the same older hash for the first time at once must each get a correct result, while the + * projection classes compile. + */ + @Test + public void concurrentFirstProjectedDecodes() throws Exception { + byte[] rowBytes = evolvingCodec(RaceV1.class).encode(raceSample()); + Supplier> rows = + Encoders.buildBeanCodec(RaceV2.class).withSchemaEvolution().build(); + byte[] arrayBytes = + Encoders.buildArrayCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get() + .encode(Arrays.asList(raceSample())); + Supplier>> arrays = + Encoders.buildArrayCodec(new TypeRef>() {}).withSchemaEvolution().build(); + Map in = new HashMap<>(); + in.put("k", raceSample()); + byte[] mapBytes = + Encoders.buildMapCodec(new TypeRef>() {}) + .withSchemaEvolution() + .build() + .get() + .encode(in); + Supplier>> maps = + Encoders.buildMapCodec(new TypeRef>() {}).withSchemaEvolution().build(); + + int threads = 8; + CyclicBarrier start = new CyclicBarrier(threads); + ExecutorService pool = Executors.newFixedThreadPool(threads); + try { + List> results = new ArrayList<>(); + for (int i = 0; i < threads; i++) { + RowEncoder row = rows.get(); + ArrayEncoder> array = arrays.get(); + MapEncoder> map = maps.get(); + results.add( + pool.submit( + () -> { + start.await(); + assertRaceProjected(row.decode(rowBytes)); + assertRaceProjected(array.decode(arrayBytes).get(0)); + assertRaceProjected(map.decode(mapBytes).get("k")); + return null; + })); + } + for (Future result : results) { + result.get(60, TimeUnit.SECONDS); + } + } finally { + pool.shutdownNow(); + } + } + + /** + * Forward reads are intentionally unsupported: a reader built against an older class cannot + * decode a payload written at a newer schema it has never seen. PersonV2 writes a v2 payload + * (with the since=2 email field); a PersonV1-built reader knows only v1's strict hash, so the v2 + * hash matches neither its current schema nor any historical projection, and decode fails loud + * rather than silently dropping the newer field. This pins the asymmetry of the + * old-writer/new-reader contract. + */ + @Test(expectedExceptions = ClassNotCompatibleException.class) + public void newerPayloadRejected() { + RowEncoder newWriter = evolvingCodec(PersonV2.class); + RowEncoder oldReader = evolvingCodec(PersonV1.class); + + PersonV2 in = new PersonV2(); + in.setName("alice"); + in.setAge(30); + in.setEmail("alice@example.com"); + + oldReader.decode(newWriter.encode(in)); + } + + @Test + public void projectedDecodeAddsNoWritersToEncoderWriter() throws Exception { + byte[] bytes = evolvingCodec(OuterV1.class).encode(sampleV1()); + BinaryRowWriter rowWriter = new BinaryRowWriter(TypeInference.inferSchema(OuterV2.class)); + RowEncoder reader = + Encoders.buildBeanCodec(OuterV2.class) + .withSchemaEvolution() + .buildForWriter() + .apply(rowWriter); + java.lang.reflect.Field children = BinaryWriter.class.getDeclaredField("children"); + children.setAccessible(true); + int before = ((List) children.get(rowWriter)).size(); + assertProjectedToV2(reader.decode(bytes)); + Assert.assertEquals(((List) children.get(rowWriter)).size(), before); + } + + private static RowEncoder evolvingCodec(Class beanClass) { + return Encoders.buildBeanCodec(beanClass).withSchemaEvolution().build().get(); + } +} diff --git a/java/fory-format/src/test/java/org/apache/fory/format/encoder/SchemaHistoryTest.java b/java/fory-format/src/test/java/org/apache/fory/format/encoder/SchemaHistoryTest.java new file mode 100644 index 0000000000..65ce072f79 --- /dev/null +++ b/java/fory-format/src/test/java/org/apache/fory/format/encoder/SchemaHistoryTest.java @@ -0,0 +1,636 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format.encoder; + +import static org.apache.fory.format.encoder.SchemaHistory.Format.COMPACT; +import static org.apache.fory.format.encoder.SchemaHistory.Format.STANDARD; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.expectThrows; + +import java.lang.reflect.TypeVariable; +import java.util.Optional; +import org.apache.fory.format.annotation.ForyRowSchema; +import org.apache.fory.format.annotation.ForyRowVersion; +import org.apache.fory.format.row.binary.writer.CompactBinaryRowWriter; +import org.apache.fory.format.type.DataTypes; +import org.apache.fory.format.type.Field; +import org.apache.fory.format.type.Schema; +import org.apache.fory.format.type.TypeInference; +import org.apache.fory.reflect.TypeRef; +import org.testng.annotations.Test; + +public class SchemaHistoryTest { + + /** + * The strict hash must be injective over nesting structure. A struct has a variable number of + * children with no boundary marker between a nested struct's last child and the parent's next + * field, so without mixing the child count {@code {a: struct, b}} and {@code {a: struct}} mix an identical byte sequence and collide. These two schemas round-trip differently, so a + * shared hash would route an old payload to the wrong projection codec. + */ + @Test + public void structBoundaryDoesNotCollide() { + Schema fieldOutsideStruct = + DataTypes.schema(DataTypes.field("a", DataTypes.struct(field("x")), false), field("b")); + Schema fieldInsideStruct = + DataTypes.schema(DataTypes.field("a", DataTypes.struct(field("x"), field("b")), false)); + + assertNotEquals( + SchemaHistory.computeStrictSchemaHash(STANDARD, fieldOutsideStruct), + SchemaHistory.computeStrictSchemaHash(STANDARD, fieldInsideStruct)); + } + + /** + * The child-count delimiter must distinguish an empty nested struct followed by a sibling from a + * nested struct that contains that sibling, the minimal form of the boundary ambiguity. + */ + @Test + public void emptyStructBoundaryDoesNotCollide() { + Schema emptyThenSibling = + DataTypes.schema(DataTypes.field("a", DataTypes.struct(), false), field("b")); + Schema siblingInsideStruct = + DataTypes.schema(DataTypes.field("a", DataTypes.struct(field("b")), false)); + + assertNotEquals( + SchemaHistory.computeStrictSchemaHash(STANDARD, emptyThenSibling), + SchemaHistory.computeStrictSchemaHash(STANDARD, siblingInsideStruct)); + } + + /** + * Structurally identical schemas must still hash equal; the delimiter must not over-discriminate. + */ + @Test + public void identicalNestedStructsHashEqual() { + assertEquals( + SchemaHistory.computeStrictSchemaHash( + STANDARD, + DataTypes.schema( + DataTypes.field("a", DataTypes.struct(field("x"), field("b")), false))), + SchemaHistory.computeStrictSchemaHash( + STANDARD, + DataTypes.schema( + DataTypes.field("a", DataTypes.struct(field("x"), field("b")), false)))); + } + + /** + * {@link DataTypes.DecimalType#name} returns a bare "decimal" with no precision or scale, so the + * strict hash must mix those in explicitly. Two decimals that differ only in precision or scale + * must hash apart, or an old payload would route to a projection codec with the wrong numeric + * layout. This guards the dedicated decimal branch in {@code hashFieldShape}. + */ + @Test + public void decimalPrecisionAndScaleDoNotCollide() { + long p10s2 = + SchemaHistory.computeStrictSchemaHash( + STANDARD, DataTypes.schema(DataTypes.field("a", DataTypes.decimal(10, 2), false))); + long p20s2 = + SchemaHistory.computeStrictSchemaHash( + STANDARD, DataTypes.schema(DataTypes.field("a", DataTypes.decimal(20, 2), false))); + long p10s4 = + SchemaHistory.computeStrictSchemaHash( + STANDARD, DataTypes.schema(DataTypes.field("a", DataTypes.decimal(10, 4), false))); + assertNotEquals(p10s2, p20s2); + assertNotEquals(p10s2, p10s4); + assertNotEquals(p20s2, p10s4); + } + + /** + * Variable-width binary and fixed-size binary share {@code TYPE_BINARY}, and two fixed-size + * binaries of different widths share the ID too, so the strict hash mixes the byte width after + * the type ID (0 for variable). Without it an old payload with a resized fixed-binary field would + * route to a projection codec with the wrong slot width. + */ + @Test + public void binaryWidthDoesNotCollide() { + long variable = + SchemaHistory.computeStrictSchemaHash( + STANDARD, DataTypes.schema(DataTypes.field("a", DataTypes.binary(), false))); + long fixed4 = + SchemaHistory.computeStrictSchemaHash( + STANDARD, DataTypes.schema(DataTypes.field("a", DataTypes.fixedWidthBinary(4), false))); + long fixed8 = + SchemaHistory.computeStrictSchemaHash( + STANDARD, DataTypes.schema(DataTypes.field("a", DataTypes.fixedWidthBinary(8), false))); + assertNotEquals(variable, fixed4); + assertNotEquals(variable, fixed8); + assertNotEquals(fixed4, fixed8); + assertEquals( + fixed4, + SchemaHistory.computeStrictSchemaHash( + STANDARD, + DataTypes.schema(DataTypes.field("a", DataTypes.fixedWidthBinary(4), false)))); + } + + /** + * The canonical cross-language field order is ascending by snake_case wire name, compared by code + * point, but Java's generated codecs bake the member-name order, compared by UTF-16 code unit. A + * character outside the Basic Multilingual Plane against one above the surrogate range makes the + * two orders diverge, so the bean cannot be emitted in canonical order and the build must reject + * it with rename guidance. + */ + @Test + public void divergentWireNameOrderIsRejected() { + IllegalStateException e = + expectThrows( + IllegalStateException.class, + () -> SchemaHistory.build(DivergentNameOrder.class, STANDARD)); + assertTrue(e.getMessage().contains("orders member"), e.getMessage()); + } + + /** + * The same divergence inside a nested bean must also be rejected. Nested-site discovery builds + * every reachable bean's history (even a single-version one, to count its versions), so the + * nested bean's own enumeration runs the order check. + */ + @Test + public void divergentWireNameOrderInNestedBeanIsRejected() { + IllegalStateException e = + expectThrows( + IllegalStateException.class, + () -> SchemaHistory.build(DivergentNestedHolder.class, STANDARD)); + assertTrue(e.getMessage().contains("orders member"), e.getMessage()); + } + + /** + * The codec finds the member of a wire name by removing its underscores, so member {@code a_} + * would read and write member {@code a}, and its own value would be lost. The build must reject + * an underscore in a live member name and in a removed-field declaration. + */ + @Test + public void underscoreMemberNameIsRejected() { + IllegalStateException live = + expectThrows( + IllegalStateException.class, + () -> SchemaHistory.build(TrailingUnderscore.class, STANDARD)); + assertTrue(live.getMessage().contains("'a_'"), live.getMessage()); + IllegalStateException removed = + expectThrows( + IllegalStateException.class, + () -> SchemaHistory.build(RemovedUnderscore.class, STANDARD)); + assertTrue(removed.getMessage().contains("'old_name'"), removed.getMessage()); + } + + @lombok.Data + public static class TrailingUnderscore { + private int a; + private int a_; + } + + @lombok.Data + @ForyRowSchema(removedFields = RemovedUnderscore.History.class) + public static class RemovedUnderscore { + private int a; + + interface History { + @ForyRowVersion(until = 2) + int old_name(); + } + } + + /** + * U+1D44E sorts before U+FF41 by UTF-16 code unit, the member-name order, and after it by code + * point, the wire-name order. + */ + @lombok.Data + public static class DivergentNameOrder { + private int a𝑎; + private int aａ; + } + + @lombok.Data + public static class DivergentNestedHolder { + private DivergentNameOrder inner; + + @ForyRowVersion(since = 2) + private int count; + } + + /** Identical decimal shapes must still hash equal; the precision/scale mix must not be noisy. */ + @Test + public void identicalDecimalsHashEqual() { + assertEquals( + SchemaHistory.computeStrictSchemaHash( + STANDARD, DataTypes.schema(DataTypes.field("a", DataTypes.decimal(10, 2), false))), + SchemaHistory.computeStrictSchemaHash( + STANDARD, DataTypes.schema(DataTypes.field("a", DataTypes.decimal(10, 2), false)))); + } + + /** + * A map header carries one combined hash for the ordered (key layout, value layout) pair, and + * {@link SchemaHistory#combineHashes} documents that it is order-sensitive. A swapped pair does + * not reach it today: the map codec skips pairs that put one bean class at two versions, and a + * key position hash differs from a value position hash in nullability. This pins the documented + * property, so the combined hash does not depend on those two facts. + */ + @Test + public void combineHashesIsOrderSensitive() { + long a = 0x0123456789abcdefL; + long b = 0x7766554433221100L; + assertNotEquals(SchemaHistory.combineHashes(a, b), SchemaHistory.combineHashes(b, a)); + } + + /** + * A field typed as a bare type variable bounded to a wrapper (the Scala 3 LTS case that {@code + * TypeInference.inferField} resolves to its bound, see issue 2439) must descend through the bound + * during evolution-site discovery. {@code getRawType} resolves the variable to {@code Optional}, + * but the bare variable carries no type arguments, so a descent that does not resolve the bound + * first fails reading the missing element. The bean inside the bound must be discovered instead. + */ + @Test + public void typeVariableBoundedToWrapperResolvesToInnerBean() { + TypeVariable boundedToOptionalBean = OptionalBeanHolder.class.getTypeParameters()[0]; + assertEquals(SchemaHistory.evolutionBean(TypeRef.of(boundedToOptionalBean)), Inner.class); + } + + /** {@code >}: a wrapper-bounded type variable for the test above. */ + private static final class OptionalBeanHolder> {} + + @lombok.Data + public static class Inner { + private String name; + } + + /** + * The writer-side hash is {@code current().strictHash()}, but the non-evolution row codec and the + * decode hot path infer the live layout straight from {@link TypeInference#inferSchema}. The two + * must produce byte-identical schemas, or a current-version payload misses the fast path. The + * history derives its per-version schema by sorting fields on the Java member name to match + * {@code inferSchema}'s {@code Descriptor.getDescriptors} order; this pins that equivalence so a + * drift in the sort key fails loudly. Fields are declared out of alphabetical order on purpose so + * the assertion has teeth. + */ + @Test + public void currentSchemaMatchesInferSchema() { + for (Class bean : + new Class[] { + OutOfOrderFields.class, NestedHolder.class, CollectionHolder.class, EvolvingBean.class + }) { + assertEquals( + SchemaHistory.build(bean, STANDARD).current().schema(), + TypeInference.inferSchema(bean), + "current() schema diverged from inferSchema for " + bean.getSimpleName()); + } + } + + /** + * The compact codec bakes its field order from {@code sortSchema(inferSchema(bean))}, while the + * compact evolution writer hashes the deep-sorted history schema. Their top-level orders must + * agree, or a current-version compact payload hashes one order and writes another. + */ + @Test + public void compactCurrentSchemaMatchesCodecOrder() { + for (Class bean : + new Class[] { + OutOfOrderFields.class, NestedHolder.class, CollectionHolder.class, EvolvingBean.class + }) { + Schema inferred = TypeInference.inferSchema(bean); + Schema history = SchemaHistory.build(bean, COMPACT).current().schema(); + assertEquals(history, CompactBinaryRowWriter.sortSchemaDeep(inferred), bean.getSimpleName()); + Schema codecOrder = CompactBinaryRowWriter.sortSchema(inferred); + for (int i = 0; i < history.numFields(); i++) { + assertEquals( + history.field(i).name(), codecOrder.field(i).name(), bean.getSimpleName() + " " + i); + } + } + } + + @lombok.Data + public static class OutOfOrderFields { + private int zebra; + private String alpha; + private long mid; + } + + @lombok.Data + public static class NestedHolder { + private Inner inner; + private int count; + } + + @lombok.Data + public static class CollectionHolder { + private java.util.List items; + private java.util.Map byName; + } + + @lombok.Data + public static class EvolvingBean { + private int base; + + @ForyRowVersion(since = 2) + private String added; + } + + /** + * The compact format sorts every struct it writes, including list elements and map values. The + * strict hash must follow that wire order whether or not the nested bean is versioned: a + * versioned bean substitutes its own sorted history schema, so an unversioned bean with the same + * layout must hash the same. + */ + @Test + public void compactHashFollowsWireOrderInsideCollections() { + SchemaHistory plain = SchemaHistory.build(PlainCollectionHolder.class, COMPACT); + SchemaHistory versioned = SchemaHistory.build(VersionedCollectionHolder.class, COMPACT); + assertEquals(plain.current().strictHash(), versioned.current().strictHash()); + + DataTypes.ListType items = + (DataTypes.ListType) plain.current().schema().getFieldByName("items").type(); + DataTypes.StructType element = (DataTypes.StructType) items.valueType(); + assertEquals(element.fields().get(0).name(), "wide"); + assertEquals(element.fields().get(1).name(), "narrow"); + } + + @lombok.Data + public static class PlainElement { + private byte narrow; + private long wide; + } + + @lombok.Data + @ForyRowSchema(removedFields = VersionedElement.History.class) + public static class VersionedElement { + private byte narrow; + private long wide; + + interface History { + @ForyRowVersion(until = 2) + String retired(); + } + } + + @lombok.Data + public static class PlainCollectionHolder { + private java.util.List items; + private java.util.Map byName; + } + + @lombok.Data + public static class VersionedCollectionHolder { + private java.util.List items; + private java.util.Map byName; + } + + /** + * Known-answer vectors for the strict hash, computed from the algorithm in {@code + * docs/specification/row_format_spec.md} independently of this implementation. The spec lists the + * same values. A change to any of them breaks every payload already written with schema + * evolution, so update them only together with a wire-format change. + */ + @Test + public void strictHashKnownAnswers() { + assertEquals(hash(DataTypes.notNullField("id", DataTypes.int32())), 0x506a3242a9f93ae2L); + assertEquals( + SchemaHistory.computeStrictSchemaHash( + COMPACT, DataTypes.schema(DataTypes.notNullField("id", DataTypes.int32()))), + 0xb7db30d6e6593b2fL); + assertEquals(hash(DataTypes.field("name", DataTypes.utf8(), true)), 0x4bd0d984e7109ef6L); + assertEquals( + hash(DataTypes.notNullField("gr\u00f6\u00dfe", DataTypes.int64())), 0xe5fcaf6a66d32a0eL); + assertEquals( + hash(DataTypes.notNullField("amount", DataTypes.decimal(10, 2))), 0x2ef05b2b90e987a9L); + assertEquals( + hash(DataTypes.notNullField("amount", DataTypes.decimal(38, -3))), 0x7e49dad4d1337a8aL); + assertEquals( + hash( + DataTypes.field("blob", DataTypes.binary(), true), + DataTypes.notNullField("digest", DataTypes.fixedWidthBinary(4))), + 0xca28ad1d8bf595e7L); + assertEquals( + hash( + DataTypes.arrayField( + "tags", DataTypes.field(DataTypes.ARRAY_ITEM_NAME, DataTypes.utf8(), true))), + 0x55f88fd672e92ea9L); + assertEquals( + hash(DataTypes.mapField("counts", DataTypes.utf8(), DataTypes.int32())), + 0x570b6dadf84868c8L); + assertEquals( + hash( + DataTypes.field( + "point", + DataTypes.struct( + DataTypes.notNullField("x", DataTypes.int32()), + DataTypes.notNullField("y", DataTypes.int32())), + true), + DataTypes.notNullField("z", DataTypes.int64())), + 0x793051ee66a15234L); + assertEquals( + hash( + DataTypes.field("day", DataTypes.date32(), true), + DataTypes.field("dur", DataTypes.duration(), true), + DataTypes.notNullField("f16", DataTypes.float16()), + DataTypes.notNullField("f32", DataTypes.float32()), + DataTypes.notNullField("f64", DataTypes.float64()), + DataTypes.field("ts", DataTypes.timestamp(), true)), + 0xf6320f76f21330bcL); + assertEquals( + SchemaHistory.combineHashes(0x506a3242a9f93ae2L, 0x4bd0d984e7109ef6L), 0x11ce9d3600e26f89L); + } + + /** + * A writer has one definition of each bean class, so a class reached at two depths (directly and + * through another versioned bean) is at the same version in both places. The history must not + * enumerate combinations that put it at two versions. + */ + @Test + public void sharedNestedClassHasOneVersionPerCombination() { + SchemaHistory history = SchemaHistory.build(SharedOuter.class, STANDARD); + assertEquals(history.versions().size(), 4); + for (SchemaHistory.VersionedSchema vs : history.versions()) { + SchemaHistory.VersionedSchema shared = vs.nestedBeanSchemas().get(SharedLeaf.class); + SchemaHistory.VersionedSchema middle = vs.nestedBeanSchemas().get(SharedMiddle.class); + assertTrue(shared == middle.nestedBeanSchemas().get(SharedLeaf.class)); + } + } + + @lombok.Data + public static class SharedLeaf { + private int a; + + @ForyRowVersion(since = 2) + private int b; + } + + @lombok.Data + public static class SharedMiddle { + private SharedLeaf leaf; + + @ForyRowVersion(since = 2) + private int extra; + } + + @lombok.Data + public static class SharedOuter { + private SharedLeaf leaf; + private SharedMiddle middle; + } + + /** + * A field removed at one version and added again with the same name and type gives two versions + * with the same layout, which collapse into one history entry. A payload with that layout cannot + * say which version wrote it, so every nested combination must treat the field the same way: as + * live, so that its value is read. + */ + @Test + public void collapsedVersionsKeepReaddedFieldLive() { + SchemaHistory history = SchemaHistory.build(ReaddedField.class, STANDARD); + assertEquals(history.versions().size(), 2); + for (SchemaHistory.VersionedSchema vs : history.versions()) { + assertTrue(vs.liveFieldNames().contains("x"), "x not live in " + vs.schema()); + } + } + + @lombok.Data + @ForyRowSchema(removedFields = ReaddedField.History.class) + public static class ReaddedField { + private SharedLeaf leaf; + + @ForyRowVersion(since = 2) + private int x; + + interface History { + @ForyRowVersion(until = 2) + int x(); + } + } + + /** + * The removed and live types of {@code pair} differ, and their codecs write the same two children + * in opposite orders. The standard format keeps each order, so the two layouts differ. The + * compact format sorts both into one order, so an older compact layout is identical to the + * current one, and the history must reject the type change. + */ + @Test + public void permutedCodecChildrenTypeChangeRejectedInCompact() { + Field wide = DataTypes.field("wide", DataTypes.int64(), false); + Field narrow = DataTypes.field("narrow", DataTypes.int32(), false); + Encoders.registerCustomCodec( + PermutedCodecField.class, WideFirst.class, new PermutedStructCodec<>(wide, narrow)); + Encoders.registerCustomCodec( + PermutedCodecField.class, NarrowFirst.class, new PermutedStructCodec<>(narrow, wide)); + + SchemaHistory.build(PermutedCodecField.class, STANDARD); + IllegalStateException e = + expectThrows( + IllegalStateException.class, + () -> SchemaHistory.build(PermutedCodecField.class, COMPACT)); + assertTrue(e.getMessage().contains("'pair'"), e.getMessage()); + } + + public static final class WideFirst {} + + public static final class NarrowFirst {} + + /** Writes a struct of the constructor's children, in the constructor's order. */ + private static final class PermutedStructCodec implements CustomCodec { + private final Field[] children; + + PermutedStructCodec(Field... children) { + this.children = children; + } + + @Override + public Field getForyField(String fieldName) { + return DataTypes.structField(fieldName, true, children); + } + + @Override + public TypeRef encodedType() { + return TypeRef.of(Object.class); + } + + @Override + public Object encode(T value) { + throw new UnsupportedOperationException(); + } + + @Override + public T decode(Object value) { + throw new UnsupportedOperationException(); + } + } + + @lombok.Data + @ForyRowSchema(removedFields = PermutedCodecField.History.class) + public static class PermutedCodecField { + @ForyRowVersion(since = 2) + private String label; + + @ForyRowVersion(since = 3) + private NarrowFirst pair; + + interface History { + @ForyRowVersion(until = 2) + WideFirst pair(); + } + } + + /** + * The removed and live types of {@code pair} differ, and their codecs write two int64 children in + * opposite orders. The compact sort is stable, so children of equal width keep the codec's order: + * the two layouts differ in both formats, the history accepts the type change, and the reader + * discards the old value. + */ + @Test + public void equalWidthCodecChildrenKeepOrderInCompact() { + Field x = DataTypes.field("x", DataTypes.int64(), false); + Field y = DataTypes.field("y", DataTypes.int64(), false); + Encoders.registerCustomCodec( + EqualWidthCodecField.class, XFirst.class, new PermutedStructCodec<>(x, y)); + Encoders.registerCustomCodec( + EqualWidthCodecField.class, YFirst.class, new PermutedStructCodec<>(y, x)); + + for (SchemaHistory.Format format : SchemaHistory.Format.values()) { + SchemaHistory history = SchemaHistory.build(EqualWidthCodecField.class, format); + assertEquals(history.versions().size(), 2, format.name()); + for (SchemaHistory.VersionedSchema vs : history.versions()) { + Field pair = vs.schema().field(0); + assertEquals(pair.name(), "pair"); + assertEquals(pair.type().fields().get(0).name(), vs.isCurrent() ? "y" : "x", format.name()); + assertEquals(vs.liveFieldNames().contains("pair"), vs.isCurrent(), format.name()); + } + } + } + + public static final class XFirst {} + + public static final class YFirst {} + + @lombok.Data + @ForyRowSchema(removedFields = EqualWidthCodecField.History.class) + public static class EqualWidthCodecField { + @ForyRowVersion(since = 2) + private YFirst pair; + + interface History { + @ForyRowVersion(until = 2) + XFirst pair(); + } + } + + private static long hash(Field... fields) { + return SchemaHistory.computeStrictSchemaHash(STANDARD, DataTypes.schema(fields)); + } + + private static Field field(String name) { + return DataTypes.field(name, DataTypes.int32(), false); + } +} diff --git a/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/RecordRowTest.java b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/RecordRowTest.java index 99c61c64ce..ccd5ebce60 100644 --- a/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/RecordRowTest.java +++ b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/RecordRowTest.java @@ -21,6 +21,8 @@ import java.time.Instant; import java.time.LocalDate; +import org.apache.fory.format.annotation.ForyRowSchema; +import org.apache.fory.format.annotation.ForyRowVersion; import org.apache.fory.format.encoder.Encoders; import org.apache.fory.format.encoder.RowEncoder; import org.apache.fory.format.row.binary.BinaryRow; @@ -86,4 +88,107 @@ public void testRecordNestedInterface() { final TestRecordNestedInterface deserializedBean = encoder.fromRow(row); Assert.assertEquals(deserializedBean.f1().f1(), bean.f1().f1()); } + + // --------------------------------------------------------------------------- + // Records with schema evolution. @ForyRowVersion on a record component propagates + // to the backing field and the accessor (its FIELD/METHOD targets), where the + // codec reads it, so a newer reader record can pick up older payloads and + // default components added later. The history interface still works because the + // bean is a record: live component names match the wire field names (record + // short-style naming). + // --------------------------------------------------------------------------- + + public record PersonV1(String name, int age) {} + + @ForyRowSchema(removedFields = PersonV2.History.class) + public record PersonV2(String name, @ForyRowVersion(since = 2) String email) { + interface History { + @ForyRowVersion(until = 2) + int age(); + } + } + + @Test + public void evolutionOlderPayload() { + RowEncoder writer = + Encoders.buildBeanCodec(PersonV1.class).withSchemaEvolution().build().get(); + RowEncoder reader = + Encoders.buildBeanCodec(PersonV2.class).withSchemaEvolution().build().get(); + PersonV2 out = reader.decode(writer.encode(new PersonV1("Luna", 7))); + Assert.assertEquals(out.name(), "Luna"); + Assert.assertNull(out.email()); + } + + @Test + public void evolutionRoundTrip() { + RowEncoder codec = + Encoders.buildBeanCodec(PersonV2.class).withSchemaEvolution().build().get(); + PersonV2 in = new PersonV2("Mars", "mars@example.com"); + Assert.assertEquals(codec.decode(codec.encode(in)), in); + } + + /** Record with a primitive added at v2: an older payload must produce the primitive default. */ + public record CounterV1(String name) {} + + public record CounterV2(String name, @ForyRowVersion(since = 2) int count) {} + + @Test + public void evolutionPrimitiveDefault() { + RowEncoder writer = + Encoders.buildBeanCodec(CounterV1.class).withSchemaEvolution().build().get(); + RowEncoder reader = + Encoders.buildBeanCodec(CounterV2.class).withSchemaEvolution().build().get(); + CounterV2 out = reader.decode(writer.encode(new CounterV1("Luna"))); + Assert.assertEquals(out.name(), "Luna"); + Assert.assertEquals(out.count(), 0); + } + + // A record component whose own type is a versioned record. The inner struct is + // inline in the outer's bytes with no per-inner hash, so the reader must pick an + // inner schema consistent with the outer's strict hash. This drives the nested + // cross-product enumeration with record-component field naming. + public record InnerV1(String name) {} + + public record InnerV2(String name, @ForyRowVersion(since = 2) String tag) {} + + public record OuterInnerV1(long id, InnerV1 inner) {} + + public record OuterInnerV2(long id, InnerV2 inner) {} + + @Test + public void evolutionNestedInnerNewerThanWriter() { + RowEncoder writer = + Encoders.buildBeanCodec(OuterInnerV1.class).withSchemaEvolution().build().get(); + RowEncoder reader = + Encoders.buildBeanCodec(OuterInnerV2.class).withSchemaEvolution().build().get(); + OuterInnerV2 out = reader.decode(writer.encode(new OuterInnerV1(42, new InnerV1("hello")))); + Assert.assertEquals(out.id(), 42); + Assert.assertEquals(out.inner().name(), "hello"); + Assert.assertNull(out.inner().tag()); + } + + // A reference component added at v2 is absent from a v1 payload, so decode supplies null + // for it and the record's canonical constructor runs with that null. A constructor that + // rejects null for the added component would throw during decode; the supported pattern is + // to tolerate the missing value, e.g. by normalizing null to a default in the constructor. + public record DefaultedV1(String name) {} + + public record DefaultedV2(String name, @ForyRowVersion(since = 2) String email) { + public DefaultedV2 { + if (email == null) { + email = "unknown"; + } + } + } + + @Test + public void evolutionConstructorDefaults() { + RowEncoder writer = + Encoders.buildBeanCodec(DefaultedV1.class).withSchemaEvolution().build().get(); + RowEncoder reader = + Encoders.buildBeanCodec(DefaultedV2.class).withSchemaEvolution().build().get(); + DefaultedV2 out = reader.decode(writer.encode(new DefaultedV1("Luna"))); + Assert.assertEquals(out.name(), "Luna"); + Assert.assertEquals(out.email(), "unknown"); + } }