From 72817de871f1b08dd500a7957a2198d8337a8c71 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Thu, 15 Jan 2026 21:17:26 +0800 Subject: [PATCH 01/44] implement streaming read/write type info --- cpp/fory/serialization/context.cc | 258 ++++++++---------- cpp/fory/serialization/context.h | 38 ++- cpp/fory/serialization/fory.h | 33 +-- cpp/fory/serialization/skip.cc | 19 +- cpp/fory/serialization/struct_serializer.h | 15 +- .../specification/xlang_serialization_spec.md | 18 +- go/fory/fory.go | 216 ++------------- go/fory/type_resolver.go | 115 ++++---- .../src/main/java/org/apache/fory/Fory.java | 60 ---- .../apache/fory/resolver/ClassResolver.java | 72 +++-- .../org/apache/fory/resolver/MetaContext.java | 14 +- .../fory/resolver/SerializationContext.java | 4 - .../apache/fory/resolver/TypeResolver.java | 74 +---- .../apache/fory/resolver/XtypeResolver.java | 40 ++- .../apache/fory/serializer/FieldGroups.java | 6 +- .../serializer/MetaSharedLayerSerializer.java | 31 +-- .../NonexistentClassSerializers.java | 8 +- .../java/org/apache/fory/type/DispatchId.java | 11 +- .../packages/fory/lib/typeMetaResolver.ts | 10 +- python/pyfory/_fory.py | 36 +-- python/pyfory/registry.py | 20 ++ python/pyfory/serialization.pyx | 35 ++- rust/fory-core/src/fory.rs | 19 +- rust/fory-core/src/resolver/context.rs | 61 ++--- rust/fory-core/src/resolver/meta_resolver.rs | 74 +++-- rust/fory-core/src/serializer/enum_.rs | 7 +- rust/fory-core/src/serializer/skip.rs | 31 ++- rust/fory-core/src/serializer/struct_.rs | 14 +- rust/fory-derive/src/object/derive_enum.rs | 10 +- 29 files changed, 521 insertions(+), 828 deletions(-) diff --git a/cpp/fory/serialization/context.cc b/cpp/fory/serialization/context.cc index 00709495d8..2a82f4a920 100644 --- a/cpp/fory/serialization/context.cc +++ b/cpp/fory/serialization/context.cc @@ -64,46 +64,33 @@ WriteContext::WriteContext(const Config &config, WriteContext::~WriteContext() = default; -Result WriteContext::push_meta(const std::type_index &type_id) { - auto it = write_type_id_index_map_.find(type_id); - if (it != write_type_id_index_map_.end()) { - return it->second; - } - - size_t index = write_type_defs_.size(); +Result +WriteContext::write_type_meta(const std::type_index &type_id) { + // Resolve type_index to TypeInfo* and delegate to the TypeInfo* version + // This ensures consistent indexing when the same type is written via + // either type_index or TypeInfo* path FORY_TRY(type_info, type_resolver_->get_type_info(type_id)); - write_type_defs_.push_back(type_info->type_def); - write_type_id_index_map_[type_id] = index; - return index; + write_type_meta(type_info); + return Result(); } -size_t WriteContext::push_meta(const TypeInfo *type_info) { +void WriteContext::write_type_meta(const TypeInfo *type_info) { auto it = write_type_info_index_map_.find(type_info); if (it != write_type_info_index_map_.end()) { - return it->second; + // Reference to previously written type: (index << 1) | 1, LSB=1 + buffer_.WriteVarUint32(static_cast((it->second << 1) | 1)); + return; } - size_t index = write_type_defs_.size(); - write_type_defs_.push_back(type_info->type_def); + // New type: index << 1, LSB=0, followed by TypeDef bytes inline + size_t index = write_type_info_index_map_.size(); + buffer_.WriteVarUint32(static_cast(index << 1)); write_type_info_index_map_[type_info] = index; - return index; -} -void WriteContext::write_meta(size_t offset) { - size_t current_pos = buffer_.writer_index(); - // Update the meta offset field (written as -1 initially) - int32_t meta_size = static_cast(current_pos - offset - 4); - buffer_.UnsafePut(offset, meta_size); - // Write all collected TypeMetas - buffer_.WriteVarUint32(static_cast(write_type_defs_.size())); - for (size_t i = 0; i < write_type_defs_.size(); ++i) { - const auto &type_def = write_type_defs_[i]; - buffer_.WriteBytes(type_def.data(), type_def.size()); - } + // Write TypeDef bytes inline + buffer_.WriteBytes(type_info->type_def.data(), type_info->type_def.size()); } -bool WriteContext::meta_empty() const { return write_type_defs_.empty(); } - /// Write pre-encoded meta string to buffer (avoids re-encoding on each write) static void write_encoded_meta_string(Buffer &buffer, const CachedMetaString &encoded) { @@ -134,9 +121,8 @@ WriteContext::write_enum_typeinfo(const std::type_index &type) { if (type_id_low == static_cast(TypeId::NAMED_ENUM)) { if (config_->compatible) { - // Write meta_index - FORY_TRY(meta_index, push_meta(type)); - buffer_.WriteVarUint32(static_cast(meta_index)); + // Write type meta inline using streaming protocol + FORY_RETURN_NOT_OK(write_type_meta(type)); } else { // Write pre-encoded namespace and type_name if (type_info->encoded_namespace && type_info->encoded_type_name) { @@ -166,9 +152,8 @@ WriteContext::write_enum_typeinfo(const TypeInfo *type_info) { if (type_id_low == static_cast(TypeId::NAMED_ENUM)) { if (config_->compatible) { - // Write meta_index using TypeInfo pointer (fast path) - size_t meta_index = push_meta(type_info); - buffer_.WriteVarUint32(static_cast(meta_index)); + // Write type meta inline using streaming protocol + write_type_meta(type_info); } else { // Write pre-encoded namespace and type_name if (type_info->encoded_namespace && type_info->encoded_type_name) { @@ -207,18 +192,16 @@ WriteContext::write_any_typeinfo(uint32_t fory_type_id, switch (type_id_low) { case static_cast(TypeId::NAMED_COMPATIBLE_STRUCT): case static_cast(TypeId::COMPATIBLE_STRUCT): { - // Write meta_index - FORY_TRY(meta_index, push_meta(concrete_type_id)); - buffer_.WriteVarUint32(static_cast(meta_index)); + // Write type meta inline using streaming protocol + FORY_RETURN_NOT_OK(write_type_meta(concrete_type_id)); break; } case static_cast(TypeId::NAMED_ENUM): case static_cast(TypeId::NAMED_EXT): case static_cast(TypeId::NAMED_STRUCT): { if (config_->compatible) { - // Write meta_index (share_meta is effectively compatible in C++) - FORY_TRY(meta_index, push_meta(concrete_type_id)); - buffer_.WriteVarUint32(static_cast(meta_index)); + // Write type meta inline using streaming protocol + FORY_RETURN_NOT_OK(write_type_meta(concrete_type_id)); } else { // Write pre-encoded namespace and type_name if (type_info->encoded_namespace && type_info->encoded_type_name) { @@ -253,16 +236,14 @@ WriteContext::write_struct_type_info(const std::type_index &type_id) { switch (type_id_low) { case static_cast(TypeId::NAMED_COMPATIBLE_STRUCT): case static_cast(TypeId::COMPATIBLE_STRUCT): { - // Write meta_index - FORY_TRY(meta_index, push_meta(type_id)); - buffer_.WriteVarUint32(static_cast(meta_index)); + // Write type meta inline using streaming protocol + FORY_RETURN_NOT_OK(write_type_meta(type_id)); break; } case static_cast(TypeId::NAMED_STRUCT): { if (config_->compatible) { - // Write meta_index - FORY_TRY(meta_index, push_meta(type_id)); - buffer_.WriteVarUint32(static_cast(meta_index)); + // Write type meta inline using streaming protocol + FORY_RETURN_NOT_OK(write_type_meta(type_id)); } else { // Write pre-encoded namespace and type_name if (type_info->encoded_namespace && type_info->encoded_type_name) { @@ -295,16 +276,14 @@ WriteContext::write_struct_type_info(const TypeInfo *type_info) { switch (type_id_low) { case static_cast(TypeId::NAMED_COMPATIBLE_STRUCT): case static_cast(TypeId::COMPATIBLE_STRUCT): { - // Write meta_index using TypeInfo pointer (fast path) - size_t meta_index = push_meta(type_info); - buffer_.WriteVarUint32(static_cast(meta_index)); + // Write type meta inline using streaming protocol + write_type_meta(type_info); break; } case static_cast(TypeId::NAMED_STRUCT): { if (config_->compatible) { - // Write meta_index using TypeInfo pointer (fast path) - size_t meta_index = push_meta(type_info); - buffer_.WriteVarUint32(static_cast(meta_index)); + // Write type meta inline using streaming protocol + write_type_meta(type_info); } else { // Write pre-encoded namespace and type_name if (type_info->encoded_namespace && type_info->encoded_type_name) { @@ -329,10 +308,7 @@ void WriteContext::reset() { // Clear error state first error_ = Error(); ref_writer_.reset(); - // Clear meta vectors/maps - they're typically small or empty - // in non-compatible mode, so clear() is efficient - write_type_defs_.clear(); - write_type_id_index_map_.clear(); + // Clear meta map for streaming TypeMeta (size is used as counter) write_type_info_index_map_.clear(); current_dyn_depth_ = 0; // Reset buffer indices for reuse - no memory operations needed @@ -385,100 +361,94 @@ ReadContext::read_enum_type_info(uint32_t base_type_id) { // Maximum number of parsed type defs to cache (avoid OOM from malicious input) static constexpr size_t kMaxParsedNumTypeDefs = 8192; -Result ReadContext::load_type_meta(int32_t meta_offset) { - size_t current_pos = buffer_->reader_index(); - size_t meta_start = current_pos + meta_offset; - buffer_->ReaderIndex(static_cast(meta_start)); - - // Load all TypeMetas +Result ReadContext::read_type_meta() { Error error; - uint32_t meta_size = buffer_->ReadVarUint32(error); + // Read the index marker + uint32_t index_marker = buffer_->ReadVarUint32(error); if (FORY_PREDICT_FALSE(!error.ok())) { return Unexpected(std::move(error)); } - reading_type_infos_.reserve(meta_size); - for (uint32_t i = 0; i < meta_size; i++) { - // Read the 8-byte header first for caching - int64_t meta_header = buffer_->ReadInt64(error); - if (FORY_PREDICT_FALSE(!error.ok())) { - return Unexpected(std::move(error)); - } + bool is_ref = (index_marker & 1) == 1; + size_t index = index_marker >> 1; - // Check if we already parsed this type meta (cache lookup by header) - auto cache_it = parsed_type_infos_.find(meta_header); - if (cache_it != parsed_type_infos_.end()) { - // Found in cache - reuse and skip the bytes - reading_type_infos_.push_back(cache_it->second); - FORY_RETURN_NOT_OK(TypeMeta::skip_bytes(*buffer_, meta_header)); - continue; - } + if (is_ref) { + // Reference to previously read type + return get_type_info_by_index(index); + } - // Not in cache - parse the TypeMeta - FORY_TRY(parsed_meta, - TypeMeta::from_bytes_with_header(*buffer_, meta_header)); - - // Find local TypeInfo to get field_id mapping (optional for schema - // evolution) - const TypeInfo *local_type_info = nullptr; - if (parsed_meta->register_by_name) { - auto result = type_resolver_->get_type_info_by_name( - parsed_meta->namespace_str, parsed_meta->type_name); - if (result.ok()) { - local_type_info = result.value(); - } - } else { - auto result = type_resolver_->get_type_info_by_id(parsed_meta->type_id); - if (result.ok()) { - local_type_info = result.value(); - } - } + // New type - read TypeMeta inline + // Read the 8-byte header first for caching + int64_t meta_header = buffer_->ReadInt64(error); + if (FORY_PREDICT_FALSE(!error.ok())) { + return Unexpected(std::move(error)); + } - // Create TypeInfo with field_ids assigned - auto type_info = std::make_unique(); - if (local_type_info) { - // Have local type - assign field_ids by comparing schemas - // Note: Extension types don't have type_meta (only structs do) - if (local_type_info->type_meta) { - TypeMeta::assign_field_ids(local_type_info->type_meta.get(), - parsed_meta->field_infos); - } - type_info->type_id = local_type_info->type_id; - type_info->type_meta = std::move(parsed_meta); - type_info->type_def = local_type_info->type_def; - // CRITICAL: Copy the harness from the registered type_info - type_info->harness = local_type_info->harness; - type_info->name_to_index = local_type_info->name_to_index; - type_info->namespace_name = local_type_info->namespace_name; - type_info->type_name = local_type_info->type_name; - type_info->register_by_name = local_type_info->register_by_name; - } else { - // No local type - create stub TypeInfo with parsed meta - type_info->type_id = parsed_meta->type_id; - type_info->type_meta = std::move(parsed_meta); + // Check if we already parsed this type meta (cache lookup by header) + auto cache_it = parsed_type_infos_.find(meta_header); + if (cache_it != parsed_type_infos_.end()) { + // Found in cache - reuse and skip the bytes + reading_type_infos_.push_back(cache_it->second); + FORY_RETURN_NOT_OK(TypeMeta::skip_bytes(*buffer_, meta_header)); + return cache_it->second; + } + + // Not in cache - parse the TypeMeta + FORY_TRY(parsed_meta, + TypeMeta::from_bytes_with_header(*buffer_, meta_header)); + + // Find local TypeInfo to get field_id mapping (optional for schema evolution) + const TypeInfo *local_type_info = nullptr; + if (parsed_meta->register_by_name) { + auto result = type_resolver_->get_type_info_by_name( + parsed_meta->namespace_str, parsed_meta->type_name); + if (result.ok()) { + local_type_info = result.value(); + } + } else { + auto result = type_resolver_->get_type_info_by_id(parsed_meta->type_id); + if (result.ok()) { + local_type_info = result.value(); } + } - // Get raw pointer before moving into storage - const TypeInfo *raw_ptr = type_info.get(); + // Create TypeInfo with field_ids assigned + auto type_info = std::make_unique(); + if (local_type_info) { + // Have local type - assign field_ids by comparing schemas + // Note: Extension types don't have type_meta (only structs do) + if (local_type_info->type_meta) { + TypeMeta::assign_field_ids(local_type_info->type_meta.get(), + parsed_meta->field_infos); + } + type_info->type_id = local_type_info->type_id; + type_info->type_meta = std::move(parsed_meta); + type_info->type_def = local_type_info->type_def; + // CRITICAL: Copy the harness from the registered type_info + type_info->harness = local_type_info->harness; + type_info->name_to_index = local_type_info->name_to_index; + type_info->namespace_name = local_type_info->namespace_name; + type_info->type_name = local_type_info->type_name; + type_info->register_by_name = local_type_info->register_by_name; + } else { + // No local type - create stub TypeInfo with parsed meta + type_info->type_id = parsed_meta->type_id; + type_info->type_meta = std::move(parsed_meta); + } - // Store in primary storage - owned_reading_type_infos_.push_back(std::move(type_info)); + // Get raw pointer before moving into storage + const TypeInfo *raw_ptr = type_info.get(); - // Cache the parsed TypeInfo (with size limit to prevent OOM) - if (parsed_type_infos_.size() < kMaxParsedNumTypeDefs) { - parsed_type_infos_[meta_header] = raw_ptr; - } + // Store in primary storage + owned_reading_type_infos_.push_back(std::move(type_info)); - reading_type_infos_.push_back(raw_ptr); + // Cache the parsed TypeInfo (with size limit to prevent OOM) + if (parsed_type_infos_.size() < kMaxParsedNumTypeDefs) { + parsed_type_infos_[meta_header] = raw_ptr; } - // Calculate size of meta section - size_t meta_end = buffer_->reader_index(); - size_t meta_section_size = meta_end - meta_start; - - // Restore buffer position - buffer_->ReaderIndex(static_cast(current_pos)); - return meta_section_size; + reading_type_infos_.push_back(raw_ptr); + return raw_ptr; } Result @@ -499,25 +469,19 @@ Result ReadContext::read_any_typeinfo() { } uint32_t type_id_low = type_id & 0xff; - // Mirror Rust's read_any_typeinfo using switch for jump table generation + // Use streaming protocol for type meta switch (type_id_low) { case static_cast(TypeId::NAMED_COMPATIBLE_STRUCT): case static_cast(TypeId::COMPATIBLE_STRUCT): { - uint32_t meta_index = buffer_->ReadVarUint32(error); - if (FORY_PREDICT_FALSE(!error.ok())) { - return Unexpected(std::move(error)); - } - return get_type_info_by_index(meta_index); + // Read type meta inline using streaming protocol + return read_type_meta(); } case static_cast(TypeId::NAMED_ENUM): case static_cast(TypeId::NAMED_EXT): case static_cast(TypeId::NAMED_STRUCT): { if (config_->compatible) { - uint32_t meta_index = buffer_->ReadVarUint32(error); - if (FORY_PREDICT_FALSE(!error.ok())) { - return Unexpected(std::move(error)); - } - return get_type_info_by_index(meta_index); + // Read type meta inline using streaming protocol + return read_type_meta(); } FORY_TRY(namespace_str, meta_string_table_.read_string(*buffer_, kNamespaceDecoder)); diff --git a/cpp/fory/serialization/context.h b/cpp/fory/serialization/context.h index faf426a53f..69ad1256bb 100644 --- a/cpp/fory/serialization/context.h +++ b/cpp/fory/serialization/context.h @@ -221,20 +221,15 @@ class WriteContext { buffer().WriteBytes(data, length); } - /// Push a TypeId's TypeMeta into the meta collection. - /// Returns the index for writing as varint. - Result push_meta(const std::type_index &type_id); + /// Write TypeMeta inline using streaming protocol. + /// First occurrence: writes (index << 1) | 0 followed by TypeDef bytes. + /// Subsequent occurrences: writes (index << 1) | 1 as reference. + Result write_type_meta(const std::type_index &type_id); - /// Push meta using TypeInfo pointer directly (avoids type_index lookup). - /// Returns the index for writing as varint. - size_t push_meta(const TypeInfo *type_info); - - /// Write all collected TypeMetas at the specified offset. - /// Updates the meta_offset field at 'offset' to point to meta section. - void write_meta(size_t offset); - - /// Check if any TypeMetas were collected. - bool meta_empty() const; + /// Write TypeMeta inline using TypeInfo pointer (fast path). + /// First occurrence: writes (index << 1) | 0 followed by TypeDef bytes. + /// Subsequent occurrences: writes (index << 1) | 1 as reference. + void write_type_meta(const TypeInfo *type_info); /// Write type information for polymorphic types. /// Handles different type categories: @@ -305,10 +300,8 @@ class WriteContext { RefWriter ref_writer_; uint32_t current_dyn_depth_; - // Meta sharing state (for compatible mode) - std::vector> write_type_defs_; - absl::flat_hash_map write_type_id_index_map_; - // Fast path: use TypeInfo pointer as key (avoids type_index hash overhead) + // Meta sharing state (for streaming inline TypeMeta) + // Maps TypeInfo* to index for reference tracking - uses map size as counter absl::flat_hash_map write_type_info_index_map_; }; @@ -543,12 +536,13 @@ class ReadContext { /// Read enum type info without type_index (fast path). Result read_enum_type_info(uint32_t base_type_id); - /// Load all TypeMetas from buffer at the specified offset. - /// After loading, the reader position is restored to where it was before. - /// @return Size of the meta section in bytes, or error - Result load_type_meta(int32_t meta_offset); + /// Read TypeMeta inline using streaming protocol. + /// Reads index marker: if LSB=0, reads new TypeDef inline; if LSB=1, returns + /// reference. + /// @return const pointer to TypeInfo, or error + Result read_type_meta(); - /// Get TypeInfo by meta index. + /// Get TypeInfo by meta index (internal use for reference lookups). /// @return const pointer to TypeInfo if found, error otherwise Result get_type_info_by_index(size_t index) const; diff --git a/cpp/fory/serialization/fory.h b/cpp/fory/serialization/fory.h index 6f68555080..716b374ac3 100644 --- a/cpp/fory/serialization/fory.h +++ b/cpp/fory/serialization/fory.h @@ -595,6 +595,7 @@ class Fory : public BaseFory { } /// Core serialization implementation. + /// TypeMeta is written inline using streaming protocol (no deferred writing). template Result serialize_impl(const T &obj, Buffer &buffer) { size_t start_pos = buffer.writer_index(); @@ -604,15 +605,9 @@ class Fory : public BaseFory { buffer.UnsafePut(buffer.writer_index(), precomputed_header_); buffer.IncreaseWriterIndex(header_length_); - // Reserve space for meta offset in compatible mode - size_t meta_start_offset = 0; - if (write_ctx_->is_compatible()) { - meta_start_offset = buffer.writer_index(); - buffer.WriteInt32(-1); // Placeholder for meta offset (fixed 4 bytes) - } - // Top-level serialization: use Tracking if ref tracking is enabled, // otherwise NullOnly for nullable handling + // TypeMeta is written inline during serialization (streaming protocol) const RefMode top_level_ref_mode = write_ctx_->track_ref() ? RefMode::Tracking : RefMode::NullOnly; Serializer::write(obj, *write_ctx_, top_level_ref_mode, true); @@ -621,32 +616,15 @@ class Fory : public BaseFory { return Unexpected(write_ctx_->take_error()); } - // Write collected TypeMetas at the end in compatible mode - if (write_ctx_->is_compatible() && !write_ctx_->meta_empty()) { - write_ctx_->write_meta(meta_start_offset); - } - return buffer.writer_index() - start_pos; } /// Core deserialization implementation. + /// TypeMeta is read inline using streaming protocol. template Result deserialize_impl(Buffer &buffer) { - // Load TypeMetas at the beginning in compatible mode - size_t bytes_to_skip = 0; - if (read_ctx_->is_compatible()) { - Error error; - int32_t meta_offset = buffer.ReadInt32(error); - if (FORY_PREDICT_FALSE(!error.ok())) { - return Unexpected(std::move(error)); - } - if (meta_offset != -1) { - FORY_TRY(meta_size, read_ctx_->load_type_meta(meta_offset)); - bytes_to_skip = meta_size; - } - } - // Top-level deserialization: use Tracking if ref tracking is enabled, // otherwise NullOnly for nullable handling + // TypeMeta is read inline during deserialization (streaming protocol) const RefMode top_level_ref_mode = read_ctx_->track_ref() ? RefMode::Tracking : RefMode::NullOnly; T result = Serializer::read(*read_ctx_, top_level_ref_mode, true); @@ -656,9 +634,6 @@ class Fory : public BaseFory { } read_ctx_->ref_reader().resolve_callbacks(); - if (bytes_to_skip > 0) { - buffer.IncreaseReaderIndex(static_cast(bytes_to_skip)); - } return result; } diff --git a/cpp/fory/serialization/skip.cc b/cpp/fory/serialization/skip.cc index 0278816422..7bc17fe0b8 100644 --- a/cpp/fory/serialization/skip.cc +++ b/cpp/fory/serialization/skip.cc @@ -226,13 +226,8 @@ void skip_struct(ReadContext &ctx, const FieldType &field_type) { if (remote_tid == TypeId::COMPATIBLE_STRUCT || remote_tid == TypeId::NAMED_COMPATIBLE_STRUCT || remote_tid == TypeId::NAMED_STRUCT) { - // These types write meta_index after type_id - uint32_t meta_index = ctx.read_varuint32(ctx.error()); - if (FORY_PREDICT_FALSE(ctx.has_error())) { - return; - } - - auto type_info_res = ctx.get_type_info_by_index(meta_index); + // These types write TypeMeta inline using streaming protocol + auto type_info_res = ctx.read_type_meta(); if (FORY_PREDICT_FALSE(!type_info_res.ok())) { ctx.set_error(std::move(type_info_res).error()); return; @@ -298,13 +293,9 @@ void skip_ext(ReadContext &ctx, const FieldType &field_type) { const TypeInfo *type_info = nullptr; if (remote_tid == TypeId::NAMED_EXT) { - // Named ext: also read meta_index - uint32_t meta_index = ctx.read_varuint32(ctx.error()); - if (FORY_PREDICT_FALSE(ctx.has_error())) { - return; - } - - auto type_info_res = ctx.get_type_info_by_index(meta_index); + // Named ext in compatible mode: read TypeMeta inline using streaming + // protocol + auto type_info_res = ctx.read_type_meta(); if (FORY_PREDICT_FALSE(!type_info_res.ok())) { ctx.set_error(std::move(type_info_res).error()); return; diff --git a/cpp/fory/serialization/struct_serializer.h b/cpp/fory/serialization/struct_serializer.h index 3844e5048f..18edfe5a13 100644 --- a/cpp/fory/serialization/struct_serializer.h +++ b/cpp/fory/serialization/struct_serializer.h @@ -2503,11 +2503,9 @@ struct Serializer>> { const TypeInfo *type_info = type_info_res.value(); ctx.write_varuint32(type_info->type_id); - // In compatible mode, always write meta index (matches Rust behavior) + // In compatible mode, write type meta inline (streaming protocol) if (ctx.is_compatible() && type_info->type_meta) { - // Use TypeInfo* overload to avoid type_index creation - size_t meta_index = ctx.push_meta(type_info); - ctx.write_varuint32(static_cast(meta_index)); + ctx.write_type_meta(type_info); } } @@ -2672,13 +2670,8 @@ struct Serializer>> { static_cast(TypeId::COMPATIBLE_STRUCT) || local_type_id_low == static_cast(TypeId::NAMED_COMPATIBLE_STRUCT)) { - // Use meta sharing: read varint index and get TypeInfo from - // meta_reader - uint32_t meta_index = ctx.read_varuint32(ctx.error()); - if (FORY_PREDICT_FALSE(ctx.has_error())) { - return T{}; - } - auto remote_type_info_res = ctx.get_type_info_by_index(meta_index); + // Read TypeMeta inline using streaming protocol + auto remote_type_info_res = ctx.read_type_meta(); if (!remote_type_info_res.ok()) { ctx.set_error(std::move(remote_type_info_res).error()); return T{}; diff --git a/docs/specification/xlang_serialization_spec.md b/docs/specification/xlang_serialization_spec.md index 23623bd37d..d5c26a98f7 100644 --- a/docs/specification/xlang_serialization_spec.md +++ b/docs/specification/xlang_serialization_spec.md @@ -290,7 +290,9 @@ All data is encoded in little-endian format. ### Meta Start Offset -If compatible mode is enabled, an uncompressed unsigned int32 (4 bytes, little endian) is appended to indicate the start offset of metadata. During serialization, this is initially written as a placeholder (e.g., `-1` or `0`), then updated after all objects are serialized and metadata is collected. +In non-streaming compatible mode, an uncompressed unsigned int32 (4 bytes, little endian) is appended to indicate the start offset of metadata. During serialization, this is initially written as a placeholder (e.g., `-1` or `0`), then updated after all objects are serialized and metadata is collected. + +**Note:** In streaming mode, the meta start offset is omitted because type metadata is written inline during serialization rather than being deferred to the end of the buffer. ## Reference Meta @@ -481,18 +483,24 @@ using one of the following mode. Which mode to use is configured when creating f - If type meta hasn't been written before, the data will be written as: ``` - | unsigned varint: 0b11111111 | type def | + | unsigned varint: index << 1 | type def bytes | ``` + The LSB=0 indicates this is a new type definition. The `index` is the sequential index + assigned to this type (starting from 0), and `type def bytes` contains the complete + TypeDef including the 8-byte global header. + - If type meta has been written before, the data will be written as: ``` - | unsigned varint: written index << 1 | + | unsigned varint: (index << 1) | 1 | ``` - `written index` is the id in `captured_type_defs`. + The LSB=1 indicates this is a reference to a previously written type. The `index` is + the same sequential index that was assigned when the type was first written. - - With this mode, `meta start offset` can be omitted. + - With this mode, `meta start offset` can be omitted since all type metadata is written + inline during serialization rather than being deferred to the end. > The normal mode and meta share mode will forbid streaming writing since it needs to look back for update the start > offset after the whole object graph writing and meta collecting is finished. Only in this way we can ensure diff --git a/go/fory/fory.go b/go/fory/fory.go index 49e3ca2c12..319868a261 100644 --- a/go/fory/fory.go +++ b/go/fory/fory.go @@ -144,7 +144,6 @@ func New(opts ...Option) *Fory { if f.config.Compatible { f.metaContext = &MetaContext{ typeMap: make(map[reflect.Type]uint32), - writingTypeDefs: make([]*TypeDef, 0), readTypeInfos: make([]*TypeInfo, 0), scopedMetaShareEnable: true, } @@ -402,30 +401,12 @@ func (f *Fory) Serialize(value any) ([]byte, error) { // WriteData protocol header writeHeader(f.writeCtx, f.config) - // In compatible mode, reserve space for meta offset (matches C++/Java) - var metaStartOffset int - if f.config.Compatible { - metaStartOffset = f.writeCtx.buffer.writerIndex - f.writeCtx.buffer.WriteInt32(-1) // Placeholder for meta offset - } - - // SerializeWithCallback the value + // Serialize the value - TypeMeta is written inline using streaming protocol f.writeCtx.WriteValue(reflect.ValueOf(value), RefModeTracking, true) if f.writeCtx.HasError() { return nil, f.writeCtx.TakeError() } - // WriteData collected TypeMetas at the end in compatible mode (matches C++/Java) - if f.config.Compatible && f.metaContext != nil && len(f.metaContext.writingTypeDefs) > 0 { - // Calculate offset from the position after meta offset field to meta section start - currentPos := f.writeCtx.buffer.writerIndex - offset := currentPos - metaStartOffset - 4 - // Update the meta offset field - f.writeCtx.buffer.PutInt32(metaStartOffset, int32(offset)) - // WriteData type definitions - f.typeResolver.writeTypeDefs(f.writeCtx.buffer, f.writeCtx.Err()) - } - return f.writeCtx.buffer.GetByteSlice(0, f.writeCtx.buffer.writerIndex), nil } @@ -435,50 +416,23 @@ func (f *Fory) Deserialize(data []byte, v any) error { defer f.resetReadState() f.readCtx.SetData(data) - metaOffset := readHeader(f.readCtx) + isNull := readHeader(f.readCtx) if f.readCtx.HasError() { return f.readCtx.TakeError() } // Check if the serialized object is null - if metaOffset == NullObjectMetaOffset { + if isNull { return nil } - // In compatible mode, load type definitions if meta offset is present - var finalPos int - if f.config.Compatible && metaOffset > 0 { - // Save current position (right after meta offset field, before object data) - dataStartPos := f.readCtx.buffer.ReaderIndex() - - // Jump to meta section and read type definitions - metaPos := dataStartPos + int(metaOffset) - f.readCtx.buffer.SetReaderIndex(metaPos) - - f.typeResolver.readTypeDefs(f.readCtx.buffer, f.readCtx.Err()) - if f.readCtx.HasError() { - return fmt.Errorf("failed to read type definitions: %w", f.readCtx.TakeError()) - } - - // Save final position (after reading TypeDefs) - finalPos = f.readCtx.buffer.ReaderIndex() - - // Return to data start position to deserialize the object - f.readCtx.buffer.SetReaderIndex(dataStartPos) - } - - // Read directly into target value + // Deserialize the value - TypeMeta is read inline using streaming protocol target := reflect.ValueOf(v).Elem() f.readCtx.ReadValue(target, RefModeTracking, true) if f.readCtx.HasError() { return f.readCtx.TakeError() } - // Restore final position if we loaded type definitions - if finalPos > 0 { - f.readCtx.buffer.SetReaderIndex(finalPos) - } - return nil } @@ -518,13 +472,6 @@ func (f *Fory) SerializeTo(buf *ByteBuffer, value any) error { // Write protocol header writeHeader(f.writeCtx, f.config) - // In compatible mode, reserve space for meta offset - var metaStartOffset int - if f.config.Compatible { - metaStartOffset = buf.writerIndex - buf.WriteInt32(-1) // Placeholder for meta offset - } - // Fast path for pointer-to-struct types (bypasses ptrToValueSerializer wrapper) rv := reflect.ValueOf(value) if rv.Kind() == reflect.Ptr && !rv.IsNil() && rv.Elem().Kind() == reflect.Struct && !f.config.TrackRef { @@ -545,32 +492,18 @@ func (f *Fory) SerializeTo(buf *ByteBuffer, value any) error { f.writeCtx.buffer = origBuffer return f.writeCtx.TakeError() } - goto finish + f.writeCtx.buffer = origBuffer + return nil } } - // Standard path + // Standard path - TypeMeta is written inline using streaming protocol f.writeCtx.WriteValue(rv, RefModeTracking, true) if f.writeCtx.HasError() { f.writeCtx.buffer = origBuffer return f.writeCtx.TakeError() } -finish: - - // Write collected TypeMetas at the end in compatible mode - if f.config.Compatible && f.metaContext != nil && len(f.metaContext.writingTypeDefs) > 0 { - // Calculate offset from the position after meta offset field to meta section start - currentPos := buf.writerIndex - offset := currentPos - metaStartOffset - 4 - - // Update the meta offset field - buf.PutInt32(metaStartOffset, int32(offset)) - - // Write type definitions - f.typeResolver.writeTypeDefs(buf, f.writeCtx.Err()) - } - // Restore original buffer f.writeCtx.buffer = origBuffer return nil @@ -587,42 +520,19 @@ func (f *Fory) DeserializeFrom(buf *ByteBuffer, v any) error { origBuffer := f.readCtx.buffer f.readCtx.buffer = buf - metaOffset := readHeader(f.readCtx) + isNull := readHeader(f.readCtx) if f.readCtx.HasError() { f.readCtx.buffer = origBuffer return f.readCtx.TakeError() } // Check if the serialized object is null - if metaOffset == NullObjectMetaOffset { + if isNull { f.readCtx.buffer = origBuffer return nil } - // In compatible mode, load type definitions if meta offset is present - var finalPos int - if f.config.Compatible && metaOffset > 0 { - // Save current position (right after meta offset field, before object data) - dataStartPos := buf.ReaderIndex() - - // Jump to meta section and read type definitions - metaPos := dataStartPos + int(metaOffset) - buf.SetReaderIndex(metaPos) - - f.typeResolver.readTypeDefs(buf, f.readCtx.Err()) - if f.readCtx.HasError() { - f.readCtx.buffer = origBuffer - return fmt.Errorf("failed to read type definitions: %w", f.readCtx.TakeError()) - } - - // Save final position (after reading TypeDefs) - finalPos = buf.ReaderIndex() - - // Return to data start position to deserialize the object - buf.SetReaderIndex(dataStartPos) - } - - // Read directly into target value + // Deserialize the value - TypeMeta is read inline using streaming protocol target := reflect.ValueOf(v).Elem() f.readCtx.ReadValue(target, RefModeTracking, true) if f.readCtx.HasError() { @@ -630,11 +540,6 @@ func (f *Fory) DeserializeFrom(buf *ByteBuffer, v any) error { return f.readCtx.TakeError() } - // Restore final position if we loaded type definitions - if finalPos > 0 { - buf.SetReaderIndex(finalPos) - } - // Restore original buffer f.readCtx.buffer = origBuffer @@ -693,30 +598,12 @@ func (f *Fory) SerializeWithCallback(buffer *ByteBuffer, v any, callback func(Bu // WriteData protocol header writeHeader(f.writeCtx, f.config) - // In compatible mode, reserve space for meta offset (matches C++/Java) - var metaStartOffset int - if f.config.Compatible { - metaStartOffset = buffer.writerIndex - buffer.WriteInt32(-1) // Placeholder for meta offset - } - - // SerializeWithCallback the value + // Serialize the value - TypeMeta is written inline using streaming protocol f.writeCtx.WriteValue(reflect.ValueOf(v), RefModeTracking, true) if f.writeCtx.HasError() { return f.writeCtx.TakeError() } - // WriteData collected TypeMetas at the end in compatible mode (matches C++/Java) - if f.config.Compatible && f.metaContext != nil && len(f.metaContext.writingTypeDefs) > 0 { - // Calculate offset from the position after meta offset field to meta section start - currentPos := buffer.writerIndex - offset := currentPos - metaStartOffset - 4 - // Update the meta offset field - buffer.PutInt32(metaStartOffset, int32(offset)) - // WriteData type definitions - f.typeResolver.writeTypeDefs(buffer, f.writeCtx.Err()) - } - return nil } @@ -738,14 +625,14 @@ func (f *Fory) DeserializeWithCallbackBuffers(buffer *ByteBuffer, v any, buffers f.readCtx.outOfBandBuffers = buffers } - // ReadData and validate header, get meta offset if present - metaOffset := readHeader(f.readCtx) + // ReadData and validate header + isNull := readHeader(f.readCtx) if f.readCtx.HasError() { return f.readCtx.TakeError() } // Check if the serialized object is null - if metaOffset == NullObjectMetaOffset { + if isNull { // v must be a pointer so we can set it to nil rv := reflect.ValueOf(v) if rv.Kind() == reflect.Ptr && !rv.IsNil() { @@ -754,29 +641,6 @@ func (f *Fory) DeserializeWithCallbackBuffers(buffer *ByteBuffer, v any, buffers return nil } - // In compatible mode, load type definitions if meta offset is present - // This matches C++ deserialize_impl: read type defs BEFORE deserializing object - var finalPos int - if f.config.Compatible && metaOffset > 0 { - // Save current position (right after meta offset field, before object data) - dataStartPos := buffer.ReaderIndex() - - // Jump to meta section and read type definitions - metaPos := dataStartPos + int(metaOffset) - buffer.SetReaderIndex(metaPos) - - f.typeResolver.readTypeDefs(buffer, f.readCtx.Err()) - if f.readCtx.HasError() { - return fmt.Errorf("failed to read type definitions: %w", f.readCtx.TakeError()) - } - - // Save final position (after reading TypeDefs) - finalPos = buffer.ReaderIndex() - - // Return to data start position to deserialize the object - buffer.SetReaderIndex(dataStartPos) - } - // v must be a pointer so we can deserialize into it if v == nil { return fmt.Errorf("v cannot be nil") @@ -788,15 +652,13 @@ func (f *Fory) DeserializeWithCallbackBuffers(buffer *ByteBuffer, v any, buffers if rv.IsNil() { return fmt.Errorf("v must be a non-nil pointer") } - // DeserializeWithCallbackBuffers directly into v + + // Deserialize the value - TypeMeta is read inline using streaming protocol f.readCtx.ReadValue(rv.Elem(), RefModeTracking, true) if f.readCtx.HasError() { return f.readCtx.TakeError() } - // Restore final position if we loaded type definitions - if finalPos > 0 { - buffer.SetReaderIndex(finalPos) - } + return nil } @@ -809,32 +671,12 @@ func (f *Fory) serializeReflectValue(value reflect.Value) ([]byte, error) { return nil, fmt.Errorf("cannot serialize struct %s directly, use pointer to struct (*%s) instead", value.Type(), value.Type()) } - // In compatible mode, reserve space for meta offset (matches C++/Java) - var metaStartOffset int - if f.config.Compatible { - metaStartOffset = f.writeCtx.buffer.writerIndex - f.writeCtx.buffer.WriteInt32(-1) // Placeholder for meta offset - } - - // SerializeWithCallback the value + // Serialize the value - TypeMeta is written inline using streaming protocol f.writeCtx.WriteValue(value, RefModeTracking, true) if f.writeCtx.HasError() { return nil, f.writeCtx.TakeError() } - // WriteData collected TypeMetas at the end in compatible mode (matches C++/Java) - if f.config.Compatible && f.metaContext != nil && len(f.metaContext.writingTypeDefs) > 0 { - // Calculate offset from the position after meta offset field to meta section start - currentPos := f.writeCtx.buffer.writerIndex - offset := currentPos - metaStartOffset - 4 - - // Update the meta offset field - f.writeCtx.buffer.PutInt32(metaStartOffset, int32(offset)) - - // WriteData type definitions - f.typeResolver.writeTypeDefs(f.writeCtx.buffer, f.writeCtx.Err()) - } - return f.writeCtx.buffer.GetByteSlice(0, f.writeCtx.buffer.writerIndex), nil } @@ -880,31 +722,23 @@ func writeNullHeader(ctx *WriteContext) { const NullObjectMetaOffset int32 = -0x7FFFFFFF // readHeader reads and validates the Fory protocol header -// Returns the meta start offset if present (0 if not present) -// Returns NullObjectMetaOffset if the serialized object is null +// Returns true if the serialized object is null // Sets error on ctx if header is invalid (use ctx.HasError() to check) -func readHeader(ctx *ReadContext) int32 { +func readHeader(ctx *ReadContext) bool { err := ctx.Err() bitmap := ctx.buffer.ReadByte(err) if ctx.HasError() { - return 0 + return false } // Check if this is a null object - only bitmap with isNilFlag was written if (bitmap & IsNilFlag) != 0 { - return NullObjectMetaOffset + return true // is null } _ = ctx.buffer.ReadByte(err) // language - // In compatible mode with meta share, Java writes a 4-byte meta offset - // We need to read it but we'll handle type defs later - if ctx.compatible { - metaOffset := ctx.buffer.ReadInt32(err) - return metaOffset - } - - return 0 + return false // not null } // ============================================================================ @@ -1080,13 +914,13 @@ func Deserialize[T any](f *Fory, data []byte, target *T) error { f.readCtx.SetData(data) // ReadData and validate header - metaOffset := readHeader(f.readCtx) + isNull := readHeader(f.readCtx) if f.readCtx.HasError() { return f.readCtx.TakeError() } // Check if the serialized object is null - if metaOffset == NullObjectMetaOffset { + if isNull { var zero T *target = zero return nil diff --git a/go/fory/type_resolver.go b/go/fory/type_resolver.go index 0828747cf0..3bda505b76 100644 --- a/go/fory/type_resolver.go +++ b/go/fory/type_resolver.go @@ -1185,12 +1185,14 @@ func (r *TypeResolver) writeSharedTypeMeta(buffer *ByteBuffer, typeInfo *TypeInf typ := typeInfo.Type if index, exists := context.typeMap[typ]; exists { - buffer.WriteVaruint32(index) + // Reference to previously written type: (index << 1) | 1, LSB=1 + buffer.WriteVaruint32((index << 1) | 1) return } + // New type: index << 1, LSB=0, followed by TypeDef bytes inline newIndex := uint32(len(context.typeMap)) - buffer.WriteVaruint32(newIndex) + buffer.WriteVaruint32(newIndex << 1) context.typeMap[typ] = newIndex // Only build TypeDef for struct types - enums don't have field definitions @@ -1204,7 +1206,8 @@ func (r *TypeResolver) writeSharedTypeMeta(buffer *ByteBuffer, typeInfo *TypeInf err.SetError(typeDefErr) return } - context.writingTypeDefs = append(context.writingTypeDefs, typeDef) + // Write TypeDef bytes inline + typeDef.writeTypeDef(buffer, err) } } @@ -1236,72 +1239,60 @@ func (r *TypeResolver) readSharedTypeMeta(buffer *ByteBuffer, err *Error) *TypeI err.SetError(fmt.Errorf("MetaContext is nil - ensure compatible mode is enabled")) return nil } - index := int32(buffer.ReadVaruint32(err)) // shared meta index id (unsigned) - if index < 0 || index >= int32(len(context.readTypeInfos)) { - err.SetError(fmt.Errorf("TypeInfo not found for index %d (have %d type infos)", index, len(context.readTypeInfos))) - return nil - } - info := context.readTypeInfos[index] - // Validate that we got a valid TypeInfo - if info.Serializer == nil { - err.SetError(fmt.Errorf("TypeInfo at index %d has nil Serializer (type=%v, typeID=%d)", index, info.Type, info.TypeID)) + // Read index marker using streaming protocol + indexMarker := buffer.ReadVaruint32(err) + if err.HasError() { return nil } - return info -} + isRef := (indexMarker & 1) == 1 + index := int32(indexMarker >> 1) -func (r *TypeResolver) writeTypeDefs(buffer *ByteBuffer, err *Error) { - context := r.fory.MetaContext() - if context == nil { - buffer.WriteVaruint32Small7(0) - return - } - sz := len(context.writingTypeDefs) - buffer.WriteVaruint32Small7(uint32(sz)) - for _, typeDef := range context.writingTypeDefs { - typeDef.writeTypeDef(buffer, err) - } - context.writingTypeDefs = nil -} + if isRef { + // Reference to previously read type + if index < 0 || index >= int32(len(context.readTypeInfos)) { + err.SetError(fmt.Errorf("TypeInfo not found for index %d (have %d type infos)", index, len(context.readTypeInfos))) + return nil + } + info := context.readTypeInfos[index] -func (r *TypeResolver) readTypeDefs(buffer *ByteBuffer, err *Error) { - numTypeDefs := int(buffer.ReadVaruint32Small7(err)) - if numTypeDefs == 0 { - return + // Validate that we got a valid TypeInfo + if info.Serializer == nil { + err.SetError(fmt.Errorf("TypeInfo at index %d has nil Serializer (type=%v, typeID=%d)", index, info.Type, info.TypeID)) + return nil + } + + return info } - context := r.fory.MetaContext() - if context == nil { - err.SetError(fmt.Errorf("MetaContext is nil but type definitions are present")) - return + + // New type - read TypeDef inline + id := buffer.ReadInt64(err) + if err.HasError() { + return nil } - for i := 0; i < numTypeDefs; i++ { - id := buffer.ReadInt64(err) - var td *TypeDef - if existingTd, exists := r.defIdToTypeDef[id]; exists { - skipTypeDef(buffer, id, err) - td = existingTd - } else { - newTd := readTypeDef(r.fory, buffer, id, err) - r.defIdToTypeDef[id] = newTd - td = newTd - // Note: We do NOT store remote TypeDef in typeToTypeDef. - // typeToTypeDef is used for WRITING and must contain locally-built TypeDefs. - // Remote TypeDefs have different field ordering/IDs based on the remote's struct. - // defIdToTypeDef caches remote TypeDefs by header hash to avoid re-parsing. - } - typeInfo, typeInfoErr := td.buildTypeInfoWithResolver(r) - if typeInfoErr != nil { - err.SetError(typeInfoErr) - return + + var td *TypeDef + if existingTd, exists := r.defIdToTypeDef[id]; exists { + skipTypeDef(buffer, id, err) + td = existingTd + } else { + newTd := readTypeDef(r.fory, buffer, id, err) + if err.HasError() { + return nil } - context.readTypeInfos = append(context.readTypeInfos, &typeInfo) - // Note: We intentionally do NOT update the original serializer's fieldDefs here. - // When serializing, Go should use its own struct definition (via initFieldsFromContext), - // not the remote TypeDef's field list. This is important for schema evolution - // where Go's struct may have different fields than the remote. + r.defIdToTypeDef[id] = newTd + td = newTd } + + typeInfo, typeInfoErr := td.buildTypeInfoWithResolver(r) + if typeInfoErr != nil { + err.SetError(typeInfoErr) + return nil + } + + context.readTypeInfos = append(context.readTypeInfos, &typeInfo) + return &typeInfo } func (r *TypeResolver) createSerializer(type_ reflect.Type, mapInStruct bool) (s Serializer, err error) { @@ -2155,9 +2146,8 @@ var ErrTypeMismatch = errors.New("fory: type ID mismatch") // MetaContext holds metadata for schema evolution and type sharing type MetaContext struct { - typeMap map[reflect.Type]uint32 - writingTypeDefs []*TypeDef - readTypeInfos []*TypeInfo + typeMap map[reflect.Type]uint32 // For writing: tracks written types + readTypeInfos []*TypeInfo // For reading: types read inline scopedMetaShareEnable bool } @@ -2169,6 +2159,5 @@ func (m *MetaContext) IsScopedMetaShareEnabled() bool { // Reset clears the meta context for reuse func (m *MetaContext) Reset() { m.typeMap = make(map[reflect.Type]uint32) - m.writingTypeDefs = nil m.readTypeInfos = nil } diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java index ada96d291e..e8bbf6ad60 100644 --- a/java/fory-core/src/main/java/org/apache/fory/Fory.java +++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java @@ -128,7 +128,6 @@ public final class Fory implements BaseFory { private int copyDepth; private final boolean copyRefTracking; private final IdentityMap originToCopyMap; - private int classDefEndOffset; public Fory(ForyBuilder builder, ClassLoader classLoader) { // Avoid set classLoader in `ForyBuilder`, which won't be clear when @@ -166,7 +165,6 @@ public Fory(ForyBuilder builder, ClassLoader classLoader) { arrayListSerializer = new ArrayListSerializer(this); hashMapSerializer = new HashMapSerializer(this); originToCopyMap = new IdentityMap<>(); - classDefEndOffset = -1; LOG.info("Created new fory {}", this); } @@ -385,37 +383,17 @@ public void resetBuffer() { } private void write(MemoryBuffer buffer, Object obj) { - int startOffset = buffer.writerIndex(); - boolean shareMeta = config.isMetaShareEnabled(); - if (shareMeta) { - buffer.writeInt32(-1); // preserve 4-byte for meta start offsets. - } // reduce caller stack if (!refResolver.writeRefOrNull(buffer, obj)) { ClassInfo classInfo = classResolver.getOrUpdateClassInfo(obj.getClass()); classResolver.writeClassInfo(buffer, classInfo); writeData(buffer, classInfo, obj); } - MetaContext metaContext = serializationContext.getMetaContext(); - if (shareMeta && metaContext != null && !metaContext.writingClassDefs.isEmpty()) { - buffer.putInt32(startOffset, buffer.writerIndex() - startOffset - 4); - classResolver.writeClassDefs(buffer); - } } private void xwrite(MemoryBuffer buffer, Object obj) { buffer.writeByte((byte) Language.JAVA.ordinal()); - int startOffset = buffer.writerIndex(); - boolean shareMeta = config.isMetaShareEnabled(); - if (shareMeta) { - buffer.writeInt32(-1); // preserve 4-byte for meta start offsets. - } xwriteRef(buffer, obj); - MetaContext metaContext = serializationContext.getMetaContext(); - if (shareMeta && metaContext != null && !metaContext.writingClassDefs.isEmpty()) { - buffer.putInt32(startOffset, buffer.writerIndex() - startOffset - 4); - classResolver.writeClassDefs(buffer); - } } /** Serialize a nullable referencable object to buffer. */ @@ -846,9 +824,6 @@ public Object deserialize(MemoryBuffer buffer, Iterable outOfBandB "outOfBandBuffers should be null when the serialized stream is " + "produced with bufferCallback null."); } - if (shareMeta) { - readClassDefs(buffer); - } Object obj; if (isTargetXLang) { obj = xreadRef(buffer); @@ -859,9 +834,6 @@ public Object deserialize(MemoryBuffer buffer, Iterable outOfBandB } catch (Throwable t) { throw ExceptionUtils.handleReadFailed(this, t); } finally { - if (classDefEndOffset != -1) { - buffer.readerIndex(classDefEndOffset); - } resetRead(); jitContext.unlock(); } @@ -1144,17 +1116,10 @@ public void serializeJavaObject(MemoryBuffer buffer, Object obj) { throwDepthSerializationException(); } if (config.isMetaShareEnabled()) { - int startOffset = buffer.writerIndex(); - buffer.writeInt32(-1); // preserve 4-byte for meta start offsets. if (!refResolver.writeRefOrNull(buffer, obj)) { ClassInfo classInfo = classResolver.getOrUpdateClassInfo(obj.getClass()); classResolver.writeClassInfo(buffer, classInfo); writeData(buffer, classInfo, obj); - MetaContext metaContext = serializationContext.getMetaContext(); - if (metaContext != null && !metaContext.writingClassDefs.isEmpty()) { - buffer.putInt32(startOffset, buffer.writerIndex() - startOffset - 4); - classResolver.writeClassDefs(buffer); - } } } else { if (!refResolver.writeRefOrNull(buffer, obj)) { @@ -1192,9 +1157,6 @@ public T deserializeJavaObject(MemoryBuffer buffer, Class cls) { if (depth > 0) { throwDepthDeserializationException(); } - if (shareMeta) { - readClassDefs(buffer); - } T obj; int nextReadRefId = refResolver.tryPreserveRefId(buffer); if (nextReadRefId >= NOT_NULL_VALUE_FLAG) { @@ -1212,9 +1174,6 @@ public T deserializeJavaObject(MemoryBuffer buffer, Class cls) { } catch (Throwable t) { throw ExceptionUtils.handleReadFailed(this, t); } finally { - if (classDefEndOffset != -1) { - buffer.readerIndex(classDefEndOffset); - } resetRead(); jitContext.unlock(); } @@ -1315,16 +1274,10 @@ public Object deserializeJavaObjectAndClass(MemoryBuffer buffer) { if (depth > 0) { throwDepthDeserializationException(); } - if (shareMeta) { - readClassDefs(buffer); - } return readRef(buffer); } catch (Throwable t) { throw ExceptionUtils.handleReadFailed(this, t); } finally { - if (classDefEndOffset != -1) { - buffer.readerIndex(classDefEndOffset); - } resetRead(); jitContext.unlock(); } @@ -1523,18 +1476,6 @@ private void serializeToStream(OutputStream outputStream, Consumer } } - private void readClassDefs(MemoryBuffer buffer) { - int relativeClassDefOffset = buffer.readInt32(); - if (relativeClassDefOffset == -1) { - return; - } - int readerIndex = buffer.readerIndex(); - buffer.readerIndex(readerIndex + relativeClassDefOffset); - classResolver.readClassDefs(buffer); - classDefEndOffset = buffer.readerIndex(); - buffer.readerIndex(readerIndex); - } - public void reset() { resetWrite(); resetRead(); @@ -1556,7 +1497,6 @@ public void resetRead() { metaStringResolver.resetRead(); serializationContext.resetRead(); peerOutOfBandEnabled = false; - classDefEndOffset = -1; depth = 0; } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index fe6c0f121a..babebd8ecb 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -1515,24 +1515,44 @@ public void writeClassInfo(MemoryBuffer buffer, ClassInfo classInfo) { } public void writeClassInfoWithMetaShare(MemoryBuffer buffer, ClassInfo classInfo) { - if (classInfo.classId != NO_CLASS_ID && !classInfo.needToWriteClassDef) { - buffer.writeVarUint32(classInfo.classId << 1); + // For NonexistentClassSerializer, the serializer handles ClassDef writing itself + // because the ClassDef comes from the object being serialized, not from the class. + // We just write a placeholder that the serializer will overwrite. + if (classInfo.serializer != null + && classInfo.serializer.getClass() + == NonexistentClassSerializers.NonexistentClassSerializer.class) { + // Write a 2-byte placeholder that NonexistentClassSerializer.writeClassDef will overwrite + buffer.writeInt16((short) 0); return; } + // For dynamically generated classes (lambdas, JDK proxies), use their stub class + // for the ClassDef since the dynamic class names cannot be loaded by name. + // The serializer will handle the actual serialization correctly. + Class classForMeta = classInfo.cls; + if (classInfo.isDynamicGeneratedClass) { + if (classInfo.classId == LAMBDA_STUB_ID) { + classForMeta = LambdaSerializer.ReplaceStub.class; + } else if (classInfo.classId == JDK_PROXY_STUB_ID) { + classForMeta = JdkProxySerializer.ReplaceStub.class; + } + } MetaContext metaContext = fory.getSerializationContext().getMetaContext(); assert metaContext != null : SET_META__CONTEXT_MSG; IdentityObjectIntMap> classMap = metaContext.classMap; int newId = classMap.size; - int id = classMap.putOrGet(classInfo.cls, newId); + int id = classMap.putOrGet(classForMeta, newId); if (id >= 0) { - buffer.writeVarUint32(id << 1 | 0b1); + // Reference to previously written type: (index << 1) | 1, LSB=1 + buffer.writeVarUint32((id << 1) | 1); } else { - buffer.writeVarUint32(newId << 1 | 0b1); - ClassDef classDef = classInfo.classDef; + // New type: index << 1, LSB=0, followed by ClassDef bytes inline + buffer.writeVarUint32(newId << 1); + ClassInfo stubClassInfo = classForMeta == classInfo.cls ? classInfo : getClassInfo(classForMeta); + ClassDef classDef = stubClassInfo.classDef; if (classDef == null) { - classDef = buildClassDef(classInfo); + classDef = buildClassDef(stubClassInfo); } - metaContext.writingClassDefs.add(classDef); + buffer.writeBytes(classDef.getEncoded()); } } @@ -1556,14 +1576,34 @@ private ClassDef buildClassDef(ClassInfo classInfo) { @Override public ClassInfo readSharedClassMeta(MemoryBuffer buffer, MetaContext metaContext) { assert metaContext != null : SET_META__CONTEXT_MSG; - int header = buffer.readVarUint32Small14(); - int id = header >>> 1; - if ((header & 0b1) == 0) { - return getOrUpdateClassInfo((short) id); - } - ClassInfo classInfo = metaContext.readClassInfos.get(id); - if (classInfo == null) { - classInfo = readSharedClassMeta(metaContext, id); + int indexMarker = buffer.readVarUint32Small14(); + boolean isRef = (indexMarker & 1) == 1; + int index = indexMarker >>> 1; + ClassInfo classInfo; + if (isRef) { + // Reference to previously read type in this stream + classInfo = metaContext.readClassInfos.get(index); + } else { + // New type in stream - but may already be known from registry + long id = buffer.readInt64(); + Tuple2 tuple2 = extRegistry.classIdToDef.get(id); + if (tuple2 != null) { + // Already known - skip the ClassDef bytes, reuse existing ClassInfo + ClassDef.skipClassDef(buffer, id); + classInfo = tuple2.f1; + if (classInfo == null) { + classInfo = buildMetaSharedClassInfo(tuple2, tuple2.f0); + } + } else { + // Unknown - read ClassDef and create ClassInfo + tuple2 = readClassDef(buffer, id); + classInfo = tuple2.f1; + if (classInfo == null) { + classInfo = buildMetaSharedClassInfo(tuple2, tuple2.f0); + } + } + // index == readClassInfos.size() since types are written sequentially + metaContext.readClassInfos.add(classInfo); } return classInfo; } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/MetaContext.java b/java/fory-core/src/main/java/org/apache/fory/resolver/MetaContext.java index 5a99e65970..83f95f49c4 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/MetaContext.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/MetaContext.java @@ -21,8 +21,6 @@ import org.apache.fory.collection.IdentityObjectIntMap; import org.apache.fory.collection.ObjectArray; -import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.meta.ClassDef; /** * Context for sharing class meta across multiple serialization. Class name, field name and field @@ -32,16 +30,6 @@ public class MetaContext { /** Classes which has sent definitions to peer. */ public final IdentityObjectIntMap> classMap = new IdentityObjectIntMap<>(1, 0.5f); - /** Class definitions read from peer. */ - public final ObjectArray readClassDefs = new ObjectArray<>(); - + /** ClassInfos read from peer for reference lookup during deserialization. */ public final ObjectArray readClassInfos = new ObjectArray<>(); - - /** - * New class definition which needs sending to peer. This will be filled up when there are new - * class definition need sending, and will be cleared after writing to buffer. - * - * @see ClassResolver#writeClassDefs(MemoryBuffer) - */ - public final ObjectArray writingClassDefs = new ObjectArray<>(); } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/SerializationContext.java b/java/fory-core/src/main/java/org/apache/fory/resolver/SerializationContext.java index 876a03dc38..852bb686da 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/SerializationContext.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/SerializationContext.java @@ -74,7 +74,6 @@ public void resetWrite() { } if (scopedMetaShareEnabled) { metaContext.classMap.clear(); - metaContext.writingClassDefs.size = 0; } else { metaContext = null; } @@ -86,7 +85,6 @@ public void resetRead() { } if (scopedMetaShareEnabled) { metaContext.readClassInfos.size = 0; - metaContext.readClassDefs.size = 0; } else { metaContext = null; } @@ -98,9 +96,7 @@ public void reset() { } if (scopedMetaShareEnabled) { metaContext.classMap.clear(); - metaContext.writingClassDefs.size = 0; metaContext.readClassInfos.size = 0; - metaContext.readClassDefs.size = 0; } else { metaContext = null; } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index e1ac536c0c..5baeea5fa2 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -280,19 +280,6 @@ public final ClassInfo readSharedClassMeta(MemoryBuffer buffer, Class targetC return classInfo; } - final ClassInfo readSharedClassMeta(MetaContext metaContext, int index) { - ClassDef classDef = metaContext.readClassDefs.get(index); - Tuple2 classDefTuple = extRegistry.classIdToDef.get(classDef.getId()); - ClassInfo classInfo; - if (classDefTuple == null || classDefTuple.f1 == null || classDefTuple.f1.serializer == null) { - classInfo = buildMetaSharedClassInfo(classDefTuple, classDef); - } else { - classInfo = classDefTuple.f1; - } - metaContext.readClassInfos.set(index, classInfo); - return classInfo; - } - final ClassInfo buildMetaSharedClassInfo( Tuple2 classDefTuple, ClassDef classDef) { ClassInfo classInfo; @@ -300,7 +287,15 @@ final ClassInfo buildMetaSharedClassInfo( classDef = classDefTuple.f0; } Class cls = loadClass(classDef.getClassSpec()); - if (!classDef.hasFieldsMeta()) { + // For nonexistent classes, always create a new ClassInfo with the correct classDef, + // even if the classDef has no fields meta. This ensures the NonexistentClassSerializer + // has access to the classDef for proper deserialization. + if (!classDef.hasFieldsMeta() + && !NonexistentClass.class.isAssignableFrom(TypeUtils.getComponentIfArray(cls))) { + classInfo = getClassInfo(cls); + } else if (ClassResolver.useReplaceResolveSerializer(cls)) { + // For classes with writeReplace/readResolve, use their natural serializer + // (ReplaceResolveSerializer) instead of MetaSharedSerializer classInfo = getClassInfo(cls); } else { classInfo = getMetaSharedClassInfo(classDef, cls); @@ -365,56 +360,7 @@ private ClassInfo getMetaSharedClassInfo(ClassDef classDef, Class clz) { return classInfo; } - /** - * Write all new class definitions meta to buffer at last, so that if some class doesn't exist on - * peer, but one of class which exists on both side are sent in this stream, the definition meta - * can still be stored in peer, and can be resolved next time when sent only an id. - */ - public final void writeClassDefs(MemoryBuffer buffer) { - MetaContext metaContext = fory.getSerializationContext().getMetaContext(); - ObjectArray writingClassDefs = metaContext.writingClassDefs; - final int size = writingClassDefs.size; - buffer.writeVarUint32Small7(size); - if (buffer.isHeapFullyWriteable()) { - writeClassDefs(buffer, writingClassDefs, size); - } else { - for (int i = 0; i < size; i++) { - writingClassDefs.get(i).writeClassDef(buffer); - } - } - metaContext.writingClassDefs.size = 0; - } - - private void writeClassDefs( - MemoryBuffer buffer, ObjectArray writingClassDefs, int size) { - for (int i = 0; i < size; i++) { - buffer.writeBytes(writingClassDefs.get(i).getEncoded()); - } - } - - /** - * Ensure all class definition are read and populated, even there are deserialization exception - * such as ClassNotFound. So next time a class def written previously identified by an id can be - * got from the meta context. - */ - public final void readClassDefs(MemoryBuffer buffer) { - MetaContext metaContext = fory.getSerializationContext().getMetaContext(); - assert metaContext != null : SET_META__CONTEXT_MSG; - int numClassDefs = buffer.readVarUint32Small7(); - for (int i = 0; i < numClassDefs; i++) { - long id = buffer.readInt64(); - Tuple2 tuple2 = extRegistry.classIdToDef.get(id); - if (tuple2 != null) { - ClassDef.skipClassDef(buffer, id); - } else { - tuple2 = readClassDef(buffer, id); - } - metaContext.readClassDefs.add(tuple2.f0); - metaContext.readClassInfos.add(tuple2.f1); - } - } - - private Tuple2 readClassDef(MemoryBuffer buffer, long header) { + protected Tuple2 readClassDef(MemoryBuffer buffer, long header) { ClassDef readClassDef = ClassDef.readClassDef(fory, buffer, header); Tuple2 tuple2 = extRegistry.classIdToDef.get(readClassDef.getId()); if (tuple2 == null) { diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java index 79983798c8..e76748004b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java @@ -749,14 +749,16 @@ public void writeSharedClassMeta(MemoryBuffer buffer, ClassInfo classInfo) { int newId = classMap.size; int id = classMap.putOrGet(classInfo.cls, newId); if (id >= 0) { - buffer.writeVarUint32(id); + // Reference to previously written type: (index << 1) | 1, LSB=1 + buffer.writeVarUint32((id << 1) | 1); } else { - buffer.writeVarUint32(newId); + // New type: index << 1, LSB=0, followed by ClassDef bytes inline + buffer.writeVarUint32(newId << 1); ClassDef classDef = classInfo.classDef; if (classDef == null) { classDef = buildClassDef(classInfo); } - metaContext.writingClassDefs.add(classDef); + buffer.writeBytes(classDef.getEncoded()); } } @@ -848,10 +850,34 @@ public ClassInfo readSharedClassMeta(MemoryBuffer buffer, MetaContext metaContex private ClassInfo readSharedClassMeta(MemoryBuffer buffer) { MetaContext metaContext = fory.getSerializationContext().getMetaContext(); assert metaContext != null : SET_META__CONTEXT_MSG; - int id = buffer.readVarUint32Small14(); - ClassInfo classInfo = metaContext.readClassInfos.get(id); - if (classInfo == null) { - classInfo = readSharedClassMeta(metaContext, id); + int indexMarker = buffer.readVarUint32Small14(); + boolean isRef = (indexMarker & 1) == 1; + int index = indexMarker >>> 1; + ClassInfo classInfo; + if (isRef) { + // Reference to previously read type in this stream + classInfo = metaContext.readClassInfos.get(index); + } else { + // New type in stream - but may already be known from registry + long id = buffer.readInt64(); + Tuple2 tuple2 = extRegistry.classIdToDef.get(id); + if (tuple2 != null) { + // Already known - skip the ClassDef bytes, reuse existing ClassInfo + ClassDef.skipClassDef(buffer, id); + classInfo = tuple2.f1; + if (classInfo == null) { + classInfo = buildMetaSharedClassInfo(tuple2, tuple2.f0); + } + } else { + // Unknown - read ClassDef and create ClassInfo + tuple2 = readClassDef(buffer, id); + classInfo = tuple2.f1; + if (classInfo == null) { + classInfo = buildMetaSharedClassInfo(tuple2, tuple2.f0); + } + } + // index == readClassInfos.size() since types are written sequentially + metaContext.readClassInfos.add(classInfo); } return classInfo; } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java index 9eb8cc74a9..c5cf0c1355 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java @@ -164,11 +164,13 @@ public static final class SerializationFieldInfo { this.qualifiedFieldName = d.getDeclaringClass() + "." + d.getName(); if (d.getField() != null) { this.fieldAccessor = FieldAccessor.createAccessor(d.getField()); - isPrimitive = d.getField().getType().isPrimitive(); } else { this.fieldAccessor = null; - isPrimitive = d.getTypeRef().getRawType().isPrimitive(); } + // Use dispatchId to determine isPrimitive for consistency with how data was written. + // This ensures correct handling in schema compatible mode where local field type + // may differ from remote (ClassDef) field type. + isPrimitive = DispatchId.isPrimitive(dispatchId); fieldConverter = d.getFieldConverter(); nullable = d.isNullable(); // descriptor.isTrackingRef() already includes the needToWriteRef check diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java index 8f01b6d694..ccc4322082 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java @@ -110,12 +110,12 @@ private void writeLayerClassMeta(MemoryBuffer buffer) { int newId = classMap.size; int id = classMap.putOrGet(layerMarkerClass, newId); if (id >= 0) { - // Already sent this layer definition - buffer.writeVarUint32(id << 1 | 0b1); + // Reference to previously written type: (index << 1) | 1, LSB=1 + buffer.writeVarUint32((id << 1) | 1); } else { - // First time, queue the layer ClassDef to be written at stream end - buffer.writeVarUint32(newId << 1 | 0b1); - metaContext.writingClassDefs.add(layerClassDef); + // New type: index << 1, LSB=0, followed by ClassDef bytes inline + buffer.writeVarUint32(newId << 1); + buffer.writeBytes(layerClassDef.getEncoded()); } } @@ -210,17 +210,16 @@ private void readLayerClassMeta(MemoryBuffer buffer) { if (metaContext == null) { return; } - int header = buffer.readVarUint32Small14(); - int id = header >>> 1; - // The class def will be read at stream end via readClassDefs() - // Here we just verify the ID is valid - if ((header & 0b1) != 0) { - // Meta share ID - either already known or will be resolved from stream end - ObjectArray readClassInfos = metaContext.readClassInfos; - if (id >= readClassInfos.size) { - // ClassDef not yet read - it will be available after readClassDefs() is called - // For now, we proceed with field reading using our local field info - } + int indexMarker = buffer.readVarUint32Small14(); + boolean isRef = (indexMarker & 1) == 1; + int index = indexMarker >>> 1; + if (isRef) { + // Reference to previously read type - already in readClassInfos, nothing to do + } else { + // New type in stream - read ClassDef inline + long id = buffer.readInt64(); + ClassDef.skipClassDef(buffer, id); + // Layer class info is managed by this serializer, not stored in readClassInfos } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java index f6ae69474d..da7d792fa2 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java @@ -98,10 +98,12 @@ private void writeClassDef(MemoryBuffer buffer, NonexistentClass.NonexistentMeta // class not exist, use class def id for identity. int id = classMap.putOrGet(value.classDef.getId(), newId); if (id >= 0) { - buffer.writeVarUint32(id << 1 | 0b1); + // Reference to previously written type: (index << 1) | 1, LSB=1 + buffer.writeVarUint32((id << 1) | 1); } else { - buffer.writeVarUint32(newId << 1 | 0b1); - metaContext.writingClassDefs.add(value.classDef); + // New type: index << 1, LSB=0, followed by ClassDef bytes inline + buffer.writeVarUint32(newId << 1); + buffer.writeBytes(value.classDef.getEncoded()); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/type/DispatchId.java b/java/fory-core/src/main/java/org/apache/fory/type/DispatchId.java index 0b12adeb2c..861dad49fe 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/DispatchId.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/DispatchId.java @@ -70,12 +70,15 @@ public static int getDispatchId(Fory fory, Descriptor d) { TypeRef typeRef = d.getTypeRef(); Class rawType = typeRef.getRawType(); TypeExtMeta typeExtMeta = typeRef.getTypeExtMeta(); - // A field is treated as primitive for dispatch only if the Java type itself is primitive. - // Boxed types with nullable=false are still dispatched as boxed types, - // but serialized without null checks. + // A field is treated as primitive for dispatch if: + // 1. The Java type itself is primitive, OR + // 2. The boxed type has nullable=false (meaning writer wrote without null flag), OR + // 3. TypeExtMeta says nullable=false and typeId is a primitive type (for schema compatible mode + // where local field may be boxed but remote wrote primitive) boolean isPrimitive = typeRef.isPrimitive() - || (rawType.isPrimitive() && typeExtMeta != null && !typeExtMeta.nullable()); + || (rawType.isPrimitive() && typeExtMeta != null && !typeExtMeta.nullable()) + || (typeExtMeta != null && !typeExtMeta.nullable() && Types.isPrimitiveType(typeId)); if (fory.isCrossLanguage()) { return xlangTypeIdToDispatchId(typeId, isPrimitive); } else { diff --git a/javascript/packages/fory/lib/typeMetaResolver.ts b/javascript/packages/fory/lib/typeMetaResolver.ts index 59562b86f5..d01f2c9d43 100644 --- a/javascript/packages/fory/lib/typeMetaResolver.ts +++ b/javascript/packages/fory/lib/typeMetaResolver.ts @@ -71,12 +71,15 @@ export class TypeMetaResolver { writeTypeMeta(typeInfo: StructTypeInfo, writer: BinaryWriter, bytes: Uint8Array) { if (typeInfo.dynamicTypeId !== -1) { - writer.varUInt32(((this.dynamicTypeId + 1) << 1) | 1); + // Reference to previously written type: (index << 1) | 1, LSB=1 + writer.varUInt32((typeInfo.dynamicTypeId << 1) | 1); } else { - typeInfo.dynamicTypeId = this.dynamicTypeId; + // New type: index << 1, LSB=0, followed by TypeMeta bytes inline + const index = this.dynamicTypeId; + typeInfo.dynamicTypeId = index; this.dynamicTypeId += 1; this.disposeTypeInfo.push(typeInfo); - writer.varUInt32(bytes.byteLength << 1); + writer.varUInt32(index << 1); writer.buffer(bytes); } } @@ -87,5 +90,6 @@ export class TypeMetaResolver { }); this.disposeTypeInfo = []; this.dynamicTypeId = 0; + this.typeMeta = []; } } diff --git a/python/pyfory/_fory.py b/python/pyfory/_fory.py index 6b091ce814..e13f8952f9 100644 --- a/python/pyfory/_fory.py +++ b/python/pyfory/_fory.py @@ -473,25 +473,12 @@ def _serialize( set_bit(buffer, mask_index, 2) else: clear_bit(buffer, mask_index, 2) - # Reserve space for type definitions offset, similar to Java implementation - type_defs_offset_pos = None - if self.serialization_context.scoped_meta_share_enabled: - type_defs_offset_pos = buffer.writer_index - buffer.write_int32(-1) # Reserve 4 bytes for type definitions offset + # Type definitions are now written inline (streaming) instead of deferred to end if self.language == Language.PYTHON: self.write_ref(buffer, obj) else: self.xwrite_ref(buffer, obj) - - # Write type definitions at the end, similar to Java implementation - if self.serialization_context.scoped_meta_share_enabled: - meta_context = self.serialization_context.meta_context - if meta_context is not None and len(meta_context.get_writing_type_defs()) > 0: - # Update the offset to point to current position - current_pos = buffer.writer_index - buffer.put_int32(type_defs_offset_pos, current_pos - type_defs_offset_pos - 4) - self.type_resolver.write_type_defs(buffer) if buffer is not self.buffer: return buffer else: @@ -618,32 +605,13 @@ def _deserialize( else: assert buffers is None, "buffers should be null when the serialized stream is produced with buffer_callback null." - # Read type definitions at the start, similar to Java implementation - end_reader_index = None - if self.serialization_context.scoped_meta_share_enabled: - relative_type_defs_offset = buffer.read_int32() - if relative_type_defs_offset != -1: - # Save current reader position - current_reader_index = buffer.reader_index - # Jump to type definitions - buffer.reader_index = current_reader_index + relative_type_defs_offset - # Read type definitions - self.type_resolver.read_type_defs(buffer) - # Save the end position (after type defs) - this is the true end of serialized data - end_reader_index = buffer.reader_index - # Jump back to continue with object deserialization - buffer.reader_index = current_reader_index + # Type definitions are now read inline (streaming) instead of at the end if is_target_x_lang: obj = self.xread_ref(buffer) else: obj = self.read_ref(buffer) - # After reading the object, position buffer at the end of serialized data - # (which is after the type definitions, not after the object data) - if end_reader_index is not None: - buffer.reader_index = end_reader_index - return obj def read_ref(self, buffer): diff --git a/python/pyfory/registry.py b/python/pyfory/registry.py index 79c16311c6..15bde5b0a5 100644 --- a/python/pyfory/registry.py +++ b/python/pyfory/registry.py @@ -837,6 +837,26 @@ def _build_type_info_from_typedef(self, type_def): ) return typeinfo + def _read_and_build_typeinfo(self, buffer): + """Read TypeDef inline from buffer and build TypeInfo. + + Used for streaming meta share where TypeDef is written inline. + """ + # Read the header (first 8 bytes) to get the type ID + header = buffer.read_int64() + # Check if we already have this TypeDef cached + type_info = self._meta_shared_typeinfo.get(header) + if type_info is not None: + # Skip the rest of the TypeDef binary for faster performance + skip_typedef(buffer, header) + else: + # Read the TypeDef and create TypeInfo + type_def = decode_typedef(buffer, self, header=header) + type_info = self._build_type_info_from_typedef(type_def) + # Cache the tuple for future use + self._meta_shared_typeinfo[header] = type_info + return type_info + def reset(self): pass diff --git a/python/pyfory/serialization.pyx b/python/pyfory/serialization.pyx index 3d3adb7e72..e960d6acb4 100644 --- a/python/pyfory/serialization.pyx +++ b/python/pyfory/serialization.pyx @@ -697,6 +697,10 @@ cdef class TypeResolver: """Read all type definitions from the buffer.""" self._resolver.read_type_defs(buffer) + cpdef inline _read_and_build_typeinfo(self, Buffer buffer): + """Read TypeDef inline from buffer and build TypeInfo.""" + return self._resolver._read_and_build_typeinfo(buffer) + cpdef inline reset(self): pass @@ -742,7 +746,7 @@ cdef class MetaContext: self._read_type_infos = [] cpdef inline void write_shared_typeinfo(self, Buffer buffer, typeinfo): - """Add a type definition to the writing queue.""" + """Write type info with streaming inline TypeDef.""" type_cls = typeinfo.cls cdef int32_t type_id = typeinfo.type_id cdef int32_t internal_type_id = type_id & 0xFF @@ -753,17 +757,20 @@ cdef class MetaContext: cdef uint64_t type_addr = type_cls cdef flat_hash_map[uint64_t, int32_t].iterator it = self._c_type_map.find(type_addr) if it != self._c_type_map.end(): - buffer.write_varuint32(deref(it).second) + # Reference to previously written type: (index << 1) | 1, LSB=1 + buffer.write_varuint32((deref(it).second << 1) | 1) return + # New type: index << 1, LSB=0, followed by TypeDef bytes inline cdef index = self._c_type_map.size() - buffer.write_varuint32(index) + buffer.write_varuint32(index << 1) self._c_type_map[type_addr] = index type_def = typeinfo.type_def if type_def is None: self.type_resolver._set_typeinfo(typeinfo) type_def = typeinfo.type_def - self._writing_type_defs.append(type_def) + # Write TypeDef bytes inline instead of deferring to end + buffer.write_bytes(type_def.encoded) cpdef inline list get_writing_type_defs(self): """Get all type definitions that need to be written.""" @@ -779,11 +786,23 @@ cdef class MetaContext: self._read_type_infos.append(type_info) cpdef inline read_shared_typeinfo(self, Buffer buffer): - """Read a type info from buffer.""" + """Read type info with streaming inline TypeDef.""" cdef type_id = buffer.read_varuint32() - if IsTypeShareMeta(type_id & 0xFF): - return self._read_type_infos[buffer.read_varuint32()] - return self.type_resolver.get_typeinfo_by_id(type_id) + if not IsTypeShareMeta(type_id & 0xFF): + return self.type_resolver.get_typeinfo_by_id(type_id) + + cdef int32_t index_marker = buffer.read_varuint32() + cdef c_bool is_ref = (index_marker & 1) == 1 + cdef int32_t index = index_marker >> 1 + + if is_ref: + # Reference to previously read type + return self._read_type_infos[index] + else: + # New type - read TypeDef inline and build TypeInfo + type_info = self.type_resolver._read_and_build_typeinfo(buffer) + self._read_type_infos.append(type_info) + return type_info cpdef inline reset_read(self): """Reset read state.""" diff --git a/rust/fory-core/src/fory.rs b/rust/fory-core/src/fory.rs index 2f1318309a..4033d0bb63 100644 --- a/rust/fory-core/src/fory.rs +++ b/rust/fory-core/src/fory.rs @@ -610,11 +610,7 @@ impl Fory { ) -> Result<(), Error> { let is_none = record.fory_is_none(); self.write_head::(is_none, &mut context.writer); - let meta_start_offset = context.writer.len(); if !is_none { - if context.is_compatible() { - context.writer.write_i32(-1); - }; // Use RefMode based on config: // - If track_ref is enabled, use RefMode::Tracking for the root object // - Otherwise, use RefMode::NullOnly which writes NOT_NULL_VALUE_FLAG @@ -623,10 +619,8 @@ impl Fory { } else { RefMode::NullOnly }; + // TypeMeta is written inline during serialization (streaming protocol) ::fory_write(record, context, ref_mode, true, false)?; - if context.is_compatible() && !context.empty() { - context.write_meta(meta_start_offset); - } } Ok(()) } @@ -997,13 +991,6 @@ impl Fory { if is_none { return Ok(T::fory_default()); } - let mut bytes_to_skip = 0; - if context.is_compatible() { - let meta_offset = context.reader.read_i32()?; - if meta_offset != -1 { - bytes_to_skip = context.load_type_meta(meta_offset as usize)?; - } - } // Use RefMode based on config: // - If track_ref is enabled, use RefMode::Tracking for the root object // - Otherwise, use RefMode::NullOnly @@ -1012,10 +999,8 @@ impl Fory { } else { RefMode::NullOnly }; + // TypeMeta is read inline during deserialization (streaming protocol) let result = ::fory_read(context, ref_mode, true); - if bytes_to_skip > 0 { - context.reader.skip(bytes_to_skip)?; - } context.ref_reader.resolve_callbacks(); result } diff --git a/rust/fory-core/src/resolver/context.rs b/rust/fory-core/src/resolver/context.rs index 701fb75c06..f099bf66a3 100644 --- a/rust/fory-core/src/resolver/context.rs +++ b/rust/fory-core/src/resolver/context.rs @@ -213,22 +213,12 @@ impl<'a> WriteContext<'a> { self.track_ref } + /// Write type meta inline using streaming protocol. + /// Writes index marker with LSB indicating new type or reference. #[inline(always)] - pub fn empty(&mut self) -> bool { - self.meta_resolver.empty() - } - - #[inline(always)] - pub fn push_meta(&mut self, type_id: std::any::TypeId) -> Result { - self.meta_resolver.push(type_id, &self.type_resolver) - } - - #[inline(always)] - pub fn write_meta(&mut self, offset: usize) { - let len = self.writer.len(); - self.writer - .set_bytes(offset, &((len - offset - 4) as u32).to_le_bytes()); - self.meta_resolver.to_bytes(&mut self.writer); + pub fn write_type_meta(&mut self, type_id: std::any::TypeId) -> Result<(), Error> { + self.meta_resolver + .write_type_meta(&mut self.writer, type_id, &self.type_resolver) } pub fn write_any_typeinfo( @@ -251,18 +241,21 @@ impl<'a> WriteContext<'a> { // should be compiled to jump table generation match fory_type_id & 0xff { types::NAMED_COMPATIBLE_STRUCT | types::COMPATIBLE_STRUCT => { - let meta_index = - self.meta_resolver - .push(concrete_type_id, &self.type_resolver)? as u32; - self.writer.write_varuint32(meta_index); + // Write type meta inline using streaming protocol + self.meta_resolver.write_type_meta( + &mut self.writer, + concrete_type_id, + &self.type_resolver, + )?; } types::NAMED_ENUM | types::NAMED_EXT | types::NAMED_STRUCT => { if self.is_share_meta() { - let meta_index = self - .meta_resolver - .push(concrete_type_id, &self.type_resolver)? - as u32; - self.writer.write_varuint32(meta_index); + // Write type meta inline using streaming protocol + self.meta_resolver.write_type_meta( + &mut self.writer, + concrete_type_id, + &self.type_resolver, + )?; } else { self.write_meta_string_bytes(namespace)?; self.write_meta_string_bytes(type_name)?; @@ -410,12 +403,12 @@ impl<'a> ReadContext<'a> { self.get_type_info_by_index(type_index) } + /// Read type meta inline using streaming protocol. + /// Returns the TypeInfo for this type. #[inline(always)] - pub fn load_type_meta(&mut self, offset: usize) -> Result { - self.meta_resolver.load( - &self.type_resolver, - &mut Reader::new(&self.reader.slice_after_cursor()[offset..]), - ) + pub fn read_type_meta(&mut self) -> Result, Error> { + self.meta_resolver + .read_type_meta(&mut self.reader, &self.type_resolver) } pub fn read_any_typeinfo(&mut self) -> Result, Error> { @@ -423,15 +416,13 @@ impl<'a> ReadContext<'a> { // should be compiled to jump table generation match fory_type_id & 0xff { types::NAMED_COMPATIBLE_STRUCT | types::COMPATIBLE_STRUCT => { - let meta_index = self.reader.read_varuint32()? as usize; - let type_info = self.get_type_info_by_index(meta_index)?.clone(); - Ok(type_info) + // Read type meta inline using streaming protocol + self.read_type_meta() } types::NAMED_ENUM | types::NAMED_EXT | types::NAMED_STRUCT => { if self.is_share_meta() { - let meta_index = self.reader.read_varuint32()? as usize; - let type_info = self.get_type_info_by_index(meta_index)?.clone(); - Ok(type_info) + // Read type meta inline using streaming protocol + self.read_type_meta() } else { let namespace = self.read_meta_string()?.to_owned(); let type_name = self.read_meta_string()?.to_owned(); diff --git a/rust/fory-core/src/resolver/meta_resolver.rs b/rust/fory-core/src/resolver/meta_resolver.rs index 7836aa30c4..19d3b386c3 100644 --- a/rust/fory-core/src/resolver/meta_resolver.rs +++ b/rust/fory-core/src/resolver/meta_resolver.rs @@ -23,9 +23,12 @@ use crate::TypeResolver; use std::collections::HashMap; use std::rc::Rc; +/// Streaming meta writer that writes TypeMeta inline during serialization. +/// Uses the streaming protocol: +/// - (index << 1) | 0 for new type definition (followed by TypeMeta bytes) +/// - (index << 1) | 1 for reference to previously written type #[derive(Default)] pub struct MetaWriterResolver { - type_defs: Vec>>, type_id_index_map: HashMap, } @@ -33,44 +36,43 @@ const MAX_PARSED_NUM_TYPE_DEFS: usize = 8192; #[allow(dead_code)] impl MetaWriterResolver { + /// Write type meta inline using streaming protocol. + /// Returns the index assigned to this type. #[inline(always)] - pub fn push( + pub fn write_type_meta( &mut self, + writer: &mut Writer, type_id: std::any::TypeId, type_resolver: &TypeResolver, - ) -> Result { + ) -> Result<(), Error> { match self.type_id_index_map.get(&type_id) { + Some(&index) => { + // Reference to previously written type: (index << 1) | 1, LSB=1 + writer.write_varuint32(((index as u32) << 1) | 1); + } None => { - let index = self.type_defs.len(); - self.type_defs - .push(type_resolver.get_type_info(&type_id)?.get_type_def()); + // New type: index << 1, LSB=0, followed by TypeMeta bytes inline + let index = self.type_id_index_map.len(); + writer.write_varuint32((index as u32) << 1); self.type_id_index_map.insert(type_id, index); - Ok(index) + // Write TypeMeta bytes inline + let type_def = type_resolver.get_type_info(&type_id)?.get_type_def(); + writer.write_bytes(&type_def); } - Some(index) => Ok(*index), } - } - - #[inline(always)] - pub fn to_bytes(&self, writer: &mut Writer) { - writer.write_varuint32(self.type_defs.len() as u32); - for item in &self.type_defs { - writer.write_bytes(item); - } - } - - #[inline(always)] - pub fn empty(&mut self) -> bool { - self.type_defs.is_empty() + Ok(()) } #[inline(always)] pub fn reset(&mut self) { - self.type_defs.clear(); self.type_id_index_map.clear(); } } +/// Streaming meta reader that reads TypeMeta inline during deserialization. +/// Uses the streaming protocol: +/// - (index << 1) | 0 for new type definition (followed by TypeMeta bytes) +/// - (index << 1) | 1 for reference to previously read type #[derive(Default)] pub struct MetaReaderResolver { pub reading_type_infos: Vec>, @@ -83,18 +85,30 @@ impl MetaReaderResolver { self.reading_type_infos.get(index) } - pub fn load( + /// Read type meta inline using streaming protocol. + /// Returns the TypeInfo for this type. + #[inline(always)] + pub fn read_type_meta( &mut self, - type_resolver: &TypeResolver, reader: &mut Reader, - ) -> Result { - let meta_size = reader.read_varuint32()?; - // self.reading_type_infos.reserve(meta_size as usize); - for _ in 0..meta_size { + type_resolver: &TypeResolver, + ) -> Result, Error> { + let index_marker = reader.read_varuint32()?; + let is_ref = (index_marker & 1) == 1; + let index = (index_marker >> 1) as usize; + + if is_ref { + // Reference to previously read type + self.reading_type_infos.get(index).cloned().ok_or_else(|| { + Error::type_error(format!("TypeInfo not found for type index: {}", index)) + }) + } else { + // New type - read TypeMeta inline let meta_header = reader.read_i64()?; if let Some(type_info) = self.parsed_type_infos.get(&meta_header) { self.reading_type_infos.push(type_info.clone()); TypeMeta::skip_bytes(reader, meta_header)?; + Ok(type_info.clone()) } else { let type_meta = Rc::new(TypeMeta::from_bytes_with_header( reader, @@ -139,10 +153,10 @@ impl MetaReaderResolver { self.parsed_type_infos .insert(meta_header, type_info.clone()); } - self.reading_type_infos.push(type_info); + self.reading_type_infos.push(type_info.clone()); + Ok(type_info) } } - Ok(reader.get_cursor()) } #[inline(always)] diff --git a/rust/fory-core/src/serializer/enum_.rs b/rust/fory-core/src/serializer/enum_.rs index d3355bc3d2..b0ed72fcd8 100644 --- a/rust/fory-core/src/serializer/enum_.rs +++ b/rust/fory-core/src/serializer/enum_.rs @@ -58,8 +58,8 @@ pub fn write_type_info(context: &mut WriteContext) -> Result<(), } let rs_type_id = std::any::TypeId::of::(); if context.is_share_meta() { - let meta_index = context.push_meta(rs_type_id)? as u32; - context.writer.write_varuint32(meta_index); + // Write type meta inline using streaming protocol + context.write_type_meta(rs_type_id)?; } else { let type_info = context.get_type_resolver().get_type_info(&rs_type_id)?; let namespace = type_info.get_namespace(); @@ -111,7 +111,8 @@ pub fn read_type_info(context: &mut ReadContext) -> Result<(), Er return Ok(()); } if context.is_share_meta() { - let _meta_index = context.reader.read_varuint32()?; + // Read type meta inline using streaming protocol + let _type_info = context.read_type_meta()?; } else { let _namespace_msb = context.read_meta_string()?; let _type_name_msb = context.read_meta_string()?; diff --git a/rust/fory-core/src/serializer/skip.rs b/rust/fory-core/src/serializer/skip.rs index 5c839fa173..980342060e 100644 --- a/rust/fory-core/src/serializer/skip.rs +++ b/rust/fory-core/src/serializer/skip.rs @@ -91,9 +91,8 @@ pub fn skip_any_value(context: &mut ReadContext, read_ref_flag: bool) -> Result< None, ), types::COMPATIBLE_STRUCT | types::NAMED_COMPATIBLE_STRUCT => { - // For compatible struct types, read meta_index to get type_info - let meta_index = context.reader.read_varuint32()? as usize; - let type_info = context.get_type_info_by_index(meta_index)?.clone(); + // For compatible struct types, read type meta inline using streaming protocol + let type_info = context.read_type_meta()?; ( FieldType { type_id, @@ -105,10 +104,9 @@ pub fn skip_any_value(context: &mut ReadContext, read_ref_flag: bool) -> Result< ) } types::STRUCT | types::NAMED_STRUCT => { - // For non-compatible struct types with share_meta enabled, read meta_index + // For non-compatible struct types with share_meta enabled, read type meta inline if context.is_share_meta() { - let meta_index = context.reader.read_varuint32()? as usize; - let type_info = context.get_type_info_by_index(meta_index)?.clone(); + let type_info = context.read_type_meta()?; ( FieldType { type_id, @@ -310,14 +308,16 @@ fn skip_struct( type_id_num: u32, type_info: &Option>, ) -> Result<(), Error> { + let type_info_rc: Option>; let type_info_value = if type_info.is_none() { let remote_type_id = context.reader.read_varuint32()?; ensure!( type_id_num == remote_type_id, Error::type_mismatch(type_id_num, remote_type_id) ); - let meta_index = context.reader.read_varuint32()?; - context.get_type_info_by_index(meta_index as usize)? + // Read type meta inline using streaming protocol + type_info_rc = Some(context.read_type_meta()?); + type_info_rc.as_ref().unwrap() } else { type_info.as_ref().unwrap() }; @@ -355,14 +355,16 @@ fn skip_ext( type_id_num: u32, type_info: &Option>, ) -> Result<(), Error> { + let type_info_rc: Option>; let type_info_value = if type_info.is_none() { let remote_type_id = context.reader.read_varuint32()?; ensure!( type_id_num == remote_type_id, Error::type_mismatch(type_id_num, remote_type_id) ); - let meta_index = context.reader.read_varuint32()?; - context.get_type_info_by_index(meta_index as usize)? + // Read type meta inline using streaming protocol + type_info_rc = Some(context.read_type_meta()?); + type_info_rc.as_ref().unwrap() } else { type_info.as_ref().unwrap() }; @@ -376,8 +378,8 @@ fn skip_ext( fn skip_user_struct(context: &mut ReadContext, _type_id_num: u32) -> Result<(), Error> { let remote_type_id = context.reader.read_varuint32()?; - let meta_index = context.reader.read_varuint32()?; - let type_info = context.get_type_info_by_index(meta_index as usize)?; + // Read type meta inline using streaming protocol + let type_info = context.read_type_meta()?; let type_meta = type_info.get_type_meta(); ensure!( type_meta.get_type_id() == remote_type_id, @@ -791,9 +793,8 @@ pub fn skip_enum_variant( let type_id = type_info.as_ref().unwrap().get_type_id(); skip_struct(context, type_id, type_info) } else { - // If no type_info provided, read it from the stream - let meta_index = context.reader.read_varuint32()?; - let type_info_rc = context.get_type_info_by_index(meta_index as usize)?.clone(); + // If no type_info provided, read it inline using streaming protocol + let type_info_rc = context.read_type_meta()?; let type_id = type_info_rc.get_type_id(); let type_info_opt = Some(type_info_rc); skip_struct(context, type_id, &type_info_opt) diff --git a/rust/fory-core/src/serializer/struct_.rs b/rust/fory-core/src/serializer/struct_.rs index 31ae7d9f5f..48caa349d8 100644 --- a/rust/fory-core/src/serializer/struct_.rs +++ b/rust/fory-core/src/serializer/struct_.rs @@ -46,8 +46,8 @@ pub fn write_type_info(context: &mut WriteContext) -> Result<(), if type_id & 0xff == TypeId::NAMED_STRUCT as u32 { if context.is_share_meta() { - let meta_index = context.push_meta(rs_type_id)? as u32; - context.writer.write_varuint32(meta_index); + // Write type meta inline using streaming protocol + context.write_type_meta(rs_type_id)?; } else { let type_info = context.get_type_resolver().get_type_info(&rs_type_id)?; let namespace = type_info.get_namespace(); @@ -58,8 +58,8 @@ pub fn write_type_info(context: &mut WriteContext) -> Result<(), } else if type_id & 0xff == TypeId::NAMED_COMPATIBLE_STRUCT as u32 || type_id & 0xff == TypeId::COMPATIBLE_STRUCT as u32 { - let meta_index = context.push_meta(rs_type_id)? as u32; - context.writer.write_varuint32(meta_index); + // Write type meta inline using streaming protocol + context.write_type_meta(rs_type_id)?; } Ok(()) } @@ -75,7 +75,8 @@ pub fn read_type_info(context: &mut ReadContext) -> Result<(), Er if local_type_id & 0xff == TypeId::NAMED_STRUCT as u32 { if context.is_share_meta() { - let _meta_index = context.reader.read_varuint32()?; + // Read type meta inline using streaming protocol + let _type_info = context.read_type_meta()?; } else { let _namespace_msb = context.read_meta_string()?; let _type_name_msb = context.read_meta_string()?; @@ -83,7 +84,8 @@ pub fn read_type_info(context: &mut ReadContext) -> Result<(), Er } else if local_type_id & 0xff == TypeId::NAMED_COMPATIBLE_STRUCT as u32 || local_type_id & 0xff == TypeId::COMPATIBLE_STRUCT as u32 { - let _meta_index = context.reader.read_varuint32(); + // Read type meta inline using streaming protocol + let _type_info = context.read_type_meta()?; } Ok(()) } diff --git a/rust/fory-derive/src/object/derive_enum.rs b/rust/fory-derive/src/object/derive_enum.rs index 07f8ba5e96..7c43565620 100644 --- a/rust/fory-derive/src/object/derive_enum.rs +++ b/rust/fory-derive/src/object/derive_enum.rs @@ -394,9 +394,8 @@ fn rust_compatible_variant_write_branches( quote! { Self::#ident { #(#field_idents),* } => { context.writer.write_varuint32((#tag_value << 2) | 0b10); - // Push named variant meta - let meta_index = context.push_meta(std::any::TypeId::of::<#meta_type_ident>())? as u32; - context.writer.write_varuint32(meta_index); + // Write type meta inline using streaming protocol + context.write_type_meta(std::any::TypeId::of::<#meta_type_ident>())?; // Write fields same as struct #(#write_fields)* } @@ -809,9 +808,8 @@ fn rust_compatible_variant_read_branches( return Ok(#default_value); } // Named variant should have variant_type == 0b10 - // Read named variant meta (peer wrote this because variant_type == 0b10) - let meta_index = context.reader.read_varuint32()? as usize; - let type_info = context.get_meta(meta_index)?.clone(); + // Read type meta inline using streaming protocol + let type_info = context.read_type_meta()?; // Use gen_read_compatible logic #compatible_read_body } From 8d0025e325500b52456463fc3e30a29f655a98fe Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 16 Jan 2026 11:41:11 +0800 Subject: [PATCH 02/44] fix java not null fields read/write --- .../fory/builder/ObjectCodecBuilder.java | 54 ++++- .../serializer/AbstractObjectSerializer.java | 18 ++ .../apache/fory/serializer/FieldGroups.java | 1 + .../serializer/MetaSharedLayerSerializer.java | 4 +- .../fory/serializer/MetaSharedSerializer.java | 35 +++- .../NonexistentClassSerializers.java | 2 +- .../fory/serializer/ObjectSerializer.java | 2 +- .../apache/fory/serializer/Serializers.java | 2 +- .../java/org/apache/fory/type/DispatchId.java | 185 +++++++++++++----- rust/fory-core/src/buffer.rs | 3 +- 10 files changed, 230 insertions(+), 76 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java index 818299e0ba..6f2feaa701 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java @@ -702,40 +702,56 @@ private List deserializeUnCompressedPrimitives( for (Descriptor descriptor : group) { int dispatchId = getNumericDescriptorDispatchId(descriptor); Expression fieldValue; - if (dispatchId == DispatchId.PRIMITIVE_BOOL || dispatchId == DispatchId.BOOL) { + if (dispatchId == DispatchId.PRIMITIVE_BOOL + || dispatchId == DispatchId.NOTNULL_BOXED_BOOL + || dispatchId == DispatchId.BOOL) { fieldValue = unsafeGetBoolean(heapBuffer, getReaderAddress(readerAddr, acc)); acc += 1; } else if (dispatchId == DispatchId.PRIMITIVE_INT8 || dispatchId == DispatchId.PRIMITIVE_UINT8 + || dispatchId == DispatchId.NOTNULL_BOXED_INT8 + || dispatchId == DispatchId.NOTNULL_BOXED_UINT8 || dispatchId == DispatchId.INT8 || dispatchId == DispatchId.UINT8) { fieldValue = unsafeGet(heapBuffer, getReaderAddress(readerAddr, acc)); acc += 1; - } else if (dispatchId == DispatchId.PRIMITIVE_CHAR || dispatchId == DispatchId.CHAR) { + } else if (dispatchId == DispatchId.PRIMITIVE_CHAR + || dispatchId == DispatchId.NOTNULL_BOXED_CHAR + || dispatchId == DispatchId.CHAR) { fieldValue = unsafeGetChar(heapBuffer, getReaderAddress(readerAddr, acc)); acc += 2; } else if (dispatchId == DispatchId.PRIMITIVE_INT16 || dispatchId == DispatchId.PRIMITIVE_UINT16 + || dispatchId == DispatchId.NOTNULL_BOXED_INT16 + || dispatchId == DispatchId.NOTNULL_BOXED_UINT16 || dispatchId == DispatchId.INT16 || dispatchId == DispatchId.UINT16) { fieldValue = unsafeGetShort(heapBuffer, getReaderAddress(readerAddr, acc)); acc += 2; } else if (dispatchId == DispatchId.PRIMITIVE_INT32 || dispatchId == DispatchId.PRIMITIVE_UINT32 + || dispatchId == DispatchId.NOTNULL_BOXED_INT32 + || dispatchId == DispatchId.NOTNULL_BOXED_UINT32 || dispatchId == DispatchId.INT32 || dispatchId == DispatchId.UINT32) { fieldValue = unsafeGetInt(heapBuffer, getReaderAddress(readerAddr, acc)); acc += 4; } else if (dispatchId == DispatchId.PRIMITIVE_INT64 || dispatchId == DispatchId.PRIMITIVE_UINT64 + || dispatchId == DispatchId.NOTNULL_BOXED_INT64 + || dispatchId == DispatchId.NOTNULL_BOXED_UINT64 || dispatchId == DispatchId.INT64 || dispatchId == DispatchId.UINT64) { fieldValue = unsafeGetLong(heapBuffer, getReaderAddress(readerAddr, acc)); acc += 8; - } else if (dispatchId == DispatchId.PRIMITIVE_FLOAT32 || dispatchId == DispatchId.FLOAT32) { + } else if (dispatchId == DispatchId.PRIMITIVE_FLOAT32 + || dispatchId == DispatchId.NOTNULL_BOXED_FLOAT32 + || dispatchId == DispatchId.FLOAT32) { fieldValue = unsafeGetFloat(heapBuffer, getReaderAddress(readerAddr, acc)); acc += 4; - } else if (dispatchId == DispatchId.PRIMITIVE_FLOAT64 || dispatchId == DispatchId.FLOAT64) { + } else if (dispatchId == DispatchId.PRIMITIVE_FLOAT64 + || dispatchId == DispatchId.NOTNULL_BOXED_FLOAT64 + || dispatchId == DispatchId.FLOAT64) { fieldValue = unsafeGetDouble(heapBuffer, getReaderAddress(readerAddr, acc)); acc += 8; } else { @@ -779,43 +795,60 @@ private List deserializeCompressedPrimitives( for (Descriptor descriptor : group) { int dispatchId = getNumericDescriptorDispatchId(descriptor); Expression fieldValue; - if (dispatchId == DispatchId.PRIMITIVE_BOOL || dispatchId == DispatchId.BOOL) { + if (dispatchId == DispatchId.PRIMITIVE_BOOL + || dispatchId == DispatchId.NOTNULL_BOXED_BOOL + || dispatchId == DispatchId.BOOL) { fieldValue = unsafeGetBoolean(heapBuffer, getReaderAddress(readerAddr, acc)); acc += 1; } else if (dispatchId == DispatchId.PRIMITIVE_INT8 || dispatchId == DispatchId.PRIMITIVE_UINT8 + || dispatchId == DispatchId.NOTNULL_BOXED_INT8 + || dispatchId == DispatchId.NOTNULL_BOXED_UINT8 || dispatchId == DispatchId.INT8 || dispatchId == DispatchId.UINT8) { fieldValue = unsafeGet(heapBuffer, getReaderAddress(readerAddr, acc)); acc += 1; - } else if (dispatchId == DispatchId.PRIMITIVE_CHAR || dispatchId == DispatchId.CHAR) { + } else if (dispatchId == DispatchId.PRIMITIVE_CHAR + || dispatchId == DispatchId.NOTNULL_BOXED_CHAR + || dispatchId == DispatchId.CHAR) { fieldValue = unsafeGetChar(heapBuffer, getReaderAddress(readerAddr, acc)); acc += 2; } else if (dispatchId == DispatchId.PRIMITIVE_INT16 || dispatchId == DispatchId.PRIMITIVE_UINT16 + || dispatchId == DispatchId.NOTNULL_BOXED_INT16 + || dispatchId == DispatchId.NOTNULL_BOXED_UINT16 || dispatchId == DispatchId.INT16 || dispatchId == DispatchId.UINT16) { fieldValue = unsafeGetShort(heapBuffer, getReaderAddress(readerAddr, acc)); acc += 2; - } else if (dispatchId == DispatchId.PRIMITIVE_FLOAT32 || dispatchId == DispatchId.FLOAT32) { + } else if (dispatchId == DispatchId.PRIMITIVE_FLOAT32 + || dispatchId == DispatchId.NOTNULL_BOXED_FLOAT32 + || dispatchId == DispatchId.FLOAT32) { fieldValue = unsafeGetFloat(heapBuffer, getReaderAddress(readerAddr, acc)); acc += 4; - } else if (dispatchId == DispatchId.PRIMITIVE_FLOAT64 || dispatchId == DispatchId.FLOAT64) { + } else if (dispatchId == DispatchId.PRIMITIVE_FLOAT64 + || dispatchId == DispatchId.NOTNULL_BOXED_FLOAT64 + || dispatchId == DispatchId.FLOAT64) { fieldValue = unsafeGetDouble(heapBuffer, getReaderAddress(readerAddr, acc)); acc += 8; } else if (dispatchId == DispatchId.PRIMITIVE_INT32 || dispatchId == DispatchId.PRIMITIVE_UINT32 + || dispatchId == DispatchId.NOTNULL_BOXED_INT32 + || dispatchId == DispatchId.NOTNULL_BOXED_UINT32 || dispatchId == DispatchId.INT32 || dispatchId == DispatchId.UINT32) { fieldValue = unsafeGetInt(heapBuffer, getReaderAddress(readerAddr, acc)); acc += 4; } else if (dispatchId == DispatchId.PRIMITIVE_INT64 || dispatchId == DispatchId.PRIMITIVE_UINT64 + || dispatchId == DispatchId.NOTNULL_BOXED_INT64 + || dispatchId == DispatchId.NOTNULL_BOXED_UINT64 || dispatchId == DispatchId.INT64 || dispatchId == DispatchId.UINT64) { fieldValue = unsafeGetLong(heapBuffer, getReaderAddress(readerAddr, acc)); acc += 8; } else if (dispatchId == DispatchId.PRIMITIVE_VARINT32 + || dispatchId == DispatchId.NOTNULL_BOXED_VARINT32 || dispatchId == DispatchId.VARINT32) { if (!compressStarted) { compressStarted = true; @@ -823,6 +856,7 @@ private List deserializeCompressedPrimitives( } fieldValue = readVarInt32(buffer); } else if (dispatchId == DispatchId.PRIMITIVE_VAR_UINT32 + || dispatchId == DispatchId.NOTNULL_BOXED_VAR_UINT32 || dispatchId == DispatchId.VAR_UINT32) { if (!compressStarted) { compressStarted = true; @@ -830,6 +864,7 @@ private List deserializeCompressedPrimitives( } fieldValue = new Invoke(buffer, "readVarUint32", PRIMITIVE_INT_TYPE); } else if (dispatchId == DispatchId.PRIMITIVE_VARINT64 + || dispatchId == DispatchId.NOTNULL_BOXED_VARINT64 || dispatchId == DispatchId.VARINT64) { if (!compressStarted) { compressStarted = true; @@ -837,6 +872,7 @@ private List deserializeCompressedPrimitives( } fieldValue = new Invoke(buffer, "readVarInt64", PRIMITIVE_LONG_TYPE); } else if (dispatchId == DispatchId.PRIMITIVE_TAGGED_INT64 + || dispatchId == DispatchId.NOTNULL_BOXED_TAGGED_INT64 || dispatchId == DispatchId.TAGGED_INT64) { if (!compressStarted) { compressStarted = true; @@ -844,6 +880,7 @@ private List deserializeCompressedPrimitives( } fieldValue = new Invoke(buffer, "readTaggedInt64", PRIMITIVE_LONG_TYPE); } else if (dispatchId == DispatchId.PRIMITIVE_VAR_UINT64 + || dispatchId == DispatchId.NOTNULL_BOXED_VAR_UINT64 || dispatchId == DispatchId.VAR_UINT64) { if (!compressStarted) { compressStarted = true; @@ -851,6 +888,7 @@ private List deserializeCompressedPrimitives( } fieldValue = new Invoke(buffer, "readVarUint64", PRIMITIVE_LONG_TYPE); } else if (dispatchId == DispatchId.PRIMITIVE_TAGGED_UINT64 + || dispatchId == DispatchId.NOTNULL_BOXED_TAGGED_UINT64 || dispatchId == DispatchId.TAGGED_UINT64) { if (!compressStarted) { compressStarted = true; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java index 6f7dd454e6..1205adde3e 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java @@ -834,66 +834,84 @@ static boolean readBasicObjectFieldValue( } return false; case DispatchId.PRIMITIVE_BOOL: + case DispatchId.NOTNULL_BOXED_BOOL: case DispatchId.BOOL: fieldAccessor.putObject(targetObject, buffer.readBoolean()); return false; case DispatchId.PRIMITIVE_INT8: case DispatchId.PRIMITIVE_UINT8: + case DispatchId.NOTNULL_BOXED_INT8: + case DispatchId.NOTNULL_BOXED_UINT8: case DispatchId.INT8: case DispatchId.UINT8: fieldAccessor.putObject(targetObject, buffer.readByte()); return false; case DispatchId.PRIMITIVE_CHAR: + case DispatchId.NOTNULL_BOXED_CHAR: case DispatchId.CHAR: fieldAccessor.putObject(targetObject, buffer.readChar()); return false; case DispatchId.PRIMITIVE_INT16: case DispatchId.PRIMITIVE_UINT16: + case DispatchId.NOTNULL_BOXED_INT16: + case DispatchId.NOTNULL_BOXED_UINT16: case DispatchId.INT16: case DispatchId.UINT16: fieldAccessor.putObject(targetObject, buffer.readInt16()); return false; case DispatchId.PRIMITIVE_INT32: case DispatchId.PRIMITIVE_UINT32: + case DispatchId.NOTNULL_BOXED_INT32: + case DispatchId.NOTNULL_BOXED_UINT32: case DispatchId.INT32: case DispatchId.UINT32: fieldAccessor.putObject(targetObject, buffer.readInt32()); return false; case DispatchId.PRIMITIVE_VARINT32: + case DispatchId.NOTNULL_BOXED_VARINT32: case DispatchId.VARINT32: fieldAccessor.putObject(targetObject, buffer.readVarInt32()); return false; case DispatchId.PRIMITIVE_VAR_UINT32: + case DispatchId.NOTNULL_BOXED_VAR_UINT32: case DispatchId.VAR_UINT32: fieldAccessor.putObject(targetObject, buffer.readVarUint32()); return false; case DispatchId.PRIMITIVE_INT64: case DispatchId.PRIMITIVE_UINT64: + case DispatchId.NOTNULL_BOXED_INT64: + case DispatchId.NOTNULL_BOXED_UINT64: case DispatchId.INT64: case DispatchId.UINT64: fieldAccessor.putObject(targetObject, buffer.readInt64()); return false; case DispatchId.PRIMITIVE_VARINT64: + case DispatchId.NOTNULL_BOXED_VARINT64: case DispatchId.VARINT64: fieldAccessor.putObject(targetObject, buffer.readVarInt64()); return false; case DispatchId.PRIMITIVE_TAGGED_INT64: + case DispatchId.NOTNULL_BOXED_TAGGED_INT64: case DispatchId.TAGGED_INT64: fieldAccessor.putObject(targetObject, buffer.readTaggedInt64()); return false; case DispatchId.PRIMITIVE_VAR_UINT64: + case DispatchId.NOTNULL_BOXED_VAR_UINT64: case DispatchId.VAR_UINT64: fieldAccessor.putObject(targetObject, buffer.readVarUint64()); return false; case DispatchId.PRIMITIVE_TAGGED_UINT64: + case DispatchId.NOTNULL_BOXED_TAGGED_UINT64: case DispatchId.TAGGED_UINT64: fieldAccessor.putObject(targetObject, buffer.readTaggedUint64()); return false; case DispatchId.PRIMITIVE_FLOAT32: + case DispatchId.NOTNULL_BOXED_FLOAT32: case DispatchId.FLOAT32: fieldAccessor.putObject(targetObject, buffer.readFloat32()); return false; case DispatchId.PRIMITIVE_FLOAT64: + case DispatchId.NOTNULL_BOXED_FLOAT64: case DispatchId.FLOAT64: fieldAccessor.putObject(targetObject, buffer.readFloat64()); return false; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java index c5cf0c1355..e0f3de50fa 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java @@ -170,6 +170,7 @@ public static final class SerializationFieldInfo { // Use dispatchId to determine isPrimitive for consistency with how data was written. // This ensures correct handling in schema compatible mode where local field type // may differ from remote (ClassDef) field type. + // Note: NOTNULL_BOXED dispatch IDs are NOT primitive - they are handled by readBasicObjectFieldValue isPrimitive = DispatchId.isPrimitive(dispatchId); fieldConverter = d.getFieldConverter(); nullable = d.isNullable(); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java index ccc4322082..9d089f9045 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java @@ -22,12 +22,10 @@ import java.util.Collection; import org.apache.fory.Fory; import org.apache.fory.collection.IdentityObjectIntMap; -import org.apache.fory.collection.ObjectArray; import org.apache.fory.collection.ObjectIntMap; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.meta.ClassDef; import org.apache.fory.reflect.FieldAccessor; -import org.apache.fory.resolver.ClassInfo; import org.apache.fory.resolver.ClassInfoHolder; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.MetaContext; @@ -453,7 +451,7 @@ private Object readFieldValueToArray(MemoryBuffer buffer, SerializationFieldInfo int dispatchId = fieldInfo.dispatchId; // Handle primitives if (DispatchId.isPrimitive(dispatchId)) { - return Serializers.readPrimitiveValue(fory, buffer, dispatchId); + return Serializers.readPrimitiveValue(buffer, dispatchId); } // Handle objects return AbstractObjectSerializer.readFinalObjectFieldValue( diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java index 5191fa7af4..286f1ef282 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java @@ -229,12 +229,13 @@ public T read(MemoryBuffer buffer) { int dispatchId = fieldInfo.dispatchId; boolean needRead = true; if (fieldInfo.isPrimitive) { - if (nullable) { - needRead = readPrimitiveNullableFieldValue(buffer, obj, fieldAccessor, dispatchId); - } else { - needRead = readPrimitiveFieldValue(buffer, obj, fieldAccessor, dispatchId); - } + // PRIMITIVE_* dispatch IDs mean non-nullable, so no null flag is read + needRead = + AbstractObjectSerializer.readPrimitiveFieldValue( + buffer, obj, fieldAccessor, dispatchId); } + // For NOTNULL_BOXED and other boxed types, let readBasicObjectFieldValue handle it + // which uses putObject for proper boxing if (needRead && (nullable ? AbstractObjectSerializer.readBasicNullableObjectFieldValue( @@ -287,7 +288,7 @@ private void compatibleRead(MemoryBuffer buffer, SerializationFieldInfo fieldInf Object fieldValue; int dispatchId = fieldInfo.dispatchId; if (DispatchId.isPrimitive(dispatchId)) { - fieldValue = Serializers.readPrimitiveValue(fory, buffer, dispatchId); + fieldValue = Serializers.readPrimitiveValue(buffer, dispatchId); } else { fieldValue = AbstractObjectSerializer.readFinalObjectFieldValue( @@ -324,7 +325,7 @@ private void readFields(MemoryBuffer buffer, Object[] fields) { int dispatchId = fieldInfo.dispatchId; // primitive field won't write null flag. if (DispatchId.isPrimitive(dispatchId)) { - fields[counter++] = Serializers.readPrimitiveValue(fory, buffer, dispatchId); + fields[counter++] = Serializers.readPrimitiveValue(buffer, dispatchId); } else { Object fieldValue = AbstractObjectSerializer.readFinalObjectFieldValue( @@ -358,57 +359,75 @@ private void readFields(MemoryBuffer buffer, Object[] fields) { } } - /** Skip primitive primitive field value since it doesn't write null flag. */ + /** Skip primitive/notnull-boxed field value since they don't write null flag. */ static boolean skipPrimitiveFieldValueFailed(Fory fory, int dispatchId, MemoryBuffer buffer) { switch (dispatchId) { case DispatchId.PRIMITIVE_BOOL: + case DispatchId.NOTNULL_BOXED_BOOL: buffer.increaseReaderIndex(1); return false; case DispatchId.PRIMITIVE_INT8: case DispatchId.PRIMITIVE_UINT8: + case DispatchId.NOTNULL_BOXED_INT8: + case DispatchId.NOTNULL_BOXED_UINT8: buffer.increaseReaderIndex(1); return false; case DispatchId.PRIMITIVE_CHAR: + case DispatchId.NOTNULL_BOXED_CHAR: buffer.increaseReaderIndex(2); return false; case DispatchId.PRIMITIVE_INT16: case DispatchId.PRIMITIVE_UINT16: + case DispatchId.NOTNULL_BOXED_INT16: + case DispatchId.NOTNULL_BOXED_UINT16: buffer.increaseReaderIndex(2); return false; case DispatchId.PRIMITIVE_INT32: + case DispatchId.NOTNULL_BOXED_INT32: buffer.increaseReaderIndex(4); return false; case DispatchId.PRIMITIVE_VARINT32: + case DispatchId.NOTNULL_BOXED_VARINT32: buffer.readVarInt32(); return false; case DispatchId.PRIMITIVE_UINT32: + case DispatchId.NOTNULL_BOXED_UINT32: buffer.increaseReaderIndex(4); return false; case DispatchId.PRIMITIVE_VAR_UINT32: + case DispatchId.NOTNULL_BOXED_VAR_UINT32: buffer.readVarUint32(); return false; case DispatchId.PRIMITIVE_INT64: + case DispatchId.NOTNULL_BOXED_INT64: buffer.increaseReaderIndex(8); return false; case DispatchId.PRIMITIVE_VARINT64: + case DispatchId.NOTNULL_BOXED_VARINT64: buffer.readVarInt64(); return false; case DispatchId.PRIMITIVE_TAGGED_INT64: + case DispatchId.NOTNULL_BOXED_TAGGED_INT64: buffer.readTaggedInt64(); return false; case DispatchId.PRIMITIVE_UINT64: + case DispatchId.NOTNULL_BOXED_UINT64: buffer.increaseReaderIndex(8); return false; case DispatchId.PRIMITIVE_VAR_UINT64: + case DispatchId.NOTNULL_BOXED_VAR_UINT64: buffer.readVarUint64(); return false; case DispatchId.PRIMITIVE_TAGGED_UINT64: + case DispatchId.NOTNULL_BOXED_TAGGED_UINT64: buffer.readTaggedUint64(); return false; case DispatchId.PRIMITIVE_FLOAT32: + case DispatchId.NOTNULL_BOXED_FLOAT32: buffer.increaseReaderIndex(4); return false; case DispatchId.PRIMITIVE_FLOAT64: + case DispatchId.NOTNULL_BOXED_FLOAT64: buffer.increaseReaderIndex(8); return false; default: diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java index da7d792fa2..c6cf0e2e97 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java @@ -198,7 +198,7 @@ public Object read(MemoryBuffer buffer) { } else { if (DispatchId.isPrimitive(fieldInfo.dispatchId)) { // Use dispatch-based read to ensure correct encoding (e.g., VARINT64 vs FIXED_INT64) - fieldValue = Serializers.readPrimitiveValue(fory, buffer, fieldInfo.dispatchId); + fieldValue = Serializers.readPrimitiveValue(buffer, fieldInfo.dispatchId); } else { fieldValue = AbstractObjectSerializer.readFinalObjectFieldValue( diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java index 7a858deebc..076c56e3ed 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java @@ -270,7 +270,7 @@ public Object[] readFields(MemoryBuffer buffer) { for (SerializationFieldInfo fieldInfo : this.buildInFields) { int dispatchId = fieldInfo.dispatchId; if (DispatchId.isPrimitive(dispatchId)) { - fieldValues[counter++] = Serializers.readPrimitiveValue(fory, buffer, dispatchId); + fieldValues[counter++] = Serializers.readPrimitiveValue(buffer, dispatchId); } else { Object fieldValue = readFinalObjectFieldValue(binding, refResolver, typeResolver, fieldInfo, buffer); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java index 46516854da..00cb404bad 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java @@ -177,7 +177,7 @@ private static Serializer createSerializer( } } - public static Object readPrimitiveValue(Fory fory, MemoryBuffer buffer, int dispatchId) { + public static Object readPrimitiveValue(MemoryBuffer buffer, int dispatchId) { switch (dispatchId) { case DispatchId.PRIMITIVE_BOOL: return buffer.readBoolean(); diff --git a/java/fory-core/src/main/java/org/apache/fory/type/DispatchId.java b/java/fory-core/src/main/java/org/apache/fory/type/DispatchId.java index 861dad49fe..1053df02f6 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/DispatchId.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/DispatchId.java @@ -45,83 +45,153 @@ public class DispatchId { public static final int PRIMITIVE_VAR_UINT64 = 17; public static final int PRIMITIVE_TAGGED_UINT64 = 18; - public static final int BOOL = 19; - public static final int INT8 = 20; - public static final int CHAR = 21; - public static final int INT16 = 22; - public static final int INT32 = 23; - public static final int VARINT32 = 24; - public static final int INT64 = 25; - public static final int VARINT64 = 26; - public static final int TAGGED_INT64 = 27; - public static final int FLOAT32 = 28; - public static final int FLOAT64 = 29; - public static final int UINT8 = 30; - public static final int UINT16 = 31; - public static final int UINT32 = 32; - public static final int VAR_UINT32 = 33; - public static final int UINT64 = 34; - public static final int VAR_UINT64 = 35; - public static final int TAGGED_UINT64 = 36; - public static final int STRING = 37; + // Non-nullable boxed types: read value directly (no null flag), box it, use putObject + // Used when remote TypeMeta has nullable=false but local field is boxed (e.g., Integer) + public static final int NOTNULL_BOXED_BOOL = 19; + public static final int NOTNULL_BOXED_INT8 = 20; + public static final int NOTNULL_BOXED_INT16 = 21; + public static final int NOTNULL_BOXED_CHAR = 22; + public static final int NOTNULL_BOXED_INT32 = 23; + public static final int NOTNULL_BOXED_VARINT32 = 24; + public static final int NOTNULL_BOXED_INT64 = 25; + public static final int NOTNULL_BOXED_VARINT64 = 26; + public static final int NOTNULL_BOXED_TAGGED_INT64 = 27; + public static final int NOTNULL_BOXED_FLOAT32 = 28; + public static final int NOTNULL_BOXED_FLOAT64 = 29; + public static final int NOTNULL_BOXED_UINT8 = 30; + public static final int NOTNULL_BOXED_UINT16 = 31; + public static final int NOTNULL_BOXED_UINT32 = 32; + public static final int NOTNULL_BOXED_VAR_UINT32 = 33; + public static final int NOTNULL_BOXED_UINT64 = 34; + public static final int NOTNULL_BOXED_VAR_UINT64 = 35; + public static final int NOTNULL_BOXED_TAGGED_UINT64 = 36; + + // Nullable boxed types: read null flag first, then box if not null + public static final int BOOL = 37; + public static final int INT8 = 38; + public static final int CHAR = 39; + public static final int INT16 = 40; + public static final int INT32 = 41; + public static final int VARINT32 = 42; + public static final int INT64 = 43; + public static final int VARINT64 = 44; + public static final int TAGGED_INT64 = 45; + public static final int FLOAT32 = 46; + public static final int FLOAT64 = 47; + public static final int UINT8 = 48; + public static final int UINT16 = 49; + public static final int UINT32 = 50; + public static final int VAR_UINT32 = 51; + public static final int UINT64 = 52; + public static final int VAR_UINT64 = 53; + public static final int TAGGED_UINT64 = 54; + public static final int STRING = 55; + + // Dispatch mode for determining how to read/write a field + private static final int MODE_PRIMITIVE = 0; // Local field is primitive, use Platform.putInt + private static final int MODE_NOTNULL_BOXED = 1; // Local is boxed, remote nullable=false, box and putObject + private static final int MODE_NULLABLE_BOXED = 2; // Local is boxed, remote nullable=true, read null flag public static int getDispatchId(Fory fory, Descriptor d) { int typeId = Types.getDescriptorTypeId(fory, d); TypeRef typeRef = d.getTypeRef(); Class rawType = typeRef.getRawType(); TypeExtMeta typeExtMeta = typeRef.getTypeExtMeta(); - // A field is treated as primitive for dispatch if: - // 1. The Java type itself is primitive, OR - // 2. The boxed type has nullable=false (meaning writer wrote without null flag), OR - // 3. TypeExtMeta says nullable=false and typeId is a primitive type (for schema compatible mode - // where local field may be boxed but remote wrote primitive) - boolean isPrimitive = - typeRef.isPrimitive() - || (rawType.isPrimitive() && typeExtMeta != null && !typeExtMeta.nullable()) - || (typeExtMeta != null && !typeExtMeta.nullable() && Types.isPrimitiveType(typeId)); + + // Determine the dispatch mode based on local field type and remote nullable flag + int mode; + boolean localIsPrimitive = rawType.isPrimitive(); + boolean remoteNullable = typeExtMeta == null || typeExtMeta.nullable(); + + if (localIsPrimitive) { + // Local field is primitive (int, long, etc.) - always use PRIMITIVE dispatch + mode = MODE_PRIMITIVE; + } else if (!remoteNullable && Types.isPrimitiveType(typeId)) { + // Local field is boxed (Integer, Long, etc.), remote wrote without null flag + // Use NOTNULL_BOXED dispatch: read value directly, box it, use putObject + mode = MODE_NOTNULL_BOXED; + } else { + // Local field is boxed, remote wrote with null flag (or non-primitive type) + mode = MODE_NULLABLE_BOXED; + } + if (fory.isCrossLanguage()) { - return xlangTypeIdToDispatchId(typeId, isPrimitive); + return xlangTypeIdToDispatchId(typeId, mode); } else { - return nativeIdToDispatchId(typeId, d, isPrimitive); + return nativeIdToDispatchId(typeId, d, mode); } } - private static int xlangTypeIdToDispatchId(int typeId, boolean isPrimitive) { + private static int xlangTypeIdToDispatchId(int typeId, int mode) { switch (typeId) { case Types.BOOL: - return isPrimitive ? PRIMITIVE_BOOL : BOOL; + return mode == MODE_PRIMITIVE + ? PRIMITIVE_BOOL + : (mode == MODE_NOTNULL_BOXED ? NOTNULL_BOXED_BOOL : BOOL); case Types.INT8: - return isPrimitive ? PRIMITIVE_INT8 : INT8; + return mode == MODE_PRIMITIVE + ? PRIMITIVE_INT8 + : (mode == MODE_NOTNULL_BOXED ? NOTNULL_BOXED_INT8 : INT8); case Types.INT16: - return isPrimitive ? PRIMITIVE_INT16 : INT16; + return mode == MODE_PRIMITIVE + ? PRIMITIVE_INT16 + : (mode == MODE_NOTNULL_BOXED ? NOTNULL_BOXED_INT16 : INT16); case Types.INT32: - return isPrimitive ? PRIMITIVE_INT32 : INT32; + return mode == MODE_PRIMITIVE + ? PRIMITIVE_INT32 + : (mode == MODE_NOTNULL_BOXED ? NOTNULL_BOXED_INT32 : INT32); case Types.VARINT32: - return isPrimitive ? PRIMITIVE_VARINT32 : VARINT32; + return mode == MODE_PRIMITIVE + ? PRIMITIVE_VARINT32 + : (mode == MODE_NOTNULL_BOXED ? NOTNULL_BOXED_VARINT32 : VARINT32); case Types.INT64: - return isPrimitive ? PRIMITIVE_INT64 : INT64; + return mode == MODE_PRIMITIVE + ? PRIMITIVE_INT64 + : (mode == MODE_NOTNULL_BOXED ? NOTNULL_BOXED_INT64 : INT64); case Types.VARINT64: - return isPrimitive ? PRIMITIVE_VARINT64 : VARINT64; + return mode == MODE_PRIMITIVE + ? PRIMITIVE_VARINT64 + : (mode == MODE_NOTNULL_BOXED ? NOTNULL_BOXED_VARINT64 : VARINT64); case Types.TAGGED_INT64: - return isPrimitive ? PRIMITIVE_TAGGED_INT64 : TAGGED_INT64; + return mode == MODE_PRIMITIVE + ? PRIMITIVE_TAGGED_INT64 + : (mode == MODE_NOTNULL_BOXED ? NOTNULL_BOXED_TAGGED_INT64 : TAGGED_INT64); case Types.UINT8: - return isPrimitive ? PRIMITIVE_UINT8 : UINT8; + return mode == MODE_PRIMITIVE + ? PRIMITIVE_UINT8 + : (mode == MODE_NOTNULL_BOXED ? NOTNULL_BOXED_UINT8 : UINT8); case Types.UINT16: - return isPrimitive ? PRIMITIVE_UINT16 : UINT16; + return mode == MODE_PRIMITIVE + ? PRIMITIVE_UINT16 + : (mode == MODE_NOTNULL_BOXED ? NOTNULL_BOXED_UINT16 : UINT16); case Types.UINT32: - return isPrimitive ? PRIMITIVE_UINT32 : UINT32; + return mode == MODE_PRIMITIVE + ? PRIMITIVE_UINT32 + : (mode == MODE_NOTNULL_BOXED ? NOTNULL_BOXED_UINT32 : UINT32); case Types.VAR_UINT32: - return isPrimitive ? PRIMITIVE_VAR_UINT32 : VAR_UINT32; + return mode == MODE_PRIMITIVE + ? PRIMITIVE_VAR_UINT32 + : (mode == MODE_NOTNULL_BOXED ? NOTNULL_BOXED_VAR_UINT32 : VAR_UINT32); case Types.UINT64: - return isPrimitive ? PRIMITIVE_UINT64 : UINT64; + return mode == MODE_PRIMITIVE + ? PRIMITIVE_UINT64 + : (mode == MODE_NOTNULL_BOXED ? NOTNULL_BOXED_UINT64 : UINT64); case Types.VAR_UINT64: - return isPrimitive ? PRIMITIVE_VAR_UINT64 : VAR_UINT64; + return mode == MODE_PRIMITIVE + ? PRIMITIVE_VAR_UINT64 + : (mode == MODE_NOTNULL_BOXED ? NOTNULL_BOXED_VAR_UINT64 : VAR_UINT64); case Types.TAGGED_UINT64: - return isPrimitive ? PRIMITIVE_TAGGED_UINT64 : TAGGED_UINT64; + return mode == MODE_PRIMITIVE + ? PRIMITIVE_TAGGED_UINT64 + : (mode == MODE_NOTNULL_BOXED ? NOTNULL_BOXED_TAGGED_UINT64 : TAGGED_UINT64); case Types.FLOAT32: - return isPrimitive ? PRIMITIVE_FLOAT32 : FLOAT32; + return mode == MODE_PRIMITIVE + ? PRIMITIVE_FLOAT32 + : (mode == MODE_NOTNULL_BOXED ? NOTNULL_BOXED_FLOAT32 : FLOAT32); case Types.FLOAT64: - return isPrimitive ? PRIMITIVE_FLOAT64 : FLOAT64; + return mode == MODE_PRIMITIVE + ? PRIMITIVE_FLOAT64 + : (mode == MODE_NOTNULL_BOXED ? NOTNULL_BOXED_FLOAT64 : FLOAT64); case Types.STRING: return STRING; default: @@ -129,13 +199,14 @@ private static int xlangTypeIdToDispatchId(int typeId, boolean isPrimitive) { } } - private static int nativeIdToDispatchId( - int nativeId, Descriptor descriptor, boolean isPrimitive) { + private static int nativeIdToDispatchId(int nativeId, Descriptor descriptor, int mode) { if (nativeId >= Types.BOOL && nativeId <= ClassResolver.NATIVE_START_ID) { - return xlangTypeIdToDispatchId(nativeId, isPrimitive); + return xlangTypeIdToDispatchId(nativeId, mode); } if (nativeId == ClassResolver.CHAR_ID) { - return isPrimitive ? PRIMITIVE_CHAR : CHAR; + return mode == MODE_PRIMITIVE + ? PRIMITIVE_CHAR + : (mode == MODE_NOTNULL_BOXED ? NOTNULL_BOXED_CHAR : CHAR); } if (nativeId == ClassResolver.PRIMITIVE_CHAR_ID) { return PRIMITIVE_CHAR; @@ -147,10 +218,18 @@ private static int nativeIdToDispatchId( "%s should use `Types.BOOL~Types.FLOAT64` with nullable meta instead, but got %s", descriptor.getField(), nativeId)); } - return xlangTypeIdToDispatchId(nativeId, isPrimitive); + return xlangTypeIdToDispatchId(nativeId, mode); } public static boolean isPrimitive(int dispatchId) { return dispatchId >= PRIMITIVE_BOOL && dispatchId <= PRIMITIVE_TAGGED_UINT64; } + + public static boolean isNotnullBoxed(int dispatchId) { + return dispatchId >= NOTNULL_BOXED_BOOL && dispatchId <= NOTNULL_BOXED_TAGGED_UINT64; + } + + public static boolean isNullableBoxed(int dispatchId) { + return dispatchId >= BOOL && dispatchId <= TAGGED_UINT64; + } } diff --git a/rust/fory-core/src/buffer.rs b/rust/fory-core/src/buffer.rs index 8d72f4f5e3..9322ca35c9 100644 --- a/rust/fory-core/src/buffer.rs +++ b/rust/fory-core/src/buffer.rs @@ -522,7 +522,8 @@ impl<'a> Reader<'a> { #[inline(always)] pub fn sub_slice(&self, start: usize, end: usize) -> Result<&[u8], Error> { - if start >= self.bf.len() || end > self.bf.len() || end < start { + // Allow start == bf.len() when end == bf.len() to support empty slices at buffer end + if start > self.bf.len() || end > self.bf.len() || end < start { Err(Error::buffer_out_of_bound( start, self.bf.len(), From 446a2e716fc5348eb68e54811de2688cd9970e3c Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 16 Jan 2026 12:29:36 +0800 Subject: [PATCH 03/44] fix go test --- go/fory/skip.go | 3 --- .../org/apache/fory/xlang/GoXlangTest.java | 27 ++++++++++++++----- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/go/fory/skip.go b/go/fory/skip.go index 3f300fa530..dcb8306f37 100644 --- a/go/fory/skip.go +++ b/go/fory/skip.go @@ -461,9 +461,6 @@ func skipMap(ctx *ReadContext, fieldDef FieldDef) { // skipStruct skips a struct value using TypeInfo // Uses context error state for deferred error checking. func skipStruct(ctx *ReadContext, info *TypeInfo) { - err := ctx.Err() - // Read struct hash (4 bytes) - _ = ctx.buffer.ReadInt32(err) if ctx.HasError() { return } diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/GoXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/GoXlangTest.java index 0b463c60d4..ecc10456bb 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/GoXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/GoXlangTest.java @@ -151,9 +151,18 @@ public void testInteger(boolean enableCodegen) throws java.io.IOException { super.testInteger(enableCodegen); } - @Test(dataProvider = "enableCodegen") - public void testItem(boolean enableCodegen) throws java.io.IOException { - super.testItem(enableCodegen); + // this test failed more frequently when refactor, create two separate tests + // to make debug more easy + @Test + public void testItemEnableCodegen() throws java.io.IOException { + super.testItem(true); + } + + // this test failed more frequently when refactor, create two separate tests + // to make debug more easy + @Test + public void testItemDisableCodegen() throws java.io.IOException { + super.testItem(false); } @Test(dataProvider = "enableCodegen") @@ -316,10 +325,14 @@ public void testNullableFieldSchemaConsistentNull(boolean enableCodegen) super.testNullableFieldSchemaConsistentNull(enableCodegen); } - @Override - @Test(dataProvider = "enableCodegen") - public void testNullableFieldCompatibleNotNull(boolean enableCodegen) throws java.io.IOException { - super.testNullableFieldCompatibleNotNull(enableCodegen); + @Test + public void testNullableFieldCompatibleNotNullEnableCodegen() throws java.io.IOException { + super.testNullableFieldCompatibleNotNull(true); + } + + @Test + public void testNullableFieldCompatibleNotNullDisableCodegen() throws java.io.IOException { + super.testNullableFieldCompatibleNotNull(false); } @Override From 1e24fcac1a8bb5352cbfd76b0e86ca4e982947fe Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 16 Jan 2026 13:49:30 +0800 Subject: [PATCH 04/44] fix metashared nullable read --- .../fory/serializer/MetaSharedSerializer.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java index 286f1ef282..f026beeb2a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java @@ -229,10 +229,17 @@ public T read(MemoryBuffer buffer) { int dispatchId = fieldInfo.dispatchId; boolean needRead = true; if (fieldInfo.isPrimitive) { - // PRIMITIVE_* dispatch IDs mean non-nullable, so no null flag is read - needRead = - AbstractObjectSerializer.readPrimitiveFieldValue( - buffer, obj, fieldAccessor, dispatchId); + if (nullable) { + // Remote wrote with null flag (e.g., Go's *int32), read null flag first + needRead = + AbstractObjectSerializer.readPrimitiveNullableFieldValue( + buffer, obj, fieldAccessor, dispatchId); + } else { + // PRIMITIVE_* dispatch IDs with nullable=false mean no null flag is read + needRead = + AbstractObjectSerializer.readPrimitiveFieldValue( + buffer, obj, fieldAccessor, dispatchId); + } } // For NOTNULL_BOXED and other boxed types, let readBasicObjectFieldValue handle it // which uses putObject for proper boxing From 3d8daa0807506ac490b49dc21ffc25b19664fbee Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 16 Jan 2026 20:18:57 +0800 Subject: [PATCH 05/44] refactor java object serializer --- .../src/main/java/org/apache/fory/Fory.java | 77 ++++++- .../apache/fory/resolver/ClassResolver.java | 4 +- .../serializer/AbstractObjectSerializer.java | 162 +------------- .../apache/fory/serializer/FieldGroups.java | 4 +- .../serializer/MetaSharedLayerSerializer.java | 205 +----------------- .../fory/serializer/MetaSharedSerializer.java | 151 +++++-------- .../NonexistentClassSerializers.java | 31 +-- .../fory/serializer/ObjectSerializer.java | 69 +----- .../fory/serializer/SerializationBinding.java | 135 ++++++++++++ 9 files changed, 295 insertions(+), 543 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java index e8bbf6ad60..9b6c1eec54 100644 --- a/java/fory-core/src/main/java/org/apache/fory/Fory.java +++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java @@ -472,6 +472,12 @@ public void writeNonRef(MemoryBuffer buffer, Object obj, Serializer serializer) depth--; } + public void writeNonRef(MemoryBuffer buffer, Object obj, ClassInfoHolder holder) { + ClassInfo classInfo = classResolver.getClassInfo(obj.getClass(), holder); + classResolver.writeClassInfo(buffer, classInfo); + writeData(buffer, classInfo, obj); + } + public void writeNonRef(MemoryBuffer buffer, Object obj, ClassInfo classInfo) { classResolver.writeClassInfo(buffer, classInfo); Serializer serializer = classInfo.getSerializer(); @@ -526,6 +532,12 @@ public void xwriteNonRef(MemoryBuffer buffer, Object obj) { xwriteData(buffer, classInfo, obj); } + public void xwriteNonRef(MemoryBuffer buffer, Object obj, ClassInfoHolder holder) { + ClassInfo classInfo = xtypeResolver.getClassInfo(obj.getClass(), holder); + xtypeResolver.writeClassInfo(buffer, obj); + xwriteData(buffer, classInfo, obj); + } + public void xwriteNonRef(MemoryBuffer buffer, Object obj, ClassInfo classInfo) { xtypeResolver.writeClassInfo(buffer, classInfo); xwriteData(buffer, classInfo, obj); @@ -879,6 +891,19 @@ public Object readRef(MemoryBuffer buffer) { } } + public Object readRef(MemoryBuffer buffer, ClassInfo classInfo) { + RefResolver refResolver = this.refResolver; + int nextReadRefId = refResolver.tryPreserveRefId(buffer); + if (nextReadRefId >= NOT_NULL_VALUE_FLAG) { + // ref value or not-null value + Object o = readDataInternal(buffer, classInfo); + refResolver.setReadObject(nextReadRefId, o); + return o; + } else { + return refResolver.getReadObject(); + } + } + public Object readRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { RefResolver refResolver = this.refResolver; int nextReadRefId = refResolver.tryPreserveRefId(buffer); @@ -996,6 +1021,38 @@ private Object readDataInternal(MemoryBuffer buffer, ClassInfo classInfo) { } } + private Object xreadDataInternal(MemoryBuffer buffer, ClassInfo classInfo) { + switch (classInfo.getClassId()) { + case Types.BOOL: + return buffer.readBoolean(); + case Types.INT8: + return buffer.readByte(); + case ClassResolver.CHAR_ID: + return buffer.readChar(); + case Types.INT16: + return buffer.readInt16(); + case Types.INT32: + if (compressInt) { + return buffer.readVarInt32(); + } else { + return buffer.readInt32(); + } + case Types.FLOAT32: + return buffer.readFloat32(); + case Types.INT64: + return LongSerializer.readInt64(buffer, longEncoding); + case Types.FLOAT64: + return buffer.readFloat64(); + case Types.STRING: + return stringSerializer.readJavaString(buffer); + default: + incReadDepth(); + Object read = classInfo.getSerializer().xread(buffer); + depth--; + return read; + } + } + public Object xreadRef(MemoryBuffer buffer) { RefResolver refResolver = this.refResolver; int nextReadRefId = refResolver.tryPreserveRefId(buffer); @@ -1008,12 +1065,25 @@ public Object xreadRef(MemoryBuffer buffer) { } } + public Object xreadRef(MemoryBuffer buffer, ClassInfo classInfo) { + RefResolver refResolver = this.refResolver; + int nextReadRefId = refResolver.tryPreserveRefId(buffer); + if (nextReadRefId >= NOT_NULL_VALUE_FLAG) { + // ref value or not-null value + Object o = xreadDataInternal(buffer, classInfo); + refResolver.setReadObject(nextReadRefId, o); + return o; + } else { + return refResolver.getReadObject(); + } + } + public Object xreadRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { RefResolver refResolver = this.refResolver; int nextReadRefId = refResolver.tryPreserveRefId(buffer); if (nextReadRefId >= NOT_NULL_VALUE_FLAG) { // ref value or not-null value - Object o = readDataInternal(buffer, xtypeResolver.readClassInfo(buffer, classInfoHolder)); + Object o = xreadDataInternal(buffer, xtypeResolver.readClassInfo(buffer, classInfoHolder)); refResolver.setReadObject(nextReadRefId, o); return o; } else { @@ -1089,6 +1159,11 @@ public Object xreadNonRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) return xreadNonRef(buffer, classInfo); } + public Object xreadNullable(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { + ClassInfo classInfo = xtypeResolver.readClassInfo(buffer, classInfoHolder); + return xreadNullable(buffer, classInfo.getSerializer()); + } + public Object xreadNullable(MemoryBuffer buffer, Serializer serializer) { byte headFlag = buffer.readByte(); if (headFlag == Fory.NULL_FLAG) { diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index babebd8ecb..3cead16341 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -710,9 +710,6 @@ public boolean isMonomorphic(Class clz) { if (!ReflectionUtils.isMonomorphic(clz)) { return false; } - if (Map.class.isAssignableFrom(clz) || Collection.class.isAssignableFrom(clz)) { - return true; - } if (clz.isArray()) { Class component = TypeUtils.getArrayComponent(clz); return isMonomorphic(component); @@ -722,6 +719,7 @@ public boolean isMonomorphic(Class clz) { if (Union.class.isAssignableFrom(clz)) { return true; } + // if internal registered and final, then taken as morphic return (isInternalRegistered(clz) || clz.isEnum()); } return ReflectionUtils.isMonomorphic(clz); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java index 1205adde3e..ad8b62a8ab 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java @@ -68,52 +68,6 @@ public AbstractObjectSerializer(Fory fory, Class type, ObjectCreator objec this.objectCreator = objectCreator; } - static void writeOtherFieldValue( - SerializationBinding binding, - MemoryBuffer buffer, - SerializationFieldInfo fieldInfo, - Object fieldValue) { - if (fieldInfo.useDeclaredTypeInfo) { - switch (fieldInfo.refMode) { - case NONE: - binding.writeNonRef(buffer, fieldValue, fieldInfo.serializer); - break; - case NULL_ONLY: - if (fieldValue == null) { - buffer.writeByte(Fory.NULL_FLAG); - } else { - buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); - binding.writeNonRef(buffer, fieldValue, fieldInfo.serializer); - } - break; - case TRACKING: - binding.writeRef(buffer, fieldValue, fieldInfo.serializer); - break; - default: - throw new IllegalStateException("Unexpected refMode: " + fieldInfo.refMode); - } - } else { - switch (fieldInfo.refMode) { - case NONE: - binding.writeNonRef(buffer, fieldValue, fieldInfo.classInfoHolder); - break; - case NULL_ONLY: - if (fieldValue == null) { - buffer.writeByte(Fory.NULL_FLAG); - } else { - buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); - binding.writeNonRef(buffer, fieldValue, fieldInfo.classInfoHolder); - } - break; - case TRACKING: - binding.writeRef(buffer, fieldValue, fieldInfo.classInfoHolder); - break; - default: - throw new IllegalStateException("Unexpected refMode: " + fieldInfo.refMode); - } - } - } - static void writeContainerFieldValue( SerializationBinding binding, RefResolver refResolver, @@ -293,7 +247,7 @@ static boolean writePrimitiveFieldValue( * * @return true if field value isn't written by this function. */ - static boolean writeBasicObjectFieldValue( + static boolean writeNotNullBasicObjectFieldValue( Fory fory, MemoryBuffer buffer, Object fieldValue, int dispatchId) { if (fieldValue == null) { throw new IllegalArgumentException( @@ -375,7 +329,7 @@ static boolean writeBasicObjectFieldValue( * @return true if dispatchId is not a basic type or ref tracking is enabled, needing further * write handling */ - static boolean writeBasicNullableObjectFieldValue( + static boolean writeNullableBasicObjectFieldValue( Fory fory, MemoryBuffer buffer, Object fieldValue, int dispatchId) { if (!fory.isBasicTypesRefIgnored()) { return true; // let common path handle this. @@ -511,65 +465,8 @@ static boolean writeBasicNullableObjectFieldValue( * because primitive field doesn't write null flag. */ static Object readFinalObjectFieldValue( - SerializationBinding binding, - RefResolver refResolver, - TypeResolver typeResolver, - SerializationFieldInfo fieldInfo, - MemoryBuffer buffer) { - Serializer serializer = fieldInfo.classInfo.getSerializer(); - binding.incReadDepth(); - Object fieldValue; - if (fieldInfo.useDeclaredTypeInfo) { - switch (fieldInfo.refMode) { - case NONE: - fieldValue = binding.read(buffer, serializer); - break; - case NULL_ONLY: - fieldValue = binding.readNullable(buffer, serializer); - break; - case TRACKING: - // whether tracking ref is recorded in `fieldInfo.serializer`, so it's still - // consistent with jit serializer. - fieldValue = binding.readRef(buffer, serializer); - break; - default: - throw new IllegalStateException("Unknown refMode: " + fieldInfo.refMode); - } - } else { - switch (fieldInfo.refMode) { - case NONE: - typeResolver.readClassInfo(buffer, fieldInfo.classInfo); - fieldValue = serializer.read(buffer); - break; - case NULL_ONLY: - { - byte headFlag = buffer.readByte(); - if (headFlag == Fory.NULL_FLAG) { - binding.decDepth(); - return null; - } - typeResolver.readClassInfo(buffer, fieldInfo.classInfo); - fieldValue = serializer.read(buffer); - } - break; - case TRACKING: - { - int nextReadRefId = refResolver.tryPreserveRefId(buffer); - if (nextReadRefId >= Fory.NOT_NULL_VALUE_FLAG) { - typeResolver.readClassInfo(buffer, fieldInfo.classInfo); - fieldValue = serializer.read(buffer); - refResolver.setReadObject(nextReadRefId, fieldValue); - } else { - fieldValue = refResolver.getReadObject(); - } - } - break; - default: - throw new IllegalStateException("Unknown refMode: " + fieldInfo.refMode); - } - } - binding.decDepth(); - return fieldValue; + SerializationBinding binding, SerializationFieldInfo fieldInfo, MemoryBuffer buffer) { + return binding.read(fieldInfo, buffer); } /** @@ -583,37 +480,7 @@ static Object readFinalObjectFieldValue( */ static Object readOtherFieldValue( SerializationBinding binding, SerializationFieldInfo fieldInfo, MemoryBuffer buffer) { - // Note: Enum has special handling for xlang compatibility - no type info for enum fields - if (fieldInfo.genericType.getCls().isEnum()) { - // Only read null flag when the field is nullable (for xlang compatibility) - if (fieldInfo.nullable && buffer.readByte() == Fory.NULL_FLAG) { - return null; - } - return fieldInfo.genericType.getSerializer(binding.typeResolver).read(buffer); - } - Object fieldValue; - switch (fieldInfo.refMode) { - case NONE: - binding.preserveRefId(-1); - fieldValue = binding.readNonRef(buffer, fieldInfo); - break; - case NULL_ONLY: - { - binding.preserveRefId(-1); - byte headFlag = buffer.readByte(); - if (headFlag == Fory.NULL_FLAG) { - return null; - } - fieldValue = binding.readNonRef(buffer, fieldInfo); - } - break; - case TRACKING: - fieldValue = binding.readRef(buffer, fieldInfo); - break; - default: - throw new IllegalStateException("Unknown refMode: " + fieldInfo.refMode); - } - return fieldValue; + return binding.read(fieldInfo, buffer); } /** @@ -789,25 +656,6 @@ private static boolean readPrimitiveFieldValue( } } - /** - * Read a nullable primitive field value from buffer. Reads the null flag first and returns early - * if null. - * - * @param buffer the buffer to read from - * @param targetObject the object to set the field value on - * @param fieldAccessor the accessor to set the field value - * @param dispatchId the class ID of the primitive type - * @return true if dispatchId is not a primitive type and needs further read handling; false if - * value was null or successfully read - */ - static boolean readPrimitiveNullableFieldValue( - MemoryBuffer buffer, Object targetObject, FieldAccessor fieldAccessor, int dispatchId) { - if (buffer.readByte() == Fory.NULL_FLAG) { - return false; - } - return readPrimitiveFieldValue(buffer, targetObject, fieldAccessor, dispatchId); - } - /** * read field value from buffer. This method handle the situation which all fields are not null. * diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java index e0f3de50fa..36327b7053 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java @@ -128,7 +128,7 @@ public static final class SerializationFieldInfo { public final RefMode refMode; public final boolean nullable; public final boolean trackingRef; - public final boolean isPrimitive; + public final boolean isPrimitiveField; // Use declared type for serialization/deserialization public final boolean useDeclaredTypeInfo; @@ -171,7 +171,7 @@ public static final class SerializationFieldInfo { // This ensures correct handling in schema compatible mode where local field type // may differ from remote (ClassDef) field type. // Note: NOTNULL_BOXED dispatch IDs are NOT primitive - they are handled by readBasicObjectFieldValue - isPrimitive = DispatchId.isPrimitive(dispatchId); + isPrimitiveField = DispatchId.isPrimitive(dispatchId); fieldConverter = d.getFieldConverter(); nullable = d.isNullable(); // descriptor.isTrackingRef() already includes the needToWriteRef check diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java index 9d089f9045..6293a3198b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java @@ -34,7 +34,6 @@ import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo; import org.apache.fory.type.Descriptor; import org.apache.fory.type.DescriptorGrouper; -import org.apache.fory.type.DispatchId; import org.apache.fory.type.Generics; /** @@ -119,8 +118,6 @@ private void writeLayerClassMeta(MemoryBuffer buffer) { private void writeFinalFields(MemoryBuffer buffer, T value) { Fory fory = this.fory; - RefResolver refResolver = this.refResolver; - boolean metaShareEnabled = fory.getConfig().isMetaShareEnabled(); for (SerializationFieldInfo fieldInfo : buildInFields) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; boolean nullable = fieldInfo.nullable; @@ -128,30 +125,11 @@ private void writeFinalFields(MemoryBuffer buffer, T value) { if (AbstractObjectSerializer.writePrimitiveFieldValue( buffer, value, fieldAccessor, dispatchId)) { Object fieldValue = fieldAccessor.getObject(value); - boolean writeBasicObjectResult = - nullable - ? AbstractObjectSerializer.writeBasicNullableObjectFieldValue( - fory, buffer, fieldValue, dispatchId) - : AbstractObjectSerializer.writeBasicObjectFieldValue( - fory, buffer, fieldValue, dispatchId); - if (writeBasicObjectResult) { - Serializer serializer = fieldInfo.classInfo.getSerializer(); - if (!metaShareEnabled || fieldInfo.useDeclaredTypeInfo) { - if (!fieldInfo.trackingRef) { - binding.writeNullable(buffer, fieldValue, serializer, nullable); - } else { - binding.writeRef(buffer, fieldValue, serializer); - } - } else { - if (fieldInfo.trackingRef && serializer.needToWriteRef()) { - if (!refResolver.writeRefOrNull(buffer, fieldValue)) { - typeResolver.writeClassInfo(buffer, fieldInfo.classInfo); - binding.write(buffer, serializer, fieldValue); - } - } else { - binding.writeNullable(buffer, fieldValue, serializer, nullable); - } - } + boolean needWrite = + nullable ? writeNullableBasicObjectFieldValue(fory, buffer, fieldValue, dispatchId) + : writeNotNullBasicObjectFieldValue(fory, buffer, fieldValue, dispatchId); + if (needWrite) { + binding.write(fieldInfo, buffer, fieldValue); } } } @@ -171,7 +149,7 @@ private void writeOtherFields(MemoryBuffer buffer, T value) { for (SerializationFieldInfo fieldInfo : otherFields) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; Object fieldValue = fieldAccessor.getObject(value); - AbstractObjectSerializer.writeOtherFieldValue(binding, buffer, fieldInfo, fieldValue); + binding.write(fieldInfo, buffer, fieldValue); } } @@ -239,18 +217,17 @@ private void readFinalFields(MemoryBuffer buffer, T obj) { fory, buffer, obj, fieldAccessor, dispatchId))) { Object fieldValue = AbstractObjectSerializer.readFinalObjectFieldValue( - binding, refResolver, classResolver, fieldInfo, buffer); + binding, fieldInfo, buffer); fieldAccessor.putObject(obj, fieldValue); } } else { // Field doesn't exist in current class - skip the value - if (MetaSharedSerializer.skipPrimitiveFieldValueFailed( - fory, fieldInfo.dispatchId, buffer)) { + if (!MetaSharedSerializer.skipPrimitiveFieldValue(fieldInfo, buffer)) { if (fieldInfo.classInfo == null) { fory.readRef(buffer, classInfoHolder); } else { AbstractObjectSerializer.readFinalObjectFieldValue( - binding, refResolver, classResolver, fieldInfo, buffer); + binding, fieldInfo, buffer); } } } @@ -289,175 +266,11 @@ public T xread(MemoryBuffer buffer) { return read(buffer); } - /** Returns the ClassDef for this layer. */ - public ClassDef getLayerClassDef() { - return layerClassDef; - } - - /** Returns the marker class used as key in metaContext.classMap. */ - public Class getLayerMarkerClass() { - return layerMarkerClass; - } - /** Returns the number of fields in this layer. */ public int getNumFields() { return buildInFields.length + containerFields.length + otherFields.length; } - /** - * Write field values from an array. Used by putFields/writeFields in ObjectStreamSerializer. - * - * @param buffer the memory buffer to write to - * @param vals array of field values in the same order as fields are declared - */ - public void writeFieldsValues(MemoryBuffer buffer, Object[] vals) { - // Write layer class meta using marker class as key (only if meta share is enabled) - if (fory.getConfig().isMetaShareEnabled()) { - writeLayerClassMeta(buffer); - } - // Write field values from array - int index = 0; - // Write final fields - for (SerializationFieldInfo fieldInfo : buildInFields) { - Object fieldValue = vals[index++]; - writeFieldValueFromArray(buffer, fieldInfo, fieldValue); - } - // Write container fields - Generics generics = fory.getGenerics(); - for (SerializationFieldInfo fieldInfo : containerFields) { - Object fieldValue = vals[index++]; - AbstractObjectSerializer.writeContainerFieldValue( - binding, refResolver, typeResolver, generics, fieldInfo, buffer, fieldValue); - } - // Write other fields - for (SerializationFieldInfo fieldInfo : otherFields) { - Object fieldValue = vals[index++]; - AbstractObjectSerializer.writeOtherFieldValue(binding, buffer, fieldInfo, fieldValue); - } - } - - private void writeFieldValueFromArray( - MemoryBuffer buffer, SerializationFieldInfo fieldInfo, Object fieldValue) { - int dispatchId = fieldInfo.dispatchId; - boolean nullable = fieldInfo.nullable; - - // Handle primitives first - switch (dispatchId) { - case DispatchId.PRIMITIVE_BOOL: - buffer.writeBoolean((Boolean) fieldValue); - return; - case DispatchId.PRIMITIVE_INT8: - case DispatchId.PRIMITIVE_UINT8: - buffer.writeByte((Byte) fieldValue); - return; - case DispatchId.PRIMITIVE_CHAR: - buffer.writeChar((Character) fieldValue); - return; - case DispatchId.PRIMITIVE_INT16: - case DispatchId.PRIMITIVE_UINT16: - buffer.writeInt16((Short) fieldValue); - return; - case DispatchId.PRIMITIVE_INT32: - buffer.writeInt32((Integer) fieldValue); - return; - case DispatchId.PRIMITIVE_VARINT32: - buffer.writeVarInt32((Integer) fieldValue); - return; - case DispatchId.PRIMITIVE_UINT32: - buffer.writeInt32((Integer) fieldValue); - return; - case DispatchId.PRIMITIVE_VAR_UINT32: - buffer.writeVarUint32((Integer) fieldValue); - return; - case DispatchId.PRIMITIVE_INT64: - buffer.writeInt64((Long) fieldValue); - return; - case DispatchId.PRIMITIVE_VARINT64: - buffer.writeVarInt64((Long) fieldValue); - return; - case DispatchId.PRIMITIVE_TAGGED_INT64: - buffer.writeTaggedInt64((Long) fieldValue); - return; - case DispatchId.PRIMITIVE_UINT64: - buffer.writeInt64((Long) fieldValue); - return; - case DispatchId.PRIMITIVE_VAR_UINT64: - buffer.writeVarUint64((Long) fieldValue); - return; - case DispatchId.PRIMITIVE_TAGGED_UINT64: - buffer.writeTaggedUint64((Long) fieldValue); - return; - case DispatchId.PRIMITIVE_FLOAT32: - buffer.writeFloat32((Float) fieldValue); - return; - case DispatchId.PRIMITIVE_FLOAT64: - buffer.writeFloat64((Double) fieldValue); - return; - default: - break; - } - - // Handle objects - boolean metaShareEnabled = fory.getConfig().isMetaShareEnabled(); - Serializer serializer = fieldInfo.classInfo.getSerializer(); - if (!metaShareEnabled || fieldInfo.useDeclaredTypeInfo) { - if (!fieldInfo.trackingRef) { - binding.writeNullable(buffer, fieldValue, serializer, nullable); - } else { - binding.writeRef(buffer, fieldValue, serializer); - } - } else { - if (fieldInfo.trackingRef && serializer.needToWriteRef()) { - if (!refResolver.writeRefOrNull(buffer, fieldValue)) { - typeResolver.writeClassInfo(buffer, fieldInfo.classInfo); - binding.write(buffer, serializer, fieldValue); - } - } else { - binding.writeNullable(buffer, fieldValue, serializer, nullable); - } - } - } - - /** - * Read field values into an array. Used by readFields in ObjectStreamSerializer. - * - * @param buffer the memory buffer to read from - * @param vals array to store field values - */ - public void readFields(MemoryBuffer buffer, Object[] vals) { - // Read and verify layer class meta (only if meta share is enabled) - if (fory.getConfig().isMetaShareEnabled()) { - readLayerClassMeta(buffer); - } - // Read field values into array - int index = 0; - // Read final fields - for (SerializationFieldInfo fieldInfo : buildInFields) { - vals[index++] = readFieldValueToArray(buffer, fieldInfo); - } - // Read container fields - Generics generics = fory.getGenerics(); - for (SerializationFieldInfo fieldInfo : containerFields) { - vals[index++] = - AbstractObjectSerializer.readContainerFieldValue(binding, generics, fieldInfo, buffer); - } - // Read other fields - for (SerializationFieldInfo fieldInfo : otherFields) { - vals[index++] = AbstractObjectSerializer.readOtherFieldValue(binding, fieldInfo, buffer); - } - } - - private Object readFieldValueToArray(MemoryBuffer buffer, SerializationFieldInfo fieldInfo) { - int dispatchId = fieldInfo.dispatchId; - // Handle primitives - if (DispatchId.isPrimitive(dispatchId)) { - return Serializers.readPrimitiveValue(buffer, dispatchId); - } - // Handle objects - return AbstractObjectSerializer.readFinalObjectFieldValue( - binding, refResolver, classResolver, fieldInfo, buffer); - } - /** * Set field values on target object from putFields data. This method maps field names from * ObjectStreamClass to actual class fields and sets matching values. diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java index f026beeb2a..2645d903c6 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java @@ -167,22 +167,6 @@ public void xwrite(MemoryBuffer buffer, T value) { @Override public T read(MemoryBuffer buffer) { - if (Utils.debugOutputEnabled()) { - LOG.info("========== MetaSharedSerializer.read() for {} ==========", type.getName()); - LOG.info("Buffer readerIndex at start: {}", buffer.readerIndex()); - LOG.info("buildInFields count: {}", buildInFields.length); - for (int i = 0; i < buildInFields.length; i++) { - SerializationFieldInfo fi = buildInFields[i]; - LOG.info( - " buildInField[{}]: name={}, dispatchId={}, nullable={}, isPrimitive={}, hasAccessor={}", - i, - fi.qualifiedFieldName, - fi.dispatchId, - fi.nullable, - fi.isPrimitive, - fi.fieldAccessor != null); - } - } if (isRecord) { Object[] fieldValues = new Object[buildInFields.length + otherFields.length + containerFields.length]; @@ -195,76 +179,38 @@ public T read(MemoryBuffer buffer) { T obj = newInstance(); Fory fory = this.fory; RefResolver refResolver = this.refResolver; - TypeResolver typeResolver = this.typeResolver; SerializationBinding binding = this.binding; refResolver.reference(obj); // read order: primitive,boxed,final,other,collection,map for (SerializationFieldInfo fieldInfo : this.buildInFields) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; boolean nullable = fieldInfo.nullable; - if (Utils.debugOutputEnabled()) { - LOG.info( - "[Java] About to read field: name={}, dispatchId={}, nullable={}, isPrimitive={}, bufferPos={}", - fieldInfo.qualifiedFieldName, - fieldInfo.dispatchId, - nullable, - fieldInfo.isPrimitive, - buffer.readerIndex()); - // Print next 16 bytes from buffer for debugging - int pos = buffer.readerIndex(); - int remaining = Math.min(16, buffer.size() - pos); - if (remaining > 0) { - byte[] peek = new byte[remaining]; - for (int i = 0; i < remaining; i++) { - peek[i] = buffer.getByte(pos + i); - } - StringBuilder hex = new StringBuilder(); - for (byte b : peek) { - hex.append(String.format("%02x", b)); - } - LOG.info("[Java] Next {} bytes at pos {}: {}", remaining, pos, hex.toString()); - } - } if (fieldAccessor != null) { int dispatchId = fieldInfo.dispatchId; - boolean needRead = true; - if (fieldInfo.isPrimitive) { - if (nullable) { - // Remote wrote with null flag (e.g., Go's *int32), read null flag first - needRead = - AbstractObjectSerializer.readPrimitiveNullableFieldValue( - buffer, obj, fieldAccessor, dispatchId); - } else { - // PRIMITIVE_* dispatch IDs with nullable=false mean no null flag is read - needRead = - AbstractObjectSerializer.readPrimitiveFieldValue( - buffer, obj, fieldAccessor, dispatchId); + if (fieldInfo.isPrimitiveField && (!nullable || buffer.readByte() != Fory.NULL_FLAG)) { + if (!readPrimitiveFieldValue(buffer, obj, fieldAccessor, dispatchId)) { + continue; } } // For NOTNULL_BOXED and other boxed types, let readBasicObjectFieldValue handle it // which uses putObject for proper boxing - if (needRead - && (nullable - ? AbstractObjectSerializer.readBasicNullableObjectFieldValue( - fory, buffer, obj, fieldAccessor, dispatchId) - : AbstractObjectSerializer.readBasicObjectFieldValue( - fory, buffer, obj, fieldAccessor, dispatchId))) { + if (nullable ? readBasicNullableObjectFieldValue(fory, buffer, obj, fieldAccessor, dispatchId) + : readBasicObjectFieldValue(fory, buffer, obj, fieldAccessor, dispatchId)) { assert fieldInfo.classInfo != null; Object fieldValue = - AbstractObjectSerializer.readFinalObjectFieldValue( - binding, refResolver, typeResolver, fieldInfo, buffer); + readFinalObjectFieldValue(binding, fieldInfo, buffer); fieldAccessor.putObject(obj, fieldValue); } } else { if (fieldInfo.fieldConverter == null) { // Skip the field value from buffer since it doesn't exist in current class - if (skipPrimitiveFieldValueFailed(fory, fieldInfo.dispatchId, buffer)) { + if (!skipPrimitiveFieldValue(fieldInfo, buffer)) { if (fieldInfo.classInfo == null) { // TODO(chaokunyang) support registered serializer in peer with ref tracking disabled. binding.readRef(buffer, classInfoHolder); } else { - AbstractObjectSerializer.readFinalObjectFieldValue( - binding, refResolver, typeResolver, fieldInfo, buffer); + readFinalObjectFieldValue( + binding, fieldInfo, buffer); } } } else { @@ -297,9 +243,7 @@ private void compatibleRead(MemoryBuffer buffer, SerializationFieldInfo fieldInf if (DispatchId.isPrimitive(dispatchId)) { fieldValue = Serializers.readPrimitiveValue(buffer, dispatchId); } else { - fieldValue = - AbstractObjectSerializer.readFinalObjectFieldValue( - binding, refResolver, classResolver, fieldInfo, buffer); + fieldValue = readFinalObjectFieldValue(binding, fieldInfo, buffer); } fieldInfo.fieldConverter.set(obj, fieldValue); } @@ -322,33 +266,24 @@ public T xread(MemoryBuffer buffer) { private void readFields(MemoryBuffer buffer, Object[] fields) { int counter = 0; Fory fory = this.fory; - RefResolver refResolver = this.refResolver; - ClassResolver classResolver = this.classResolver; SerializationBinding binding = this.binding; // read order: primitive,boxed,final,other,collection,map for (SerializationFieldInfo fieldInfo : this.buildInFields) { if (fieldInfo.fieldAccessor != null) { assert fieldInfo.classInfo != null; int dispatchId = fieldInfo.dispatchId; - // primitive field won't write null flag. if (DispatchId.isPrimitive(dispatchId)) { fields[counter++] = Serializers.readPrimitiveValue(buffer, dispatchId); } else { Object fieldValue = - AbstractObjectSerializer.readFinalObjectFieldValue( - binding, refResolver, classResolver, fieldInfo, buffer); + readFinalObjectFieldValue( + binding, fieldInfo, buffer); fields[counter++] = fieldValue; } } else { // Skip the field value from buffer since it doesn't exist in current class - if (skipPrimitiveFieldValueFailed(fory, fieldInfo.dispatchId, buffer)) { - if (fieldInfo.classInfo == null) { - // TODO(chaokunyang) support registered serializer in peer with ref tracking disabled. - fory.readRef(buffer, classInfoHolder); - } else { - AbstractObjectSerializer.readFinalObjectFieldValue( - binding, refResolver, classResolver, fieldInfo, buffer); - } + if (!skipPrimitiveFieldValue(fieldInfo, buffer)) { + skipObjectFieldValue(buffer, fieldInfo); } // remapping will handle those extra fields from peers. fields[counter++] = null; @@ -366,79 +301,99 @@ private void readFields(MemoryBuffer buffer, Object[] fields) { } } + private void skipObjectFieldValue(MemoryBuffer buffer, SerializationFieldInfo fieldInfo) { + if (fieldInfo.classInfo == null) { + switch (fieldInfo.refMode) { + case NONE: + binding.readNonRef(buffer, fieldInfo); + break; + case NULL_ONLY: + binding.readNullable(buffer, fieldInfo); + break; + case TRACKING: + binding.readRef(buffer, fieldInfo); + break; + default: + } + } else { + readFinalObjectFieldValue( + binding, fieldInfo, buffer); + } + } + /** Skip primitive/notnull-boxed field value since they don't write null flag. */ - static boolean skipPrimitiveFieldValueFailed(Fory fory, int dispatchId, MemoryBuffer buffer) { - switch (dispatchId) { + static boolean skipPrimitiveFieldValue(SerializationFieldInfo fieldInfo, MemoryBuffer buffer) { + switch (fieldInfo.dispatchId) { case DispatchId.PRIMITIVE_BOOL: case DispatchId.NOTNULL_BOXED_BOOL: buffer.increaseReaderIndex(1); - return false; + return true; case DispatchId.PRIMITIVE_INT8: case DispatchId.PRIMITIVE_UINT8: case DispatchId.NOTNULL_BOXED_INT8: case DispatchId.NOTNULL_BOXED_UINT8: buffer.increaseReaderIndex(1); - return false; + return true; case DispatchId.PRIMITIVE_CHAR: case DispatchId.NOTNULL_BOXED_CHAR: buffer.increaseReaderIndex(2); - return false; + return true; case DispatchId.PRIMITIVE_INT16: case DispatchId.PRIMITIVE_UINT16: case DispatchId.NOTNULL_BOXED_INT16: case DispatchId.NOTNULL_BOXED_UINT16: buffer.increaseReaderIndex(2); - return false; + return true; case DispatchId.PRIMITIVE_INT32: case DispatchId.NOTNULL_BOXED_INT32: buffer.increaseReaderIndex(4); - return false; + return true; case DispatchId.PRIMITIVE_VARINT32: case DispatchId.NOTNULL_BOXED_VARINT32: buffer.readVarInt32(); - return false; + return true; case DispatchId.PRIMITIVE_UINT32: case DispatchId.NOTNULL_BOXED_UINT32: buffer.increaseReaderIndex(4); - return false; + return true; case DispatchId.PRIMITIVE_VAR_UINT32: case DispatchId.NOTNULL_BOXED_VAR_UINT32: buffer.readVarUint32(); - return false; + return true; case DispatchId.PRIMITIVE_INT64: case DispatchId.NOTNULL_BOXED_INT64: buffer.increaseReaderIndex(8); - return false; + return true; case DispatchId.PRIMITIVE_VARINT64: case DispatchId.NOTNULL_BOXED_VARINT64: buffer.readVarInt64(); - return false; + return true; case DispatchId.PRIMITIVE_TAGGED_INT64: case DispatchId.NOTNULL_BOXED_TAGGED_INT64: buffer.readTaggedInt64(); - return false; + return true; case DispatchId.PRIMITIVE_UINT64: case DispatchId.NOTNULL_BOXED_UINT64: buffer.increaseReaderIndex(8); - return false; + return true; case DispatchId.PRIMITIVE_VAR_UINT64: case DispatchId.NOTNULL_BOXED_VAR_UINT64: buffer.readVarUint64(); - return false; + return true; case DispatchId.PRIMITIVE_TAGGED_UINT64: case DispatchId.NOTNULL_BOXED_TAGGED_UINT64: buffer.readTaggedUint64(); - return false; + return true; case DispatchId.PRIMITIVE_FLOAT32: case DispatchId.NOTNULL_BOXED_FLOAT32: buffer.increaseReaderIndex(4); - return false; + return true; case DispatchId.PRIMITIVE_FLOAT64: case DispatchId.NOTNULL_BOXED_FLOAT64: buffer.increaseReaderIndex(8); - return false; - default: return true; + default: + return false; } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java index c6cf0e2e97..88b1f97132 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java @@ -19,7 +19,6 @@ package org.apache.fory.serializer; -import static org.apache.fory.serializer.AbstractObjectSerializer.writeOtherFieldValue; import static org.apache.fory.serializer.SerializationUtils.getTypeResolver; import java.util.ArrayList; @@ -69,14 +68,12 @@ private ClassFieldsInfo(FieldGroups fieldGroups, int classVersionHash) { public static final class NonexistentClassSerializer extends Serializer { private final ClassDef classDef; - private final ClassInfoHolder classInfoHolder; private final LongMap fieldsInfoMap; private final SerializationBinding binding; public NonexistentClassSerializer(Fory fory, ClassDef classDef) { super(fory, NonexistentClass.NonexistentMetaShared.class); this.classDef = classDef; - classInfoHolder = fory.getClassResolver().nilClassInfoHolder(); fieldsInfoMap = new LongMap<>(); binding = SerializationBinding.createBinding(fory); Preconditions.checkArgument(fory.getConfig().isMetaShareEnabled()); @@ -136,14 +133,7 @@ public void write(MemoryBuffer buffer, Object v) { // Use dispatch-based write to ensure correct encoding (e.g., VARINT64 vs FIXED_INT64) Serializers.writePrimitiveValue(buffer, fieldValue, fieldInfo.dispatchId); } else { - if (fieldInfo.useDeclaredTypeInfo) { - // whether tracking ref is recorded in `fieldInfo.serializer`, so it's still - // consistent with jit serializer. - Serializer serializer = classInfo.getSerializer(); - binding.writeRef(buffer, fieldValue, serializer); - } else { - binding.writeRef(buffer, fieldValue, classInfo); - } + binding.write(fieldInfo, buffer, fieldValue); } } Generics generics = fory.getGenerics(); @@ -154,7 +144,7 @@ public void write(MemoryBuffer buffer, Object v) { } for (SerializationFieldInfo fieldInfo : fieldsInfo.otherFields) { Object fieldValue = value.get(fieldInfo.qualifiedFieldName); - writeOtherFieldValue(binding, buffer, fieldInfo, fieldValue); + binding.write(fieldInfo, buffer, fieldValue); } } @@ -185,27 +175,12 @@ public Object read(MemoryBuffer buffer) { new NonexistentClass.NonexistentMetaShared(classDef); Fory fory = this.fory; RefResolver refResolver = fory.getRefResolver(); - ClassResolver classResolver = fory.getClassResolver(); refResolver.reference(obj); List entries = new ArrayList<>(); // read order: primitive,boxed,final,other,collection,map ClassFieldsInfo fieldsInfo = getClassFieldsInfo(classDef); for (SerializationFieldInfo fieldInfo : fieldsInfo.buildInFields) { - Object fieldValue; - if (fieldInfo.classInfo == null) { - // TODO(chaokunyang) support registered serializer in peer with ref tracking disabled. - fieldValue = fory.readRef(buffer, classInfoHolder); - } else { - if (DispatchId.isPrimitive(fieldInfo.dispatchId)) { - // Use dispatch-based read to ensure correct encoding (e.g., VARINT64 vs FIXED_INT64) - fieldValue = Serializers.readPrimitiveValue(buffer, fieldInfo.dispatchId); - } else { - fieldValue = - AbstractObjectSerializer.readFinalObjectFieldValue( - binding, refResolver, classResolver, fieldInfo, buffer); - } - } - entries.add(new MapEntry(fieldInfo.qualifiedFieldName, fieldValue)); + entries.add(new MapEntry(fieldInfo.qualifiedFieldName, binding.read(fieldInfo, buffer))); } Generics generics = fory.getGenerics(); for (SerializationFieldInfo fieldInfo : fieldsInfo.containerFields) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java index 076c56e3ed..6367e5ab31 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java @@ -130,6 +130,11 @@ public ObjectSerializer(Fory fory, Class cls, boolean resolveParent) { containerFields = fieldGroups.containerFields; } + @Override + public void xwrite(MemoryBuffer buffer, T value) { + write(buffer, value); + } + @Override public void write(MemoryBuffer buffer, T value) { Fory fory = this.fory; @@ -154,18 +159,12 @@ private void writeOtherFields(MemoryBuffer buffer, T value) { for (SerializationFieldInfo fieldInfo : otherFields) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; Object fieldValue = fieldAccessor.getObject(value); - writeOtherFieldValue(binding, buffer, fieldInfo, fieldValue); + binding.write(fieldInfo, buffer, fieldValue); } } - @Override - public void xwrite(MemoryBuffer buffer, T value) { - write(buffer, value); - } - private void writeBuildInFields( MemoryBuffer buffer, T value, Fory fory, RefResolver refResolver, TypeResolver typeResolver) { - boolean metaShareEnabled = fory.getConfig().isMetaShareEnabled(); for (SerializationFieldInfo fieldInfo : this.buildInFields) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; boolean nullable = fieldInfo.nullable; @@ -174,52 +173,10 @@ private void writeBuildInFields( Object fieldValue = fieldAccessor.getObject(value); boolean needWrite = nullable - ? writeBasicNullableObjectFieldValue(fory, buffer, fieldValue, dispatchId) - : writeBasicObjectFieldValue(fory, buffer, fieldValue, dispatchId); + ? writeNullableBasicObjectFieldValue(fory, buffer, fieldValue, dispatchId) + : writeNotNullBasicObjectFieldValue(fory, buffer, fieldValue, dispatchId); if (needWrite) { - Serializer serializer = fieldInfo.classInfo.getSerializer(); - if (!metaShareEnabled || fieldInfo.useDeclaredTypeInfo) { - switch (fieldInfo.refMode) { - case NONE: - binding.write(buffer, serializer, fieldValue); - break; - case NULL_ONLY: - binding.writeNullable(buffer, fieldValue, serializer); - break; - case TRACKING: - // whether tracking ref is recorded in `fieldInfo.serializer`, so it's still - // consistent with jit serializer. - binding.writeRef(buffer, fieldValue, serializer); - break; - default: - throw new IllegalStateException("Unexpected refMode: " + fieldInfo.refMode); - } - } else { - switch (fieldInfo.refMode) { - case NONE: - typeResolver.writeClassInfo(buffer, fieldInfo.classInfo); - binding.write(buffer, serializer, fieldValue); - break; - case NULL_ONLY: - if (fieldValue == null) { - buffer.writeByte(Fory.NULL_FLAG); - } else { - buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); - typeResolver.writeClassInfo(buffer, fieldInfo.classInfo); - binding.write(buffer, serializer, fieldValue); - } - break; - case TRACKING: - if (!refResolver.writeRefOrNull(buffer, fieldValue)) { - typeResolver.writeClassInfo(buffer, fieldInfo.classInfo); - // No generics for field, no need to update `depth`. - binding.write(buffer, serializer, fieldValue); - } - break; - default: - throw new IllegalStateException("Unexpected refMode: " + fieldInfo.refMode); - } - } + binding.write(fieldInfo, buffer, fieldValue); } } } @@ -257,8 +214,6 @@ public T xread(MemoryBuffer buffer) { public Object[] readFields(MemoryBuffer buffer) { Fory fory = this.fory; - RefResolver refResolver = this.refResolver; - TypeResolver typeResolver = this.typeResolver; if (fory.checkClassVersion()) { int hash = buffer.readInt32(); checkClassVersion(type, hash, classVersionHash); @@ -273,7 +228,7 @@ public Object[] readFields(MemoryBuffer buffer) { fieldValues[counter++] = Serializers.readPrimitiveValue(buffer, dispatchId); } else { Object fieldValue = - readFinalObjectFieldValue(binding, refResolver, typeResolver, fieldInfo, buffer); + readFinalObjectFieldValue(binding, fieldInfo, buffer); fieldValues[counter++] = fieldValue; } } @@ -291,8 +246,6 @@ public Object[] readFields(MemoryBuffer buffer) { public T readAndSetFields(MemoryBuffer buffer, T obj) { Fory fory = this.fory; - RefResolver refResolver = this.refResolver; - TypeResolver typeResolver = this.typeResolver; if (fory.checkClassVersion()) { int hash = buffer.readInt32(); checkClassVersion(type, hash, classVersionHash); @@ -307,7 +260,7 @@ public T readAndSetFields(MemoryBuffer buffer, T obj) { ? readBasicNullableObjectFieldValue(fory, buffer, obj, fieldAccessor, dispatchId) : readBasicObjectFieldValue(fory, buffer, obj, fieldAccessor, dispatchId))) { Object fieldValue = - readFinalObjectFieldValue(binding, refResolver, typeResolver, fieldInfo, buffer); + readFinalObjectFieldValue(binding, fieldInfo, buffer); fieldAccessor.putObject(obj, fieldValue); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java index 4ce0ebaf0c..5b4b53ba60 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java @@ -28,6 +28,7 @@ import org.apache.fory.resolver.ClassInfo; import org.apache.fory.resolver.ClassInfoHolder; import org.apache.fory.resolver.ClassResolver; +import org.apache.fory.resolver.RefMode; import org.apache.fory.resolver.RefResolver; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.resolver.XtypeResolver; @@ -81,8 +82,12 @@ abstract void writeNullable( abstract void writeContainerFieldValue( MemoryBuffer buffer, Object fieldValue, ClassInfo classInfo); + abstract void write(SerializationFieldInfo fieldInfo, MemoryBuffer buffer, Object value); + abstract void write(MemoryBuffer buffer, Serializer serializer, Object value); + abstract Object read(SerializationFieldInfo fieldInfo, MemoryBuffer buffer); + abstract Object read(MemoryBuffer buffer, Serializer serializer); abstract T readRef(MemoryBuffer buffer, Serializer serializer); @@ -101,6 +106,8 @@ abstract void writeContainerFieldValue( abstract Object readNullable(MemoryBuffer buffer, Serializer serializer); + abstract Object readNullable(MemoryBuffer buffer, SerializationFieldInfo field); + abstract Object readNullable( MemoryBuffer buffer, Serializer serializer, boolean nullable); @@ -201,6 +208,14 @@ public Object readNonRef(MemoryBuffer buffer, SerializationFieldInfo field) { return fory.readNonRef(buffer, field.classInfoHolder); } + @Override + public Object readNullable(MemoryBuffer buffer, SerializationFieldInfo field) { + if (field.useDeclaredTypeInfo) { + return fory.readNullable(buffer, field.classInfo.getSerializer()); + } + return fory.readNullable(buffer, field.classInfoHolder); + } + @Override public Object readNullable(MemoryBuffer buffer, Serializer serializer) { return fory.readNullable(buffer, serializer); @@ -241,6 +256,28 @@ public void write(MemoryBuffer buffer, Serializer serializer, Object value) { serializer.write(buffer, value); } + @Override + Object read(SerializationFieldInfo fieldInfo, MemoryBuffer buffer) { + if (fieldInfo.useDeclaredTypeInfo) { + if (fieldInfo.refMode == RefMode.TRACKING) { + return fory.readRef(buffer, fieldInfo.classInfo); + } else { + if (fieldInfo.refMode != RefMode.NULL_ONLY || buffer.readByte() != Fory.NULL_FLAG) { + return fory.readNonRef(buffer, fieldInfo.classInfo); + } + } + } else { + if (fieldInfo.refMode == RefMode.TRACKING) { + return fory.readRef(buffer, fieldInfo.classInfoHolder); + } else { + if (fieldInfo.refMode != RefMode.NULL_ONLY || buffer.readByte() != Fory.NULL_FLAG) { + return fory.readNonRef(buffer, fieldInfo.classInfoHolder); + } + } + } + return null; + } + @Override public Object read(MemoryBuffer buffer, Serializer serializer) { return serializer.read(buffer); @@ -326,6 +363,38 @@ public void writeContainerFieldValue( MemoryBuffer buffer, Object fieldValue, ClassInfo classInfo) { fory.writeNonRef(buffer, fieldValue, classInfo); } + + @Override + void write(SerializationFieldInfo fieldInfo, MemoryBuffer buffer, Object fieldValue) { + if (fieldInfo.useDeclaredTypeInfo) { + Serializer serializer = fieldInfo.classInfo.getSerializer(); + if (fieldInfo.refMode == RefMode.TRACKING) { + if (!refResolver.writeRefOrNull(buffer, fieldValue)) { + serializer.write(buffer, fieldValue); + } + } else { + if (fieldInfo.refMode == RefMode.NULL_ONLY) { + if (fieldValue == null) { + buffer.writeByte(Fory.NULL_FLAG); + return; + } + serializer.write(buffer, fieldValue); + } + } + } else { + if (fieldInfo.refMode == RefMode.TRACKING) { + fory.writeRef(buffer, fieldValue, fieldInfo.classInfoHolder); + } else { + if (fieldInfo.refMode == RefMode.NULL_ONLY) { + if (fieldValue == null) { + buffer.writeByte(Fory.NULL_FLAG); + return; + } + fory.writeNonRef(buffer, fieldValue, fieldInfo.classInfoHolder); + } + } + } + } } static final class XlangSerializationBinding extends SerializationBinding { @@ -336,6 +405,42 @@ static final class XlangSerializationBinding extends SerializationBinding { xtypeResolver = fory.getXtypeResolver(); } + @Override + void write(SerializationFieldInfo fieldInfo, MemoryBuffer buffer, Object fieldValue) { + if (fieldInfo.useDeclaredTypeInfo) { + Serializer serializer = fieldInfo.classInfo.getSerializer(); + if (fieldInfo.refMode == RefMode.TRACKING) { + if (!refResolver.writeRefOrNull(buffer, fieldValue)) { + serializer.xwrite(buffer, fieldValue); + } + } else if (fieldInfo.refMode == RefMode.NULL_ONLY) { + if (fieldValue == null) { + buffer.writeByte(Fory.NULL_FLAG); + return; + } + buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); + serializer.xwrite(buffer, fieldValue); + } else { + // RefMode.NONE: not nullable, no ref tracking - just write value directly + serializer.xwrite(buffer, fieldValue); + } + } else { + if (fieldInfo.refMode == RefMode.TRACKING) { + fory.xwriteRef(buffer, fieldValue, fieldInfo.classInfoHolder); + } else if (fieldInfo.refMode == RefMode.NULL_ONLY) { + if (fieldValue == null) { + buffer.writeByte(Fory.NULL_FLAG); + return; + } + buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); + fory.xwriteNonRef(buffer, fieldValue, fieldInfo.classInfoHolder); + } else { + // RefMode.NONE: not nullable, no ref tracking - just write value directly + fory.xwriteNonRef(buffer, fieldValue, fieldInfo.classInfoHolder); + } + } + } + @Override public void writeRef(MemoryBuffer buffer, T obj) { fory.xwriteRef(buffer, obj); @@ -423,6 +528,14 @@ public Object readNonRef(MemoryBuffer buffer, SerializationFieldInfo field) { } } + @Override + public Object readNullable(MemoryBuffer buffer, SerializationFieldInfo field) { + if (field.useDeclaredTypeInfo) { + return fory.xreadNullable(buffer, field.classInfo.getSerializer()); + } + return fory.xreadNullable(buffer, field.classInfoHolder); + } + @Override public Object readNullable(MemoryBuffer buffer, Serializer serializer) { return fory.xreadNullable(buffer, serializer); @@ -460,6 +573,28 @@ public void write(MemoryBuffer buffer, Serializer serializer, Object value) { serializer.xwrite(buffer, value); } + @Override + Object read(SerializationFieldInfo fieldInfo, MemoryBuffer buffer) { + if (fieldInfo.useDeclaredTypeInfo) { + if (fieldInfo.refMode == RefMode.TRACKING) { + return fory.xreadRef(buffer, fieldInfo.classInfo); + } else { + if (fieldInfo.refMode != RefMode.NULL_ONLY || buffer.readByte() != Fory.NULL_FLAG) { + return fory.xreadNonRef(buffer, fieldInfo.classInfo); + } + } + } else { + if (fieldInfo.refMode == RefMode.TRACKING) { + return fory.xreadRef(buffer, fieldInfo.classInfoHolder); + } else { + if (fieldInfo.refMode != RefMode.NULL_ONLY || buffer.readByte() != Fory.NULL_FLAG) { + return fory.xreadNonRef(buffer, fieldInfo.classInfoHolder); + } + } + } + return null; + } + @Override public Object read(MemoryBuffer buffer, Serializer serializer) { return serializer.xread(buffer); From 1d9f76ed4a79ad7e342cfdb97ae7ef909e43db21 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 16 Jan 2026 20:20:36 +0800 Subject: [PATCH 06/44] simplify debug output guard --- .../org/apache/fory/builder/MetaSharedCodecBuilder.java | 2 +- .../java/org/apache/fory/builder/ObjectCodecBuilder.java | 2 +- .../src/main/java/org/apache/fory/logging/LogLevel.java | 2 +- .../main/java/org/apache/fory/meta/TypeDefDecoder.java | 4 ++-- .../main/java/org/apache/fory/meta/TypeDefEncoder.java | 2 +- .../org/apache/fory/serializer/MetaSharedSerializer.java | 4 ++-- .../java/org/apache/fory/serializer/ObjectSerializer.java | 4 ++-- .../src/main/java/org/apache/fory/util/Utils.java | 8 ++------ 8 files changed, 12 insertions(+), 16 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java index 865ded99cc..dde01b76c2 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java @@ -94,7 +94,7 @@ public MetaSharedCodecBuilder(TypeRef beanType, Fory fory, ClassDef classDef) f -> MetaSharedSerializer.consolidateFields(f._getTypeResolver(), beanClass, classDef)); DescriptorGrouper grouper = typeResolver(r -> r.createDescriptorGrouper(descriptors, false)); List sortedDescriptors = grouper.getSortedDescriptors(); - if (org.apache.fory.util.Utils.debugOutputEnabled()) { + if (org.apache.fory.util.Utils.DEBUG_OUTPUT_ENABLED) { LOG.info("========== sorted descriptors for {} ==========", classDef.getClassName()); for (Descriptor d : sortedDescriptors) { LOG.info( diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java index 6f2feaa701..21613cabf9 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java @@ -104,7 +104,7 @@ public ObjectCodecBuilder(Class beanClass, Fory fory) { } Collection p = descriptors; DescriptorGrouper grouper = typeResolver(r -> r.createDescriptorGrouper(p, false)); - if (org.apache.fory.util.Utils.debugOutputEnabled()) { + if (org.apache.fory.util.Utils.DEBUG_OUTPUT_ENABLED) { LOG.info("========== sorted descriptors for {} ==========", beanClass.getSimpleName()); List sortedDescriptors = grouper.getSortedDescriptors(); for (Descriptor d : sortedDescriptors) { diff --git a/java/fory-core/src/main/java/org/apache/fory/logging/LogLevel.java b/java/fory-core/src/main/java/org/apache/fory/logging/LogLevel.java index 0778799379..3747f54359 100644 --- a/java/fory-core/src/main/java/org/apache/fory/logging/LogLevel.java +++ b/java/fory-core/src/main/java/org/apache/fory/logging/LogLevel.java @@ -34,7 +34,7 @@ public class LogLevel { public static final int DEFAULT_LEVEL; static { - if (Utils.debugOutputEnabled()) { + if (Utils.DEBUG_OUTPUT_ENABLED) { DEFAULT_LEVEL = DEBUG_LEVEL; } else { DEFAULT_LEVEL = INFO_LEVEL; diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefDecoder.java b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefDecoder.java index e0d56b50f1..3f02c62dfa 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefDecoder.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefDecoder.java @@ -62,7 +62,7 @@ public static ClassDef decodeClassDef(XtypeResolver resolver, MemoryBuffer input if ((header & REGISTER_BY_NAME_FLAG) != 0) { String namespace = readPkgName(buffer); String typeName = readTypeName(buffer); - if (Utils.debugOutputEnabled()) { + if (Utils.DEBUG_OUTPUT_ENABLED) { LOG.info("Decode class {} using namespace {}", typeName, namespace); } ClassInfo userTypeInfo = resolver.getUserTypeInfo(namespace, typeName); @@ -84,7 +84,7 @@ public static ClassDef decodeClassDef(XtypeResolver resolver, MemoryBuffer input readFieldsInfo(buffer, resolver, classSpec.entireClassName, numFields); boolean hasFieldsMeta = (id & HAS_FIELDS_META_FLAG) != 0; ClassDef classDef = new ClassDef(classSpec, classFields, hasFieldsMeta, id, decoded.f1); - if (Utils.debugOutputEnabled()) { + if (Utils.DEBUG_OUTPUT_ENABLED) { LOG.info("[Java TypeDef DECODED] " + classDef); // Compute and print diff with local TypeDef Class cls = classSpec.type; diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefEncoder.java b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefEncoder.java index 511a6d4bf1..854b41c018 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefEncoder.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefEncoder.java @@ -121,7 +121,7 @@ static ClassDef buildClassDefWithFieldInfos( true, encodeClassDef.getInt64(0), classDefBytes); - if (Utils.debugOutputEnabled()) { + if (Utils.DEBUG_OUTPUT_ENABLED) { LOG.info("[Java TypeDef BUILT] " + classDef); } return classDef; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java index 2645d903c6..8aeaf5a4ff 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java @@ -86,7 +86,7 @@ public MetaSharedSerializer(Fory fory, Class type, ClassDef classDef) { "Class version check should be disabled when compatible mode is enabled."); Preconditions.checkArgument( fory.getConfig().isMetaShareEnabled(), "Meta share must be enabled."); - if (Utils.debugOutputEnabled()) { + if (Utils.DEBUG_OUTPUT_ENABLED) { LOG.info("========== MetaSharedSerializer ClassDef for {} ==========", type.getName()); LOG.info("ClassDef fieldsInfo count: {}", classDef.getFieldsInfo().size()); for (int i = 0; i < classDef.getFieldsInfo().size(); i++) { @@ -96,7 +96,7 @@ public MetaSharedSerializer(Fory fory, Class type, ClassDef classDef) { Collection descriptors = consolidateFields(fory._getTypeResolver(), type, classDef); DescriptorGrouper descriptorGrouper = fory._getTypeResolver().createDescriptorGrouper(descriptors, false); - if (Utils.debugOutputEnabled()) { + if (Utils.DEBUG_OUTPUT_ENABLED) { LOG.info( "========== MetaSharedSerializer sorted descriptors for {} ==========", type.getName()); for (Descriptor d : descriptorGrouper.getSortedDescriptors()) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java index 6367e5ab31..e545e4a792 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java @@ -88,7 +88,7 @@ public ObjectSerializer(Fory fory, Class cls, boolean resolveParent) { boolean shareMeta = fory.getConfig().isMetaShareEnabled(); if (shareMeta) { ClassDef classDef = typeResolver.getTypeDef(cls, resolveParent); - if (Utils.debugOutputEnabled()) { + if (Utils.DEBUG_OUTPUT_ENABLED) { LOG.info("========== ObjectSerializer ClassDef for {} ==========", cls.getName()); LOG.info("ClassDef fieldsInfo count: {}", classDef.getFieldsInfo().size()); for (int i = 0; i < classDef.getFieldsInfo().size(); i++) { @@ -101,7 +101,7 @@ public ObjectSerializer(Fory fory, Class cls, boolean resolveParent) { } DescriptorGrouper grouper = typeResolver.createDescriptorGrouper(descriptors, false); descriptors = grouper.getSortedDescriptors(); - if (Utils.debugOutputEnabled()) { + if (Utils.DEBUG_OUTPUT_ENABLED) { LOG.info("========== ObjectSerializer sorted descriptors for {} ==========", cls.getName()); for (Descriptor d : descriptors) { LOG.info( diff --git a/java/fory-core/src/main/java/org/apache/fory/util/Utils.java b/java/fory-core/src/main/java/org/apache/fory/util/Utils.java index 3592e3ff71..6687619f77 100644 --- a/java/fory-core/src/main/java/org/apache/fory/util/Utils.java +++ b/java/fory-core/src/main/java/org/apache/fory/util/Utils.java @@ -21,14 +21,10 @@ /** Misc common utils. */ public class Utils { - private static final boolean DEBUG_OUTPUT_ENABLED; + /** Checks if ENABLE_FORY_DEBUG_OUTPUT env var is set to "1". */ + public static final boolean DEBUG_OUTPUT_ENABLED; static { DEBUG_OUTPUT_ENABLED = "1".equals(System.getenv("ENABLE_FORY_DEBUG_OUTPUT")); } - - /** Checks if ENABLE_FORY_DEBUG_OUTPUT env var is set to "1". */ - public static boolean debugOutputEnabled() { - return DEBUG_OUTPUT_ENABLED; - } } From 021eaff52b5b2882a1b8cf5446693742409221f2 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 16 Jan 2026 20:36:07 +0800 Subject: [PATCH 07/44] refine mode code --- .../src/main/java/org/apache/fory/Fory.java | 1 - .../apache/fory/reflect/FieldAccessor.java | 4 +- .../apache/fory/resolver/ClassResolver.java | 3 +- .../apache/fory/resolver/TypeResolver.java | 1 - .../serializer/AbstractObjectSerializer.java | 25 +------ .../apache/fory/serializer/FieldGroups.java | 3 +- .../serializer/MetaSharedLayerSerializer.java | 17 ++--- .../fory/serializer/MetaSharedSerializer.java | 67 ++++++++----------- .../NonexistentClassSerializers.java | 4 +- .../fory/serializer/ObjectSerializer.java | 15 ++--- .../java/org/apache/fory/type/DispatchId.java | 6 +- 11 files changed, 53 insertions(+), 93 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java index 9b6c1eec54..79b55226ae 100644 --- a/java/fory-core/src/main/java/org/apache/fory/Fory.java +++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java @@ -53,7 +53,6 @@ import org.apache.fory.resolver.ClassInfoHolder; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.MapRefResolver; -import org.apache.fory.resolver.MetaContext; import org.apache.fory.resolver.MetaStringResolver; import org.apache.fory.resolver.NoRefResolver; import org.apache.fory.resolver.RefResolver; diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java index aa55765e17..bb62a27a66 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java @@ -497,14 +497,14 @@ public Object get(Object obj) { } } - static class GeneratedAccessor extends FieldAccessor { + static final class GeneratedAccessor extends FieldAccessor { private static final ClassValueCache>> cache = ClassValueCache.newClassKeyCache(8); private final MethodHandle getter; private final MethodHandle setter; - protected GeneratedAccessor(Field field) { + GeneratedAccessor(Field field) { super(field, -1); ConcurrentMap> map = cache.get(field.getDeclaringClass(), ConcurrentHashMap::new); diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 3cead16341..c74e1a5791 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -1545,7 +1545,8 @@ public void writeClassInfoWithMetaShare(MemoryBuffer buffer, ClassInfo classInfo } else { // New type: index << 1, LSB=0, followed by ClassDef bytes inline buffer.writeVarUint32(newId << 1); - ClassInfo stubClassInfo = classForMeta == classInfo.cls ? classInfo : getClassInfo(classForMeta); + ClassInfo stubClassInfo = + classForMeta == classInfo.cls ? classInfo : getClassInfo(classForMeta); ClassDef classDef = stubClassInfo.classDef; if (classDef == null) { classDef = buildClassDef(stubClassInfo); diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index 5baeea5fa2..852d88f3b6 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -50,7 +50,6 @@ import org.apache.fory.codegen.Expression.Invoke; import org.apache.fory.collection.IdentityMap; import org.apache.fory.collection.LongMap; -import org.apache.fory.collection.ObjectArray; import org.apache.fory.collection.Tuple2; import org.apache.fory.config.CompatibleMode; import org.apache.fory.exception.ForyException; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java index ad8b62a8ab..b6cad6b5ea 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java @@ -190,6 +190,7 @@ static boolean writePrimitiveFieldValue( if (fieldOffset != -1) { return writePrimitiveFieldValue(buffer, targetObject, fieldOffset, dispatchId); } + // graalvm use GeneratedAccessor, which will be this code path. switch (dispatchId) { case DispatchId.PRIMITIVE_BOOL: buffer.writeBoolean((Boolean) fieldAccessor.get(targetObject)); @@ -460,29 +461,6 @@ static boolean writeNullableBasicObjectFieldValue( } } - /** - * Read final object field value. Note that primitive field value can't be read by this method, - * because primitive field doesn't write null flag. - */ - static Object readFinalObjectFieldValue( - SerializationBinding binding, SerializationFieldInfo fieldInfo, MemoryBuffer buffer) { - return binding.read(fieldInfo, buffer); - } - - /** - * Read a non-container field value that is not a final type. Handles enum types, reference - * tracking, and nullable fields according to xlang serialization protocol. - * - * @param binding the serialization binding for read operations - * @param fieldInfo the field metadata including type info and nullability - * @param buffer the buffer to read from - * @return the deserialized field value, or null if the field is nullable and was null - */ - static Object readOtherFieldValue( - SerializationBinding binding, SerializationFieldInfo fieldInfo, MemoryBuffer buffer) { - return binding.read(fieldInfo, buffer); - } - /** * Read a container field value (Collection or Map). Handles reference tracking, nullable fields, * and pushes/pops generic type information for proper deserialization of parameterized types. @@ -541,6 +519,7 @@ static boolean readPrimitiveFieldValue( if (fieldOffset != -1) { return readPrimitiveFieldValue(buffer, targetObject, fieldOffset, dispatchId); } + // graalvm use GeneratedAccessor, which will be this code path. switch (dispatchId) { case DispatchId.PRIMITIVE_BOOL: fieldAccessor.set(targetObject, buffer.readBoolean()); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java index 36327b7053..edfcd285d1 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java @@ -170,7 +170,8 @@ public static final class SerializationFieldInfo { // Use dispatchId to determine isPrimitive for consistency with how data was written. // This ensures correct handling in schema compatible mode where local field type // may differ from remote (ClassDef) field type. - // Note: NOTNULL_BOXED dispatch IDs are NOT primitive - they are handled by readBasicObjectFieldValue + // Note: NOTNULL_BOXED dispatch IDs are NOT primitive - they are handled by + // readBasicObjectFieldValue isPrimitiveField = DispatchId.isPrimitive(dispatchId); fieldConverter = d.getFieldConverter(); nullable = d.isNullable(); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java index 6293a3198b..436226a124 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java @@ -27,9 +27,7 @@ import org.apache.fory.meta.ClassDef; import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.resolver.ClassInfoHolder; -import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.MetaContext; -import org.apache.fory.resolver.RefResolver; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo; import org.apache.fory.type.Descriptor; @@ -126,7 +124,8 @@ private void writeFinalFields(MemoryBuffer buffer, T value) { buffer, value, fieldAccessor, dispatchId)) { Object fieldValue = fieldAccessor.getObject(value); boolean needWrite = - nullable ? writeNullableBasicObjectFieldValue(fory, buffer, fieldValue, dispatchId) + nullable + ? writeNullableBasicObjectFieldValue(fory, buffer, fieldValue, dispatchId) : writeNotNullBasicObjectFieldValue(fory, buffer, fieldValue, dispatchId); if (needWrite) { binding.write(fieldInfo, buffer, fieldValue); @@ -201,9 +200,6 @@ private void readLayerClassMeta(MemoryBuffer buffer) { private void readFinalFields(MemoryBuffer buffer, T obj) { Fory fory = this.fory; - RefResolver refResolver = this.refResolver; - ClassResolver classResolver = this.classResolver; - for (SerializationFieldInfo fieldInfo : buildInFields) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; if (fieldAccessor != null) { @@ -215,9 +211,7 @@ private void readFinalFields(MemoryBuffer buffer, T obj) { fory, buffer, obj, fieldAccessor, dispatchId) : AbstractObjectSerializer.readBasicObjectFieldValue( fory, buffer, obj, fieldAccessor, dispatchId))) { - Object fieldValue = - AbstractObjectSerializer.readFinalObjectFieldValue( - binding, fieldInfo, buffer); + Object fieldValue = binding.read(fieldInfo, buffer); fieldAccessor.putObject(obj, fieldValue); } } else { @@ -226,8 +220,7 @@ private void readFinalFields(MemoryBuffer buffer, T obj) { if (fieldInfo.classInfo == null) { fory.readRef(buffer, classInfoHolder); } else { - AbstractObjectSerializer.readFinalObjectFieldValue( - binding, fieldInfo, buffer); + binding.read(fieldInfo, buffer); } } } @@ -248,7 +241,7 @@ private void readContainerFields(MemoryBuffer buffer, T obj) { private void readUserTypeFields(MemoryBuffer buffer, T obj) { for (SerializationFieldInfo fieldInfo : otherFields) { - Object fieldValue = AbstractObjectSerializer.readOtherFieldValue(binding, fieldInfo, buffer); + Object fieldValue = binding.read(fieldInfo, buffer); FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; if (fieldAccessor != null) { fieldAccessor.putObject(obj, fieldValue); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java index 8aeaf5a4ff..7b77e3c489 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java @@ -34,7 +34,6 @@ import org.apache.fory.meta.ClassDef; import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.resolver.ClassInfoHolder; -import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.RefResolver; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo; @@ -165,6 +164,21 @@ public void xwrite(MemoryBuffer buffer, T value) { write(buffer, value); } + private T newInstance() { + if (!hasDefaultValues) { + return newBean(); + } + T obj = GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE ? newBean() : Platform.newInstance(type); + // Set default values for missing fields in Scala case classes + DefaultValueUtils.setDefaultValues(obj, defaultValueFields); + return obj; + } + + @Override + public T xread(MemoryBuffer buffer) { + return read(buffer); + } + @Override public T read(MemoryBuffer buffer) { if (isRecord) { @@ -194,24 +208,18 @@ public T read(MemoryBuffer buffer) { } // For NOTNULL_BOXED and other boxed types, let readBasicObjectFieldValue handle it // which uses putObject for proper boxing - if (nullable ? readBasicNullableObjectFieldValue(fory, buffer, obj, fieldAccessor, dispatchId) - : readBasicObjectFieldValue(fory, buffer, obj, fieldAccessor, dispatchId)) { + if (nullable + ? readBasicNullableObjectFieldValue(fory, buffer, obj, fieldAccessor, dispatchId) + : readBasicObjectFieldValue(fory, buffer, obj, fieldAccessor, dispatchId)) { assert fieldInfo.classInfo != null; - Object fieldValue = - readFinalObjectFieldValue(binding, fieldInfo, buffer); + Object fieldValue = binding.read(fieldInfo, buffer); fieldAccessor.putObject(obj, fieldValue); } } else { if (fieldInfo.fieldConverter == null) { // Skip the field value from buffer since it doesn't exist in current class if (!skipPrimitiveFieldValue(fieldInfo, buffer)) { - if (fieldInfo.classInfo == null) { - // TODO(chaokunyang) support registered serializer in peer with ref tracking disabled. - binding.readRef(buffer, classInfoHolder); - } else { - readFinalObjectFieldValue( - binding, fieldInfo, buffer); - } + binding.read(fieldInfo, buffer); } } else { compatibleRead(buffer, fieldInfo, obj); @@ -228,7 +236,7 @@ public T read(MemoryBuffer buffer) { } } for (SerializationFieldInfo fieldInfo : otherFields) { - Object fieldValue = AbstractObjectSerializer.readOtherFieldValue(binding, fieldInfo, buffer); + Object fieldValue = binding.read(fieldInfo, buffer); FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; if (fieldAccessor != null) { fieldAccessor.putObject(obj, fieldValue); @@ -243,26 +251,12 @@ private void compatibleRead(MemoryBuffer buffer, SerializationFieldInfo fieldInf if (DispatchId.isPrimitive(dispatchId)) { fieldValue = Serializers.readPrimitiveValue(buffer, dispatchId); } else { - fieldValue = readFinalObjectFieldValue(binding, fieldInfo, buffer); + fieldValue = binding.read(fieldInfo, buffer); + ; } fieldInfo.fieldConverter.set(obj, fieldValue); } - private T newInstance() { - if (!hasDefaultValues) { - return newBean(); - } - T obj = GraalvmSupport.IN_GRAALVM_NATIVE_IMAGE ? newBean() : Platform.newInstance(type); - // Set default values for missing fields in Scala case classes - DefaultValueUtils.setDefaultValues(obj, defaultValueFields); - return obj; - } - - @Override - public T xread(MemoryBuffer buffer) { - return read(buffer); - } - private void readFields(MemoryBuffer buffer, Object[] fields) { int counter = 0; Fory fory = this.fory; @@ -275,9 +269,7 @@ private void readFields(MemoryBuffer buffer, Object[] fields) { if (DispatchId.isPrimitive(dispatchId)) { fields[counter++] = Serializers.readPrimitiveValue(buffer, dispatchId); } else { - Object fieldValue = - readFinalObjectFieldValue( - binding, fieldInfo, buffer); + Object fieldValue = binding.read(fieldInfo, buffer); fields[counter++] = fieldValue; } } else { @@ -289,16 +281,16 @@ private void readFields(MemoryBuffer buffer, Object[] fields) { fields[counter++] = null; } } - for (SerializationFieldInfo fieldInfo : otherFields) { - Object fieldValue = AbstractObjectSerializer.readOtherFieldValue(binding, fieldInfo, buffer); - fields[counter++] = fieldValue; - } Generics generics = fory.getGenerics(); for (SerializationFieldInfo fieldInfo : containerFields) { Object fieldValue = AbstractObjectSerializer.readContainerFieldValue(binding, generics, fieldInfo, buffer); fields[counter++] = fieldValue; } + for (SerializationFieldInfo fieldInfo : otherFields) { + Object fieldValue = binding.read(fieldInfo, buffer); + fields[counter++] = fieldValue; + } } private void skipObjectFieldValue(MemoryBuffer buffer, SerializationFieldInfo fieldInfo) { @@ -316,8 +308,7 @@ private void skipObjectFieldValue(MemoryBuffer buffer, SerializationFieldInfo fi default: } } else { - readFinalObjectFieldValue( - binding, fieldInfo, buffer); + binding.read(fieldInfo, buffer); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java index 88b1f97132..d78e82d021 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java @@ -33,7 +33,6 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.meta.ClassDef; import org.apache.fory.resolver.ClassInfo; -import org.apache.fory.resolver.ClassInfoHolder; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.MetaContext; import org.apache.fory.resolver.MetaStringResolver; @@ -189,8 +188,7 @@ public Object read(MemoryBuffer buffer) { entries.add(new MapEntry(fieldInfo.qualifiedFieldName, fieldValue)); } for (SerializationFieldInfo fieldInfo : fieldsInfo.otherFields) { - Object fieldValue = - AbstractObjectSerializer.readOtherFieldValue(binding, fieldInfo, buffer); + Object fieldValue = binding.read(fieldInfo, buffer); entries.add(new MapEntry(fieldInfo.qualifiedFieldName, fieldValue)); } obj.setEntries(entries); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java index e545e4a792..f4a383caba 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java @@ -150,7 +150,7 @@ public void write(MemoryBuffer buffer, T value) { buffer.writeInt32(classVersionHash); } // write order: primitive,boxed,final,other,collection,map - writeBuildInFields(buffer, value, fory, refResolver, typeResolver); + writeBuildInFields(buffer, value, fory); writeContainerFields(buffer, value, fory, refResolver, typeResolver); writeOtherFields(buffer, value); } @@ -163,8 +163,7 @@ private void writeOtherFields(MemoryBuffer buffer, T value) { } } - private void writeBuildInFields( - MemoryBuffer buffer, T value, Fory fory, RefResolver refResolver, TypeResolver typeResolver) { + private void writeBuildInFields(MemoryBuffer buffer, T value, Fory fory) { for (SerializationFieldInfo fieldInfo : this.buildInFields) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; boolean nullable = fieldInfo.nullable; @@ -227,8 +226,7 @@ public Object[] readFields(MemoryBuffer buffer) { if (DispatchId.isPrimitive(dispatchId)) { fieldValues[counter++] = Serializers.readPrimitiveValue(buffer, dispatchId); } else { - Object fieldValue = - readFinalObjectFieldValue(binding, fieldInfo, buffer); + Object fieldValue = binding.read(fieldInfo, buffer); fieldValues[counter++] = fieldValue; } } @@ -238,7 +236,7 @@ public Object[] readFields(MemoryBuffer buffer) { fieldValues[counter++] = fieldValue; } for (SerializationFieldInfo fieldInfo : otherFields) { - Object fieldValue = readOtherFieldValue(binding, fieldInfo, buffer); + Object fieldValue = binding.read(fieldInfo, buffer); fieldValues[counter++] = fieldValue; } return fieldValues; @@ -259,8 +257,7 @@ public T readAndSetFields(MemoryBuffer buffer, T obj) { && (nullable ? readBasicNullableObjectFieldValue(fory, buffer, obj, fieldAccessor, dispatchId) : readBasicObjectFieldValue(fory, buffer, obj, fieldAccessor, dispatchId))) { - Object fieldValue = - readFinalObjectFieldValue(binding, fieldInfo, buffer); + Object fieldValue = binding.read(fieldInfo, buffer); fieldAccessor.putObject(obj, fieldValue); } } @@ -271,7 +268,7 @@ public T readAndSetFields(MemoryBuffer buffer, T obj) { fieldAccessor.putObject(obj, fieldValue); } for (SerializationFieldInfo fieldInfo : otherFields) { - Object fieldValue = readOtherFieldValue(binding, fieldInfo, buffer); + Object fieldValue = binding.read(fieldInfo, buffer); FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; fieldAccessor.putObject(obj, fieldValue); } diff --git a/java/fory-core/src/main/java/org/apache/fory/type/DispatchId.java b/java/fory-core/src/main/java/org/apache/fory/type/DispatchId.java index 1053df02f6..85118b2985 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/DispatchId.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/DispatchId.java @@ -89,8 +89,10 @@ public class DispatchId { // Dispatch mode for determining how to read/write a field private static final int MODE_PRIMITIVE = 0; // Local field is primitive, use Platform.putInt - private static final int MODE_NOTNULL_BOXED = 1; // Local is boxed, remote nullable=false, box and putObject - private static final int MODE_NULLABLE_BOXED = 2; // Local is boxed, remote nullable=true, read null flag + private static final int MODE_NOTNULL_BOXED = + 1; // Local is boxed, remote nullable=false, box and putObject + private static final int MODE_NULLABLE_BOXED = + 2; // Local is boxed, remote nullable=true, read null flag public static int getDispatchId(Fory fory, Descriptor d) { int typeId = Types.getDescriptorTypeId(fory, d); From 59dc47dc4147eecaa6e82a0da09aab51b652c3a2 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 17 Jan 2026 00:00:01 +0800 Subject: [PATCH 08/44] fix tests --- AGENTS.md | 1 + .../src/main/java/org/apache/fory/Fory.java | 4 - .../fory/builder/MetaSharedCodecBuilder.java | 2 +- .../builder/MetaSharedLayerCodecBuilder.java | 2 +- .../fory/builder/ObjectCodecBuilder.java | 2 +- .../java/org/apache/fory/config/Config.java | 15 - .../org/apache/fory/config/ForyBuilder.java | 7 - .../java/org/apache/fory/meta/FieldInfo.java | 3 +- .../apache/fory/reflect/FieldAccessor.java | 8 +- .../serializer/AbstractObjectSerializer.java | 708 +++++++++++++++--- .../apache/fory/serializer/FieldSkipper.java | 235 ++++++ .../serializer/MetaSharedLayerSerializer.java | 54 +- .../fory/serializer/MetaSharedSerializer.java | 166 +--- .../NonexistentClassSerializers.java | 28 +- .../fory/serializer/ObjectSerializer.java | 62 +- .../fory/serializer/PrimitiveSerializers.java | 16 +- .../fory/serializer/SerializationBinding.java | 29 +- .../apache/fory/serializer/Serializer.java | 4 +- .../apache/fory/serializer/Serializers.java | 95 --- .../apache/fory/type/DescriptorGrouper.java | 26 +- .../java/org/apache/fory/type/DispatchId.java | 32 +- .../org/apache/fory/util/StringUtils.java | 4 + .../fory/builder/ObjectCodecBuilderTest.java | 3 - .../org/apache/fory/xlang/RustXlangTest.java | 11 +- .../org/apache/fory/xlang/XlangTestBase.java | 33 +- rust/tests/tests/test_cross_language.rs | 2 +- 26 files changed, 999 insertions(+), 553 deletions(-) create mode 100644 java/fory-core/src/main/java/org/apache/fory/serializer/FieldSkipper.java diff --git a/AGENTS.md b/AGENTS.md index 58c1666a31..1ab1a72655 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,6 +14,7 @@ While working on Fory, please remember: - **Cross-Language Consistency**: Maintain consistency across language implementations while respecting language-specific idioms. - **Graalvm Support using fory codegen**: For graalvm, please use `fory codegen` to generate the serializer when building graalvm native image, do not use graallvm reflect-related configuration unless for JDK `proxy`. - **Xlang Type System**: Java `native mode(xlang=false)` shares same type systems between type id from `Types.BOOL~Types.STRING` with `xlang mode(xlang=true)`, but for other types, java `native mode` has different type ids. +- **Remote git repository**: `git@github.com:apache/fory.git` is remote repository, do not use other remote repository when you want to check code under `main` branch. ## Build and Development Commands diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java index 79b55226ae..c702cb4859 100644 --- a/java/fory-core/src/main/java/org/apache/fory/Fory.java +++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java @@ -1719,10 +1719,6 @@ public boolean isStringRefIgnored() { return config.isStringRefIgnored(); } - public boolean isBasicTypesRefIgnored() { - return config.isBasicTypesRefIgnored(); - } - public boolean checkClassVersion() { return config.checkClassVersion(); } diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java index dde01b76c2..92d8e733bc 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java @@ -106,7 +106,7 @@ public MetaSharedCodecBuilder(TypeRef beanType, Fory fory, ClassDef classDef) } } objectCodecOptimizer = - new ObjectCodecOptimizer(beanClass, grouper, !fory.isBasicTypesRefIgnored(), ctx); + new ObjectCodecOptimizer(beanClass, grouper, false, ctx); String defaultValueLanguage = "None"; DefaultValueUtils.DefaultValueField[] defaultValueFields = diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedLayerCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedLayerCodecBuilder.java index a3dd6cd9f2..a09081fed1 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedLayerCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedLayerCodecBuilder.java @@ -69,7 +69,7 @@ public MetaSharedLayerCodecBuilder( Collection descriptors = layerClassDef.getDescriptors(typeResolver, beanClass); DescriptorGrouper grouper = typeResolver(r -> r.createDescriptorGrouper(descriptors, false)); objectCodecOptimizer = - new ObjectCodecOptimizer(beanClass, grouper, !fory.isBasicTypesRefIgnored(), ctx); + new ObjectCodecOptimizer(beanClass, grouper, false, ctx); } // Must be static to be shared across the whole process life. diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java index 21613cabf9..4b4a8bca1b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java @@ -121,7 +121,7 @@ public ObjectCodecBuilder(Class beanClass, Fory fory) { ? new Literal(ObjectSerializer.computeStructHash(fory, grouper), PRIMITIVE_INT_TYPE) : null; objectCodecOptimizer = - new ObjectCodecOptimizer(beanClass, grouper, !fory.isBasicTypesRefIgnored(), ctx); + new ObjectCodecOptimizer(beanClass, grouper, false, ctx); if (isRecord) { if (!recordCtrAccessible) { buildRecordComponentDefaultValues(); diff --git a/java/fory-core/src/main/java/org/apache/fory/config/Config.java b/java/fory-core/src/main/java/org/apache/fory/config/Config.java index e81ed4e5f5..148deb2139 100644 --- a/java/fory-core/src/main/java/org/apache/fory/config/Config.java +++ b/java/fory-core/src/main/java/org/apache/fory/config/Config.java @@ -36,7 +36,6 @@ public class Config implements Serializable { private final String name; private final Language language; private final boolean trackingRef; - private final boolean basicTypesRefIgnored; private final boolean stringRefIgnored; private final boolean timeRefIgnored; private final boolean copyRef; @@ -67,15 +66,11 @@ public class Config implements Serializable { private final int bufferSizeLimitBytes; private final int maxDepth; private final float mapRefLoadFactor; - boolean foryDebugOutputEnabled = - "1".equals(System.getenv("ENABLE_FORY_DEBUG_OUTPUT")) - || "true".equals(System.getenv("ENABLE_FORY_DEBUG_OUTPUT")); public Config(ForyBuilder builder) { name = builder.name; language = builder.language; trackingRef = builder.trackingRef; - basicTypesRefIgnored = !trackingRef || builder.basicTypesRefIgnored; stringRefIgnored = !trackingRef || builder.stringRefIgnored; timeRefIgnored = !trackingRef || builder.timeRefIgnored; copyRef = builder.copyRef; @@ -139,10 +134,6 @@ public boolean copyRef() { return copyRef; } - public boolean isBasicTypesRefIgnored() { - return basicTypesRefIgnored; - } - public boolean isStringRefIgnored() { return stringRefIgnored; } @@ -299,10 +290,6 @@ public boolean isScalaOptimizationEnabled() { return scalaOptimizationEnabled; } - public boolean isForyDebugOutputEnabled() { - return foryDebugOutputEnabled; - } - @Override public boolean equals(Object o) { if (this == o) { @@ -315,7 +302,6 @@ public boolean equals(Object o) { return name == config.name && trackingRef == config.trackingRef && mapRefLoadFactor == config.mapRefLoadFactor - && basicTypesRefIgnored == config.basicTypesRefIgnored && stringRefIgnored == config.stringRefIgnored && timeRefIgnored == config.timeRefIgnored && copyRef == config.copyRef @@ -351,7 +337,6 @@ public int hashCode() { language, mapRefLoadFactor, trackingRef, - basicTypesRefIgnored, stringRefIgnored, timeRefIgnored, copyRef, diff --git a/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java b/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java index 4a2b60b077..fded0c5ab9 100644 --- a/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java @@ -62,7 +62,6 @@ public final class ForyBuilder { Language language = Language.JAVA; boolean trackingRef = false; boolean copyRef = false; - boolean basicTypesRefIgnored = true; boolean stringRefIgnored = true; boolean timeRefIgnored = true; ClassLoader classLoader; @@ -126,12 +125,6 @@ public ForyBuilder withRefCopy(boolean copyRef) { return this; } - /** Whether ignore basic types shared reference. */ - public ForyBuilder ignoreBasicTypesRef(boolean ignoreBasicTypesRef) { - this.basicTypesRefIgnored = ignoreBasicTypesRef; - return this; - } - /** Whether ignore string shared reference. */ public ForyBuilder ignoreStringRef(boolean ignoreStringRef) { this.stringRefIgnored = ignoreStringRef; diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java b/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java index 076b9d7cf6..00a3b78684 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java @@ -30,6 +30,7 @@ import org.apache.fory.type.Descriptor; import org.apache.fory.type.DescriptorBuilder; import org.apache.fory.type.Types; +import org.apache.fory.util.StringUtils; /** * FieldInfo contains all necessary info of a field to execute serialization/deserialization logic. @@ -178,7 +179,7 @@ public int hashCode() { public String toString() { return "FieldInfo{" + "fieldName='" - + fieldName + + StringUtils.lowerCamelToLowerUnderscore(fieldName) + '\'' + ", definedClass='" + definedClass diff --git a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java index bb62a27a66..5b0ba43e07 100644 --- a/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java +++ b/java/fory-core/src/main/java/org/apache/fory/reflect/FieldAccessor.java @@ -83,7 +83,9 @@ public Field getField() { } public final void putObject(Object targetObject, Object object) { - if (fieldOffset != -1) { + // For primitive fields, we must use set() which calls the correct Platform.putXxx method. + // Platform.putObject writes object references, not primitive values. + if (fieldOffset != -1 && !field.getType().isPrimitive()) { Platform.putObject(targetObject, fieldOffset, object); } else { set(targetObject, object); @@ -91,7 +93,9 @@ public final void putObject(Object targetObject, Object object) { } public final Object getObject(Object targetObject) { - if (fieldOffset != -1) { + // For primitive fields, we must use get() which calls the correct Platform.getXxx method + // and returns the boxed value. Platform.getObject interprets primitive bytes as object refs. + if (fieldOffset != -1 && !field.getType().isPrimitive()) { return Platform.getObject(targetObject, fieldOffset); } else { return get(targetObject); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java index b6cad6b5ea..9000c4f44e 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java @@ -25,6 +25,7 @@ import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; + import org.apache.fory.Fory; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.Platform; @@ -33,8 +34,8 @@ import org.apache.fory.reflect.ObjectCreators; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.reflect.TypeRef; -import org.apache.fory.resolver.ClassInfo; import org.apache.fory.resolver.ClassResolver; +import org.apache.fory.resolver.RefMode; import org.apache.fory.resolver.RefResolver; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo; @@ -42,11 +43,14 @@ import org.apache.fory.type.DescriptorGrouper; import org.apache.fory.type.DispatchId; import org.apache.fory.type.Generics; +import org.apache.fory.logging.Logger; +import org.apache.fory.logging.LoggerFactory; import org.apache.fory.util.record.RecordComponent; import org.apache.fory.util.record.RecordInfo; import org.apache.fory.util.record.RecordUtils; public abstract class AbstractObjectSerializer extends Serializer { + private static final Logger LOG = LoggerFactory.getLogger(AbstractObjectSerializer.class); protected final RefResolver refResolver; protected final ClassResolver classResolver; protected final TypeResolver typeResolver; @@ -68,60 +72,140 @@ public AbstractObjectSerializer(Fory fory, Class type, ObjectCreator objec this.objectCreator = objectCreator; } - static void writeContainerFieldValue( + /** + * Write field value to buffer by reading from the object via fieldAccessor. Handles primitive + * types, unsigned/compressed numbers, and common types like String with optimized fast paths. + * + *

This method reads the field value from the object using the fieldAccessor in fieldInfo, + * then writes it to the buffer. It is the write counterpart of {@link + * #readBuildInFieldValue(SerializationBinding, SerializationFieldInfo, MemoryBuffer, Object)}. + * + * @param binding the serialization binding for write operations + * @param fieldInfo the field metadata including type, nullability info, and field accessor + * @param buffer the buffer to write to + * @param obj the object to read the field value from + */ + static void writeBuildInField( + SerializationBinding binding, + SerializationFieldInfo fieldInfo, + MemoryBuffer buffer, + Object obj) { + Fory fory = binding.fory; + FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; + int dispatchId = fieldInfo.dispatchId; + // The dispatch ID already encodes whether the data should have a null flag prefix: + // - PRIMITIVE_* and NOTNULL_BOXED_* dispatch IDs have no null flag prefix + // - Nullable dispatch IDs (STRING, BOOL, INT8, etc.) have null flag prefix + // So we write based on dispatch ID, not the local nullable setting. + if (writePrimitiveFieldValue(buffer, obj, fieldAccessor, dispatchId)) { + Object fieldValue = fieldAccessor.getObject(obj); + boolean needWrite = + writeBasicObjectFieldValue(fory, buffer, fieldInfo, fieldValue, dispatchId) + && writeNullableBasicObjectFieldValue(fory, buffer, fieldValue, dispatchId); + if (needWrite) { + binding.writeField(fieldInfo, buffer, fieldValue); + } + } + } + + /** + * Write field value to buffer. Handles primitive types, unsigned/compressed numbers, and common + * types like String with optimized fast paths. + * + *

This method is the write counterpart of {@link #readBuildInFieldValue(SerializationBinding, + * SerializationFieldInfo, MemoryBuffer)}. + * + * @param binding the serialization binding for write operations + * @param fieldInfo the field metadata including type and nullability info + * @param buffer the buffer to write to + * @param fieldValue the value to write + */ + static void writeBuildInFieldValue( SerializationBinding binding, - RefResolver refResolver, - TypeResolver typeResolver, - Generics generics, SerializationFieldInfo fieldInfo, MemoryBuffer buffer, Object fieldValue) { - switch (fieldInfo.refMode) { - case NONE: - generics.pushGenericType(fieldInfo.genericType); - binding.writeContainerFieldValue( - buffer, - fieldValue, - typeResolver.getClassInfo(fieldValue.getClass(), fieldInfo.classInfoHolder)); - generics.popGenericType(); - break; - case NULL_ONLY: - if (fieldValue == null) { - buffer.writeByte(Fory.NULL_FLAG); - } else { - buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); - generics.pushGenericType(fieldInfo.genericType); - binding.writeContainerFieldValue( - buffer, - fieldValue, - typeResolver.getClassInfo(fieldValue.getClass(), fieldInfo.classInfoHolder)); - generics.popGenericType(); - } - break; - case TRACKING: - if (!refResolver.writeRefOrNull(buffer, fieldValue)) { - ClassInfo classInfo = - typeResolver.getClassInfo(fieldValue.getClass(), fieldInfo.classInfoHolder); - generics.pushGenericType(fieldInfo.genericType); - binding.writeContainerFieldValue(buffer, fieldValue, classInfo); - generics.popGenericType(); - } - break; + Fory fory = binding.fory; + boolean nullable = fieldInfo.nullable; + int dispatchId = fieldInfo.dispatchId; + // Fast path for primitive types + if (writePrimitiveValue(buffer, fieldValue, dispatchId)) { + return; + } + boolean needWrite = + nullable + ? writeNullableBasicObjectFieldValue(fory, buffer, fieldValue, dispatchId) + : writeNotNullBasicObjectFieldValue(fory, buffer, fieldValue, dispatchId); + if (!needWrite) { + return; + } + // Fall back to binding.write for complex types + binding.writeField(fieldInfo, buffer, fieldValue); + } + + private static boolean writePrimitiveValue(MemoryBuffer buffer, Object value, int dispatchId) { + switch (dispatchId) { + case DispatchId.PRIMITIVE_BOOL: + buffer.writeBoolean((Boolean) value); + return true; + case DispatchId.PRIMITIVE_INT8: + case DispatchId.PRIMITIVE_UINT8: + buffer.writeByte((Byte) value); + return true; + case DispatchId.PRIMITIVE_CHAR: + buffer.writeChar((Character) value); + return true; + case DispatchId.PRIMITIVE_INT16: + case DispatchId.PRIMITIVE_UINT16: + buffer.writeInt16((Short) value); + return true; + case DispatchId.PRIMITIVE_INT32: + case DispatchId.PRIMITIVE_UINT32: + buffer.writeInt32((Integer) value); + return true; + case DispatchId.PRIMITIVE_VARINT32: + buffer.writeVarInt32((Integer) value); + return true; + case DispatchId.PRIMITIVE_VAR_UINT32: + buffer.writeVarUint32((Integer) value); + return true; + case DispatchId.PRIMITIVE_INT64: + case DispatchId.PRIMITIVE_UINT64: + buffer.writeInt64((Long) value); + return true; + case DispatchId.PRIMITIVE_VARINT64: + buffer.writeVarInt64((Long) value); + return true; + case DispatchId.PRIMITIVE_TAGGED_INT64: + buffer.writeTaggedInt64((Long) value); + return true; + case DispatchId.PRIMITIVE_VAR_UINT64: + buffer.writeVarUint64((Long) value); + return true; + case DispatchId.PRIMITIVE_TAGGED_UINT64: + buffer.writeTaggedUint64((Long) value); + return true; + case DispatchId.PRIMITIVE_FLOAT32: + buffer.writeFloat32((Float) value); + return true; + case DispatchId.PRIMITIVE_FLOAT64: + buffer.writeFloat64((Double) value); + return true; default: - throw new IllegalStateException("Unexpected refMode: " + fieldInfo.refMode); + return false; } } /** * Write a primitive field value to buffer using direct memory offset access. * - * @param buffer the buffer to write to + * @param buffer the buffer to write to * @param targetObject the object containing the field - * @param fieldOffset the memory offset of the field - * @param dispatchId the class ID of the primitive type + * @param fieldOffset the memory offset of the field + * @param dispatchId the class ID of the primitive type * @return true if dispatchId is not a primitive type and needs further write handling */ - static boolean writePrimitiveFieldValue( + private static boolean writePrimitiveFieldValue( MemoryBuffer buffer, Object targetObject, long fieldOffset, int dispatchId) { switch (dispatchId) { case DispatchId.PRIMITIVE_BOOL: @@ -178,10 +262,10 @@ static boolean writePrimitiveFieldValue( /** * Write a primitive field value to buffer using the field accessor. * - * @param buffer the buffer to write to - * @param targetObject the object containing the field + * @param buffer the buffer to write to + * @param targetObject the object containing the field * @param fieldAccessor the accessor to get the field value - * @param dispatchId the class ID of the primitive type + * @param dispatchId the class ID of the primitive type * @return true if dispatchId is not a primitive type and needs further write handling */ static boolean writePrimitiveFieldValue( @@ -255,9 +339,6 @@ static boolean writeNotNullBasicObjectFieldValue( "Non-nullable field has null value. In xlang mode, fields are non-nullable by default. " + "Use @ForyField(nullable=true) to allow null values."); } - if (!fory.isBasicTypesRefIgnored()) { - return true; // let common path handle this. - } // add time types serialization here. switch (dispatchId) { case DispatchId.STRING: // fastpath for string. @@ -323,18 +404,15 @@ static boolean writeNotNullBasicObjectFieldValue( * Write a nullable boxed primitive or String field value to buffer. Writes null flag before value * if the field is null. * - * @param fory the fory instance for compression and ref tracking settings - * @param buffer the buffer to write to + * @param fory the fory instance for compression and ref tracking settings + * @param buffer the buffer to write to * @param fieldValue the field value to write (may be null) * @param dispatchId the class ID of the boxed type * @return true if dispatchId is not a basic type or ref tracking is enabled, needing further - * write handling + * write handling */ static boolean writeNullableBasicObjectFieldValue( Fory fory, MemoryBuffer buffer, Object fieldValue, int dispatchId) { - if (!fory.isBasicTypesRefIgnored()) { - return true; // let common path handle this. - } // add time types serialization here. switch (dispatchId) { case DispatchId.STRING: // fastpath for string. @@ -461,14 +539,143 @@ static boolean writeNullableBasicObjectFieldValue( } } + /** + * Write field value to buffer for PRIMITIVE_* and NOTNULL_BOXED_* dispatch IDs, plus STRING with + * proper refMode handling. This method handles dispatch IDs that don't have a null flag prefix. + * + *

Note: This method only handles PRIMITIVE_*, NOTNULL_BOXED_* dispatch IDs, and STRING. + * Nullable dispatch IDs (BOOL, INT8, etc.) must be handled by {@link + * #writeNullableBasicObjectFieldValue} since they require a null flag prefix. + * + * @return true if field value isn't written by this function and needs further handling. + */ + private static boolean writeBasicObjectFieldValue( + Fory fory, + MemoryBuffer buffer, + SerializationFieldInfo fieldInfo, + Object fieldValue, + int dispatchId) { + // Only handle PRIMITIVE_*, NOTNULL_BOXED_* dispatch IDs, and STRING here. + // Nullable dispatch IDs (BOOL, INT8, etc.) require a null flag prefix + // and must be handled by writeNullableBasicObjectFieldValue. + switch (dispatchId) { + case DispatchId.PRIMITIVE_BOOL: + case DispatchId.NOTNULL_BOXED_BOOL: + buffer.writeBoolean((Boolean) fieldValue); + return false; + case DispatchId.PRIMITIVE_INT8: + case DispatchId.PRIMITIVE_UINT8: + case DispatchId.NOTNULL_BOXED_INT8: + case DispatchId.NOTNULL_BOXED_UINT8: + buffer.writeByte((Byte) fieldValue); + return false; + case DispatchId.PRIMITIVE_CHAR: + case DispatchId.NOTNULL_BOXED_CHAR: + buffer.writeChar((Character) fieldValue); + return false; + case DispatchId.PRIMITIVE_INT16: + case DispatchId.PRIMITIVE_UINT16: + case DispatchId.NOTNULL_BOXED_INT16: + case DispatchId.NOTNULL_BOXED_UINT16: + buffer.writeInt16((Short) fieldValue); + return false; + case DispatchId.PRIMITIVE_INT32: + case DispatchId.PRIMITIVE_UINT32: + case DispatchId.NOTNULL_BOXED_INT32: + case DispatchId.NOTNULL_BOXED_UINT32: + buffer.writeInt32((Integer) fieldValue); + return false; + case DispatchId.PRIMITIVE_VARINT32: + case DispatchId.NOTNULL_BOXED_VARINT32: + buffer.writeVarInt32((Integer) fieldValue); + return false; + case DispatchId.PRIMITIVE_VAR_UINT32: + case DispatchId.NOTNULL_BOXED_VAR_UINT32: + buffer.writeVarUint32((Integer) fieldValue); + return false; + case DispatchId.PRIMITIVE_INT64: + case DispatchId.PRIMITIVE_UINT64: + case DispatchId.NOTNULL_BOXED_INT64: + case DispatchId.NOTNULL_BOXED_UINT64: + buffer.writeInt64((Long) fieldValue); + return false; + case DispatchId.PRIMITIVE_VARINT64: + case DispatchId.NOTNULL_BOXED_VARINT64: + buffer.writeVarInt64((Long) fieldValue); + return false; + case DispatchId.PRIMITIVE_TAGGED_INT64: + case DispatchId.NOTNULL_BOXED_TAGGED_INT64: + buffer.writeTaggedInt64((Long) fieldValue); + return false; + case DispatchId.PRIMITIVE_VAR_UINT64: + case DispatchId.NOTNULL_BOXED_VAR_UINT64: + buffer.writeVarUint64((Long) fieldValue); + return false; + case DispatchId.PRIMITIVE_TAGGED_UINT64: + case DispatchId.NOTNULL_BOXED_TAGGED_UINT64: + buffer.writeTaggedUint64((Long) fieldValue); + return false; + case DispatchId.PRIMITIVE_FLOAT32: + case DispatchId.NOTNULL_BOXED_FLOAT32: + buffer.writeFloat32((Float) fieldValue); + return false; + case DispatchId.PRIMITIVE_FLOAT64: + case DispatchId.NOTNULL_BOXED_FLOAT64: + buffer.writeFloat64((Double) fieldValue); + return false; + case DispatchId.STRING: + // Handle STRING with proper refMode handling + if (fieldInfo.refMode == RefMode.TRACKING) { + fory.writeJavaStringRef(buffer, (String) fieldValue); + } else { + if (fieldInfo.refMode == RefMode.NULL_ONLY) { + if (fieldValue == null) { + buffer.writeByte(Fory.NULL_FLAG); + } else { + buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); + fory.writeJavaString(buffer, (String) fieldValue); + } + } else { + fory.writeJavaString(buffer, (String) fieldValue); + } + } + return false; + default: + return true; + } + } + + static void writeContainerFieldValue( + SerializationBinding binding, + RefResolver refResolver, + Generics generics, + SerializationFieldInfo fieldInfo, + MemoryBuffer buffer, + Object fieldValue) { + if (fieldInfo.refMode == RefMode.TRACKING) { + if (refResolver.writeRefOrNull(buffer, fieldValue)) { + return; + } + } else if (fieldInfo.refMode == RefMode.NULL_ONLY) { + if (fieldValue == null) { + buffer.writeByte(Fory.NULL_FLAG); + return; + } + buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); + } + generics.pushGenericType(fieldInfo.genericType); + binding.writeContainerFieldValue(fieldInfo, buffer, fieldValue); + generics.popGenericType(); + } + /** * Read a container field value (Collection or Map). Handles reference tracking, nullable fields, * and pushes/pops generic type information for proper deserialization of parameterized types. * - * @param binding the serialization binding for read operations - * @param generics the generics context for tracking parameterized types + * @param binding the serialization binding for read operations + * @param generics the generics context for tracking parameterized types * @param fieldInfo the field metadata including generic type info and nullability - * @param buffer the buffer to read from + * @param buffer the buffer to read from * @return the deserialized container field value, or null if the field is nullable and was null */ static Object readContainerFieldValue( @@ -484,18 +691,17 @@ static Object readContainerFieldValue( fieldValue = binding.readContainerFieldValue(buffer, fieldInfo); generics.popGenericType(); break; - case NULL_ONLY: - { - binding.preserveRefId(-1); - byte headFlag = buffer.readByte(); - if (headFlag == Fory.NULL_FLAG) { - return null; - } - generics.pushGenericType(fieldInfo.genericType); - fieldValue = binding.readContainerFieldValue(buffer, fieldInfo); - generics.popGenericType(); + case NULL_ONLY: { + binding.preserveRefId(-1); + byte headFlag = buffer.readByte(); + if (headFlag == Fory.NULL_FLAG) { + return null; } - break; + generics.pushGenericType(fieldInfo.genericType); + fieldValue = binding.readContainerFieldValue(buffer, fieldInfo); + generics.popGenericType(); + } + break; case TRACKING: generics.pushGenericType(fieldInfo.genericType); fieldValue = binding.readContainerFieldValueRef(buffer, fieldInfo); @@ -507,19 +713,263 @@ static Object readContainerFieldValue( return fieldValue; } + /** + * Sentinel object to indicate the dispatch ID was not handled by optimized path. + */ + private static final Object UNHANDLED_SENTINEL = new Object(); + + /** + * Read field value from buffer and return it. Handles primitive types, unsigned/compressed + * numbers, and common types like String with optimized fast paths. + * + *

This method is similar to {@link #readBuildInFieldValue(SerializationBinding, + * SerializationFieldInfo, MemoryBuffer, Object)}, but returns the field value instead of setting + * it into the target object. Useful for record types where field values need to be collected into + * an array before constructing the object. + * + *

Note: The dispatch ID from fieldInfo determines the actual data format in the buffer + * (whether there's a null flag prefix or not), regardless of the local field's nullable setting. + * This is important for schema compatibility when peer's field definition differs from local. + * + * @param binding the serialization binding for read operations + * @param fieldInfo the field metadata including type and nullability info + * @param buffer the buffer to read from + * @return the deserialized field value, or null if the field is nullable and was null + * @see #readBuildInFieldValue(SerializationBinding, SerializationFieldInfo, MemoryBuffer, Object) + */ + static Object readBuildInFieldValue( + SerializationBinding binding, SerializationFieldInfo fieldInfo, MemoryBuffer buffer) { + Fory fory = binding.fory; + int dispatchId = fieldInfo.dispatchId; + // Try optimized path for basic types (primitives, boxed, string) + // The dispatch ID already encodes whether the data has a null flag prefix: + // - PRIMITIVE_* and NOTNULL_BOXED_* dispatch IDs have no null flag prefix + // - Nullable dispatch IDs (BOOL, INT8, etc.) have null flag prefix + // So we try both paths based on dispatch ID, not the local nullable setting. + // Try primitive and non-null boxed types first (PRIMITIVE_*, NOTNULL_BOXED_*, STRING) + Object value = readBasicObjectValue(fory, buffer, fieldInfo, dispatchId); + if (value != UNHANDLED_SENTINEL) { + return value; + } + // Try nullable types (BOOL, INT8, etc. with null flag prefix) + value = readBasicNullableObjectValue(buffer, dispatchId); + if (value != UNHANDLED_SENTINEL) { + return value; + } + // Fall back to binding.read for complex types + return binding.readField(fieldInfo, buffer); + } + + /** + * Read a non-nullable basic object value from buffer and return it. Handles PRIMITIVE_*, + * NOTNULL_BOXED_*, and STRING dispatch IDs with optimized fast paths. + * + *

Note: Nullable dispatch IDs (BOOL, INT8, etc.) must be handled by + * {@link #readBasicNullableObjectValue} since they have a null flag prefix in the serialized + * data. + * + * @return the value if handled, or {@link #UNHANDLED_SENTINEL} if not a basic type + */ + private static Object readBasicObjectValue( + Fory fory, MemoryBuffer buffer, SerializationFieldInfo fieldInfo, int dispatchId) { + switch (dispatchId) { + case DispatchId.PRIMITIVE_BOOL: + case DispatchId.NOTNULL_BOXED_BOOL: + return buffer.readBoolean(); + case DispatchId.PRIMITIVE_INT8: + case DispatchId.PRIMITIVE_UINT8: + case DispatchId.NOTNULL_BOXED_INT8: + case DispatchId.NOTNULL_BOXED_UINT8: + return buffer.readByte(); + case DispatchId.PRIMITIVE_CHAR: + case DispatchId.NOTNULL_BOXED_CHAR: + return buffer.readChar(); + case DispatchId.PRIMITIVE_INT16: + case DispatchId.PRIMITIVE_UINT16: + case DispatchId.NOTNULL_BOXED_INT16: + case DispatchId.NOTNULL_BOXED_UINT16: + return buffer.readInt16(); + case DispatchId.PRIMITIVE_INT32: + case DispatchId.PRIMITIVE_UINT32: + case DispatchId.NOTNULL_BOXED_INT32: + case DispatchId.NOTNULL_BOXED_UINT32: + return buffer.readInt32(); + case DispatchId.PRIMITIVE_VARINT32: + case DispatchId.NOTNULL_BOXED_VARINT32: + return buffer.readVarInt32(); + case DispatchId.PRIMITIVE_VAR_UINT32: + case DispatchId.NOTNULL_BOXED_VAR_UINT32: + return buffer.readVarUint32(); + case DispatchId.PRIMITIVE_INT64: + case DispatchId.PRIMITIVE_UINT64: + case DispatchId.NOTNULL_BOXED_INT64: + case DispatchId.NOTNULL_BOXED_UINT64: + return buffer.readInt64(); + case DispatchId.PRIMITIVE_VARINT64: + case DispatchId.NOTNULL_BOXED_VARINT64: + return buffer.readVarInt64(); + case DispatchId.PRIMITIVE_TAGGED_INT64: + case DispatchId.NOTNULL_BOXED_TAGGED_INT64: + return buffer.readTaggedInt64(); + case DispatchId.PRIMITIVE_VAR_UINT64: + case DispatchId.NOTNULL_BOXED_VAR_UINT64: + return buffer.readVarUint64(); + case DispatchId.PRIMITIVE_TAGGED_UINT64: + case DispatchId.NOTNULL_BOXED_TAGGED_UINT64: + return buffer.readTaggedUint64(); + case DispatchId.PRIMITIVE_FLOAT32: + case DispatchId.NOTNULL_BOXED_FLOAT32: + return buffer.readFloat32(); + case DispatchId.PRIMITIVE_FLOAT64: + case DispatchId.NOTNULL_BOXED_FLOAT64: + return buffer.readFloat64(); + case DispatchId.STRING: + if (fieldInfo.refMode == RefMode.TRACKING) { + return fory.readJavaStringRef(buffer); + } else if (fieldInfo.refMode == RefMode.NULL_ONLY) { + if (buffer.readByte() == Fory.NULL_FLAG) { + return null; + } + return fory.readJavaString(buffer); + } else { + // RefMode.NONE - read string directly without null flag + return fory.readJavaString(buffer); + } + default: + return UNHANDLED_SENTINEL; + } + } + + /** + * Read a nullable basic object value from buffer and return it. Reads null flag before value for + * nullable boxed types (BOOL, INT8, etc.). + * + *

Note: STRING is handled by {@link #readBasicObjectValue} with proper refMode check. + * + * @return the value (possibly null) if handled, or {@link #UNHANDLED_SENTINEL} if not a basic + * type + */ + private static Object readBasicNullableObjectValue(MemoryBuffer buffer, int dispatchId) { + switch (dispatchId) { + case DispatchId.BOOL: + if (buffer.readByte() == Fory.NULL_FLAG) { + return null; + } + return buffer.readBoolean(); + case DispatchId.INT8: + case DispatchId.UINT8: + if (buffer.readByte() == Fory.NULL_FLAG) { + return null; + } + return buffer.readByte(); + case DispatchId.CHAR: + if (buffer.readByte() == Fory.NULL_FLAG) { + return null; + } + return buffer.readChar(); + case DispatchId.INT16: + case DispatchId.UINT16: + if (buffer.readByte() == Fory.NULL_FLAG) { + return null; + } + return buffer.readInt16(); + case DispatchId.INT32: + case DispatchId.UINT32: + if (buffer.readByte() == Fory.NULL_FLAG) { + return null; + } + return buffer.readInt32(); + case DispatchId.VARINT32: + if (buffer.readByte() == Fory.NULL_FLAG) { + return null; + } + return buffer.readVarInt32(); + case DispatchId.VAR_UINT32: + if (buffer.readByte() == Fory.NULL_FLAG) { + return null; + } + return buffer.readVarUint32(); + case DispatchId.INT64: + case DispatchId.UINT64: + if (buffer.readByte() == Fory.NULL_FLAG) { + return null; + } + return buffer.readInt64(); + case DispatchId.VARINT64: + if (buffer.readByte() == Fory.NULL_FLAG) { + return null; + } + return buffer.readVarInt64(); + case DispatchId.TAGGED_INT64: + if (buffer.readByte() == Fory.NULL_FLAG) { + return null; + } + return buffer.readTaggedInt64(); + case DispatchId.VAR_UINT64: + if (buffer.readByte() == Fory.NULL_FLAG) { + return null; + } + return buffer.readVarUint64(); + case DispatchId.TAGGED_UINT64: + if (buffer.readByte() == Fory.NULL_FLAG) { + return null; + } + return buffer.readTaggedUint64(); + case DispatchId.FLOAT32: + if (buffer.readByte() == Fory.NULL_FLAG) { + return null; + } + return buffer.readFloat32(); + case DispatchId.FLOAT64: + if (buffer.readByte() == Fory.NULL_FLAG) { + return null; + } + return buffer.readFloat64(); + default: + return UNHANDLED_SENTINEL; + } + } + + /** + * Handle all numeric fields read include unsigned and compressed numbers. + * It also include fastpath for common type such as String. + * + *

Note: The dispatch ID from fieldInfo determines the actual data format in the buffer + * (whether there's a null flag prefix or not), regardless of the local field's nullable setting. + * This is important for schema compatibility when peer's field definition differs from local. + */ + static void readBuildInFieldValue( + SerializationBinding binding, SerializationFieldInfo fieldInfo, MemoryBuffer buffer, Object targetObject) { + Fory fory = binding.fory; + FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; + int dispatchId = fieldInfo.dispatchId; + // The dispatch ID already encodes whether the data has a null flag prefix: + // - PRIMITIVE_* and NOTNULL_BOXED_* dispatch IDs have no null flag prefix + // - Nullable dispatch IDs (BOOL, INT8, etc.) have null flag prefix + // So we try both paths based on dispatch ID, not the local nullable setting. + boolean needRead = readPrimitiveFieldValue(buffer, targetObject, fieldAccessor, dispatchId) + && readBasicObjectFieldValue(fory, buffer, targetObject, fieldInfo, dispatchId) + && readBasicNullableObjectFieldValue(fory, buffer, targetObject, fieldAccessor, dispatchId); + if (needRead) { + Object fieldValue = binding.readField(fieldInfo, buffer); + fieldAccessor.putObject(targetObject, fieldValue); + } + } + /** * Read a primitive value from buffer and set it to field referenced by fieldAccessor * of targetObject. * * @return true if classId is not a primitive type id. */ - static boolean readPrimitiveFieldValue( + private static boolean readPrimitiveFieldValue( MemoryBuffer buffer, Object targetObject, FieldAccessor fieldAccessor, int dispatchId) { long fieldOffset = fieldAccessor.getFieldOffset(); if (fieldOffset != -1) { return readPrimitiveFieldValue(buffer, targetObject, fieldOffset, dispatchId); } // graalvm use GeneratedAccessor, which will be this code path. + // we still need `PRIMITIVE` cases since peer may send switch (dispatchId) { case DispatchId.PRIMITIVE_BOOL: fieldAccessor.set(targetObject, buffer.readBoolean()); @@ -575,10 +1025,10 @@ static boolean readPrimitiveFieldValue( /** * Read a primitive field value from buffer and set it using direct memory offset access. * - * @param buffer the buffer to read from + * @param buffer the buffer to read from * @param targetObject the object to set the field value on - * @param fieldOffset the memory offset of the field - * @param dispatchId the dispatch ID of the primitive type + * @param fieldOffset the memory offset of the field + * @param dispatchId the dispatch ID of the primitive type * @return true if classId is not a primitive type and needs further read handling */ private static boolean readPrimitiveFieldValue( @@ -636,7 +1086,13 @@ private static boolean readPrimitiveFieldValue( } /** - * read field value from buffer. This method handle the situation which all fields are not null. + * Read field value from buffer and set it on the target object. This method handles PRIMITIVE_* + * and NOTNULL_BOXED_* dispatch IDs where null values are not allowed. + * + *

Note: This method only handles PRIMITIVE_* and NOTNULL_BOXED_* dispatch IDs. Nullable + * dispatch IDs (BOOL, INT8, etc.) must be handled by {@link #readBasicNullableObjectFieldValue} + * since they have a null flag prefix in the serialized data, regardless of the local field's + * nullable setting. * * @return true if field value isn't read by this function. */ @@ -644,104 +1100,86 @@ static boolean readBasicObjectFieldValue( Fory fory, MemoryBuffer buffer, Object targetObject, - FieldAccessor fieldAccessor, + SerializationFieldInfo fieldInfo, int dispatchId) { - if (!fory.isBasicTypesRefIgnored()) { - return true; // let common path handle this. - } - // add time types serialization here. - // Handle both primitive and nullable dispatchIds for schema compatible mode - // where Java field is boxed but ClassDef says non-nullable (primitive encoding) + FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; + // Only handle PRIMITIVE_* and NOTNULL_BOXED_* dispatch IDs here. + // Nullable dispatch IDs (STRING, BOOL, INT8, etc.) have a null flag prefix in the serialized data + // and must be handled by readBasicNullableObjectFieldValue. switch (dispatchId) { - case DispatchId.STRING: // fastpath for string. - if (fory.getStringSerializer().needToWriteRef()) { - fieldAccessor.putObject(targetObject, fory.readJavaStringRef(buffer)); - } else { - fieldAccessor.putObject(targetObject, fory.readString(buffer)); - } - return false; case DispatchId.PRIMITIVE_BOOL: case DispatchId.NOTNULL_BOXED_BOOL: - case DispatchId.BOOL: fieldAccessor.putObject(targetObject, buffer.readBoolean()); return false; case DispatchId.PRIMITIVE_INT8: case DispatchId.PRIMITIVE_UINT8: case DispatchId.NOTNULL_BOXED_INT8: case DispatchId.NOTNULL_BOXED_UINT8: - case DispatchId.INT8: - case DispatchId.UINT8: fieldAccessor.putObject(targetObject, buffer.readByte()); return false; case DispatchId.PRIMITIVE_CHAR: case DispatchId.NOTNULL_BOXED_CHAR: - case DispatchId.CHAR: fieldAccessor.putObject(targetObject, buffer.readChar()); return false; case DispatchId.PRIMITIVE_INT16: case DispatchId.PRIMITIVE_UINT16: case DispatchId.NOTNULL_BOXED_INT16: case DispatchId.NOTNULL_BOXED_UINT16: - case DispatchId.INT16: - case DispatchId.UINT16: fieldAccessor.putObject(targetObject, buffer.readInt16()); return false; case DispatchId.PRIMITIVE_INT32: case DispatchId.PRIMITIVE_UINT32: case DispatchId.NOTNULL_BOXED_INT32: case DispatchId.NOTNULL_BOXED_UINT32: - case DispatchId.INT32: - case DispatchId.UINT32: fieldAccessor.putObject(targetObject, buffer.readInt32()); return false; case DispatchId.PRIMITIVE_VARINT32: case DispatchId.NOTNULL_BOXED_VARINT32: - case DispatchId.VARINT32: fieldAccessor.putObject(targetObject, buffer.readVarInt32()); return false; case DispatchId.PRIMITIVE_VAR_UINT32: case DispatchId.NOTNULL_BOXED_VAR_UINT32: - case DispatchId.VAR_UINT32: fieldAccessor.putObject(targetObject, buffer.readVarUint32()); return false; case DispatchId.PRIMITIVE_INT64: case DispatchId.PRIMITIVE_UINT64: case DispatchId.NOTNULL_BOXED_INT64: case DispatchId.NOTNULL_BOXED_UINT64: - case DispatchId.INT64: - case DispatchId.UINT64: fieldAccessor.putObject(targetObject, buffer.readInt64()); return false; case DispatchId.PRIMITIVE_VARINT64: case DispatchId.NOTNULL_BOXED_VARINT64: - case DispatchId.VARINT64: fieldAccessor.putObject(targetObject, buffer.readVarInt64()); return false; case DispatchId.PRIMITIVE_TAGGED_INT64: case DispatchId.NOTNULL_BOXED_TAGGED_INT64: - case DispatchId.TAGGED_INT64: fieldAccessor.putObject(targetObject, buffer.readTaggedInt64()); return false; case DispatchId.PRIMITIVE_VAR_UINT64: case DispatchId.NOTNULL_BOXED_VAR_UINT64: - case DispatchId.VAR_UINT64: fieldAccessor.putObject(targetObject, buffer.readVarUint64()); return false; case DispatchId.PRIMITIVE_TAGGED_UINT64: case DispatchId.NOTNULL_BOXED_TAGGED_UINT64: - case DispatchId.TAGGED_UINT64: fieldAccessor.putObject(targetObject, buffer.readTaggedUint64()); return false; case DispatchId.PRIMITIVE_FLOAT32: case DispatchId.NOTNULL_BOXED_FLOAT32: - case DispatchId.FLOAT32: fieldAccessor.putObject(targetObject, buffer.readFloat32()); return false; case DispatchId.PRIMITIVE_FLOAT64: case DispatchId.NOTNULL_BOXED_FLOAT64: - case DispatchId.FLOAT64: fieldAccessor.putObject(targetObject, buffer.readFloat64()); return false; + case DispatchId.STRING: + if (fieldInfo.refMode == RefMode.TRACKING) { + fieldAccessor.putObject(targetObject, fory.readJavaStringRef(buffer)); + } else { + if (fieldInfo.refMode != RefMode.NULL_ONLY || buffer.readByte() != Fory.NULL_FLAG) { + fieldAccessor.putObject(targetObject, fory.readJavaString(buffer)); + } + } + return false; default: return true; } @@ -750,29 +1188,25 @@ static boolean readBasicObjectFieldValue( /** * Read a nullable boxed primitive or String field value from buffer and set it on the target * object. Reads the null flag before value for nullable types. + * Note that this method must handle all unsigned/compressed int encodings, since the fallback code path + * are type based, and won't handle such cases. * - * @param fory the fory instance for compression and ref tracking settings - * @param buffer the buffer to read from - * @param targetObject the object to set the field value on + * @param fory the fory instance for compression and ref tracking settings + * @param buffer the buffer to read from + * @param targetObject the object to set the field value on * @param fieldAccessor the accessor to set the field value - * @param dispatchId the class ID of the boxed type + * @param dispatchId the class ID of the boxed type * @return true if dispatchId is not a basic type or ref tracking is enabled, needing further read - * handling + * handling */ - static boolean readBasicNullableObjectFieldValue( + private static boolean readBasicNullableObjectFieldValue( Fory fory, MemoryBuffer buffer, Object targetObject, FieldAccessor fieldAccessor, int dispatchId) { - if (!fory.isBasicTypesRefIgnored()) { - return true; // let common path handle this. - } // add time types serialization here. switch (dispatchId) { - case DispatchId.STRING: // fastpath for string. - fieldAccessor.putObject(targetObject, fory.readJavaStringRef(buffer)); - return false; case DispatchId.BOOL: if (buffer.readByte() == Fory.NULL_FLAG) { fieldAccessor.putObject(targetObject, null); @@ -875,11 +1309,53 @@ static boolean readBasicNullableObjectFieldValue( fieldAccessor.putObject(targetObject, buffer.readFloat64()); } return false; + // string is handled in readBasicObjectFieldValue default: return true; } } + static Object readPrimitiveValue(MemoryBuffer buffer, int dispatchId) { + switch (dispatchId) { + case DispatchId.PRIMITIVE_BOOL: + return buffer.readBoolean(); + case DispatchId.PRIMITIVE_INT8: + case DispatchId.PRIMITIVE_UINT8: + return buffer.readByte(); + case DispatchId.PRIMITIVE_CHAR: + return buffer.readChar(); + case DispatchId.PRIMITIVE_INT16: + case DispatchId.PRIMITIVE_UINT16: + return buffer.readInt16(); + case DispatchId.PRIMITIVE_INT32: + return buffer.readInt32(); + case DispatchId.PRIMITIVE_VARINT32: + return buffer.readVarInt32(); + case DispatchId.PRIMITIVE_UINT32: + return buffer.readInt32(); + case DispatchId.PRIMITIVE_VAR_UINT32: + return buffer.readVarUint32(); + case DispatchId.PRIMITIVE_INT64: + return buffer.readInt64(); + case DispatchId.PRIMITIVE_VARINT64: + return buffer.readVarInt64(); + case DispatchId.PRIMITIVE_TAGGED_INT64: + return buffer.readTaggedInt64(); + case DispatchId.PRIMITIVE_UINT64: + return buffer.readInt64(); + case DispatchId.PRIMITIVE_VAR_UINT64: + return buffer.readVarUint64(); + case DispatchId.PRIMITIVE_TAGGED_UINT64: + return buffer.readTaggedUint64(); + case DispatchId.PRIMITIVE_FLOAT32: + return buffer.readFloat32(); + case DispatchId.PRIMITIVE_FLOAT64: + return buffer.readFloat64(); + default: + return UNHANDLED_SENTINEL; + } + } + @Override public T copy(T originObj) { if (immutable) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldSkipper.java b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldSkipper.java new file mode 100644 index 0000000000..5776c0418e --- /dev/null +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldSkipper.java @@ -0,0 +1,235 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.serializer; + +import org.apache.fory.Fory; +import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo; +import org.apache.fory.type.DispatchId; + +/** + * Utility class for skipping field values in the buffer when a field doesn't exist in the current + * class. This is used for schema compatibility when deserializing data from peers with different + * class definitions. + */ +public class FieldSkipper { + + /** + * Skip a field value in the buffer. Handles all dispatch IDs including primitive types, + * non-null boxed types, nullable boxed types, and compressed number encodings. + * + * @param binding the serialization binding for fallback reads + * @param fieldInfo the field metadata + * @param buffer the buffer to skip from + */ + static void skipField( + SerializationBinding binding, SerializationFieldInfo fieldInfo, MemoryBuffer buffer) { + if (!skipFieldValue(fieldInfo, buffer)) { + // Fall back to binding.read for complex types (objects, collections, etc.) + binding.readField(fieldInfo, buffer); + } + } + + /** + * Skip a field value based on its dispatch ID. Handles primitive types, non-null boxed types, + * and nullable boxed types with their specific encodings (including compressed numbers). + * + * @return true if the field was skipped, false if it needs fallback handling + */ + private static boolean skipFieldValue(SerializationFieldInfo fieldInfo, MemoryBuffer buffer) { + switch (fieldInfo.dispatchId) { + // ============ Primitive types (no null flag) ============ + case DispatchId.PRIMITIVE_BOOL: + buffer.increaseReaderIndex(1); + return true; + case DispatchId.PRIMITIVE_INT8: + case DispatchId.PRIMITIVE_UINT8: + buffer.increaseReaderIndex(1); + return true; + case DispatchId.PRIMITIVE_CHAR: + buffer.increaseReaderIndex(2); + return true; + case DispatchId.PRIMITIVE_INT16: + case DispatchId.PRIMITIVE_UINT16: + buffer.increaseReaderIndex(2); + return true; + case DispatchId.PRIMITIVE_INT32: + case DispatchId.PRIMITIVE_UINT32: + buffer.increaseReaderIndex(4); + return true; + case DispatchId.PRIMITIVE_VARINT32: + buffer.readVarInt32(); + return true; + case DispatchId.PRIMITIVE_VAR_UINT32: + buffer.readVarUint32(); + return true; + case DispatchId.PRIMITIVE_INT64: + case DispatchId.PRIMITIVE_UINT64: + buffer.increaseReaderIndex(8); + return true; + case DispatchId.PRIMITIVE_VARINT64: + buffer.readVarInt64(); + return true; + case DispatchId.PRIMITIVE_TAGGED_INT64: + buffer.readTaggedInt64(); + return true; + case DispatchId.PRIMITIVE_VAR_UINT64: + buffer.readVarUint64(); + return true; + case DispatchId.PRIMITIVE_TAGGED_UINT64: + buffer.readTaggedUint64(); + return true; + case DispatchId.PRIMITIVE_FLOAT32: + buffer.increaseReaderIndex(4); + return true; + case DispatchId.PRIMITIVE_FLOAT64: + buffer.increaseReaderIndex(8); + return true; + + // ============ Non-null boxed types (no null flag) ============ + case DispatchId.NOTNULL_BOXED_BOOL: + buffer.increaseReaderIndex(1); + return true; + case DispatchId.NOTNULL_BOXED_INT8: + case DispatchId.NOTNULL_BOXED_UINT8: + buffer.increaseReaderIndex(1); + return true; + case DispatchId.NOTNULL_BOXED_CHAR: + buffer.increaseReaderIndex(2); + return true; + case DispatchId.NOTNULL_BOXED_INT16: + case DispatchId.NOTNULL_BOXED_UINT16: + buffer.increaseReaderIndex(2); + return true; + case DispatchId.NOTNULL_BOXED_INT32: + case DispatchId.NOTNULL_BOXED_UINT32: + buffer.increaseReaderIndex(4); + return true; + case DispatchId.NOTNULL_BOXED_VARINT32: + buffer.readVarInt32(); + return true; + case DispatchId.NOTNULL_BOXED_VAR_UINT32: + buffer.readVarUint32(); + return true; + case DispatchId.NOTNULL_BOXED_INT64: + case DispatchId.NOTNULL_BOXED_UINT64: + buffer.increaseReaderIndex(8); + return true; + case DispatchId.NOTNULL_BOXED_VARINT64: + buffer.readVarInt64(); + return true; + case DispatchId.NOTNULL_BOXED_TAGGED_INT64: + buffer.readTaggedInt64(); + return true; + case DispatchId.NOTNULL_BOXED_VAR_UINT64: + buffer.readVarUint64(); + return true; + case DispatchId.NOTNULL_BOXED_TAGGED_UINT64: + buffer.readTaggedUint64(); + return true; + case DispatchId.NOTNULL_BOXED_FLOAT32: + buffer.increaseReaderIndex(4); + return true; + case DispatchId.NOTNULL_BOXED_FLOAT64: + buffer.increaseReaderIndex(8); + return true; + + // ============ Nullable boxed types (with null flag) ============ + case DispatchId.BOOL: + if (buffer.readByte() != Fory.NULL_FLAG) { + buffer.increaseReaderIndex(1); + } + return true; + case DispatchId.INT8: + case DispatchId.UINT8: + if (buffer.readByte() != Fory.NULL_FLAG) { + buffer.increaseReaderIndex(1); + } + return true; + case DispatchId.CHAR: + if (buffer.readByte() != Fory.NULL_FLAG) { + buffer.increaseReaderIndex(2); + } + return true; + case DispatchId.INT16: + case DispatchId.UINT16: + if (buffer.readByte() != Fory.NULL_FLAG) { + buffer.increaseReaderIndex(2); + } + return true; + case DispatchId.INT32: + case DispatchId.UINT32: + if (buffer.readByte() != Fory.NULL_FLAG) { + buffer.increaseReaderIndex(4); + } + return true; + case DispatchId.VARINT32: + if (buffer.readByte() != Fory.NULL_FLAG) { + buffer.readVarInt32(); + } + return true; + case DispatchId.VAR_UINT32: + if (buffer.readByte() != Fory.NULL_FLAG) { + buffer.readVarUint32(); + } + return true; + case DispatchId.INT64: + case DispatchId.UINT64: + if (buffer.readByte() != Fory.NULL_FLAG) { + buffer.increaseReaderIndex(8); + } + return true; + case DispatchId.VARINT64: + if (buffer.readByte() != Fory.NULL_FLAG) { + buffer.readVarInt64(); + } + return true; + case DispatchId.TAGGED_INT64: + if (buffer.readByte() != Fory.NULL_FLAG) { + buffer.readTaggedInt64(); + } + return true; + case DispatchId.VAR_UINT64: + if (buffer.readByte() != Fory.NULL_FLAG) { + buffer.readVarUint64(); + } + return true; + case DispatchId.TAGGED_UINT64: + if (buffer.readByte() != Fory.NULL_FLAG) { + buffer.readTaggedUint64(); + } + return true; + case DispatchId.FLOAT32: + if (buffer.readByte() != Fory.NULL_FLAG) { + buffer.increaseReaderIndex(4); + } + return true; + case DispatchId.FLOAT64: + if (buffer.readByte() != Fory.NULL_FLAG) { + buffer.increaseReaderIndex(8); + } + return true; + + default: + // Complex types (String, objects, collections, etc.) need fallback handling + return false; + } + } +} diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java index 436226a124..73462868cc 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java @@ -26,7 +26,6 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.meta.ClassDef; import org.apache.fory.reflect.FieldAccessor; -import org.apache.fory.resolver.ClassInfoHolder; import org.apache.fory.resolver.MetaContext; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo; @@ -47,14 +46,13 @@ * @see ObjectStreamSerializer * @see org.apache.fory.builder.LayerMarkerClassGenerator */ -@SuppressWarnings({"unchecked"}) +@SuppressWarnings({"unchecked", "rawtypes"}) public class MetaSharedLayerSerializer extends MetaSharedLayerSerializerBase { private final ClassDef layerClassDef; private final Class layerMarkerClass; private final SerializationFieldInfo[] buildInFields; private final SerializationFieldInfo[] otherFields; private final SerializationFieldInfo[] containerFields; - private final ClassInfoHolder classInfoHolder; private final SerializationBinding binding; private final TypeResolver typeResolver; @@ -73,7 +71,6 @@ public MetaSharedLayerSerializer( this.layerMarkerClass = layerMarkerClass; this.typeResolver = fory._getTypeResolver(); this.binding = SerializationBinding.createBinding(fory); - this.classInfoHolder = classResolver.nilClassInfoHolder(); // Build field infos from layerClassDef Collection descriptors = layerClassDef.getDescriptors(typeResolver, type); @@ -115,22 +112,8 @@ private void writeLayerClassMeta(MemoryBuffer buffer) { } private void writeFinalFields(MemoryBuffer buffer, T value) { - Fory fory = this.fory; for (SerializationFieldInfo fieldInfo : buildInFields) { - FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; - boolean nullable = fieldInfo.nullable; - int dispatchId = fieldInfo.dispatchId; - if (AbstractObjectSerializer.writePrimitiveFieldValue( - buffer, value, fieldAccessor, dispatchId)) { - Object fieldValue = fieldAccessor.getObject(value); - boolean needWrite = - nullable - ? writeNullableBasicObjectFieldValue(fory, buffer, fieldValue, dispatchId) - : writeNotNullBasicObjectFieldValue(fory, buffer, fieldValue, dispatchId); - if (needWrite) { - binding.write(fieldInfo, buffer, fieldValue); - } - } + AbstractObjectSerializer.writeBuildInField(binding, fieldInfo, buffer, value); } } @@ -140,7 +123,7 @@ private void writeContainerFields(MemoryBuffer buffer, T value) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; Object fieldValue = fieldAccessor.getObject(value); AbstractObjectSerializer.writeContainerFieldValue( - binding, refResolver, typeResolver, generics, fieldInfo, buffer, fieldValue); + binding, refResolver, generics, fieldInfo, buffer, fieldValue); } } @@ -148,7 +131,7 @@ private void writeOtherFields(MemoryBuffer buffer, T value) { for (SerializationFieldInfo fieldInfo : otherFields) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; Object fieldValue = fieldAccessor.getObject(value); - binding.write(fieldInfo, buffer, fieldValue); + binding.writeField(fieldInfo, buffer, fieldValue); } } @@ -198,31 +181,14 @@ private void readLayerClassMeta(MemoryBuffer buffer) { } } - private void readFinalFields(MemoryBuffer buffer, T obj) { - Fory fory = this.fory; + private void readFinalFields(MemoryBuffer buffer, T targetObject) { for (SerializationFieldInfo fieldInfo : buildInFields) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; if (fieldAccessor != null) { - boolean nullable = fieldInfo.nullable; - int dispatchId = fieldInfo.dispatchId; - if (AbstractObjectSerializer.readPrimitiveFieldValue(buffer, obj, fieldAccessor, dispatchId) - && (nullable - ? AbstractObjectSerializer.readBasicNullableObjectFieldValue( - fory, buffer, obj, fieldAccessor, dispatchId) - : AbstractObjectSerializer.readBasicObjectFieldValue( - fory, buffer, obj, fieldAccessor, dispatchId))) { - Object fieldValue = binding.read(fieldInfo, buffer); - fieldAccessor.putObject(obj, fieldValue); - } + AbstractObjectSerializer.readBuildInFieldValue(binding, fieldInfo, buffer, targetObject); } else { // Field doesn't exist in current class - skip the value - if (!MetaSharedSerializer.skipPrimitiveFieldValue(fieldInfo, buffer)) { - if (fieldInfo.classInfo == null) { - fory.readRef(buffer, classInfoHolder); - } else { - binding.read(fieldInfo, buffer); - } - } + FieldSkipper.skipField(binding, fieldInfo, buffer); } } } @@ -241,7 +207,7 @@ private void readContainerFields(MemoryBuffer buffer, T obj) { private void readUserTypeFields(MemoryBuffer buffer, T obj) { for (SerializationFieldInfo fieldInfo : otherFields) { - Object fieldValue = binding.read(fieldInfo, buffer); + Object fieldValue = binding.readField(fieldInfo, buffer); FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; if (fieldAccessor != null) { fieldAccessor.putObject(obj, fieldValue); @@ -306,9 +272,7 @@ private void setFieldValuesFromPutFields( * @return array of field values in putFields order */ @Override - @SuppressWarnings("rawtypes") - public Object[] getFieldValuesForPutFields( - Object obj, org.apache.fory.collection.ObjectIntMap fieldIndexMap, int arraySize) { + public Object[] getFieldValuesForPutFields(Object obj, ObjectIntMap fieldIndexMap, int arraySize) { Object[] vals = new Object[arraySize]; // Get final fields getFieldValuesForPutFields(obj, fieldIndexMap, vals, buildInFields); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java index 7b77e3c489..b8046c7990 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java @@ -33,7 +33,6 @@ import org.apache.fory.memory.Platform; import org.apache.fory.meta.ClassDef; import org.apache.fory.reflect.FieldAccessor; -import org.apache.fory.resolver.ClassInfoHolder; import org.apache.fory.resolver.RefResolver; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo; @@ -73,7 +72,6 @@ public class MetaSharedSerializer extends AbstractObjectSerializer { private final SerializationFieldInfo[] otherFields; private final RecordInfo recordInfo; private Serializer serializer; - private final ClassInfoHolder classInfoHolder; private final SerializationBinding binding; private final boolean hasDefaultValues; private final DefaultValueUtils.DefaultValueField[] defaultValueFields; @@ -113,7 +111,6 @@ public MetaSharedSerializer(Fory fory, Class type, ClassDef classDef) { buildInFields = fieldGroups.buildInFields; containerFields = fieldGroups.containerFields; otherFields = fieldGroups.userTypeFields; - classInfoHolder = this.classResolver.nilClassInfoHolder(); if (isRecord) { List fieldNames = descriptorGrouper.getSortedDescriptors().stream() @@ -149,6 +146,11 @@ public MetaSharedSerializer(Fory fory, Class type, ClassDef classDef) { this.defaultValueFields = defaultValueFields; } + @Override + public void xwrite(MemoryBuffer buffer, T value) { + write(buffer, value); + } + @Override public void write(MemoryBuffer buffer, T value) { if (serializer == null) { @@ -159,11 +161,6 @@ public void write(MemoryBuffer buffer, T value) { serializer.write(buffer, value); } - @Override - public void xwrite(MemoryBuffer buffer, T value) { - write(buffer, value); - } - private T newInstance() { if (!hasDefaultValues) { return newBean(); @@ -190,39 +187,22 @@ public T read(MemoryBuffer buffer) { Arrays.fill(recordInfo.getRecordComponents(), null); return t; } - T obj = newInstance(); + T targetObject = newInstance(); Fory fory = this.fory; RefResolver refResolver = this.refResolver; SerializationBinding binding = this.binding; - refResolver.reference(obj); + refResolver.reference(targetObject); // read order: primitive,boxed,final,other,collection,map for (SerializationFieldInfo fieldInfo : this.buildInFields) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; - boolean nullable = fieldInfo.nullable; if (fieldAccessor != null) { - int dispatchId = fieldInfo.dispatchId; - if (fieldInfo.isPrimitiveField && (!nullable || buffer.readByte() != Fory.NULL_FLAG)) { - if (!readPrimitiveFieldValue(buffer, obj, fieldAccessor, dispatchId)) { - continue; - } - } - // For NOTNULL_BOXED and other boxed types, let readBasicObjectFieldValue handle it - // which uses putObject for proper boxing - if (nullable - ? readBasicNullableObjectFieldValue(fory, buffer, obj, fieldAccessor, dispatchId) - : readBasicObjectFieldValue(fory, buffer, obj, fieldAccessor, dispatchId)) { - assert fieldInfo.classInfo != null; - Object fieldValue = binding.read(fieldInfo, buffer); - fieldAccessor.putObject(obj, fieldValue); - } + AbstractObjectSerializer.readBuildInFieldValue(binding, fieldInfo, buffer, targetObject); } else { if (fieldInfo.fieldConverter == null) { // Skip the field value from buffer since it doesn't exist in current class - if (!skipPrimitiveFieldValue(fieldInfo, buffer)) { - binding.read(fieldInfo, buffer); - } + FieldSkipper.skipField(binding, fieldInfo, buffer); } else { - compatibleRead(buffer, fieldInfo, obj); + compatibleRead(buffer, fieldInfo, targetObject); } } } @@ -232,26 +212,26 @@ public T read(MemoryBuffer buffer) { AbstractObjectSerializer.readContainerFieldValue(binding, generics, fieldInfo, buffer); FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; if (fieldAccessor != null) { - fieldAccessor.putObject(obj, fieldValue); + fieldAccessor.putObject(targetObject, fieldValue); } } for (SerializationFieldInfo fieldInfo : otherFields) { - Object fieldValue = binding.read(fieldInfo, buffer); + Object fieldValue = binding.readField(fieldInfo, buffer); FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; if (fieldAccessor != null) { - fieldAccessor.putObject(obj, fieldValue); + fieldAccessor.putObject(targetObject, fieldValue); } } - return obj; + return targetObject; } private void compatibleRead(MemoryBuffer buffer, SerializationFieldInfo fieldInfo, Object obj) { Object fieldValue; int dispatchId = fieldInfo.dispatchId; if (DispatchId.isPrimitive(dispatchId)) { - fieldValue = Serializers.readPrimitiveValue(buffer, dispatchId); + fieldValue = AbstractObjectSerializer.readPrimitiveValue(buffer, dispatchId); } else { - fieldValue = binding.read(fieldInfo, buffer); + fieldValue = binding.readField(fieldInfo, buffer); ; } fieldInfo.fieldConverter.set(obj, fieldValue); @@ -264,19 +244,12 @@ private void readFields(MemoryBuffer buffer, Object[] fields) { // read order: primitive,boxed,final,other,collection,map for (SerializationFieldInfo fieldInfo : this.buildInFields) { if (fieldInfo.fieldAccessor != null) { - assert fieldInfo.classInfo != null; - int dispatchId = fieldInfo.dispatchId; - if (DispatchId.isPrimitive(dispatchId)) { - fields[counter++] = Serializers.readPrimitiveValue(buffer, dispatchId); - } else { - Object fieldValue = binding.read(fieldInfo, buffer); - fields[counter++] = fieldValue; - } + fields[counter++] = AbstractObjectSerializer.readBuildInFieldValue(binding, fieldInfo, buffer); } else { - // Skip the field value from buffer since it doesn't exist in current class - if (!skipPrimitiveFieldValue(fieldInfo, buffer)) { - skipObjectFieldValue(buffer, fieldInfo); - } + // Skip the field value from buffer since it doesn't exist in current class. + // For records, fieldConverter can't be used since records are immutable and + // constructed all at once. We just read to advance buffer position. + FieldSkipper.skipField(binding, fieldInfo, buffer); // remapping will handle those extra fields from peers. fields[counter++] = null; } @@ -288,106 +261,11 @@ private void readFields(MemoryBuffer buffer, Object[] fields) { fields[counter++] = fieldValue; } for (SerializationFieldInfo fieldInfo : otherFields) { - Object fieldValue = binding.read(fieldInfo, buffer); + Object fieldValue = binding.readField(fieldInfo, buffer); fields[counter++] = fieldValue; } } - private void skipObjectFieldValue(MemoryBuffer buffer, SerializationFieldInfo fieldInfo) { - if (fieldInfo.classInfo == null) { - switch (fieldInfo.refMode) { - case NONE: - binding.readNonRef(buffer, fieldInfo); - break; - case NULL_ONLY: - binding.readNullable(buffer, fieldInfo); - break; - case TRACKING: - binding.readRef(buffer, fieldInfo); - break; - default: - } - } else { - binding.read(fieldInfo, buffer); - } - } - - /** Skip primitive/notnull-boxed field value since they don't write null flag. */ - static boolean skipPrimitiveFieldValue(SerializationFieldInfo fieldInfo, MemoryBuffer buffer) { - switch (fieldInfo.dispatchId) { - case DispatchId.PRIMITIVE_BOOL: - case DispatchId.NOTNULL_BOXED_BOOL: - buffer.increaseReaderIndex(1); - return true; - case DispatchId.PRIMITIVE_INT8: - case DispatchId.PRIMITIVE_UINT8: - case DispatchId.NOTNULL_BOXED_INT8: - case DispatchId.NOTNULL_BOXED_UINT8: - buffer.increaseReaderIndex(1); - return true; - case DispatchId.PRIMITIVE_CHAR: - case DispatchId.NOTNULL_BOXED_CHAR: - buffer.increaseReaderIndex(2); - return true; - case DispatchId.PRIMITIVE_INT16: - case DispatchId.PRIMITIVE_UINT16: - case DispatchId.NOTNULL_BOXED_INT16: - case DispatchId.NOTNULL_BOXED_UINT16: - buffer.increaseReaderIndex(2); - return true; - case DispatchId.PRIMITIVE_INT32: - case DispatchId.NOTNULL_BOXED_INT32: - buffer.increaseReaderIndex(4); - return true; - case DispatchId.PRIMITIVE_VARINT32: - case DispatchId.NOTNULL_BOXED_VARINT32: - buffer.readVarInt32(); - return true; - case DispatchId.PRIMITIVE_UINT32: - case DispatchId.NOTNULL_BOXED_UINT32: - buffer.increaseReaderIndex(4); - return true; - case DispatchId.PRIMITIVE_VAR_UINT32: - case DispatchId.NOTNULL_BOXED_VAR_UINT32: - buffer.readVarUint32(); - return true; - case DispatchId.PRIMITIVE_INT64: - case DispatchId.NOTNULL_BOXED_INT64: - buffer.increaseReaderIndex(8); - return true; - case DispatchId.PRIMITIVE_VARINT64: - case DispatchId.NOTNULL_BOXED_VARINT64: - buffer.readVarInt64(); - return true; - case DispatchId.PRIMITIVE_TAGGED_INT64: - case DispatchId.NOTNULL_BOXED_TAGGED_INT64: - buffer.readTaggedInt64(); - return true; - case DispatchId.PRIMITIVE_UINT64: - case DispatchId.NOTNULL_BOXED_UINT64: - buffer.increaseReaderIndex(8); - return true; - case DispatchId.PRIMITIVE_VAR_UINT64: - case DispatchId.NOTNULL_BOXED_VAR_UINT64: - buffer.readVarUint64(); - return true; - case DispatchId.PRIMITIVE_TAGGED_UINT64: - case DispatchId.NOTNULL_BOXED_TAGGED_UINT64: - buffer.readTaggedUint64(); - return true; - case DispatchId.PRIMITIVE_FLOAT32: - case DispatchId.NOTNULL_BOXED_FLOAT32: - buffer.increaseReaderIndex(4); - return true; - case DispatchId.PRIMITIVE_FLOAT64: - case DispatchId.NOTNULL_BOXED_FLOAT64: - buffer.increaseReaderIndex(8); - return true; - default: - return false; - } - } - public static Collection consolidateFields( TypeResolver resolver, Class cls, ClassDef classDef) { return classDef.getDescriptors(resolver, cls); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java index d78e82d021..edbc79d86b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java @@ -32,7 +32,6 @@ import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.meta.ClassDef; -import org.apache.fory.resolver.ClassInfo; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.MetaContext; import org.apache.fory.resolver.MetaStringResolver; @@ -43,7 +42,6 @@ import org.apache.fory.serializer.Serializers.CrossLanguageCompatibleSerializer; import org.apache.fory.type.Descriptor; import org.apache.fory.type.DescriptorGrouper; -import org.apache.fory.type.DispatchId; import org.apache.fory.type.Generics; import org.apache.fory.util.Preconditions; @@ -118,32 +116,17 @@ public void write(MemoryBuffer buffer, Object v) { // write order: primitive,boxed,final,other,collection,map for (SerializationFieldInfo fieldInfo : fieldsInfo.buildInFields) { Object fieldValue = value.get(fieldInfo.qualifiedFieldName); - ClassInfo classInfo = fieldInfo.classInfo; - if (fory.getConfig().isForyDebugOutputEnabled()) { - LOG.info( - "NonexistentClassSerializer.write: field={}, dispatchId={}, isPrimitive={}, value={}, serializer={}", - fieldInfo.qualifiedFieldName, - fieldInfo.dispatchId, - DispatchId.isPrimitive(fieldInfo.dispatchId), - fieldValue, - classInfo != null ? classInfo.getSerializer() : null); - } - if (DispatchId.isPrimitive(fieldInfo.dispatchId)) { - // Use dispatch-based write to ensure correct encoding (e.g., VARINT64 vs FIXED_INT64) - Serializers.writePrimitiveValue(buffer, fieldValue, fieldInfo.dispatchId); - } else { - binding.write(fieldInfo, buffer, fieldValue); - } + AbstractObjectSerializer.writeBuildInFieldValue(binding, fieldInfo, buffer, fieldValue); } Generics generics = fory.getGenerics(); for (SerializationFieldInfo fieldInfo : fieldsInfo.containerFields) { Object fieldValue = value.get(fieldInfo.qualifiedFieldName); AbstractObjectSerializer.writeContainerFieldValue( - binding, refResolver, classResolver, generics, fieldInfo, buffer, fieldValue); + binding, refResolver, generics, fieldInfo, buffer, fieldValue); } for (SerializationFieldInfo fieldInfo : fieldsInfo.otherFields) { Object fieldValue = value.get(fieldInfo.qualifiedFieldName); - binding.write(fieldInfo, buffer, fieldValue); + binding.writeField(fieldInfo, buffer, fieldValue); } } @@ -179,7 +162,8 @@ public Object read(MemoryBuffer buffer) { // read order: primitive,boxed,final,other,collection,map ClassFieldsInfo fieldsInfo = getClassFieldsInfo(classDef); for (SerializationFieldInfo fieldInfo : fieldsInfo.buildInFields) { - entries.add(new MapEntry(fieldInfo.qualifiedFieldName, binding.read(fieldInfo, buffer))); + Object fieldValue = AbstractObjectSerializer.readBuildInFieldValue(binding, fieldInfo, buffer); + entries.add(new MapEntry(fieldInfo.qualifiedFieldName, fieldValue)); } Generics generics = fory.getGenerics(); for (SerializationFieldInfo fieldInfo : fieldsInfo.containerFields) { @@ -188,7 +172,7 @@ public Object read(MemoryBuffer buffer) { entries.add(new MapEntry(fieldInfo.qualifiedFieldName, fieldValue)); } for (SerializationFieldInfo fieldInfo : fieldsInfo.otherFields) { - Object fieldValue = binding.read(fieldInfo, buffer); + Object fieldValue = binding.readField(fieldInfo, buffer); entries.add(new MapEntry(fieldInfo.qualifiedFieldName, fieldValue)); } obj.setEntries(entries); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java index f4a383caba..e90c197a7a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java @@ -140,18 +140,11 @@ public void write(MemoryBuffer buffer, T value) { Fory fory = this.fory; RefResolver refResolver = this.refResolver; if (fory.checkClassVersion()) { - if (fory.getConfig().isForyDebugOutputEnabled()) { - LOG.info( - "[Java][fory-debug] Writing struct hash for {} at position {}: hash={}", - type.getSimpleName(), - buffer.writerIndex(), - classVersionHash); - } buffer.writeInt32(classVersionHash); } // write order: primitive,boxed,final,other,collection,map writeBuildInFields(buffer, value, fory); - writeContainerFields(buffer, value, fory, refResolver, typeResolver); + writeContainerFields(buffer, value, fory, refResolver); writeOtherFields(buffer, value); } @@ -159,36 +152,24 @@ private void writeOtherFields(MemoryBuffer buffer, T value) { for (SerializationFieldInfo fieldInfo : otherFields) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; Object fieldValue = fieldAccessor.getObject(value); - binding.write(fieldInfo, buffer, fieldValue); + binding.writeField(fieldInfo, buffer, fieldValue); } } private void writeBuildInFields(MemoryBuffer buffer, T value, Fory fory) { for (SerializationFieldInfo fieldInfo : this.buildInFields) { - FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; - boolean nullable = fieldInfo.nullable; - int dispatchId = fieldInfo.dispatchId; - if (writePrimitiveFieldValue(buffer, value, fieldAccessor, dispatchId)) { - Object fieldValue = fieldAccessor.getObject(value); - boolean needWrite = - nullable - ? writeNullableBasicObjectFieldValue(fory, buffer, fieldValue, dispatchId) - : writeNotNullBasicObjectFieldValue(fory, buffer, fieldValue, dispatchId); - if (needWrite) { - binding.write(fieldInfo, buffer, fieldValue); - } - } + AbstractObjectSerializer.writeBuildInField(binding, fieldInfo, buffer, value); } } private void writeContainerFields( - MemoryBuffer buffer, T value, Fory fory, RefResolver refResolver, TypeResolver typeResolver) { + MemoryBuffer buffer, T value, Fory fory, RefResolver refResolver) { Generics generics = fory.getGenerics(); for (SerializationFieldInfo fieldInfo : containerFields) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; Object fieldValue = fieldAccessor.getObject(value); writeContainerFieldValue( - binding, refResolver, typeResolver, generics, fieldInfo, buffer, fieldValue); + binding, refResolver, generics, fieldInfo, buffer, fieldValue); } } @@ -224,9 +205,9 @@ public Object[] readFields(MemoryBuffer buffer) { for (SerializationFieldInfo fieldInfo : this.buildInFields) { int dispatchId = fieldInfo.dispatchId; if (DispatchId.isPrimitive(dispatchId)) { - fieldValues[counter++] = Serializers.readPrimitiveValue(buffer, dispatchId); + fieldValues[counter++] = AbstractObjectSerializer.readPrimitiveValue(buffer, dispatchId); } else { - Object fieldValue = binding.read(fieldInfo, buffer); + Object fieldValue = binding.readField(fieldInfo, buffer); fieldValues[counter++] = fieldValue; } } @@ -236,7 +217,7 @@ public Object[] readFields(MemoryBuffer buffer) { fieldValues[counter++] = fieldValue; } for (SerializationFieldInfo fieldInfo : otherFields) { - Object fieldValue = binding.read(fieldInfo, buffer); + Object fieldValue = binding.readField(fieldInfo, buffer); fieldValues[counter++] = fieldValue; } return fieldValues; @@ -251,15 +232,8 @@ public T readAndSetFields(MemoryBuffer buffer, T obj) { // read order: primitive,boxed,final,other,collection,map for (SerializationFieldInfo fieldInfo : this.buildInFields) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; - boolean nullable = fieldInfo.nullable; - int dispatchId = fieldInfo.dispatchId; - if (readPrimitiveFieldValue(buffer, obj, fieldAccessor, dispatchId) - && (nullable - ? readBasicNullableObjectFieldValue(fory, buffer, obj, fieldAccessor, dispatchId) - : readBasicObjectFieldValue(fory, buffer, obj, fieldAccessor, dispatchId))) { - Object fieldValue = binding.read(fieldInfo, buffer); - fieldAccessor.putObject(obj, fieldValue); - } + // a numeric type can have only three kinds: primitive, not_null_boxed, nullable_boxed + readBuildInFieldValue(binding, fieldInfo, buffer, obj); } Generics generics = fory.getGenerics(); for (SerializationFieldInfo fieldInfo : containerFields) { @@ -268,7 +242,7 @@ public T readAndSetFields(MemoryBuffer buffer, T obj) { fieldAccessor.putObject(obj, fieldValue); } for (SerializationFieldInfo fieldInfo : otherFields) { - Object fieldValue = binding.read(fieldInfo, buffer); + Object fieldValue = binding.readField(fieldInfo, buffer); FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; fieldAccessor.putObject(obj, fieldValue); } @@ -280,19 +254,7 @@ public static int computeStructHash(Fory fory, DescriptorGrouper grouper) { String fingerprint = Fingerprint.computeStructFingerprint(fory, sorted); byte[] bytes = fingerprint.getBytes(StandardCharsets.UTF_8); long hashLong = MurmurHash3.murmurhash3_x64_128(bytes, 0, bytes.length, 47)[0]; - int hash = (int) (hashLong & 0xffffffffL); - if (fory.getConfig().isForyDebugOutputEnabled()) { - String className = - sorted.isEmpty() ? "" : String.valueOf(sorted.get(0).getDeclaringClass()); - LOG.info( - "[Java][fory-debug] struct " - + className - + " version fingerprint=\"" - + fingerprint - + "\" version hash=" - + hash); - } - return hash; + return (int) (hashLong & 0xffffffffL); } public static void checkClassVersion(Class cls, int readHash, int classVersionHash) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/PrimitiveSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/PrimitiveSerializers.java index 0affca6517..34ff88938e 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/PrimitiveSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/PrimitiveSerializers.java @@ -36,7 +36,7 @@ public class PrimitiveSerializers { public static final class BooleanSerializer extends Serializers.CrossLanguageCompatibleSerializer { public BooleanSerializer(Fory fory, Class cls) { - super(fory, (Class) cls, !(cls.isPrimitive() || fory.isBasicTypesRefIgnored()), true); + super(fory, (Class) cls, false, true); } @Override @@ -53,7 +53,7 @@ public Boolean read(MemoryBuffer buffer) { public static final class ByteSerializer extends Serializers.CrossLanguageCompatibleSerializer { public ByteSerializer(Fory fory, Class cls) { - super(fory, (Class) cls, !(cls.isPrimitive() || fory.isBasicTypesRefIgnored()), true); + super(fory, (Class) cls, false, true); } @Override @@ -105,7 +105,7 @@ public Integer xread(MemoryBuffer buffer) { public static final class CharSerializer extends ImmutableSerializer { public CharSerializer(Fory fory, Class cls) { - super(fory, (Class) cls, !(cls.isPrimitive() || fory.isBasicTypesRefIgnored())); + super(fory, (Class) cls, false); } @Override @@ -122,7 +122,7 @@ public Character read(MemoryBuffer buffer) { public static final class ShortSerializer extends Serializers.CrossLanguageCompatibleSerializer { public ShortSerializer(Fory fory, Class cls) { - super(fory, (Class) cls, !(cls.isPrimitive() || fory.isBasicTypesRefIgnored()), true); + super(fory, (Class) cls, false, true); } @Override @@ -141,7 +141,7 @@ public static final class IntSerializer private final boolean compressNumber; public IntSerializer(Fory fory, Class cls) { - super(fory, (Class) cls, !(cls.isPrimitive() || fory.isBasicTypesRefIgnored()), true); + super(fory, (Class) cls, false, true); compressNumber = fory.compressInt(); } @@ -180,7 +180,7 @@ public static final class LongSerializer private final LongEncoding longEncoding; public LongSerializer(Fory fory, Class cls) { - super(fory, (Class) cls, !(cls.isPrimitive() || fory.isBasicTypesRefIgnored()), true); + super(fory, (Class) cls, false, true); longEncoding = fory.longEncoding(); } @@ -261,7 +261,7 @@ public Long xread(MemoryBuffer buffer) { public static final class FloatSerializer extends Serializers.CrossLanguageCompatibleSerializer { public FloatSerializer(Fory fory, Class cls) { - super(fory, (Class) cls, !(cls.isPrimitive() || fory.isBasicTypesRefIgnored()), true); + super(fory, (Class) cls, false, true); } @Override @@ -278,7 +278,7 @@ public Float read(MemoryBuffer buffer) { public static final class DoubleSerializer extends Serializers.CrossLanguageCompatibleSerializer { public DoubleSerializer(Fory fory, Class cls) { - super(fory, (Class) cls, !(cls.isPrimitive() || fory.isBasicTypesRefIgnored()), true); + super(fory, (Class) cls, false, true); } @Override diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java index 5b4b53ba60..93ac076f90 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java @@ -80,13 +80,13 @@ abstract void writeNullable( MemoryBuffer buffer, Object obj, Serializer serializer, boolean nullable); abstract void writeContainerFieldValue( - MemoryBuffer buffer, Object fieldValue, ClassInfo classInfo); + SerializationFieldInfo fieldInfo, MemoryBuffer buffer, Object fieldValue); - abstract void write(SerializationFieldInfo fieldInfo, MemoryBuffer buffer, Object value); + abstract void writeField(SerializationFieldInfo fieldInfo, MemoryBuffer buffer, Object fieldValue); abstract void write(MemoryBuffer buffer, Serializer serializer, Object value); - abstract Object read(SerializationFieldInfo fieldInfo, MemoryBuffer buffer); + abstract Object readField(SerializationFieldInfo fieldInfo, MemoryBuffer buffer); abstract Object read(MemoryBuffer buffer, Serializer serializer); @@ -257,7 +257,7 @@ public void write(MemoryBuffer buffer, Serializer serializer, Object value) { } @Override - Object read(SerializationFieldInfo fieldInfo, MemoryBuffer buffer) { + Object readField(SerializationFieldInfo fieldInfo, MemoryBuffer buffer) { if (fieldInfo.useDeclaredTypeInfo) { if (fieldInfo.refMode == RefMode.TRACKING) { return fory.readRef(buffer, fieldInfo.classInfo); @@ -360,12 +360,18 @@ public void writeNullable( @Override public void writeContainerFieldValue( - MemoryBuffer buffer, Object fieldValue, ClassInfo classInfo) { - fory.writeNonRef(buffer, fieldValue, classInfo); + SerializationFieldInfo fieldInfo, MemoryBuffer buffer, Object fieldValue) { + if (fieldInfo.useDeclaredTypeInfo) { + ClassInfo classInfo = + typeResolver.getClassInfo(fieldValue.getClass(), fieldInfo.classInfoHolder); + fory.writeNonRef(buffer, fieldValue, classInfo); + } else { + fory.writeNonRef(buffer, fieldValue, fieldInfo.classInfoHolder); + } } @Override - void write(SerializationFieldInfo fieldInfo, MemoryBuffer buffer, Object fieldValue) { + void writeField(SerializationFieldInfo fieldInfo, MemoryBuffer buffer, Object fieldValue) { if (fieldInfo.useDeclaredTypeInfo) { Serializer serializer = fieldInfo.classInfo.getSerializer(); if (fieldInfo.refMode == RefMode.TRACKING) { @@ -406,7 +412,7 @@ static final class XlangSerializationBinding extends SerializationBinding { } @Override - void write(SerializationFieldInfo fieldInfo, MemoryBuffer buffer, Object fieldValue) { + void writeField(SerializationFieldInfo fieldInfo, MemoryBuffer buffer, Object fieldValue) { if (fieldInfo.useDeclaredTypeInfo) { Serializer serializer = fieldInfo.classInfo.getSerializer(); if (fieldInfo.refMode == RefMode.TRACKING) { @@ -574,7 +580,7 @@ public void write(MemoryBuffer buffer, Serializer serializer, Object value) { } @Override - Object read(SerializationFieldInfo fieldInfo, MemoryBuffer buffer) { + Object readField(SerializationFieldInfo fieldInfo, MemoryBuffer buffer) { if (fieldInfo.useDeclaredTypeInfo) { if (fieldInfo.refMode == RefMode.TRACKING) { return fory.xreadRef(buffer, fieldInfo.classInfo); @@ -677,7 +683,10 @@ public void writeNullable( @Override public void writeContainerFieldValue( - MemoryBuffer buffer, Object fieldValue, ClassInfo classInfo) { + SerializationFieldInfo fieldInfo, MemoryBuffer buffer, Object fieldValue) { + assert fieldInfo.useDeclaredTypeInfo; + ClassInfo classInfo = + typeResolver.getClassInfo(fieldValue.getClass(), fieldInfo.classInfoHolder); fory.xwriteData(buffer, classInfo, fieldValue); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializer.java index 8a3e7fd2e5..40acdc1542 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializer.java @@ -75,7 +75,7 @@ public Serializer(Fory fory, Class type) { this.type = type; this.isJava = !fory.isCrossLanguage(); if (fory.trackingRef()) { - needToWriteRef = !TypeUtils.isBoxed(TypeUtils.wrap(type)) || !fory.isBasicTypesRefIgnored(); + needToWriteRef = !TypeUtils.isBoxed(TypeUtils.wrap(type)) || false; } else { needToWriteRef = false; } @@ -88,7 +88,7 @@ public Serializer(Fory fory, Class type, boolean immutable) { this.type = type; this.isJava = !fory.isCrossLanguage(); if (fory.trackingRef()) { - needToWriteRef = !TypeUtils.isBoxed(TypeUtils.wrap(type)) || !fory.isBasicTypesRefIgnored(); + needToWriteRef = !TypeUtils.isBoxed(TypeUtils.wrap(type)) || false; } else { needToWriteRef = false; } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java index 00cb404bad..020ac2efc4 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java @@ -50,7 +50,6 @@ import org.apache.fory.meta.ClassDef; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.resolver.ClassResolver; -import org.apache.fory.type.DispatchId; import org.apache.fory.util.ExceptionUtils; import org.apache.fory.util.GraalvmSupport; import org.apache.fory.util.GraalvmSupport.GraalvmSerializerHolder; @@ -177,100 +176,6 @@ private static Serializer createSerializer( } } - public static Object readPrimitiveValue(MemoryBuffer buffer, int dispatchId) { - switch (dispatchId) { - case DispatchId.PRIMITIVE_BOOL: - return buffer.readBoolean(); - case DispatchId.PRIMITIVE_INT8: - case DispatchId.PRIMITIVE_UINT8: - return buffer.readByte(); - case DispatchId.PRIMITIVE_CHAR: - return buffer.readChar(); - case DispatchId.PRIMITIVE_INT16: - case DispatchId.PRIMITIVE_UINT16: - return buffer.readInt16(); - case DispatchId.PRIMITIVE_INT32: - return buffer.readInt32(); - case DispatchId.PRIMITIVE_VARINT32: - return buffer.readVarInt32(); - case DispatchId.PRIMITIVE_UINT32: - return buffer.readInt32(); - case DispatchId.PRIMITIVE_VAR_UINT32: - return buffer.readVarUint32(); - case DispatchId.PRIMITIVE_INT64: - return buffer.readInt64(); - case DispatchId.PRIMITIVE_VARINT64: - return buffer.readVarInt64(); - case DispatchId.PRIMITIVE_TAGGED_INT64: - return buffer.readTaggedInt64(); - case DispatchId.PRIMITIVE_UINT64: - return buffer.readInt64(); - case DispatchId.PRIMITIVE_VAR_UINT64: - return buffer.readVarUint64(); - case DispatchId.PRIMITIVE_TAGGED_UINT64: - return buffer.readTaggedUint64(); - case DispatchId.PRIMITIVE_FLOAT32: - return buffer.readFloat32(); - case DispatchId.PRIMITIVE_FLOAT64: - return buffer.readFloat64(); - default: - throw new IllegalStateException("unreachable"); - } - } - - public static void writePrimitiveValue(MemoryBuffer buffer, Object value, int dispatchId) { - switch (dispatchId) { - case DispatchId.PRIMITIVE_BOOL: - buffer.writeBoolean((Boolean) value); - break; - case DispatchId.PRIMITIVE_INT8: - case DispatchId.PRIMITIVE_UINT8: - buffer.writeByte((Byte) value); - break; - case DispatchId.PRIMITIVE_CHAR: - buffer.writeChar((Character) value); - break; - case DispatchId.PRIMITIVE_INT16: - case DispatchId.PRIMITIVE_UINT16: - buffer.writeInt16((Short) value); - break; - case DispatchId.PRIMITIVE_INT32: - case DispatchId.PRIMITIVE_UINT32: - buffer.writeInt32((Integer) value); - break; - case DispatchId.PRIMITIVE_VARINT32: - buffer.writeVarInt32((Integer) value); - break; - case DispatchId.PRIMITIVE_VAR_UINT32: - buffer.writeVarUint32((Integer) value); - break; - case DispatchId.PRIMITIVE_INT64: - case DispatchId.PRIMITIVE_UINT64: - buffer.writeInt64((Long) value); - break; - case DispatchId.PRIMITIVE_VARINT64: - buffer.writeVarInt64((Long) value); - break; - case DispatchId.PRIMITIVE_TAGGED_INT64: - buffer.writeTaggedInt64((Long) value); - break; - case DispatchId.PRIMITIVE_VAR_UINT64: - buffer.writeVarUint64((Long) value); - break; - case DispatchId.PRIMITIVE_TAGGED_UINT64: - buffer.writeTaggedUint64((Long) value); - break; - case DispatchId.PRIMITIVE_FLOAT32: - buffer.writeFloat32((Float) value); - break; - case DispatchId.PRIMITIVE_FLOAT64: - buffer.writeFloat64((Double) value); - break; - default: - throw new IllegalStateException("unreachable dispatchId: " + dispatchId); - } - } - public abstract static class CrossLanguageCompatibleSerializer extends Serializer { public CrossLanguageCompatibleSerializer(Fory fory, Class cls) { diff --git a/java/fory-core/src/main/java/org/apache/fory/type/DescriptorGrouper.java b/java/fory-core/src/main/java/org/apache/fory/type/DescriptorGrouper.java index 2d232c784c..c499c91f8f 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/DescriptorGrouper.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/DescriptorGrouper.java @@ -32,12 +32,26 @@ /** * A utility class to group class fields into groups. - *
  • primitive fields - *
  • boxed primitive fields - *
  • final fields - *
  • collection fields - *
  • map fields - *
  • other fields + * + *
      + *
    • primitive fields + *
    • boxed primitive fields + *
    • final fields + *
    • collection fields + *
    • map fields + *
    • other fields + *
    + * + *

    IMPORTANT: Resorting fields is mandatory in cross-language (xlang) serialization. The + * Fory protocol specification requires that both serialization peers (e.g., Java, Rust, Go, Python) + * use exactly the same sorting algorithm to determine field order. The in-flight byte order of + * fields is not guaranteed to match any particular peer's original declaration order. Instead, each + * peer must independently sort fields using the same algorithm to ensure consistent + * serialization/deserialization. + * + *

    The sorting groups fields by type category (primitives, boxed, collections, maps, etc.) and + * then sorts by field name within each category. Both reader and writer must apply this sorting to + * produce identical field ordering. */ public class DescriptorGrouper { diff --git a/java/fory-core/src/main/java/org/apache/fory/type/DispatchId.java b/java/fory-core/src/main/java/org/apache/fory/type/DispatchId.java index 85118b2985..6955bb9f95 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/DispatchId.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/DispatchId.java @@ -103,17 +103,29 @@ public static int getDispatchId(Fory fory, Descriptor d) { // Determine the dispatch mode based on local field type and remote nullable flag int mode; boolean localIsPrimitive = rawType.isPrimitive(); - boolean remoteNullable = typeExtMeta == null || typeExtMeta.nullable(); - - if (localIsPrimitive) { - // Local field is primitive (int, long, etc.) - always use PRIMITIVE dispatch - mode = MODE_PRIMITIVE; - } else if (!remoteNullable && Types.isPrimitiveType(typeId)) { - // Local field is boxed (Integer, Long, etc.), remote wrote without null flag - // Use NOTNULL_BOXED dispatch: read value directly, box it, use putObject - mode = MODE_NOTNULL_BOXED; + + // Determine remote nullable: if typeExtMeta exists, use remote info; otherwise use local type + // For consistent schema (no typeExtMeta), primitive fields never have null flag + boolean remoteNullable; + if (typeExtMeta != null) { + remoteNullable = typeExtMeta.nullable(); + } else { + // No remote info (consistent schema) - use local field type to determine nullable + // Primitive types are never nullable, boxed/reference types are nullable + remoteNullable = !localIsPrimitive; + } + + if (!remoteNullable) { + // Remote wrote without null flag (or no remote info and local is primitive) + if (localIsPrimitive) { + mode = MODE_PRIMITIVE; + } else if (Types.isPrimitiveType(typeId)) { + mode = MODE_NOTNULL_BOXED; + } else { + mode = MODE_NULLABLE_BOXED; + } } else { - // Local field is boxed, remote wrote with null flag (or non-primitive type) + // Remote wrote with null flag - MUST read null flag regardless of local field type mode = MODE_NULLABLE_BOXED; } diff --git a/java/fory-core/src/main/java/org/apache/fory/util/StringUtils.java b/java/fory-core/src/main/java/org/apache/fory/util/StringUtils.java index 1024733658..5861f7cad8 100644 --- a/java/fory-core/src/main/java/org/apache/fory/util/StringUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/util/StringUtils.java @@ -252,6 +252,10 @@ public static String lowerUnderscoreToLowerCamelCase(String lowerUnderscore) { } // example: "variableName" -> "variable_name" + public static String toSnakeCase(String lowerCamel) { + return lowerCamelToLowerUnderscore(lowerCamel); + } + public static String lowerCamelToLowerUnderscore(String lowerCamel) { StringBuilder builder = new StringBuilder(); int length = lowerCamel.length(); diff --git a/java/fory-core/src/test/java/org/apache/fory/builder/ObjectCodecBuilderTest.java b/java/fory-core/src/test/java/org/apache/fory/builder/ObjectCodecBuilderTest.java index 003c3d1305..c3d6d7f86f 100644 --- a/java/fory-core/src/test/java/org/apache/fory/builder/ObjectCodecBuilderTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/builder/ObjectCodecBuilderTest.java @@ -99,7 +99,6 @@ public void testDefaultPackage() throws Exception { public static Object[][] codecConfig() { return Sets.cartesianProduct( ImmutableSet.of(true, false), // referenceTracking - ImmutableSet.of(true, false), // basicTypesReferenceIgnored ImmutableSet.of(true, false), // compressNumber ImmutableSet.of(1, 4, 7) // structFieldsRepeat ) @@ -111,7 +110,6 @@ public static Object[][] codecConfig() { @Test(dataProvider = "codecConfig") public void testSeqCodec( boolean referenceTracking, - boolean basicTypesRefIgnored, boolean compressNumber, int fieldsRepeat) { Class structClass = Struct.createStructClass("Struct" + fieldsRepeat, fieldsRepeat); @@ -120,7 +118,6 @@ public void testSeqCodec( .withLanguage(Language.JAVA) .withRefTracking(referenceTracking) .withClassLoader(structClass.getClassLoader()) - .ignoreBasicTypesRef(basicTypesRefIgnored) .withNumberCompressed(compressNumber) .requireClassRegistration(false) .build(); diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/RustXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/RustXlangTest.java index 56f1061257..6e404bfb0f 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/RustXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/RustXlangTest.java @@ -121,9 +121,14 @@ public void testSimpleStruct(boolean enableCodegen) throws java.io.IOException { super.testSimpleStruct(enableCodegen); } - @Test(dataProvider = "enableCodegen") - public void testSimpleNamedStruct(boolean enableCodegen) throws java.io.IOException { - super.testSimpleNamedStruct(enableCodegen); + @Test + public void testSimpleNamedStructCodegenEnabled() throws java.io.IOException { + super.testSimpleNamedStruct(false); + } + + @Test + public void testSimpleNamedStructCodegenDisabled() throws java.io.IOException { + super.testSimpleNamedStruct(false); } @Test(dataProvider = "enableCodegen") diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java b/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java index 31ee15caad..2a6f92d1d6 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java @@ -581,6 +581,8 @@ public void testSimpleStruct(boolean enableCodegen) throws java.io.IOException { obj.f8 = 41; obj.last = 42; + serDeCheck(fory, obj); + MemoryBuffer buffer = MemoryUtils.buffer(64); fory.serialize(buffer, obj); @@ -619,6 +621,7 @@ public void testSimpleNamedStruct(boolean enableCodegen) throws java.io.IOExcept obj.f8 = 41; obj.last = 42; + serDeCheck(fory, obj); MemoryBuffer buffer = MemoryUtils.buffer(64); fory.serialize(buffer, obj); @@ -654,6 +657,11 @@ public void testList(boolean enableCodegen) throws java.io.IOException { fory.serialize(buffer, itemList); fory.serialize(buffer, itemList2); + serDeCheck(fory, strList); + serDeCheck(fory, strList2); + serDeCheck(fory, itemList); + serDeCheck(fory, itemList2); + ExecutionContext ctx = prepareExecution(caseName, buffer.getBytes(0, buffer.writerIndex())); runPeer(ctx); MemoryBuffer buffer2 = readBuffer(ctx.dataFile()); @@ -690,6 +698,10 @@ public void testMap(boolean enableCodegen) throws java.io.IOException { itemMap.put(null, item2); itemMap.put("k3", null); itemMap.put("k4", item3); + + serDeCheck(fory, strMap); + serDeCheck(fory, itemMap); + fory.serialize(buffer, strMap); fory.serialize(buffer, itemMap); ExecutionContext ctx = prepareExecution(caseName, buffer.getBytes(0, buffer.writerIndex())); @@ -782,6 +794,10 @@ public void testItem(boolean enableCodegen) throws java.io.IOException { // Go strings are always non-nil (empty string for "no value") item3.name = ""; + serDeCheck(fory, item1); + serDeCheck(fory, item2); + serDeCheck(fory, item3); + MemoryBuffer buffer = MemoryUtils.buffer(64); fory.serialize(buffer, item1); fory.serialize(buffer, item2); @@ -1022,6 +1038,9 @@ private void _testSkipCustom(Fory fory1, Fory fory2, String caseName) throws IOE MyStruct myStruct = new MyStruct(42); wrapper.myExt = new MyExt(43); wrapper.myStruct = myStruct; + + serDeCheck(fory1, wrapper); + byte[] serialize = fory1.serialize(wrapper); ExecutionContext ctx = prepareExecution(caseName, serialize); runPeer(ctx); @@ -1153,6 +1172,8 @@ public void testStructVersionCheck(boolean enableCodegen) throws java.io.IOExcep obj.f2 = "test"; obj.f3 = 3.2; + serDeCheck(fory, obj); + MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(32); fory.serialize(buffer, obj); byte[] bytes = buffer.getBytes(0, buffer.writerIndex()); @@ -1798,7 +1819,7 @@ public void testNullableFieldSchemaConsistentNotNull(boolean enableCodegen) obj.nullableMap.put("nk1", "nv1"); // First verify Java serialization works - Assert.assertEquals(xserDe(fory, obj), obj); + serDeCheck(fory, obj); MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(512); fory.serialize(buffer, obj); @@ -1856,7 +1877,7 @@ public void testNullableFieldSchemaConsistentNull(boolean enableCodegen) obj.nullableMap = null; // First verify Java serialization works - Assert.assertEquals(xserDe(fory, obj), obj); + serDeCheck(fory, obj); MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(512); fory.serialize(buffer, obj); @@ -1990,7 +2011,7 @@ public void testNullableFieldCompatibleNotNull(boolean enableCodegen) throws jav obj.nullableMap2.put("nk1", "nv1"); // First verify Java serialization works - Assert.assertEquals(xserDe(fory, obj), obj); + serDeCheck(fory, obj); MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(1024); fory.serialize(buffer, obj); @@ -2055,7 +2076,7 @@ public void testNullableFieldCompatibleNull(boolean enableCodegen) throws java.i obj.nullableMap2 = null; // First verify Java serialization works - Assert.assertEquals(xserDe(fory, obj), obj); + serDeCheck(fory, obj); MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(1024); fory.serialize(buffer, obj); @@ -2596,7 +2617,7 @@ public void testUnsignedSchemaConsistent(boolean enableCodegen) throws java.io.I obj.u64TaggedNullableField = 500000000L; // First verify Java serialization works - Assert.assertEquals(xserDe(fory, obj), obj); + serDeCheck(fory, obj); MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(512); fory.serialize(buffer, obj); @@ -2714,7 +2735,7 @@ public void testUnsignedSchemaCompatible(boolean enableCodegen) throws java.io.I obj.u64TaggedField2 = 500000000L; // First verify Java serialization works - Assert.assertEquals(xserDe(fory, obj), obj); + serDeCheck(fory, obj); MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(1024); fory.serialize(buffer, obj); diff --git a/rust/tests/tests/test_cross_language.rs b/rust/tests/tests/test_cross_language.rs index 6022673828..c23b6f6b43 100644 --- a/rust/tests/tests/test_cross_language.rs +++ b/rust/tests/tests/test_cross_language.rs @@ -389,7 +389,7 @@ fn test_simple_struct() { #[test] #[ignore] -fn test_simple_named_struct() { +fn test_named_simple_struct() { let data_file_path = get_data_file(); let bytes = fs::read(&data_file_path).unwrap(); let mut fory = Fory::default().compatible(true).xlang(true); From 6a1d48f954ecccb0d8418c43bc8d27cbae438f9a Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 17 Jan 2026 03:34:54 +0800 Subject: [PATCH 09/44] refactor dispatch logic --- .../fory/builder/BaseObjectCodecBuilder.java | 87 +- .../fory/builder/MetaSharedCodecBuilder.java | 3 +- .../builder/MetaSharedLayerCodecBuilder.java | 3 +- .../fory/builder/ObjectCodecBuilder.java | 207 +-- .../java/org/apache/fory/meta/FieldInfo.java | 11 +- .../serializer/AbstractObjectSerializer.java | 1354 +++++------------ .../apache/fory/serializer/FieldGroups.java | 23 +- .../apache/fory/serializer/FieldSkipper.java | 188 +-- .../serializer/MetaSharedLayerSerializer.java | 3 +- .../fory/serializer/MetaSharedSerializer.java | 13 +- .../NonexistentClassSerializers.java | 5 +- .../fory/serializer/ObjectSerializer.java | 14 +- .../fory/serializer/SerializationBinding.java | 84 +- .../java/org/apache/fory/type/DispatchId.java | 228 +-- .../fory/builder/ObjectCodecBuilderTest.java | 5 +- .../org/apache/fory/xlang/XlangTestBase.java | 6 +- 16 files changed, 632 insertions(+), 1602 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java index 53a338ec30..ce887481e0 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java @@ -480,54 +480,36 @@ private Expression serializePrimitiveField( Expression inputObject, Expression buffer, Descriptor descriptor) { int dispatchId = getNumericDescriptorDispatchId(descriptor); switch (dispatchId) { - case DispatchId.PRIMITIVE_BOOL: case DispatchId.BOOL: return new Invoke(buffer, "writeBoolean", inputObject); - case DispatchId.PRIMITIVE_INT8: - case DispatchId.PRIMITIVE_UINT8: case DispatchId.INT8: case DispatchId.UINT8: return new Invoke(buffer, "writeByte", inputObject); - case DispatchId.PRIMITIVE_CHAR: case DispatchId.CHAR: return new Invoke(buffer, "writeChar", inputObject); - case DispatchId.PRIMITIVE_INT16: - case DispatchId.PRIMITIVE_UINT16: case DispatchId.INT16: case DispatchId.UINT16: return new Invoke(buffer, "writeInt16", inputObject); - case DispatchId.PRIMITIVE_INT32: - case DispatchId.PRIMITIVE_UINT32: case DispatchId.INT32: case DispatchId.UINT32: return new Invoke(buffer, "writeInt32", inputObject); - case DispatchId.PRIMITIVE_VARINT32: case DispatchId.VARINT32: return new Invoke(buffer, "writeVarInt32", inputObject); - case DispatchId.PRIMITIVE_VAR_UINT32: case DispatchId.VAR_UINT32: return new Invoke(buffer, "writeVarUint32", inputObject); - case DispatchId.PRIMITIVE_INT64: - case DispatchId.PRIMITIVE_UINT64: case DispatchId.INT64: case DispatchId.UINT64: return new Invoke(buffer, "writeInt64", inputObject); - case DispatchId.PRIMITIVE_VARINT64: case DispatchId.VARINT64: return new Invoke(buffer, "writeVarInt64", inputObject); - case DispatchId.PRIMITIVE_TAGGED_INT64: case DispatchId.TAGGED_INT64: return new Invoke(buffer, "writeTaggedInt64", inputObject); - case DispatchId.PRIMITIVE_VAR_UINT64: case DispatchId.VAR_UINT64: return new Invoke(buffer, "writeVarUint64", inputObject); - case DispatchId.PRIMITIVE_TAGGED_UINT64: case DispatchId.TAGGED_UINT64: return new Invoke(buffer, "writeTaggedUint64", inputObject); - case DispatchId.PRIMITIVE_FLOAT32: case DispatchId.FLOAT32: return new Invoke(buffer, "writeFloat32", inputObject); - case DispatchId.PRIMITIVE_FLOAT64: case DispatchId.FLOAT64: return new Invoke(buffer, "writeFloat64", inputObject); default: @@ -1971,71 +1953,48 @@ private Expression deserializeForNotNullForField( private Expression deserializePrimitiveField(Expression buffer, Descriptor descriptor) { int dispatchId = getNumericDescriptorDispatchId(descriptor); + boolean isPrimitive = descriptor.getRawType().isPrimitive(); switch (dispatchId) { - case DispatchId.PRIMITIVE_BOOL: - return new Invoke(buffer, "readBoolean", PRIMITIVE_BOOLEAN_TYPE); case DispatchId.BOOL: - return new Invoke(buffer, "readBoolean", BOOLEAN_TYPE); - case DispatchId.PRIMITIVE_INT8: - case DispatchId.PRIMITIVE_UINT8: - return new Invoke(buffer, "readByte", PRIMITIVE_BYTE_TYPE); + return new Invoke( + buffer, "readBoolean", isPrimitive ? PRIMITIVE_BOOLEAN_TYPE : BOOLEAN_TYPE); case DispatchId.INT8: case DispatchId.UINT8: - return new Invoke(buffer, "readByte", BYTE_TYPE); - case DispatchId.PRIMITIVE_CHAR: - return readChar(buffer); + return new Invoke(buffer, "readByte", isPrimitive ? PRIMITIVE_BYTE_TYPE : BYTE_TYPE); case DispatchId.CHAR: - return new Invoke(buffer, "readChar", CHAR_TYPE); - case DispatchId.PRIMITIVE_INT16: - case DispatchId.PRIMITIVE_UINT16: - return readInt16(buffer); + return isPrimitive ? readChar(buffer) : new Invoke(buffer, "readChar", CHAR_TYPE); case DispatchId.INT16: case DispatchId.UINT16: - return new Invoke(buffer, readInt16Func(), SHORT_TYPE); - case DispatchId.PRIMITIVE_INT32: - case DispatchId.PRIMITIVE_UINT32: - return readInt32(buffer); + return isPrimitive ? readInt16(buffer) : new Invoke(buffer, readInt16Func(), SHORT_TYPE); case DispatchId.INT32: case DispatchId.UINT32: - return new Invoke(buffer, readIntFunc(), INT_TYPE); - case DispatchId.PRIMITIVE_VARINT32: - return readVarInt32(buffer); + return isPrimitive ? readInt32(buffer) : new Invoke(buffer, readIntFunc(), INT_TYPE); case DispatchId.VARINT32: - return new Invoke(buffer, readVarInt32Func(), INT_TYPE); - case DispatchId.PRIMITIVE_VAR_UINT32: - return new Invoke(buffer, "readVarUint32", PRIMITIVE_INT_TYPE); + return isPrimitive + ? readVarInt32(buffer) + : new Invoke(buffer, readVarInt32Func(), INT_TYPE); case DispatchId.VAR_UINT32: - return new Invoke(buffer, "readVarUint32", INT_TYPE); - case DispatchId.PRIMITIVE_INT64: - case DispatchId.PRIMITIVE_UINT64: - return readInt64(buffer); + return new Invoke(buffer, "readVarUint32", isPrimitive ? PRIMITIVE_INT_TYPE : INT_TYPE); case DispatchId.INT64: case DispatchId.UINT64: - return new Invoke(buffer, readLongFunc(), LONG_TYPE); - case DispatchId.PRIMITIVE_VARINT64: - return new Invoke(buffer, "readVarInt64", PRIMITIVE_LONG_TYPE); + return isPrimitive ? readInt64(buffer) : new Invoke(buffer, readLongFunc(), LONG_TYPE); case DispatchId.VARINT64: - return new Invoke(buffer, "readVarInt64", LONG_TYPE); - case DispatchId.PRIMITIVE_TAGGED_INT64: - return new Invoke(buffer, "readTaggedInt64", PRIMITIVE_LONG_TYPE); + return new Invoke(buffer, "readVarInt64", isPrimitive ? PRIMITIVE_LONG_TYPE : LONG_TYPE); case DispatchId.TAGGED_INT64: - return new Invoke(buffer, "readTaggedInt64", LONG_TYPE); - case DispatchId.PRIMITIVE_VAR_UINT64: - return new Invoke(buffer, "readVarUint64", PRIMITIVE_LONG_TYPE); + return new Invoke(buffer, "readTaggedInt64", isPrimitive ? PRIMITIVE_LONG_TYPE : LONG_TYPE); case DispatchId.VAR_UINT64: - return new Invoke(buffer, "readVarUint64", LONG_TYPE); - case DispatchId.PRIMITIVE_TAGGED_UINT64: - return new Invoke(buffer, "readTaggedUint64", PRIMITIVE_LONG_TYPE); + return new Invoke(buffer, "readVarUint64", isPrimitive ? PRIMITIVE_LONG_TYPE : LONG_TYPE); case DispatchId.TAGGED_UINT64: - return new Invoke(buffer, "readTaggedUint64", LONG_TYPE); - case DispatchId.PRIMITIVE_FLOAT32: - return readFloat32(buffer); + return new Invoke( + buffer, "readTaggedUint64", isPrimitive ? PRIMITIVE_LONG_TYPE : LONG_TYPE); case DispatchId.FLOAT32: - return new Invoke(buffer, readFloat32Func(), FLOAT_TYPE); - case DispatchId.PRIMITIVE_FLOAT64: - return readFloat64(buffer); + return isPrimitive + ? readFloat32(buffer) + : new Invoke(buffer, readFloat32Func(), FLOAT_TYPE); case DispatchId.FLOAT64: - return new Invoke(buffer, readFloat64Func(), DOUBLE_TYPE); + return isPrimitive + ? readFloat64(buffer) + : new Invoke(buffer, readFloat64Func(), DOUBLE_TYPE); default: throw new IllegalStateException("Unsupported dispatchId: " + dispatchId); } diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java index 92d8e733bc..235b2fd7b2 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java @@ -105,8 +105,7 @@ public MetaSharedCodecBuilder(TypeRef beanType, Fory fory, ClassDef classDef) d.isNullable()); } } - objectCodecOptimizer = - new ObjectCodecOptimizer(beanClass, grouper, false, ctx); + objectCodecOptimizer = new ObjectCodecOptimizer(beanClass, grouper, false, ctx); String defaultValueLanguage = "None"; DefaultValueUtils.DefaultValueField[] defaultValueFields = diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedLayerCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedLayerCodecBuilder.java index a09081fed1..800de0bd33 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedLayerCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedLayerCodecBuilder.java @@ -68,8 +68,7 @@ public MetaSharedLayerCodecBuilder( this.layerMarkerClass = layerMarkerClass; Collection descriptors = layerClassDef.getDescriptors(typeResolver, beanClass); DescriptorGrouper grouper = typeResolver(r -> r.createDescriptorGrouper(descriptors, false)); - objectCodecOptimizer = - new ObjectCodecOptimizer(beanClass, grouper, false, ctx); + objectCodecOptimizer = new ObjectCodecOptimizer(beanClass, grouper, false, ctx); } // Must be static to be shared across the whole process life. diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java index 4b4a8bca1b..67e5851a7e 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java @@ -120,8 +120,7 @@ public ObjectCodecBuilder(Class beanClass, Fory fory) { fory.checkClassVersion() ? new Literal(ObjectSerializer.computeStructHash(fory, grouper), PRIMITIVE_INT_TYPE) : null; - objectCodecOptimizer = - new ObjectCodecOptimizer(beanClass, grouper, false, ctx); + objectCodecOptimizer = new ObjectCodecOptimizer(beanClass, grouper, false, ctx); if (isRecord) { if (!recordCtrAccessible) { buildRecordComponentDefaultValues(); @@ -273,40 +272,28 @@ private List serializePrimitivesUnCompressed( if (fieldValue instanceof Inlineable) { ((Inlineable) fieldValue).inline(); } - if (dispatchId == DispatchId.PRIMITIVE_BOOL || dispatchId == DispatchId.BOOL) { + if (dispatchId == DispatchId.BOOL) { groupExpressions.add(unsafePutBoolean(base, getWriterPos(writerAddr, acc), fieldValue)); acc += 1; - } else if (dispatchId == DispatchId.PRIMITIVE_INT8 - || dispatchId == DispatchId.PRIMITIVE_UINT8 - || dispatchId == DispatchId.INT8 - || dispatchId == DispatchId.UINT8) { + } else if (dispatchId == DispatchId.INT8 || dispatchId == DispatchId.UINT8) { groupExpressions.add(unsafePut(base, getWriterPos(writerAddr, acc), fieldValue)); acc += 1; - } else if (dispatchId == DispatchId.PRIMITIVE_CHAR || dispatchId == DispatchId.CHAR) { + } else if (dispatchId == DispatchId.CHAR) { groupExpressions.add(unsafePutChar(base, getWriterPos(writerAddr, acc), fieldValue)); acc += 2; - } else if (dispatchId == DispatchId.PRIMITIVE_INT16 - || dispatchId == DispatchId.PRIMITIVE_UINT16 - || dispatchId == DispatchId.INT16 - || dispatchId == DispatchId.UINT16) { + } else if (dispatchId == DispatchId.INT16 || dispatchId == DispatchId.UINT16) { groupExpressions.add(unsafePutShort(base, getWriterPos(writerAddr, acc), fieldValue)); acc += 2; - } else if (dispatchId == DispatchId.PRIMITIVE_INT32 - || dispatchId == DispatchId.PRIMITIVE_UINT32 - || dispatchId == DispatchId.INT32 - || dispatchId == DispatchId.UINT32) { + } else if (dispatchId == DispatchId.INT32 || dispatchId == DispatchId.UINT32) { groupExpressions.add(unsafePutInt(base, getWriterPos(writerAddr, acc), fieldValue)); acc += 4; - } else if (dispatchId == DispatchId.PRIMITIVE_INT64 - || dispatchId == DispatchId.PRIMITIVE_UINT64 - || dispatchId == DispatchId.INT64 - || dispatchId == DispatchId.UINT64) { + } else if (dispatchId == DispatchId.INT64 || dispatchId == DispatchId.UINT64) { groupExpressions.add(unsafePutLong(base, getWriterPos(writerAddr, acc), fieldValue)); acc += 8; - } else if (dispatchId == DispatchId.PRIMITIVE_FLOAT32 || dispatchId == DispatchId.FLOAT32) { + } else if (dispatchId == DispatchId.FLOAT32) { groupExpressions.add(unsafePutFloat(base, getWriterPos(writerAddr, acc), fieldValue)); acc += 4; - } else if (dispatchId == DispatchId.PRIMITIVE_FLOAT64 || dispatchId == DispatchId.FLOAT64) { + } else if (dispatchId == DispatchId.FLOAT64) { groupExpressions.add(unsafePutDouble(base, getWriterPos(writerAddr, acc), fieldValue)); acc += 8; } else { @@ -338,24 +325,18 @@ private List serializePrimitivesCompressed( for (List group : primitiveGroups) { for (Descriptor d : group) { int id = getNumericDescriptorDispatchId(d); - if (id == DispatchId.PRIMITIVE_INT32 - || id == DispatchId.PRIMITIVE_VARINT32 - || id == DispatchId.PRIMITIVE_VAR_UINT32 - || id == DispatchId.INT32 + if (id == DispatchId.INT32 || id == DispatchId.VARINT32 - || id == DispatchId.VAR_UINT32) { + || id == DispatchId.VAR_UINT32 + || id == DispatchId.UINT32) { // varint may be written as 5bytes, use 8bytes for written as long to reduce cost. extraSize += 4; - } else if (id == DispatchId.PRIMITIVE_INT64 - || id == DispatchId.PRIMITIVE_VARINT64 - || id == DispatchId.PRIMITIVE_TAGGED_INT64 - || id == DispatchId.PRIMITIVE_VAR_UINT64 - || id == DispatchId.PRIMITIVE_TAGGED_UINT64 - || id == DispatchId.INT64 + } else if (id == DispatchId.INT64 || id == DispatchId.VARINT64 || id == DispatchId.TAGGED_INT64 || id == DispatchId.VAR_UINT64 - || id == DispatchId.TAGGED_UINT64) { + || id == DispatchId.TAGGED_UINT64 + || id == DispatchId.UINT64) { extraSize += 1; // long use 1~9 bytes. } } @@ -381,79 +362,61 @@ private List serializePrimitivesCompressed( if (fieldValue instanceof Inlineable) { ((Inlineable) fieldValue).inline(); } - if (dispatchId == DispatchId.PRIMITIVE_BOOL || dispatchId == DispatchId.BOOL) { + if (dispatchId == DispatchId.BOOL) { groupExpressions.add(unsafePutBoolean(base, getWriterPos(writerAddr, acc), fieldValue)); acc += 1; - } else if (dispatchId == DispatchId.PRIMITIVE_INT8 - || dispatchId == DispatchId.PRIMITIVE_UINT8 - || dispatchId == DispatchId.INT8 - || dispatchId == DispatchId.UINT8) { + } else if (dispatchId == DispatchId.INT8 || dispatchId == DispatchId.UINT8) { groupExpressions.add(unsafePut(base, getWriterPos(writerAddr, acc), fieldValue)); acc += 1; - } else if (dispatchId == DispatchId.PRIMITIVE_CHAR || dispatchId == DispatchId.CHAR) { + } else if (dispatchId == DispatchId.CHAR) { groupExpressions.add(unsafePutChar(base, getWriterPos(writerAddr, acc), fieldValue)); acc += 2; - } else if (dispatchId == DispatchId.PRIMITIVE_INT16 - || dispatchId == DispatchId.PRIMITIVE_UINT16 - || dispatchId == DispatchId.INT16 - || dispatchId == DispatchId.UINT16) { + } else if (dispatchId == DispatchId.INT16 || dispatchId == DispatchId.UINT16) { groupExpressions.add(unsafePutShort(base, getWriterPos(writerAddr, acc), fieldValue)); acc += 2; - } else if (dispatchId == DispatchId.PRIMITIVE_FLOAT32 || dispatchId == DispatchId.FLOAT32) { + } else if (dispatchId == DispatchId.FLOAT32) { groupExpressions.add(unsafePutFloat(base, getWriterPos(writerAddr, acc), fieldValue)); acc += 4; - } else if (dispatchId == DispatchId.PRIMITIVE_FLOAT64 || dispatchId == DispatchId.FLOAT64) { + } else if (dispatchId == DispatchId.FLOAT64) { groupExpressions.add(unsafePutDouble(base, getWriterPos(writerAddr, acc), fieldValue)); acc += 8; - } else if (dispatchId == DispatchId.PRIMITIVE_INT32 - || dispatchId == DispatchId.PRIMITIVE_UINT32 - || dispatchId == DispatchId.INT32 - || dispatchId == DispatchId.UINT32) { + } else if (dispatchId == DispatchId.INT32 || dispatchId == DispatchId.UINT32) { groupExpressions.add(unsafePutInt(base, getWriterPos(writerAddr, acc), fieldValue)); acc += 4; - } else if (dispatchId == DispatchId.PRIMITIVE_INT64 - || dispatchId == DispatchId.PRIMITIVE_UINT64 - || dispatchId == DispatchId.INT64 - || dispatchId == DispatchId.UINT64) { + } else if (dispatchId == DispatchId.INT64 || dispatchId == DispatchId.UINT64) { groupExpressions.add(unsafePutLong(base, getWriterPos(writerAddr, acc), fieldValue)); acc += 8; - } else if (dispatchId == DispatchId.PRIMITIVE_VARINT32 - || dispatchId == DispatchId.VARINT32) { + } else if (dispatchId == DispatchId.VARINT32) { if (!compressStarted) { addIncWriterIndexExpr(groupExpressions, buffer, acc); compressStarted = true; } groupExpressions.add(new Invoke(buffer, "_unsafeWriteVarInt32", fieldValue)); - } else if (dispatchId == DispatchId.PRIMITIVE_VAR_UINT32 - || dispatchId == DispatchId.VAR_UINT32) { + } else if (dispatchId == DispatchId.VAR_UINT32) { if (!compressStarted) { addIncWriterIndexExpr(groupExpressions, buffer, acc); compressStarted = true; } groupExpressions.add(new Invoke(buffer, "_unsafeWriteVarUint32", fieldValue)); - } else if (dispatchId == DispatchId.PRIMITIVE_VARINT64 - || dispatchId == DispatchId.VARINT64) { + } else if (dispatchId == DispatchId.VARINT64) { if (!compressStarted) { addIncWriterIndexExpr(groupExpressions, buffer, acc); compressStarted = true; } groupExpressions.add(new Invoke(buffer, "writeVarInt64", fieldValue)); - } else if (dispatchId == DispatchId.PRIMITIVE_TAGGED_INT64 - || dispatchId == DispatchId.TAGGED_INT64) { + } else if (dispatchId == DispatchId.TAGGED_INT64) { if (!compressStarted) { addIncWriterIndexExpr(groupExpressions, buffer, acc); compressStarted = true; } groupExpressions.add(new Invoke(buffer, "writeTaggedInt64", fieldValue)); - } else if (dispatchId == DispatchId.PRIMITIVE_VAR_UINT64 - || dispatchId == DispatchId.VAR_UINT64) { + } else if (dispatchId == DispatchId.VAR_UINT64) { if (!compressStarted) { addIncWriterIndexExpr(groupExpressions, buffer, acc); compressStarted = true; } groupExpressions.add(new Invoke(buffer, "writeVarUint64", fieldValue)); - } else if (dispatchId == DispatchId.PRIMITIVE_TAGGED_UINT64 - || dispatchId == DispatchId.TAGGED_UINT64) { + } else if (dispatchId == DispatchId.TAGGED_UINT64) { if (!compressStarted) { addIncWriterIndexExpr(groupExpressions, buffer, acc); compressStarted = true; @@ -702,56 +665,28 @@ private List deserializeUnCompressedPrimitives( for (Descriptor descriptor : group) { int dispatchId = getNumericDescriptorDispatchId(descriptor); Expression fieldValue; - if (dispatchId == DispatchId.PRIMITIVE_BOOL - || dispatchId == DispatchId.NOTNULL_BOXED_BOOL - || dispatchId == DispatchId.BOOL) { + if (dispatchId == DispatchId.BOOL) { fieldValue = unsafeGetBoolean(heapBuffer, getReaderAddress(readerAddr, acc)); acc += 1; - } else if (dispatchId == DispatchId.PRIMITIVE_INT8 - || dispatchId == DispatchId.PRIMITIVE_UINT8 - || dispatchId == DispatchId.NOTNULL_BOXED_INT8 - || dispatchId == DispatchId.NOTNULL_BOXED_UINT8 - || dispatchId == DispatchId.INT8 - || dispatchId == DispatchId.UINT8) { + } else if (dispatchId == DispatchId.INT8 || dispatchId == DispatchId.UINT8) { fieldValue = unsafeGet(heapBuffer, getReaderAddress(readerAddr, acc)); acc += 1; - } else if (dispatchId == DispatchId.PRIMITIVE_CHAR - || dispatchId == DispatchId.NOTNULL_BOXED_CHAR - || dispatchId == DispatchId.CHAR) { + } else if (dispatchId == DispatchId.CHAR) { fieldValue = unsafeGetChar(heapBuffer, getReaderAddress(readerAddr, acc)); acc += 2; - } else if (dispatchId == DispatchId.PRIMITIVE_INT16 - || dispatchId == DispatchId.PRIMITIVE_UINT16 - || dispatchId == DispatchId.NOTNULL_BOXED_INT16 - || dispatchId == DispatchId.NOTNULL_BOXED_UINT16 - || dispatchId == DispatchId.INT16 - || dispatchId == DispatchId.UINT16) { + } else if (dispatchId == DispatchId.INT16 || dispatchId == DispatchId.UINT16) { fieldValue = unsafeGetShort(heapBuffer, getReaderAddress(readerAddr, acc)); acc += 2; - } else if (dispatchId == DispatchId.PRIMITIVE_INT32 - || dispatchId == DispatchId.PRIMITIVE_UINT32 - || dispatchId == DispatchId.NOTNULL_BOXED_INT32 - || dispatchId == DispatchId.NOTNULL_BOXED_UINT32 - || dispatchId == DispatchId.INT32 - || dispatchId == DispatchId.UINT32) { + } else if (dispatchId == DispatchId.INT32 || dispatchId == DispatchId.UINT32) { fieldValue = unsafeGetInt(heapBuffer, getReaderAddress(readerAddr, acc)); acc += 4; - } else if (dispatchId == DispatchId.PRIMITIVE_INT64 - || dispatchId == DispatchId.PRIMITIVE_UINT64 - || dispatchId == DispatchId.NOTNULL_BOXED_INT64 - || dispatchId == DispatchId.NOTNULL_BOXED_UINT64 - || dispatchId == DispatchId.INT64 - || dispatchId == DispatchId.UINT64) { + } else if (dispatchId == DispatchId.INT64 || dispatchId == DispatchId.UINT64) { fieldValue = unsafeGetLong(heapBuffer, getReaderAddress(readerAddr, acc)); acc += 8; - } else if (dispatchId == DispatchId.PRIMITIVE_FLOAT32 - || dispatchId == DispatchId.NOTNULL_BOXED_FLOAT32 - || dispatchId == DispatchId.FLOAT32) { + } else if (dispatchId == DispatchId.FLOAT32) { fieldValue = unsafeGetFloat(heapBuffer, getReaderAddress(readerAddr, acc)); acc += 4; - } else if (dispatchId == DispatchId.PRIMITIVE_FLOAT64 - || dispatchId == DispatchId.NOTNULL_BOXED_FLOAT64 - || dispatchId == DispatchId.FLOAT64) { + } else if (dispatchId == DispatchId.FLOAT64) { fieldValue = unsafeGetDouble(heapBuffer, getReaderAddress(readerAddr, acc)); acc += 8; } else { @@ -795,101 +730,61 @@ private List deserializeCompressedPrimitives( for (Descriptor descriptor : group) { int dispatchId = getNumericDescriptorDispatchId(descriptor); Expression fieldValue; - if (dispatchId == DispatchId.PRIMITIVE_BOOL - || dispatchId == DispatchId.NOTNULL_BOXED_BOOL - || dispatchId == DispatchId.BOOL) { + if (dispatchId == DispatchId.BOOL) { fieldValue = unsafeGetBoolean(heapBuffer, getReaderAddress(readerAddr, acc)); acc += 1; - } else if (dispatchId == DispatchId.PRIMITIVE_INT8 - || dispatchId == DispatchId.PRIMITIVE_UINT8 - || dispatchId == DispatchId.NOTNULL_BOXED_INT8 - || dispatchId == DispatchId.NOTNULL_BOXED_UINT8 - || dispatchId == DispatchId.INT8 - || dispatchId == DispatchId.UINT8) { + } else if (dispatchId == DispatchId.INT8 || dispatchId == DispatchId.UINT8) { fieldValue = unsafeGet(heapBuffer, getReaderAddress(readerAddr, acc)); acc += 1; - } else if (dispatchId == DispatchId.PRIMITIVE_CHAR - || dispatchId == DispatchId.NOTNULL_BOXED_CHAR - || dispatchId == DispatchId.CHAR) { + } else if (dispatchId == DispatchId.CHAR) { fieldValue = unsafeGetChar(heapBuffer, getReaderAddress(readerAddr, acc)); acc += 2; - } else if (dispatchId == DispatchId.PRIMITIVE_INT16 - || dispatchId == DispatchId.PRIMITIVE_UINT16 - || dispatchId == DispatchId.NOTNULL_BOXED_INT16 - || dispatchId == DispatchId.NOTNULL_BOXED_UINT16 - || dispatchId == DispatchId.INT16 - || dispatchId == DispatchId.UINT16) { + } else if (dispatchId == DispatchId.INT16 || dispatchId == DispatchId.UINT16) { fieldValue = unsafeGetShort(heapBuffer, getReaderAddress(readerAddr, acc)); acc += 2; - } else if (dispatchId == DispatchId.PRIMITIVE_FLOAT32 - || dispatchId == DispatchId.NOTNULL_BOXED_FLOAT32 - || dispatchId == DispatchId.FLOAT32) { + } else if (dispatchId == DispatchId.FLOAT32) { fieldValue = unsafeGetFloat(heapBuffer, getReaderAddress(readerAddr, acc)); acc += 4; - } else if (dispatchId == DispatchId.PRIMITIVE_FLOAT64 - || dispatchId == DispatchId.NOTNULL_BOXED_FLOAT64 - || dispatchId == DispatchId.FLOAT64) { + } else if (dispatchId == DispatchId.FLOAT64) { fieldValue = unsafeGetDouble(heapBuffer, getReaderAddress(readerAddr, acc)); acc += 8; - } else if (dispatchId == DispatchId.PRIMITIVE_INT32 - || dispatchId == DispatchId.PRIMITIVE_UINT32 - || dispatchId == DispatchId.NOTNULL_BOXED_INT32 - || dispatchId == DispatchId.NOTNULL_BOXED_UINT32 - || dispatchId == DispatchId.INT32 - || dispatchId == DispatchId.UINT32) { + } else if (dispatchId == DispatchId.INT32 || dispatchId == DispatchId.UINT32) { fieldValue = unsafeGetInt(heapBuffer, getReaderAddress(readerAddr, acc)); acc += 4; - } else if (dispatchId == DispatchId.PRIMITIVE_INT64 - || dispatchId == DispatchId.PRIMITIVE_UINT64 - || dispatchId == DispatchId.NOTNULL_BOXED_INT64 - || dispatchId == DispatchId.NOTNULL_BOXED_UINT64 - || dispatchId == DispatchId.INT64 - || dispatchId == DispatchId.UINT64) { + } else if (dispatchId == DispatchId.INT64 || dispatchId == DispatchId.UINT64) { fieldValue = unsafeGetLong(heapBuffer, getReaderAddress(readerAddr, acc)); acc += 8; - } else if (dispatchId == DispatchId.PRIMITIVE_VARINT32 - || dispatchId == DispatchId.NOTNULL_BOXED_VARINT32 - || dispatchId == DispatchId.VARINT32) { + } else if (dispatchId == DispatchId.VARINT32) { if (!compressStarted) { compressStarted = true; addIncReaderIndexExpr(groupExpressions, buffer, acc); } fieldValue = readVarInt32(buffer); - } else if (dispatchId == DispatchId.PRIMITIVE_VAR_UINT32 - || dispatchId == DispatchId.NOTNULL_BOXED_VAR_UINT32 - || dispatchId == DispatchId.VAR_UINT32) { + } else if (dispatchId == DispatchId.VAR_UINT32) { if (!compressStarted) { compressStarted = true; addIncReaderIndexExpr(groupExpressions, buffer, acc); } fieldValue = new Invoke(buffer, "readVarUint32", PRIMITIVE_INT_TYPE); - } else if (dispatchId == DispatchId.PRIMITIVE_VARINT64 - || dispatchId == DispatchId.NOTNULL_BOXED_VARINT64 - || dispatchId == DispatchId.VARINT64) { + } else if (dispatchId == DispatchId.VARINT64) { if (!compressStarted) { compressStarted = true; addIncReaderIndexExpr(groupExpressions, buffer, acc); } fieldValue = new Invoke(buffer, "readVarInt64", PRIMITIVE_LONG_TYPE); - } else if (dispatchId == DispatchId.PRIMITIVE_TAGGED_INT64 - || dispatchId == DispatchId.NOTNULL_BOXED_TAGGED_INT64 - || dispatchId == DispatchId.TAGGED_INT64) { + } else if (dispatchId == DispatchId.TAGGED_INT64) { if (!compressStarted) { compressStarted = true; addIncReaderIndexExpr(groupExpressions, buffer, acc); } fieldValue = new Invoke(buffer, "readTaggedInt64", PRIMITIVE_LONG_TYPE); - } else if (dispatchId == DispatchId.PRIMITIVE_VAR_UINT64 - || dispatchId == DispatchId.NOTNULL_BOXED_VAR_UINT64 - || dispatchId == DispatchId.VAR_UINT64) { + } else if (dispatchId == DispatchId.VAR_UINT64) { if (!compressStarted) { compressStarted = true; addIncReaderIndexExpr(groupExpressions, buffer, acc); } fieldValue = new Invoke(buffer, "readVarUint64", PRIMITIVE_LONG_TYPE); - } else if (dispatchId == DispatchId.PRIMITIVE_TAGGED_UINT64 - || dispatchId == DispatchId.NOTNULL_BOXED_TAGGED_UINT64 - || dispatchId == DispatchId.TAGGED_UINT64) { + } else if (dispatchId == DispatchId.TAGGED_UINT64) { if (!compressStarted) { compressStarted = true; addIncReaderIndexExpr(groupExpressions, buffer, acc); diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java b/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java index 00a3b78684..ba0f3f82b5 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java @@ -130,8 +130,15 @@ Descriptor toDescriptor(TypeResolver resolver, Descriptor descriptor) { boolean typeMismatch = !typeRef.equals(declared); if (typeMismatch) { if (!declared.getRawType().isAssignableFrom(rawType)) { - // boxed/primitive are handled automatically - if (!declared.unwrap().isPrimitive() || !typeRef.unwrap().isPrimitive()) { + // Check if both are primitives (or boxed primitives) of the same underlying type + // For example: Integer <-> int is OK, but Long -> int is NOT OK + Class declaredPrimitive = declared.unwrap().getRawType(); + Class remotePrimitive = typeRef.unwrap().getRawType(); + boolean bothPrimitives = declaredPrimitive.isPrimitive() && remotePrimitive.isPrimitive(); + boolean samePrimitiveType = + bothPrimitives && declaredPrimitive.equals(remotePrimitive); + // Set field to null if types are incompatible (not the same primitive type) + if (!samePrimitiveType) { builder.field(null); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java index 9000c4f44e..24079aece4 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java @@ -25,8 +25,9 @@ import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; - import org.apache.fory.Fory; +import org.apache.fory.logging.Logger; +import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.Platform; import org.apache.fory.reflect.FieldAccessor; @@ -43,8 +44,6 @@ import org.apache.fory.type.DescriptorGrouper; import org.apache.fory.type.DispatchId; import org.apache.fory.type.Generics; -import org.apache.fory.logging.Logger; -import org.apache.fory.logging.LoggerFactory; import org.apache.fory.util.record.RecordComponent; import org.apache.fory.util.record.RecordInfo; import org.apache.fory.util.record.RecordUtils; @@ -76,8 +75,8 @@ public AbstractObjectSerializer(Fory fory, Class type, ObjectCreator objec * Write field value to buffer by reading from the object via fieldAccessor. Handles primitive * types, unsigned/compressed numbers, and common types like String with optimized fast paths. * - *

    This method reads the field value from the object using the fieldAccessor in fieldInfo, - * then writes it to the buffer. It is the write counterpart of {@link + *

    This method reads the field value from the object using the fieldAccessor in fieldInfo, then + * writes it to the buffer. It is the write counterpart of {@link * #readBuildInFieldValue(SerializationBinding, SerializationFieldInfo, MemoryBuffer, Object)}. * * @param binding the serialization binding for write operations @@ -90,22 +89,14 @@ static void writeBuildInField( SerializationFieldInfo fieldInfo, MemoryBuffer buffer, Object obj) { - Fory fory = binding.fory; - FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; - int dispatchId = fieldInfo.dispatchId; - // The dispatch ID already encodes whether the data should have a null flag prefix: - // - PRIMITIVE_* and NOTNULL_BOXED_* dispatch IDs have no null flag prefix - // - Nullable dispatch IDs (STRING, BOOL, INT8, etc.) have null flag prefix - // So we write based on dispatch ID, not the local nullable setting. - if (writePrimitiveFieldValue(buffer, obj, fieldAccessor, dispatchId)) { - Object fieldValue = fieldAccessor.getObject(obj); - boolean needWrite = - writeBasicObjectFieldValue(fory, buffer, fieldInfo, fieldValue, dispatchId) - && writeNullableBasicObjectFieldValue(fory, buffer, fieldValue, dispatchId); - if (needWrite) { - binding.writeField(fieldInfo, buffer, fieldValue); - } + // Handle primitive fields with direct memory access + if (fieldInfo.isPrimitiveField) { + writePrimitiveFieldValue(buffer, obj, fieldInfo.fieldAccessor, fieldInfo.dispatchId); + return; } + // Handle non-primitive fields based on refMode + Object fieldValue = fieldInfo.fieldAccessor.getObject(obj); + writeBuildInFieldValue(binding, fieldInfo, buffer, fieldValue); } /** @@ -125,133 +116,81 @@ static void writeBuildInFieldValue( SerializationFieldInfo fieldInfo, MemoryBuffer buffer, Object fieldValue) { - Fory fory = binding.fory; - boolean nullable = fieldInfo.nullable; - int dispatchId = fieldInfo.dispatchId; - // Fast path for primitive types - if (writePrimitiveValue(buffer, fieldValue, dispatchId)) { - return; - } - boolean needWrite = - nullable - ? writeNullableBasicObjectFieldValue(fory, buffer, fieldValue, dispatchId) - : writeNotNullBasicObjectFieldValue(fory, buffer, fieldValue, dispatchId); - if (!needWrite) { - return; - } - // Fall back to binding.write for complex types - binding.writeField(fieldInfo, buffer, fieldValue); - } - - private static boolean writePrimitiveValue(MemoryBuffer buffer, Object value, int dispatchId) { - switch (dispatchId) { - case DispatchId.PRIMITIVE_BOOL: - buffer.writeBoolean((Boolean) value); - return true; - case DispatchId.PRIMITIVE_INT8: - case DispatchId.PRIMITIVE_UINT8: - buffer.writeByte((Byte) value); - return true; - case DispatchId.PRIMITIVE_CHAR: - buffer.writeChar((Character) value); - return true; - case DispatchId.PRIMITIVE_INT16: - case DispatchId.PRIMITIVE_UINT16: - buffer.writeInt16((Short) value); - return true; - case DispatchId.PRIMITIVE_INT32: - case DispatchId.PRIMITIVE_UINT32: - buffer.writeInt32((Integer) value); - return true; - case DispatchId.PRIMITIVE_VARINT32: - buffer.writeVarInt32((Integer) value); - return true; - case DispatchId.PRIMITIVE_VAR_UINT32: - buffer.writeVarUint32((Integer) value); - return true; - case DispatchId.PRIMITIVE_INT64: - case DispatchId.PRIMITIVE_UINT64: - buffer.writeInt64((Long) value); - return true; - case DispatchId.PRIMITIVE_VARINT64: - buffer.writeVarInt64((Long) value); - return true; - case DispatchId.PRIMITIVE_TAGGED_INT64: - buffer.writeTaggedInt64((Long) value); - return true; - case DispatchId.PRIMITIVE_VAR_UINT64: - buffer.writeVarUint64((Long) value); - return true; - case DispatchId.PRIMITIVE_TAGGED_UINT64: - buffer.writeTaggedUint64((Long) value); - return true; - case DispatchId.PRIMITIVE_FLOAT32: - buffer.writeFloat32((Float) value); - return true; - case DispatchId.PRIMITIVE_FLOAT64: - buffer.writeFloat64((Double) value); - return true; - default: - return false; + RefMode refMode = fieldInfo.refMode; + // Handle non-primitive fields based on refMode + if (refMode == RefMode.NONE) { + // No null flag - write value directly + writeNotPrimitiveFieldValue(binding, buffer, fieldValue, fieldInfo); + } else if (refMode == RefMode.NULL_ONLY) { + // Write null flag, then value if not null + if (fieldValue == null) { + buffer.writeByte(Fory.NULL_FLAG); + return; + } + buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); + writeNotPrimitiveFieldValue(binding, buffer, fieldValue, fieldInfo); + } else { + // RefMode.TRACKING - use binding for reference tracking + binding.writeField(fieldInfo, buffer, fieldValue); } } /** * Write a primitive field value to buffer using direct memory offset access. * - * @param buffer the buffer to write to + * @param buffer the buffer to write to * @param targetObject the object containing the field - * @param fieldOffset the memory offset of the field - * @param dispatchId the class ID of the primitive type + * @param fieldOffset the memory offset of the field + * @param dispatchId the class ID of the primitive type * @return true if dispatchId is not a primitive type and needs further write handling */ private static boolean writePrimitiveFieldValue( MemoryBuffer buffer, Object targetObject, long fieldOffset, int dispatchId) { switch (dispatchId) { - case DispatchId.PRIMITIVE_BOOL: + case DispatchId.BOOL: buffer.writeBoolean(Platform.getBoolean(targetObject, fieldOffset)); return false; - case DispatchId.PRIMITIVE_INT8: - case DispatchId.PRIMITIVE_UINT8: + case DispatchId.INT8: + case DispatchId.UINT8: buffer.writeByte(Platform.getByte(targetObject, fieldOffset)); return false; - case DispatchId.PRIMITIVE_CHAR: + case DispatchId.CHAR: buffer.writeChar(Platform.getChar(targetObject, fieldOffset)); return false; - case DispatchId.PRIMITIVE_INT16: - case DispatchId.PRIMITIVE_UINT16: + case DispatchId.INT16: + case DispatchId.UINT16: buffer.writeInt16(Platform.getShort(targetObject, fieldOffset)); return false; - case DispatchId.PRIMITIVE_INT32: - case DispatchId.PRIMITIVE_UINT32: + case DispatchId.INT32: + case DispatchId.UINT32: buffer.writeInt32(Platform.getInt(targetObject, fieldOffset)); return false; - case DispatchId.PRIMITIVE_VARINT32: + case DispatchId.VARINT32: buffer.writeVarInt32(Platform.getInt(targetObject, fieldOffset)); return false; - case DispatchId.PRIMITIVE_VAR_UINT32: + case DispatchId.VAR_UINT32: buffer.writeVarUint32(Platform.getInt(targetObject, fieldOffset)); return false; - case DispatchId.PRIMITIVE_FLOAT32: + case DispatchId.FLOAT32: buffer.writeFloat32(Platform.getFloat(targetObject, fieldOffset)); return false; - case DispatchId.PRIMITIVE_INT64: - case DispatchId.PRIMITIVE_UINT64: + case DispatchId.INT64: + case DispatchId.UINT64: buffer.writeInt64(Platform.getLong(targetObject, fieldOffset)); return false; - case DispatchId.PRIMITIVE_VARINT64: + case DispatchId.VARINT64: buffer.writeVarInt64(Platform.getLong(targetObject, fieldOffset)); return false; - case DispatchId.PRIMITIVE_TAGGED_INT64: + case DispatchId.TAGGED_INT64: buffer.writeTaggedInt64(Platform.getLong(targetObject, fieldOffset)); return false; - case DispatchId.PRIMITIVE_VAR_UINT64: + case DispatchId.VAR_UINT64: buffer.writeVarUint64(Platform.getLong(targetObject, fieldOffset)); return false; - case DispatchId.PRIMITIVE_TAGGED_UINT64: + case DispatchId.TAGGED_UINT64: buffer.writeTaggedUint64(Platform.getLong(targetObject, fieldOffset)); return false; - case DispatchId.PRIMITIVE_FLOAT64: + case DispatchId.FLOAT64: buffer.writeFloat64(Platform.getDouble(targetObject, fieldOffset)); return false; default: @@ -262,10 +201,10 @@ private static boolean writePrimitiveFieldValue( /** * Write a primitive field value to buffer using the field accessor. * - * @param buffer the buffer to write to - * @param targetObject the object containing the field + * @param buffer the buffer to write to + * @param targetObject the object containing the field * @param fieldAccessor the accessor to get the field value - * @param dispatchId the class ID of the primitive type + * @param dispatchId the class ID of the primitive type * @return true if dispatchId is not a primitive type and needs further write handling */ static boolean writePrimitiveFieldValue( @@ -276,50 +215,50 @@ static boolean writePrimitiveFieldValue( } // graalvm use GeneratedAccessor, which will be this code path. switch (dispatchId) { - case DispatchId.PRIMITIVE_BOOL: + case DispatchId.BOOL: buffer.writeBoolean((Boolean) fieldAccessor.get(targetObject)); return false; - case DispatchId.PRIMITIVE_INT8: - case DispatchId.PRIMITIVE_UINT8: + case DispatchId.INT8: + case DispatchId.UINT8: buffer.writeByte((Byte) fieldAccessor.get(targetObject)); return false; - case DispatchId.PRIMITIVE_CHAR: + case DispatchId.CHAR: buffer.writeChar((Character) fieldAccessor.get(targetObject)); return false; - case DispatchId.PRIMITIVE_INT16: - case DispatchId.PRIMITIVE_UINT16: + case DispatchId.INT16: + case DispatchId.UINT16: buffer.writeInt16((Short) fieldAccessor.get(targetObject)); return false; - case DispatchId.PRIMITIVE_INT32: - case DispatchId.PRIMITIVE_UINT32: + case DispatchId.INT32: + case DispatchId.UINT32: buffer.writeInt32((Integer) fieldAccessor.get(targetObject)); return false; - case DispatchId.PRIMITIVE_VARINT32: + case DispatchId.VARINT32: buffer.writeVarInt32((Integer) fieldAccessor.get(targetObject)); return false; - case DispatchId.PRIMITIVE_VAR_UINT32: + case DispatchId.VAR_UINT32: buffer.writeVarUint32((Integer) fieldAccessor.get(targetObject)); return false; - case DispatchId.PRIMITIVE_FLOAT32: + case DispatchId.FLOAT32: buffer.writeFloat32((Float) fieldAccessor.get(targetObject)); return false; - case DispatchId.PRIMITIVE_INT64: - case DispatchId.PRIMITIVE_UINT64: + case DispatchId.INT64: + case DispatchId.UINT64: buffer.writeInt64((Long) fieldAccessor.get(targetObject)); return false; - case DispatchId.PRIMITIVE_VARINT64: + case DispatchId.VARINT64: buffer.writeVarInt64((Long) fieldAccessor.get(targetObject)); return false; - case DispatchId.PRIMITIVE_TAGGED_INT64: + case DispatchId.TAGGED_INT64: buffer.writeTaggedInt64((Long) fieldAccessor.get(targetObject)); return false; - case DispatchId.PRIMITIVE_VAR_UINT64: + case DispatchId.VAR_UINT64: buffer.writeVarUint64((Long) fieldAccessor.get(targetObject)); return false; - case DispatchId.PRIMITIVE_TAGGED_UINT64: + case DispatchId.TAGGED_UINT64: buffer.writeTaggedUint64((Long) fieldAccessor.get(targetObject)); return false; - case DispatchId.PRIMITIVE_FLOAT64: + case DispatchId.FLOAT64: buffer.writeFloat64((Double) fieldAccessor.get(targetObject)); return false; default: @@ -329,319 +268,70 @@ static boolean writePrimitiveFieldValue( /** * Write field value to buffer. This method handle the situation which all fields are not null. - * - * @return true if field value isn't written by this function. */ - static boolean writeNotNullBasicObjectFieldValue( - Fory fory, MemoryBuffer buffer, Object fieldValue, int dispatchId) { + static void writeNotPrimitiveFieldValue( + SerializationBinding binding, + MemoryBuffer buffer, + Object fieldValue, + SerializationFieldInfo fieldInfo) { if (fieldValue == null) { throw new IllegalArgumentException( "Non-nullable field has null value. In xlang mode, fields are non-nullable by default. " + "Use @ForyField(nullable=true) to allow null values."); } // add time types serialization here. - switch (dispatchId) { + switch (fieldInfo.dispatchId) { case DispatchId.STRING: // fastpath for string. - String stringValue = (String) (fieldValue); - if (fory.getStringSerializer().needToWriteRef()) { - fory.writeJavaStringRef(buffer, stringValue); - } else { - fory.writeString(buffer, stringValue); - } - return false; + binding.fory.writeString(buffer, (String) fieldValue); + return; case DispatchId.BOOL: buffer.writeBoolean((Boolean) fieldValue); - return false; + return; case DispatchId.INT8: case DispatchId.UINT8: buffer.writeByte((Byte) fieldValue); - return false; + return; case DispatchId.CHAR: buffer.writeChar((Character) fieldValue); - return false; + return; case DispatchId.INT16: case DispatchId.UINT16: buffer.writeInt16((Short) fieldValue); - return false; + return; case DispatchId.INT32: case DispatchId.UINT32: buffer.writeInt32((Integer) fieldValue); - return false; + return; case DispatchId.VARINT32: buffer.writeVarInt32((Integer) fieldValue); - return false; + return; case DispatchId.VAR_UINT32: buffer.writeVarUint32((Integer) fieldValue); - return false; + return; case DispatchId.INT64: case DispatchId.UINT64: buffer.writeInt64((Long) fieldValue); - return false; + return; case DispatchId.VARINT64: buffer.writeVarInt64((Long) fieldValue); - return false; + return; case DispatchId.TAGGED_INT64: buffer.writeTaggedInt64((Long) fieldValue); - return false; + return; case DispatchId.VAR_UINT64: buffer.writeVarUint64((Long) fieldValue); - return false; + return; case DispatchId.TAGGED_UINT64: buffer.writeTaggedUint64((Long) fieldValue); - return false; + return; case DispatchId.FLOAT32: buffer.writeFloat32((Float) fieldValue); - return false; - case DispatchId.FLOAT64: - buffer.writeFloat64((Double) fieldValue); - return false; - default: - return true; - } - } - - /** - * Write a nullable boxed primitive or String field value to buffer. Writes null flag before value - * if the field is null. - * - * @param fory the fory instance for compression and ref tracking settings - * @param buffer the buffer to write to - * @param fieldValue the field value to write (may be null) - * @param dispatchId the class ID of the boxed type - * @return true if dispatchId is not a basic type or ref tracking is enabled, needing further - * write handling - */ - static boolean writeNullableBasicObjectFieldValue( - Fory fory, MemoryBuffer buffer, Object fieldValue, int dispatchId) { - // add time types serialization here. - switch (dispatchId) { - case DispatchId.STRING: // fastpath for string. - fory.writeJavaStringRef(buffer, (String) (fieldValue)); - return false; - case DispatchId.BOOL: - if (fieldValue == null) { - buffer.writeByte(Fory.NULL_FLAG); - } else { - buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); - buffer.writeBoolean((Boolean) (fieldValue)); - } - return false; - case DispatchId.INT8: - case DispatchId.UINT8: - if (fieldValue == null) { - buffer.writeByte(Fory.NULL_FLAG); - } else { - buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); - buffer.writeByte((Byte) (fieldValue)); - } - return false; - case DispatchId.CHAR: - if (fieldValue == null) { - buffer.writeByte(Fory.NULL_FLAG); - } else { - buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); - buffer.writeChar((Character) (fieldValue)); - } - return false; - case DispatchId.INT16: - case DispatchId.UINT16: - if (fieldValue == null) { - buffer.writeByte(Fory.NULL_FLAG); - } else { - buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); - buffer.writeInt16((Short) (fieldValue)); - } - return false; - case DispatchId.INT32: - case DispatchId.UINT32: - if (fieldValue == null) { - buffer.writeByte(Fory.NULL_FLAG); - } else { - buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); - buffer.writeInt32((Integer) (fieldValue)); - } - return false; - case DispatchId.VARINT32: - if (fieldValue == null) { - buffer.writeByte(Fory.NULL_FLAG); - } else { - buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); - buffer.writeVarInt32((Integer) (fieldValue)); - } - return false; - case DispatchId.VAR_UINT32: - if (fieldValue == null) { - buffer.writeByte(Fory.NULL_FLAG); - } else { - buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); - buffer.writeVarUint32((Integer) (fieldValue)); - } - return false; - case DispatchId.FLOAT32: - if (fieldValue == null) { - buffer.writeByte(Fory.NULL_FLAG); - } else { - buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); - buffer.writeFloat32((Float) (fieldValue)); - } - return false; - case DispatchId.INT64: - case DispatchId.UINT64: - if (fieldValue == null) { - buffer.writeByte(Fory.NULL_FLAG); - } else { - buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); - buffer.writeInt64((Long) fieldValue); - } - return false; - case DispatchId.VARINT64: - if (fieldValue == null) { - buffer.writeByte(Fory.NULL_FLAG); - } else { - buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); - buffer.writeVarInt64((Long) fieldValue); - } - return false; - case DispatchId.TAGGED_INT64: - if (fieldValue == null) { - buffer.writeByte(Fory.NULL_FLAG); - } else { - buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); - buffer.writeTaggedInt64((Long) fieldValue); - } - return false; - case DispatchId.VAR_UINT64: - if (fieldValue == null) { - buffer.writeByte(Fory.NULL_FLAG); - } else { - buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); - buffer.writeVarUint64((Long) fieldValue); - } - return false; - case DispatchId.TAGGED_UINT64: - if (fieldValue == null) { - buffer.writeByte(Fory.NULL_FLAG); - } else { - buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); - buffer.writeTaggedUint64((Long) fieldValue); - } - return false; + return; case DispatchId.FLOAT64: - if (fieldValue == null) { - buffer.writeByte(Fory.NULL_FLAG); - } else { - buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); - buffer.writeFloat64((Double) (fieldValue)); - } - return false; - default: - return true; - } - } - - /** - * Write field value to buffer for PRIMITIVE_* and NOTNULL_BOXED_* dispatch IDs, plus STRING with - * proper refMode handling. This method handles dispatch IDs that don't have a null flag prefix. - * - *

    Note: This method only handles PRIMITIVE_*, NOTNULL_BOXED_* dispatch IDs, and STRING. - * Nullable dispatch IDs (BOOL, INT8, etc.) must be handled by {@link - * #writeNullableBasicObjectFieldValue} since they require a null flag prefix. - * - * @return true if field value isn't written by this function and needs further handling. - */ - private static boolean writeBasicObjectFieldValue( - Fory fory, - MemoryBuffer buffer, - SerializationFieldInfo fieldInfo, - Object fieldValue, - int dispatchId) { - // Only handle PRIMITIVE_*, NOTNULL_BOXED_* dispatch IDs, and STRING here. - // Nullable dispatch IDs (BOOL, INT8, etc.) require a null flag prefix - // and must be handled by writeNullableBasicObjectFieldValue. - switch (dispatchId) { - case DispatchId.PRIMITIVE_BOOL: - case DispatchId.NOTNULL_BOXED_BOOL: - buffer.writeBoolean((Boolean) fieldValue); - return false; - case DispatchId.PRIMITIVE_INT8: - case DispatchId.PRIMITIVE_UINT8: - case DispatchId.NOTNULL_BOXED_INT8: - case DispatchId.NOTNULL_BOXED_UINT8: - buffer.writeByte((Byte) fieldValue); - return false; - case DispatchId.PRIMITIVE_CHAR: - case DispatchId.NOTNULL_BOXED_CHAR: - buffer.writeChar((Character) fieldValue); - return false; - case DispatchId.PRIMITIVE_INT16: - case DispatchId.PRIMITIVE_UINT16: - case DispatchId.NOTNULL_BOXED_INT16: - case DispatchId.NOTNULL_BOXED_UINT16: - buffer.writeInt16((Short) fieldValue); - return false; - case DispatchId.PRIMITIVE_INT32: - case DispatchId.PRIMITIVE_UINT32: - case DispatchId.NOTNULL_BOXED_INT32: - case DispatchId.NOTNULL_BOXED_UINT32: - buffer.writeInt32((Integer) fieldValue); - return false; - case DispatchId.PRIMITIVE_VARINT32: - case DispatchId.NOTNULL_BOXED_VARINT32: - buffer.writeVarInt32((Integer) fieldValue); - return false; - case DispatchId.PRIMITIVE_VAR_UINT32: - case DispatchId.NOTNULL_BOXED_VAR_UINT32: - buffer.writeVarUint32((Integer) fieldValue); - return false; - case DispatchId.PRIMITIVE_INT64: - case DispatchId.PRIMITIVE_UINT64: - case DispatchId.NOTNULL_BOXED_INT64: - case DispatchId.NOTNULL_BOXED_UINT64: - buffer.writeInt64((Long) fieldValue); - return false; - case DispatchId.PRIMITIVE_VARINT64: - case DispatchId.NOTNULL_BOXED_VARINT64: - buffer.writeVarInt64((Long) fieldValue); - return false; - case DispatchId.PRIMITIVE_TAGGED_INT64: - case DispatchId.NOTNULL_BOXED_TAGGED_INT64: - buffer.writeTaggedInt64((Long) fieldValue); - return false; - case DispatchId.PRIMITIVE_VAR_UINT64: - case DispatchId.NOTNULL_BOXED_VAR_UINT64: - buffer.writeVarUint64((Long) fieldValue); - return false; - case DispatchId.PRIMITIVE_TAGGED_UINT64: - case DispatchId.NOTNULL_BOXED_TAGGED_UINT64: - buffer.writeTaggedUint64((Long) fieldValue); - return false; - case DispatchId.PRIMITIVE_FLOAT32: - case DispatchId.NOTNULL_BOXED_FLOAT32: - buffer.writeFloat32((Float) fieldValue); - return false; - case DispatchId.PRIMITIVE_FLOAT64: - case DispatchId.NOTNULL_BOXED_FLOAT64: buffer.writeFloat64((Double) fieldValue); - return false; - case DispatchId.STRING: - // Handle STRING with proper refMode handling - if (fieldInfo.refMode == RefMode.TRACKING) { - fory.writeJavaStringRef(buffer, (String) fieldValue); - } else { - if (fieldInfo.refMode == RefMode.NULL_ONLY) { - if (fieldValue == null) { - buffer.writeByte(Fory.NULL_FLAG); - } else { - buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); - fory.writeJavaString(buffer, (String) fieldValue); - } - } else { - fory.writeJavaString(buffer, (String) fieldValue); - } - } - return false; + return; default: - return true; + binding.writeField(fieldInfo, RefMode.NONE, buffer, fieldValue); } } @@ -672,10 +362,10 @@ static void writeContainerFieldValue( * Read a container field value (Collection or Map). Handles reference tracking, nullable fields, * and pushes/pops generic type information for proper deserialization of parameterized types. * - * @param binding the serialization binding for read operations - * @param generics the generics context for tracking parameterized types + * @param binding the serialization binding for read operations + * @param generics the generics context for tracking parameterized types * @param fieldInfo the field metadata including generic type info and nullability - * @param buffer the buffer to read from + * @param buffer the buffer to read from * @return the deserialized container field value, or null if the field is nullable and was null */ static Object readContainerFieldValue( @@ -691,17 +381,18 @@ static Object readContainerFieldValue( fieldValue = binding.readContainerFieldValue(buffer, fieldInfo); generics.popGenericType(); break; - case NULL_ONLY: { - binding.preserveRefId(-1); - byte headFlag = buffer.readByte(); - if (headFlag == Fory.NULL_FLAG) { - return null; + case NULL_ONLY: + { + binding.preserveRefId(-1); + byte headFlag = buffer.readByte(); + if (headFlag == Fory.NULL_FLAG) { + return null; + } + generics.pushGenericType(fieldInfo.genericType); + fieldValue = binding.readContainerFieldValue(buffer, fieldInfo); + generics.popGenericType(); } - generics.pushGenericType(fieldInfo.genericType); - fieldValue = binding.readContainerFieldValue(buffer, fieldInfo); - generics.popGenericType(); - } - break; + break; case TRACKING: generics.pushGenericType(fieldInfo.genericType); fieldValue = binding.readContainerFieldValueRef(buffer, fieldInfo); @@ -713,11 +404,6 @@ static Object readContainerFieldValue( return fieldValue; } - /** - * Sentinel object to indicate the dispatch ID was not handled by optimized path. - */ - private static final Object UNHANDLED_SENTINEL = new Object(); - /** * Read field value from buffer and return it. Handles primitive types, unsigned/compressed * numbers, and common types like String with optimized fast paths. @@ -727,230 +413,111 @@ static Object readContainerFieldValue( * it into the target object. Useful for record types where field values need to be collected into * an array before constructing the object. * - *

    Note: The dispatch ID from fieldInfo determines the actual data format in the buffer - * (whether there's a null flag prefix or not), regardless of the local field's nullable setting. - * This is important for schema compatibility when peer's field definition differs from local. + *

    The refMode determines how to read the value from buffer: - RefMode.NONE: read directly + * without null flag - RefMode.NULL_ONLY: read null flag first, then value if not null - + * RefMode.TRACKING: use reference tracking * - * @param binding the serialization binding for read operations + * @param binding the serialization binding for read operations * @param fieldInfo the field metadata including type and nullability info - * @param buffer the buffer to read from + * @param buffer the buffer to read from * @return the deserialized field value, or null if the field is nullable and was null * @see #readBuildInFieldValue(SerializationBinding, SerializationFieldInfo, MemoryBuffer, Object) */ static Object readBuildInFieldValue( SerializationBinding binding, SerializationFieldInfo fieldInfo, MemoryBuffer buffer) { - Fory fory = binding.fory; int dispatchId = fieldInfo.dispatchId; - // Try optimized path for basic types (primitives, boxed, string) - // The dispatch ID already encodes whether the data has a null flag prefix: - // - PRIMITIVE_* and NOTNULL_BOXED_* dispatch IDs have no null flag prefix - // - Nullable dispatch IDs (BOOL, INT8, etc.) have null flag prefix - // So we try both paths based on dispatch ID, not the local nullable setting. - // Try primitive and non-null boxed types first (PRIMITIVE_*, NOTNULL_BOXED_*, STRING) - Object value = readBasicObjectValue(fory, buffer, fieldInfo, dispatchId); - if (value != UNHANDLED_SENTINEL) { - return value; - } - // Try nullable types (BOOL, INT8, etc. with null flag prefix) - value = readBasicNullableObjectValue(buffer, dispatchId); - if (value != UNHANDLED_SENTINEL) { - return value; + RefMode refMode = fieldInfo.refMode; + // Use refMode to determine if there's a null flag prefix in the stream + if (refMode == RefMode.NONE) { + // No null flag in stream - read directly + return readNotNullBuildInFieldValue(binding, buffer, fieldInfo, dispatchId); + } else if (refMode == RefMode.NULL_ONLY) { + // Read null flag from buffer + if (buffer.readByte() == Fory.NULL_FLAG) { + return null; + } + return readNotNullBuildInFieldValue(binding, buffer, fieldInfo, dispatchId); } - // Fall back to binding.read for complex types + // Fall back to binding.read for complex types or TRACKING mode return binding.readField(fieldInfo, buffer); } /** - * Read a non-nullable basic object value from buffer and return it. Handles PRIMITIVE_*, - * NOTNULL_BOXED_*, and STRING dispatch IDs with optimized fast paths. - * - *

    Note: Nullable dispatch IDs (BOOL, INT8, etc.) must be handled by - * {@link #readBasicNullableObjectValue} since they have a null flag prefix in the serialized - * data. - * - * @return the value if handled, or {@link #UNHANDLED_SENTINEL} if not a basic type + * Read a non-nullable basic object value from buffer and return it. Handles PRIMITIVE_*, and + * STRING dispatch IDs with optimized fast paths. */ - private static Object readBasicObjectValue( - Fory fory, MemoryBuffer buffer, SerializationFieldInfo fieldInfo, int dispatchId) { + private static Object readNotNullBuildInFieldValue( + SerializationBinding binding, + MemoryBuffer buffer, + SerializationFieldInfo fieldInfo, + int dispatchId) { switch (dispatchId) { - case DispatchId.PRIMITIVE_BOOL: - case DispatchId.NOTNULL_BOXED_BOOL: - return buffer.readBoolean(); - case DispatchId.PRIMITIVE_INT8: - case DispatchId.PRIMITIVE_UINT8: - case DispatchId.NOTNULL_BOXED_INT8: - case DispatchId.NOTNULL_BOXED_UINT8: - return buffer.readByte(); - case DispatchId.PRIMITIVE_CHAR: - case DispatchId.NOTNULL_BOXED_CHAR: - return buffer.readChar(); - case DispatchId.PRIMITIVE_INT16: - case DispatchId.PRIMITIVE_UINT16: - case DispatchId.NOTNULL_BOXED_INT16: - case DispatchId.NOTNULL_BOXED_UINT16: - return buffer.readInt16(); - case DispatchId.PRIMITIVE_INT32: - case DispatchId.PRIMITIVE_UINT32: - case DispatchId.NOTNULL_BOXED_INT32: - case DispatchId.NOTNULL_BOXED_UINT32: - return buffer.readInt32(); - case DispatchId.PRIMITIVE_VARINT32: - case DispatchId.NOTNULL_BOXED_VARINT32: - return buffer.readVarInt32(); - case DispatchId.PRIMITIVE_VAR_UINT32: - case DispatchId.NOTNULL_BOXED_VAR_UINT32: - return buffer.readVarUint32(); - case DispatchId.PRIMITIVE_INT64: - case DispatchId.PRIMITIVE_UINT64: - case DispatchId.NOTNULL_BOXED_INT64: - case DispatchId.NOTNULL_BOXED_UINT64: - return buffer.readInt64(); - case DispatchId.PRIMITIVE_VARINT64: - case DispatchId.NOTNULL_BOXED_VARINT64: - return buffer.readVarInt64(); - case DispatchId.PRIMITIVE_TAGGED_INT64: - case DispatchId.NOTNULL_BOXED_TAGGED_INT64: - return buffer.readTaggedInt64(); - case DispatchId.PRIMITIVE_VAR_UINT64: - case DispatchId.NOTNULL_BOXED_VAR_UINT64: - return buffer.readVarUint64(); - case DispatchId.PRIMITIVE_TAGGED_UINT64: - case DispatchId.NOTNULL_BOXED_TAGGED_UINT64: - return buffer.readTaggedUint64(); - case DispatchId.PRIMITIVE_FLOAT32: - case DispatchId.NOTNULL_BOXED_FLOAT32: - return buffer.readFloat32(); - case DispatchId.PRIMITIVE_FLOAT64: - case DispatchId.NOTNULL_BOXED_FLOAT64: - return buffer.readFloat64(); - case DispatchId.STRING: - if (fieldInfo.refMode == RefMode.TRACKING) { - return fory.readJavaStringRef(buffer); - } else if (fieldInfo.refMode == RefMode.NULL_ONLY) { - if (buffer.readByte() == Fory.NULL_FLAG) { - return null; - } - return fory.readJavaString(buffer); - } else { - // RefMode.NONE - read string directly without null flag - return fory.readJavaString(buffer); - } - default: - return UNHANDLED_SENTINEL; - } - } - - /** - * Read a nullable basic object value from buffer and return it. Reads null flag before value for - * nullable boxed types (BOOL, INT8, etc.). - * - *

    Note: STRING is handled by {@link #readBasicObjectValue} with proper refMode check. - * - * @return the value (possibly null) if handled, or {@link #UNHANDLED_SENTINEL} if not a basic - * type - */ - private static Object readBasicNullableObjectValue(MemoryBuffer buffer, int dispatchId) { - switch (dispatchId) { - case DispatchId.BOOL: - if (buffer.readByte() == Fory.NULL_FLAG) { - return null; - } + case DispatchId.BOOL: return buffer.readBoolean(); case DispatchId.INT8: case DispatchId.UINT8: - if (buffer.readByte() == Fory.NULL_FLAG) { - return null; - } return buffer.readByte(); case DispatchId.CHAR: - if (buffer.readByte() == Fory.NULL_FLAG) { - return null; - } return buffer.readChar(); case DispatchId.INT16: case DispatchId.UINT16: - if (buffer.readByte() == Fory.NULL_FLAG) { - return null; - } return buffer.readInt16(); case DispatchId.INT32: case DispatchId.UINT32: - if (buffer.readByte() == Fory.NULL_FLAG) { - return null; - } return buffer.readInt32(); case DispatchId.VARINT32: - if (buffer.readByte() == Fory.NULL_FLAG) { - return null; - } return buffer.readVarInt32(); case DispatchId.VAR_UINT32: - if (buffer.readByte() == Fory.NULL_FLAG) { - return null; - } return buffer.readVarUint32(); case DispatchId.INT64: case DispatchId.UINT64: - if (buffer.readByte() == Fory.NULL_FLAG) { - return null; - } return buffer.readInt64(); case DispatchId.VARINT64: - if (buffer.readByte() == Fory.NULL_FLAG) { - return null; - } return buffer.readVarInt64(); case DispatchId.TAGGED_INT64: - if (buffer.readByte() == Fory.NULL_FLAG) { - return null; - } return buffer.readTaggedInt64(); case DispatchId.VAR_UINT64: - if (buffer.readByte() == Fory.NULL_FLAG) { - return null; - } return buffer.readVarUint64(); case DispatchId.TAGGED_UINT64: - if (buffer.readByte() == Fory.NULL_FLAG) { - return null; - } return buffer.readTaggedUint64(); case DispatchId.FLOAT32: - if (buffer.readByte() == Fory.NULL_FLAG) { - return null; - } return buffer.readFloat32(); case DispatchId.FLOAT64: - if (buffer.readByte() == Fory.NULL_FLAG) { - return null; - } return buffer.readFloat64(); + case DispatchId.STRING: + return binding.fory.readJavaString(buffer); default: - return UNHANDLED_SENTINEL; + return binding.readField(fieldInfo, RefMode.NONE, buffer); } } /** - * Handle all numeric fields read include unsigned and compressed numbers. - * It also include fastpath for common type such as String. - * - *

    Note: The dispatch ID from fieldInfo determines the actual data format in the buffer - * (whether there's a null flag prefix or not), regardless of the local field's nullable setting. - * This is important for schema compatibility when peer's field definition differs from local. + * Handle all numeric fields read include unsigned and compressed numbers. It also include + * fastpath for common type such as String. */ static void readBuildInFieldValue( - SerializationBinding binding, SerializationFieldInfo fieldInfo, MemoryBuffer buffer, Object targetObject) { - Fory fory = binding.fory; + SerializationBinding binding, + SerializationFieldInfo fieldInfo, + MemoryBuffer buffer, + Object targetObject) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; int dispatchId = fieldInfo.dispatchId; - // The dispatch ID already encodes whether the data has a null flag prefix: - // - PRIMITIVE_* and NOTNULL_BOXED_* dispatch IDs have no null flag prefix - // - Nullable dispatch IDs (BOOL, INT8, etc.) have null flag prefix - // So we try both paths based on dispatch ID, not the local nullable setting. - boolean needRead = readPrimitiveFieldValue(buffer, targetObject, fieldAccessor, dispatchId) - && readBasicObjectFieldValue(fory, buffer, targetObject, fieldInfo, dispatchId) - && readBasicNullableObjectFieldValue(fory, buffer, targetObject, fieldAccessor, dispatchId); - if (needRead) { + if (fieldInfo.refMode == RefMode.NONE) { + if (fieldInfo.isPrimitiveField) { + readPrimitiveFieldValue(buffer, targetObject, fieldAccessor, dispatchId); + } else { + readNotPrimitiveFieldValue(binding, buffer, targetObject, fieldInfo, dispatchId); + } + } else if (fieldInfo.refMode == RefMode.NULL_ONLY) { + if (buffer.readByte() == Fory.NULL_FLAG) { + return; + } + if (fieldInfo.isPrimitiveField) { + readPrimitiveFieldValue(buffer, targetObject, fieldAccessor, dispatchId); + } else { + readNotPrimitiveFieldValue(binding, buffer, targetObject, fieldInfo, dispatchId); + } + } else { Object fieldValue = binding.readField(fieldInfo, buffer); fieldAccessor.putObject(targetObject, fieldValue); } @@ -959,400 +526,195 @@ && readBasicObjectFieldValue(fory, buffer, targetObject, fieldInfo, dispatchId) /** * Read a primitive value from buffer and set it to field referenced by fieldAccessor * of targetObject. - * - * @return true if classId is not a primitive type id. */ - private static boolean readPrimitiveFieldValue( + private static void readPrimitiveFieldValue( MemoryBuffer buffer, Object targetObject, FieldAccessor fieldAccessor, int dispatchId) { long fieldOffset = fieldAccessor.getFieldOffset(); if (fieldOffset != -1) { - return readPrimitiveFieldValue(buffer, targetObject, fieldOffset, dispatchId); + readPrimitiveFieldValue(buffer, targetObject, fieldOffset, dispatchId); + return; } // graalvm use GeneratedAccessor, which will be this code path. // we still need `PRIMITIVE` cases since peer may send switch (dispatchId) { - case DispatchId.PRIMITIVE_BOOL: + case DispatchId.BOOL: fieldAccessor.set(targetObject, buffer.readBoolean()); - return false; - case DispatchId.PRIMITIVE_INT8: - case DispatchId.PRIMITIVE_UINT8: + return; + case DispatchId.INT8: + case DispatchId.UINT8: fieldAccessor.set(targetObject, buffer.readByte()); - return false; - case DispatchId.PRIMITIVE_CHAR: + return; + case DispatchId.CHAR: fieldAccessor.set(targetObject, buffer.readChar()); - return false; - case DispatchId.PRIMITIVE_INT16: - case DispatchId.PRIMITIVE_UINT16: + return; + case DispatchId.INT16: + case DispatchId.UINT16: fieldAccessor.set(targetObject, buffer.readInt16()); - return false; - case DispatchId.PRIMITIVE_INT32: - case DispatchId.PRIMITIVE_UINT32: + return; + case DispatchId.INT32: + case DispatchId.UINT32: fieldAccessor.set(targetObject, buffer.readInt32()); - return false; - case DispatchId.PRIMITIVE_VARINT32: + return; + case DispatchId.VARINT32: fieldAccessor.set(targetObject, buffer.readVarInt32()); - return false; - case DispatchId.PRIMITIVE_VAR_UINT32: + return; + case DispatchId.VAR_UINT32: fieldAccessor.set(targetObject, buffer.readVarUint32()); - return false; - case DispatchId.PRIMITIVE_FLOAT32: + return; + case DispatchId.FLOAT32: fieldAccessor.set(targetObject, buffer.readFloat32()); - return false; - case DispatchId.PRIMITIVE_INT64: - case DispatchId.PRIMITIVE_UINT64: + return; + case DispatchId.INT64: + case DispatchId.UINT64: fieldAccessor.set(targetObject, buffer.readInt64()); - return false; - case DispatchId.PRIMITIVE_VARINT64: + return; + case DispatchId.VARINT64: fieldAccessor.set(targetObject, buffer.readVarInt64()); - return false; - case DispatchId.PRIMITIVE_TAGGED_INT64: + return; + case DispatchId.TAGGED_INT64: fieldAccessor.set(targetObject, buffer.readTaggedInt64()); - return false; - case DispatchId.PRIMITIVE_VAR_UINT64: + return; + case DispatchId.VAR_UINT64: fieldAccessor.set(targetObject, buffer.readVarUint64()); - return false; - case DispatchId.PRIMITIVE_TAGGED_UINT64: + return; + case DispatchId.TAGGED_UINT64: fieldAccessor.set(targetObject, buffer.readTaggedUint64()); - return false; - case DispatchId.PRIMITIVE_FLOAT64: + return; + case DispatchId.FLOAT64: fieldAccessor.set(targetObject, buffer.readFloat64()); - return false; + return; default: - return true; + throw new IllegalArgumentException("Unsupported dispatch id " + dispatchId); } } /** * Read a primitive field value from buffer and set it using direct memory offset access. * - * @param buffer the buffer to read from + * @param buffer the buffer to read from * @param targetObject the object to set the field value on - * @param fieldOffset the memory offset of the field - * @param dispatchId the dispatch ID of the primitive type - * @return true if classId is not a primitive type and needs further read handling + * @param fieldOffset the memory offset of the field + * @param dispatchId the dispatch ID of the primitive type */ - private static boolean readPrimitiveFieldValue( + private static void readPrimitiveFieldValue( MemoryBuffer buffer, Object targetObject, long fieldOffset, int dispatchId) { switch (dispatchId) { - case DispatchId.PRIMITIVE_BOOL: + case DispatchId.BOOL: Platform.putBoolean(targetObject, fieldOffset, buffer.readBoolean()); - return false; - case DispatchId.PRIMITIVE_INT8: - case DispatchId.PRIMITIVE_UINT8: + return; + case DispatchId.INT8: + case DispatchId.UINT8: Platform.putByte(targetObject, fieldOffset, buffer.readByte()); - return false; - case DispatchId.PRIMITIVE_CHAR: + return; + case DispatchId.CHAR: Platform.putChar(targetObject, fieldOffset, buffer.readChar()); - return false; - case DispatchId.PRIMITIVE_INT16: - case DispatchId.PRIMITIVE_UINT16: + return; + case DispatchId.INT16: + case DispatchId.UINT16: Platform.putShort(targetObject, fieldOffset, buffer.readInt16()); - return false; - case DispatchId.PRIMITIVE_INT32: - case DispatchId.PRIMITIVE_UINT32: + return; + case DispatchId.INT32: + case DispatchId.UINT32: Platform.putInt(targetObject, fieldOffset, buffer.readInt32()); - return false; - case DispatchId.PRIMITIVE_VARINT32: + return; + case DispatchId.VARINT32: Platform.putInt(targetObject, fieldOffset, buffer.readVarInt32()); - return false; - case DispatchId.PRIMITIVE_VAR_UINT32: + return; + case DispatchId.VAR_UINT32: Platform.putInt(targetObject, fieldOffset, buffer.readVarUint32()); - return false; - case DispatchId.PRIMITIVE_FLOAT32: + return; + case DispatchId.FLOAT32: Platform.putFloat(targetObject, fieldOffset, buffer.readFloat32()); - return false; - case DispatchId.PRIMITIVE_INT64: - case DispatchId.PRIMITIVE_UINT64: + return; + case DispatchId.INT64: + case DispatchId.UINT64: Platform.putLong(targetObject, fieldOffset, buffer.readInt64()); - return false; - case DispatchId.PRIMITIVE_VARINT64: + return; + case DispatchId.VARINT64: Platform.putLong(targetObject, fieldOffset, buffer.readVarInt64()); - return false; - case DispatchId.PRIMITIVE_TAGGED_INT64: + return; + case DispatchId.TAGGED_INT64: Platform.putLong(targetObject, fieldOffset, buffer.readTaggedInt64()); - return false; - case DispatchId.PRIMITIVE_VAR_UINT64: + return; + case DispatchId.VAR_UINT64: Platform.putLong(targetObject, fieldOffset, buffer.readVarUint64()); - return false; - case DispatchId.PRIMITIVE_TAGGED_UINT64: + return; + case DispatchId.TAGGED_UINT64: Platform.putLong(targetObject, fieldOffset, buffer.readTaggedUint64()); - return false; - case DispatchId.PRIMITIVE_FLOAT64: + return; + case DispatchId.FLOAT64: Platform.putDouble(targetObject, fieldOffset, buffer.readFloat64()); - return false; + return; default: - return true; + throw new IllegalArgumentException("Unsupported dispatch id " + dispatchId); } } /** * Read field value from buffer and set it on the target object. This method handles PRIMITIVE_* * and NOTNULL_BOXED_* dispatch IDs where null values are not allowed. - * - *

    Note: This method only handles PRIMITIVE_* and NOTNULL_BOXED_* dispatch IDs. Nullable - * dispatch IDs (BOOL, INT8, etc.) must be handled by {@link #readBasicNullableObjectFieldValue} - * since they have a null flag prefix in the serialized data, regardless of the local field's - * nullable setting. - * - * @return true if field value isn't read by this function. */ - static boolean readBasicObjectFieldValue( - Fory fory, + private static void readNotPrimitiveFieldValue( + SerializationBinding binding, MemoryBuffer buffer, Object targetObject, SerializationFieldInfo fieldInfo, int dispatchId) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; - // Only handle PRIMITIVE_* and NOTNULL_BOXED_* dispatch IDs here. - // Nullable dispatch IDs (STRING, BOOL, INT8, etc.) have a null flag prefix in the serialized data - // and must be handled by readBasicNullableObjectFieldValue. - switch (dispatchId) { - case DispatchId.PRIMITIVE_BOOL: - case DispatchId.NOTNULL_BOXED_BOOL: - fieldAccessor.putObject(targetObject, buffer.readBoolean()); - return false; - case DispatchId.PRIMITIVE_INT8: - case DispatchId.PRIMITIVE_UINT8: - case DispatchId.NOTNULL_BOXED_INT8: - case DispatchId.NOTNULL_BOXED_UINT8: - fieldAccessor.putObject(targetObject, buffer.readByte()); - return false; - case DispatchId.PRIMITIVE_CHAR: - case DispatchId.NOTNULL_BOXED_CHAR: - fieldAccessor.putObject(targetObject, buffer.readChar()); - return false; - case DispatchId.PRIMITIVE_INT16: - case DispatchId.PRIMITIVE_UINT16: - case DispatchId.NOTNULL_BOXED_INT16: - case DispatchId.NOTNULL_BOXED_UINT16: - fieldAccessor.putObject(targetObject, buffer.readInt16()); - return false; - case DispatchId.PRIMITIVE_INT32: - case DispatchId.PRIMITIVE_UINT32: - case DispatchId.NOTNULL_BOXED_INT32: - case DispatchId.NOTNULL_BOXED_UINT32: - fieldAccessor.putObject(targetObject, buffer.readInt32()); - return false; - case DispatchId.PRIMITIVE_VARINT32: - case DispatchId.NOTNULL_BOXED_VARINT32: - fieldAccessor.putObject(targetObject, buffer.readVarInt32()); - return false; - case DispatchId.PRIMITIVE_VAR_UINT32: - case DispatchId.NOTNULL_BOXED_VAR_UINT32: - fieldAccessor.putObject(targetObject, buffer.readVarUint32()); - return false; - case DispatchId.PRIMITIVE_INT64: - case DispatchId.PRIMITIVE_UINT64: - case DispatchId.NOTNULL_BOXED_INT64: - case DispatchId.NOTNULL_BOXED_UINT64: - fieldAccessor.putObject(targetObject, buffer.readInt64()); - return false; - case DispatchId.PRIMITIVE_VARINT64: - case DispatchId.NOTNULL_BOXED_VARINT64: - fieldAccessor.putObject(targetObject, buffer.readVarInt64()); - return false; - case DispatchId.PRIMITIVE_TAGGED_INT64: - case DispatchId.NOTNULL_BOXED_TAGGED_INT64: - fieldAccessor.putObject(targetObject, buffer.readTaggedInt64()); - return false; - case DispatchId.PRIMITIVE_VAR_UINT64: - case DispatchId.NOTNULL_BOXED_VAR_UINT64: - fieldAccessor.putObject(targetObject, buffer.readVarUint64()); - return false; - case DispatchId.PRIMITIVE_TAGGED_UINT64: - case DispatchId.NOTNULL_BOXED_TAGGED_UINT64: - fieldAccessor.putObject(targetObject, buffer.readTaggedUint64()); - return false; - case DispatchId.PRIMITIVE_FLOAT32: - case DispatchId.NOTNULL_BOXED_FLOAT32: - fieldAccessor.putObject(targetObject, buffer.readFloat32()); - return false; - case DispatchId.PRIMITIVE_FLOAT64: - case DispatchId.NOTNULL_BOXED_FLOAT64: - fieldAccessor.putObject(targetObject, buffer.readFloat64()); - return false; - case DispatchId.STRING: - if (fieldInfo.refMode == RefMode.TRACKING) { - fieldAccessor.putObject(targetObject, fory.readJavaStringRef(buffer)); - } else { - if (fieldInfo.refMode != RefMode.NULL_ONLY || buffer.readByte() != Fory.NULL_FLAG) { - fieldAccessor.putObject(targetObject, fory.readJavaString(buffer)); - } - } - return false; - default: - return true; - } - } - - /** - * Read a nullable boxed primitive or String field value from buffer and set it on the target - * object. Reads the null flag before value for nullable types. - * Note that this method must handle all unsigned/compressed int encodings, since the fallback code path - * are type based, and won't handle such cases. - * - * @param fory the fory instance for compression and ref tracking settings - * @param buffer the buffer to read from - * @param targetObject the object to set the field value on - * @param fieldAccessor the accessor to set the field value - * @param dispatchId the class ID of the boxed type - * @return true if dispatchId is not a basic type or ref tracking is enabled, needing further read - * handling - */ - private static boolean readBasicNullableObjectFieldValue( - Fory fory, - MemoryBuffer buffer, - Object targetObject, - FieldAccessor fieldAccessor, - int dispatchId) { - // add time types serialization here. switch (dispatchId) { case DispatchId.BOOL: - if (buffer.readByte() == Fory.NULL_FLAG) { - fieldAccessor.putObject(targetObject, null); - } else { - fieldAccessor.putObject(targetObject, buffer.readBoolean()); - } - return false; + fieldAccessor.putObject(targetObject, buffer.readBoolean()); + return; case DispatchId.INT8: case DispatchId.UINT8: - if (buffer.readByte() == Fory.NULL_FLAG) { - fieldAccessor.putObject(targetObject, null); - } else { - fieldAccessor.putObject(targetObject, buffer.readByte()); - } - return false; + fieldAccessor.putObject(targetObject, buffer.readByte()); + return; case DispatchId.CHAR: - if (buffer.readByte() == Fory.NULL_FLAG) { - fieldAccessor.putObject(targetObject, null); - } else { - fieldAccessor.putObject(targetObject, buffer.readChar()); - } - return false; + fieldAccessor.putObject(targetObject, buffer.readChar()); + return; case DispatchId.INT16: case DispatchId.UINT16: - if (buffer.readByte() == Fory.NULL_FLAG) { - fieldAccessor.putObject(targetObject, null); - } else { - fieldAccessor.putObject(targetObject, buffer.readInt16()); - } - return false; + fieldAccessor.putObject(targetObject, buffer.readInt16()); + return; case DispatchId.INT32: case DispatchId.UINT32: - if (buffer.readByte() == Fory.NULL_FLAG) { - fieldAccessor.putObject(targetObject, null); - } else { - fieldAccessor.putObject(targetObject, buffer.readInt32()); - } - return false; + fieldAccessor.putObject(targetObject, buffer.readInt32()); + return; case DispatchId.VARINT32: - if (buffer.readByte() == Fory.NULL_FLAG) { - fieldAccessor.putObject(targetObject, null); - } else { - fieldAccessor.putObject(targetObject, buffer.readVarInt32()); - } - return false; + fieldAccessor.putObject(targetObject, buffer.readVarInt32()); + return; case DispatchId.VAR_UINT32: - if (buffer.readByte() == Fory.NULL_FLAG) { - fieldAccessor.putObject(targetObject, null); - } else { - fieldAccessor.putObject(targetObject, buffer.readVarUint32()); - } - return false; + fieldAccessor.putObject(targetObject, buffer.readVarUint32()); + return; case DispatchId.INT64: case DispatchId.UINT64: - if (buffer.readByte() == Fory.NULL_FLAG) { - fieldAccessor.putObject(targetObject, null); - } else { - fieldAccessor.putObject(targetObject, buffer.readInt64()); - } - return false; + fieldAccessor.putObject(targetObject, buffer.readInt64()); + return; case DispatchId.VARINT64: - if (buffer.readByte() == Fory.NULL_FLAG) { - fieldAccessor.putObject(targetObject, null); - } else { - fieldAccessor.putObject(targetObject, buffer.readVarInt64()); - } - return false; + fieldAccessor.putObject(targetObject, buffer.readVarInt64()); + return; case DispatchId.TAGGED_INT64: - if (buffer.readByte() == Fory.NULL_FLAG) { - fieldAccessor.putObject(targetObject, null); - } else { - fieldAccessor.putObject(targetObject, buffer.readTaggedInt64()); - } - return false; + fieldAccessor.putObject(targetObject, buffer.readTaggedInt64()); + return; case DispatchId.VAR_UINT64: - if (buffer.readByte() == Fory.NULL_FLAG) { - fieldAccessor.putObject(targetObject, null); - } else { - fieldAccessor.putObject(targetObject, buffer.readVarUint64()); - } - return false; + fieldAccessor.putObject(targetObject, buffer.readVarUint64()); + return; case DispatchId.TAGGED_UINT64: - if (buffer.readByte() == Fory.NULL_FLAG) { - fieldAccessor.putObject(targetObject, null); - } else { - fieldAccessor.putObject(targetObject, buffer.readTaggedUint64()); - } - return false; + fieldAccessor.putObject(targetObject, buffer.readTaggedUint64()); + return; case DispatchId.FLOAT32: - if (buffer.readByte() == Fory.NULL_FLAG) { - fieldAccessor.putObject(targetObject, null); - } else { - fieldAccessor.putObject(targetObject, buffer.readFloat32()); - } - return false; + fieldAccessor.putObject(targetObject, buffer.readFloat32()); + return; case DispatchId.FLOAT64: - if (buffer.readByte() == Fory.NULL_FLAG) { - fieldAccessor.putObject(targetObject, null); - } else { - fieldAccessor.putObject(targetObject, buffer.readFloat64()); - } - return false; - // string is handled in readBasicObjectFieldValue - default: - return true; - } - } - - static Object readPrimitiveValue(MemoryBuffer buffer, int dispatchId) { - switch (dispatchId) { - case DispatchId.PRIMITIVE_BOOL: - return buffer.readBoolean(); - case DispatchId.PRIMITIVE_INT8: - case DispatchId.PRIMITIVE_UINT8: - return buffer.readByte(); - case DispatchId.PRIMITIVE_CHAR: - return buffer.readChar(); - case DispatchId.PRIMITIVE_INT16: - case DispatchId.PRIMITIVE_UINT16: - return buffer.readInt16(); - case DispatchId.PRIMITIVE_INT32: - return buffer.readInt32(); - case DispatchId.PRIMITIVE_VARINT32: - return buffer.readVarInt32(); - case DispatchId.PRIMITIVE_UINT32: - return buffer.readInt32(); - case DispatchId.PRIMITIVE_VAR_UINT32: - return buffer.readVarUint32(); - case DispatchId.PRIMITIVE_INT64: - return buffer.readInt64(); - case DispatchId.PRIMITIVE_VARINT64: - return buffer.readVarInt64(); - case DispatchId.PRIMITIVE_TAGGED_INT64: - return buffer.readTaggedInt64(); - case DispatchId.PRIMITIVE_UINT64: - return buffer.readInt64(); - case DispatchId.PRIMITIVE_VAR_UINT64: - return buffer.readVarUint64(); - case DispatchId.PRIMITIVE_TAGGED_UINT64: - return buffer.readTaggedUint64(); - case DispatchId.PRIMITIVE_FLOAT32: - return buffer.readFloat32(); - case DispatchId.PRIMITIVE_FLOAT64: - return buffer.readFloat64(); + fieldAccessor.putObject(targetObject, buffer.readFloat64()); + return; + case DispatchId.STRING: + fieldAccessor.putObject(targetObject, binding.fory.readJavaString(buffer)); + return; default: - return UNHANDLED_SENTINEL; + // Use RefMode.NONE because null flag was already handled by caller + Object fieldValue = binding.readField(fieldInfo, RefMode.NONE, buffer); + fieldAccessor.putObject(targetObject, fieldValue); } } @@ -1396,7 +758,11 @@ private Object[] copyFields(T originObj) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; long fieldOffset = fieldAccessor.getFieldOffset(); if (fieldOffset != -1) { - fieldValues[i] = copyField(originObj, fieldOffset, fieldInfo.dispatchId); + if (fieldInfo.isPrimitiveField) { + fieldValues[i] = copyPrimitiveField(originObj, fieldOffset, fieldInfo.dispatchId); + } else { + fieldValues[i] = copyNotPrimitiveField(originObj, fieldOffset, fieldInfo.dispatchId); + } } else { // field in record class has offset -1 Object fieldValue = fieldAccessor.get(originObj); @@ -1421,97 +787,121 @@ public static void copyFields( long fieldOffset = fieldAccessor.getFieldOffset(); // record class won't go to this path; assert fieldOffset != -1; - switch (fieldInfo.dispatchId) { - case DispatchId.PRIMITIVE_BOOL: - Platform.putBoolean(newObj, fieldOffset, Platform.getBoolean(originObj, fieldOffset)); - break; - case DispatchId.PRIMITIVE_INT8: - case DispatchId.PRIMITIVE_UINT8: - Platform.putByte(newObj, fieldOffset, Platform.getByte(originObj, fieldOffset)); - break; - case DispatchId.PRIMITIVE_CHAR: - Platform.putChar(newObj, fieldOffset, Platform.getChar(originObj, fieldOffset)); - break; - case DispatchId.PRIMITIVE_INT16: - case DispatchId.PRIMITIVE_UINT16: - Platform.putShort(newObj, fieldOffset, Platform.getShort(originObj, fieldOffset)); - break; - case DispatchId.PRIMITIVE_INT32: - case DispatchId.PRIMITIVE_VARINT32: - case DispatchId.PRIMITIVE_UINT32: - case DispatchId.PRIMITIVE_VAR_UINT32: - Platform.putInt(newObj, fieldOffset, Platform.getInt(originObj, fieldOffset)); - break; - case DispatchId.PRIMITIVE_INT64: - case DispatchId.PRIMITIVE_VARINT64: - case DispatchId.PRIMITIVE_TAGGED_INT64: - case DispatchId.PRIMITIVE_UINT64: - case DispatchId.PRIMITIVE_VAR_UINT64: - case DispatchId.PRIMITIVE_TAGGED_UINT64: - Platform.putLong(newObj, fieldOffset, Platform.getLong(originObj, fieldOffset)); - break; - case DispatchId.PRIMITIVE_FLOAT32: - Platform.putFloat(newObj, fieldOffset, Platform.getFloat(originObj, fieldOffset)); - break; - case DispatchId.PRIMITIVE_FLOAT64: - Platform.putDouble(newObj, fieldOffset, Platform.getDouble(originObj, fieldOffset)); - break; - case DispatchId.BOOL: - case DispatchId.INT8: - case DispatchId.UINT8: - case DispatchId.CHAR: - case DispatchId.INT16: - case DispatchId.UINT16: - case DispatchId.INT32: - case DispatchId.VARINT32: - case DispatchId.UINT32: - case DispatchId.VAR_UINT32: - case DispatchId.INT64: - case DispatchId.VARINT64: - case DispatchId.TAGGED_INT64: - case DispatchId.UINT64: - case DispatchId.VAR_UINT64: - case DispatchId.TAGGED_UINT64: - case DispatchId.FLOAT32: - case DispatchId.FLOAT64: - case DispatchId.STRING: - Platform.putObject(newObj, fieldOffset, Platform.getObject(originObj, fieldOffset)); - break; - default: - Platform.putObject( - newObj, fieldOffset, fory.copyObject(Platform.getObject(originObj, fieldOffset))); + if (fieldInfo.isPrimitiveField) { + copySetPrimitiveField(originObj, newObj, fieldOffset, fieldInfo.dispatchId); + } else { + copySetNotPrimitiveField(fory, originObj, newObj, fieldOffset, fieldInfo.dispatchId); } } } - private Object copyField(Object targetObject, long fieldOffset, int typeId) { + private static void copySetPrimitiveField( + Object originObj, Object newObj, long fieldOffset, int typeId) { + switch (typeId) { + case DispatchId.BOOL: + Platform.putBoolean(newObj, fieldOffset, Platform.getBoolean(originObj, fieldOffset)); + break; + case DispatchId.INT8: + case DispatchId.UINT8: + Platform.putByte(newObj, fieldOffset, Platform.getByte(originObj, fieldOffset)); + break; + case DispatchId.CHAR: + Platform.putChar(newObj, fieldOffset, Platform.getChar(originObj, fieldOffset)); + break; + case DispatchId.INT16: + case DispatchId.UINT16: + Platform.putShort(newObj, fieldOffset, Platform.getShort(originObj, fieldOffset)); + break; + case DispatchId.INT32: + case DispatchId.VARINT32: + case DispatchId.UINT32: + case DispatchId.VAR_UINT32: + Platform.putInt(newObj, fieldOffset, Platform.getInt(originObj, fieldOffset)); + break; + case DispatchId.INT64: + case DispatchId.VARINT64: + case DispatchId.TAGGED_INT64: + case DispatchId.UINT64: + case DispatchId.VAR_UINT64: + case DispatchId.TAGGED_UINT64: + Platform.putLong(newObj, fieldOffset, Platform.getLong(originObj, fieldOffset)); + break; + case DispatchId.FLOAT32: + Platform.putFloat(newObj, fieldOffset, Platform.getFloat(originObj, fieldOffset)); + break; + case DispatchId.FLOAT64: + Platform.putDouble(newObj, fieldOffset, Platform.getDouble(originObj, fieldOffset)); + break; + default: + throw new RuntimeException("Unknown primitive type: " + typeId); + } + } + + private static void copySetNotPrimitiveField( + Fory fory, Object originObj, Object newObj, long fieldOffset, int typeId) { + switch (typeId) { + case DispatchId.BOOL: + case DispatchId.INT8: + case DispatchId.UINT8: + case DispatchId.CHAR: + case DispatchId.INT16: + case DispatchId.UINT16: + case DispatchId.INT32: + case DispatchId.VARINT32: + case DispatchId.UINT32: + case DispatchId.VAR_UINT32: + case DispatchId.INT64: + case DispatchId.VARINT64: + case DispatchId.TAGGED_INT64: + case DispatchId.UINT64: + case DispatchId.VAR_UINT64: + case DispatchId.TAGGED_UINT64: + case DispatchId.FLOAT32: + case DispatchId.FLOAT64: + case DispatchId.STRING: + Platform.putObject(newObj, fieldOffset, Platform.getObject(originObj, fieldOffset)); + break; + default: + Platform.putObject( + newObj, fieldOffset, fory.copyObject(Platform.getObject(originObj, fieldOffset))); + } + } + + private Object copyPrimitiveField(Object targetObject, long fieldOffset, int typeId) { switch (typeId) { - case DispatchId.PRIMITIVE_BOOL: + case DispatchId.BOOL: return Platform.getBoolean(targetObject, fieldOffset); - case DispatchId.PRIMITIVE_INT8: - case DispatchId.PRIMITIVE_UINT8: + case DispatchId.INT8: + case DispatchId.UINT8: return Platform.getByte(targetObject, fieldOffset); - case DispatchId.PRIMITIVE_CHAR: + case DispatchId.CHAR: return Platform.getChar(targetObject, fieldOffset); - case DispatchId.PRIMITIVE_INT16: - case DispatchId.PRIMITIVE_UINT16: + case DispatchId.INT16: + case DispatchId.UINT16: return Platform.getShort(targetObject, fieldOffset); - case DispatchId.PRIMITIVE_INT32: - case DispatchId.PRIMITIVE_VARINT32: - case DispatchId.PRIMITIVE_UINT32: - case DispatchId.PRIMITIVE_VAR_UINT32: + case DispatchId.INT32: + case DispatchId.VARINT32: + case DispatchId.UINT32: + case DispatchId.VAR_UINT32: return Platform.getInt(targetObject, fieldOffset); - case DispatchId.PRIMITIVE_FLOAT32: + case DispatchId.FLOAT32: return Platform.getFloat(targetObject, fieldOffset); - case DispatchId.PRIMITIVE_INT64: - case DispatchId.PRIMITIVE_VARINT64: - case DispatchId.PRIMITIVE_TAGGED_INT64: - case DispatchId.PRIMITIVE_UINT64: - case DispatchId.PRIMITIVE_VAR_UINT64: - case DispatchId.PRIMITIVE_TAGGED_UINT64: + case DispatchId.INT64: + case DispatchId.VARINT64: + case DispatchId.TAGGED_INT64: + case DispatchId.UINT64: + case DispatchId.VAR_UINT64: + case DispatchId.TAGGED_UINT64: return Platform.getLong(targetObject, fieldOffset); - case DispatchId.PRIMITIVE_FLOAT64: + case DispatchId.FLOAT64: return Platform.getDouble(targetObject, fieldOffset); + default: + throw new RuntimeException("Unknown primitive type: " + typeId); + } + } + + private Object copyNotPrimitiveField(Object targetObject, long fieldOffset, int typeId) { + switch (typeId) { case DispatchId.BOOL: case DispatchId.INT8: case DispatchId.UINT8: diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java index edfcd285d1..73b3f598bc 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java @@ -26,6 +26,7 @@ import java.util.Collection; import java.util.List; import org.apache.fory.Fory; +import org.apache.fory.meta.TypeExtMeta; import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.reflect.TypeRef; import org.apache.fory.resolver.ClassInfo; @@ -167,16 +168,20 @@ public static final class SerializationFieldInfo { } else { this.fieldAccessor = null; } - // Use dispatchId to determine isPrimitive for consistency with how data was written. - // This ensures correct handling in schema compatible mode where local field type - // may differ from remote (ClassDef) field type. - // Note: NOTNULL_BOXED dispatch IDs are NOT primitive - they are handled by - // readBasicObjectFieldValue - isPrimitiveField = DispatchId.isPrimitive(dispatchId); + // Use local field type to determine if field is primitive. + // This determines how to write the value to the object (Platform.putInt vs putObject). + isPrimitiveField = typeRef.getRawType().isPrimitive(); fieldConverter = d.getFieldConverter(); - nullable = d.isNullable(); - // descriptor.isTrackingRef() already includes the needToWriteRef check - trackingRef = d.isTrackingRef(); + // For xlang compatibility, check TypeExtMeta first (from remote peer's type meta) + // This ensures we read data correctly when remote's nullable differs from local + TypeExtMeta extMeta = typeRef.getTypeExtMeta(); + if (extMeta != null) { + nullable = extMeta.nullable(); + trackingRef = extMeta.trackingRef(); + } else { + nullable = d.isNullable(); + trackingRef = d.isTrackingRef(); + } refMode = RefMode.of(trackingRef, nullable); GenericType t = resolver.buildGenericType(typeRef); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldSkipper.java b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldSkipper.java index 5776c0418e..e353274dd0 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldSkipper.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldSkipper.java @@ -21,6 +21,7 @@ import org.apache.fory.Fory; import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.resolver.RefMode; import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo; import org.apache.fory.type.DispatchId; @@ -32,8 +33,8 @@ public class FieldSkipper { /** - * Skip a field value in the buffer. Handles all dispatch IDs including primitive types, - * non-null boxed types, nullable boxed types, and compressed number encodings. + * Skip a field value in the buffer. Handles all dispatch IDs including basic types and complex + * types. Whether to read a null flag is determined by fieldInfo.refMode. * * @param binding the serialization binding for fallback reads * @param fieldInfo the field metadata @@ -41,194 +42,89 @@ public class FieldSkipper { */ static void skipField( SerializationBinding binding, SerializationFieldInfo fieldInfo, MemoryBuffer buffer) { - if (!skipFieldValue(fieldInfo, buffer)) { + if (!skipFieldValue(binding, fieldInfo, buffer)) { // Fall back to binding.read for complex types (objects, collections, etc.) binding.readField(fieldInfo, buffer); } } /** - * Skip a field value based on its dispatch ID. Handles primitive types, non-null boxed types, - * and nullable boxed types with their specific encodings (including compressed numbers). + * Skip a field value based on its dispatch ID and refMode. The refMode determines whether a null + * flag exists in the stream. * * @return true if the field was skipped, false if it needs fallback handling */ - private static boolean skipFieldValue(SerializationFieldInfo fieldInfo, MemoryBuffer buffer) { - switch (fieldInfo.dispatchId) { - // ============ Primitive types (no null flag) ============ - case DispatchId.PRIMITIVE_BOOL: - buffer.increaseReaderIndex(1); - return true; - case DispatchId.PRIMITIVE_INT8: - case DispatchId.PRIMITIVE_UINT8: - buffer.increaseReaderIndex(1); - return true; - case DispatchId.PRIMITIVE_CHAR: - buffer.increaseReaderIndex(2); - return true; - case DispatchId.PRIMITIVE_INT16: - case DispatchId.PRIMITIVE_UINT16: - buffer.increaseReaderIndex(2); - return true; - case DispatchId.PRIMITIVE_INT32: - case DispatchId.PRIMITIVE_UINT32: - buffer.increaseReaderIndex(4); - return true; - case DispatchId.PRIMITIVE_VARINT32: - buffer.readVarInt32(); - return true; - case DispatchId.PRIMITIVE_VAR_UINT32: - buffer.readVarUint32(); - return true; - case DispatchId.PRIMITIVE_INT64: - case DispatchId.PRIMITIVE_UINT64: - buffer.increaseReaderIndex(8); - return true; - case DispatchId.PRIMITIVE_VARINT64: - buffer.readVarInt64(); - return true; - case DispatchId.PRIMITIVE_TAGGED_INT64: - buffer.readTaggedInt64(); - return true; - case DispatchId.PRIMITIVE_VAR_UINT64: - buffer.readVarUint64(); - return true; - case DispatchId.PRIMITIVE_TAGGED_UINT64: - buffer.readTaggedUint64(); - return true; - case DispatchId.PRIMITIVE_FLOAT32: - buffer.increaseReaderIndex(4); - return true; - case DispatchId.PRIMITIVE_FLOAT64: - buffer.increaseReaderIndex(8); - return true; + private static boolean skipFieldValue( + SerializationBinding binding, SerializationFieldInfo fieldInfo, MemoryBuffer buffer) { + int dispatchId = fieldInfo.dispatchId; + RefMode refMode = fieldInfo.refMode; - // ============ Non-null boxed types (no null flag) ============ - case DispatchId.NOTNULL_BOXED_BOOL: - buffer.increaseReaderIndex(1); - return true; - case DispatchId.NOTNULL_BOXED_INT8: - case DispatchId.NOTNULL_BOXED_UINT8: - buffer.increaseReaderIndex(1); - return true; - case DispatchId.NOTNULL_BOXED_CHAR: - buffer.increaseReaderIndex(2); - return true; - case DispatchId.NOTNULL_BOXED_INT16: - case DispatchId.NOTNULL_BOXED_UINT16: - buffer.increaseReaderIndex(2); - return true; - case DispatchId.NOTNULL_BOXED_INT32: - case DispatchId.NOTNULL_BOXED_UINT32: - buffer.increaseReaderIndex(4); - return true; - case DispatchId.NOTNULL_BOXED_VARINT32: - buffer.readVarInt32(); - return true; - case DispatchId.NOTNULL_BOXED_VAR_UINT32: - buffer.readVarUint32(); - return true; - case DispatchId.NOTNULL_BOXED_INT64: - case DispatchId.NOTNULL_BOXED_UINT64: - buffer.increaseReaderIndex(8); - return true; - case DispatchId.NOTNULL_BOXED_VARINT64: - buffer.readVarInt64(); - return true; - case DispatchId.NOTNULL_BOXED_TAGGED_INT64: - buffer.readTaggedInt64(); - return true; - case DispatchId.NOTNULL_BOXED_VAR_UINT64: - buffer.readVarUint64(); - return true; - case DispatchId.NOTNULL_BOXED_TAGGED_UINT64: - buffer.readTaggedUint64(); - return true; - case DispatchId.NOTNULL_BOXED_FLOAT32: - buffer.increaseReaderIndex(4); - return true; - case DispatchId.NOTNULL_BOXED_FLOAT64: - buffer.increaseReaderIndex(8); - return true; + // For non-basic types, let the binding handle it + if (!DispatchId.isBasicType(dispatchId)) { + return false; + } + + // If refMode is not NONE, we need to check for null flag first + if (refMode != RefMode.NONE) { + if (buffer.readByte() == Fory.NULL_FLAG) { + return true; // Field is null, nothing more to skip + } + } - // ============ Nullable boxed types (with null flag) ============ + // Now skip the actual value bytes based on dispatch ID + switch (dispatchId) { case DispatchId.BOOL: - if (buffer.readByte() != Fory.NULL_FLAG) { - buffer.increaseReaderIndex(1); - } + buffer.increaseReaderIndex(1); return true; case DispatchId.INT8: case DispatchId.UINT8: - if (buffer.readByte() != Fory.NULL_FLAG) { - buffer.increaseReaderIndex(1); - } + buffer.increaseReaderIndex(1); return true; case DispatchId.CHAR: - if (buffer.readByte() != Fory.NULL_FLAG) { - buffer.increaseReaderIndex(2); - } + buffer.increaseReaderIndex(2); return true; case DispatchId.INT16: case DispatchId.UINT16: - if (buffer.readByte() != Fory.NULL_FLAG) { - buffer.increaseReaderIndex(2); - } + buffer.increaseReaderIndex(2); return true; case DispatchId.INT32: case DispatchId.UINT32: - if (buffer.readByte() != Fory.NULL_FLAG) { - buffer.increaseReaderIndex(4); - } + buffer.increaseReaderIndex(4); return true; case DispatchId.VARINT32: - if (buffer.readByte() != Fory.NULL_FLAG) { - buffer.readVarInt32(); - } + buffer.readVarInt32(); return true; case DispatchId.VAR_UINT32: - if (buffer.readByte() != Fory.NULL_FLAG) { - buffer.readVarUint32(); - } + buffer.readVarUint32(); return true; case DispatchId.INT64: case DispatchId.UINT64: - if (buffer.readByte() != Fory.NULL_FLAG) { - buffer.increaseReaderIndex(8); - } + buffer.increaseReaderIndex(8); return true; case DispatchId.VARINT64: - if (buffer.readByte() != Fory.NULL_FLAG) { - buffer.readVarInt64(); - } + buffer.readVarInt64(); return true; case DispatchId.TAGGED_INT64: - if (buffer.readByte() != Fory.NULL_FLAG) { - buffer.readTaggedInt64(); - } + buffer.readTaggedInt64(); return true; case DispatchId.VAR_UINT64: - if (buffer.readByte() != Fory.NULL_FLAG) { - buffer.readVarUint64(); - } + buffer.readVarUint64(); return true; case DispatchId.TAGGED_UINT64: - if (buffer.readByte() != Fory.NULL_FLAG) { - buffer.readTaggedUint64(); - } + buffer.readTaggedUint64(); return true; case DispatchId.FLOAT32: - if (buffer.readByte() != Fory.NULL_FLAG) { - buffer.increaseReaderIndex(4); - } + buffer.increaseReaderIndex(4); return true; case DispatchId.FLOAT64: - if (buffer.readByte() != Fory.NULL_FLAG) { - buffer.increaseReaderIndex(8); - } + buffer.increaseReaderIndex(8); + return true; + case DispatchId.STRING: + // Read and discard the string - no class info in stream for STRING + binding.fory.readJavaString(buffer); return true; - default: - // Complex types (String, objects, collections, etc.) need fallback handling + // Other types need fallback handling return false; } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java index 73462868cc..3ecf8580d5 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java @@ -272,7 +272,8 @@ private void setFieldValuesFromPutFields( * @return array of field values in putFields order */ @Override - public Object[] getFieldValuesForPutFields(Object obj, ObjectIntMap fieldIndexMap, int arraySize) { + public Object[] getFieldValuesForPutFields( + Object obj, ObjectIntMap fieldIndexMap, int arraySize) { Object[] vals = new Object[arraySize]; // Get final fields getFieldValuesForPutFields(obj, fieldIndexMap, vals, buildInFields); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java index b8046c7990..6f8123e2e0 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java @@ -38,7 +38,6 @@ import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo; import org.apache.fory.type.Descriptor; import org.apache.fory.type.DescriptorGrouper; -import org.apache.fory.type.DispatchId; import org.apache.fory.type.Generics; import org.apache.fory.type.Types; import org.apache.fory.util.DefaultValueUtils; @@ -226,14 +225,7 @@ public T read(MemoryBuffer buffer) { } private void compatibleRead(MemoryBuffer buffer, SerializationFieldInfo fieldInfo, Object obj) { - Object fieldValue; - int dispatchId = fieldInfo.dispatchId; - if (DispatchId.isPrimitive(dispatchId)) { - fieldValue = AbstractObjectSerializer.readPrimitiveValue(buffer, dispatchId); - } else { - fieldValue = binding.readField(fieldInfo, buffer); - ; - } + Object fieldValue = AbstractObjectSerializer.readBuildInFieldValue(binding, fieldInfo, buffer); fieldInfo.fieldConverter.set(obj, fieldValue); } @@ -244,7 +236,8 @@ private void readFields(MemoryBuffer buffer, Object[] fields) { // read order: primitive,boxed,final,other,collection,map for (SerializationFieldInfo fieldInfo : this.buildInFields) { if (fieldInfo.fieldAccessor != null) { - fields[counter++] = AbstractObjectSerializer.readBuildInFieldValue(binding, fieldInfo, buffer); + fields[counter++] = + AbstractObjectSerializer.readBuildInFieldValue(binding, fieldInfo, buffer); } else { // Skip the field value from buffer since it doesn't exist in current class. // For records, fieldConverter can't be used since records are immutable and diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java index edbc79d86b..b4ea521061 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java @@ -32,7 +32,6 @@ import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.meta.ClassDef; -import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.MetaContext; import org.apache.fory.resolver.MetaStringResolver; import org.apache.fory.resolver.RefResolver; @@ -109,7 +108,6 @@ public void write(MemoryBuffer buffer, Object v) { ClassFieldsInfo fieldsInfo = getClassFieldsInfo(classDef); Fory fory = this.fory; RefResolver refResolver = fory.getRefResolver(); - ClassResolver classResolver = fory.getClassResolver(); if (fory.checkClassVersion()) { buffer.writeInt32(fieldsInfo.classVersionHash); } @@ -162,7 +160,8 @@ public Object read(MemoryBuffer buffer) { // read order: primitive,boxed,final,other,collection,map ClassFieldsInfo fieldsInfo = getClassFieldsInfo(classDef); for (SerializationFieldInfo fieldInfo : fieldsInfo.buildInFields) { - Object fieldValue = AbstractObjectSerializer.readBuildInFieldValue(binding, fieldInfo, buffer); + Object fieldValue = + AbstractObjectSerializer.readBuildInFieldValue(binding, fieldInfo, buffer); entries.add(new MapEntry(fieldInfo.qualifiedFieldName, fieldValue)); } Generics generics = fory.getGenerics(); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java index e90c197a7a..e12d870ffb 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java @@ -19,6 +19,8 @@ package org.apache.fory.serializer; +import static org.apache.fory.serializer.AbstractObjectSerializer.readBuildInFieldValue; + import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Collection; @@ -37,7 +39,6 @@ import org.apache.fory.serializer.struct.Fingerprint; import org.apache.fory.type.Descriptor; import org.apache.fory.type.DescriptorGrouper; -import org.apache.fory.type.DispatchId; import org.apache.fory.type.Generics; import org.apache.fory.util.MurmurHash3; import org.apache.fory.util.Utils; @@ -168,8 +169,7 @@ private void writeContainerFields( for (SerializationFieldInfo fieldInfo : containerFields) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; Object fieldValue = fieldAccessor.getObject(value); - writeContainerFieldValue( - binding, refResolver, generics, fieldInfo, buffer, fieldValue); + writeContainerFieldValue(binding, refResolver, generics, fieldInfo, buffer, fieldValue); } } @@ -203,13 +203,7 @@ public Object[] readFields(MemoryBuffer buffer) { int counter = 0; // read order: primitive,boxed,final,other,collection,map for (SerializationFieldInfo fieldInfo : this.buildInFields) { - int dispatchId = fieldInfo.dispatchId; - if (DispatchId.isPrimitive(dispatchId)) { - fieldValues[counter++] = AbstractObjectSerializer.readPrimitiveValue(buffer, dispatchId); - } else { - Object fieldValue = binding.readField(fieldInfo, buffer); - fieldValues[counter++] = fieldValue; - } + fieldValues[counter++] = readBuildInFieldValue(binding, fieldInfo, buffer); } Generics generics = fory.getGenerics(); for (SerializationFieldInfo fieldInfo : containerFields) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java index 93ac076f90..6101e560ab 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java @@ -82,11 +82,21 @@ abstract void writeNullable( abstract void writeContainerFieldValue( SerializationFieldInfo fieldInfo, MemoryBuffer buffer, Object fieldValue); - abstract void writeField(SerializationFieldInfo fieldInfo, MemoryBuffer buffer, Object fieldValue); + public final void writeField( + SerializationFieldInfo fieldInfo, MemoryBuffer buffer, Object fieldValue) { + writeField(fieldInfo, fieldInfo.refMode, buffer, fieldValue); + } + + abstract void writeField( + SerializationFieldInfo fieldInfo, RefMode refMode, MemoryBuffer buffer, Object fieldValue); abstract void write(MemoryBuffer buffer, Serializer serializer, Object value); - abstract Object readField(SerializationFieldInfo fieldInfo, MemoryBuffer buffer); + public final Object readField(SerializationFieldInfo fieldInfo, MemoryBuffer buffer) { + return readField(fieldInfo, fieldInfo.refMode, buffer); + } + + abstract Object readField(SerializationFieldInfo fieldInfo, RefMode mode, MemoryBuffer buffer); abstract Object read(MemoryBuffer buffer, Serializer serializer); @@ -257,20 +267,20 @@ public void write(MemoryBuffer buffer, Serializer serializer, Object value) { } @Override - Object readField(SerializationFieldInfo fieldInfo, MemoryBuffer buffer) { + Object readField(SerializationFieldInfo fieldInfo, RefMode refMode, MemoryBuffer buffer) { if (fieldInfo.useDeclaredTypeInfo) { - if (fieldInfo.refMode == RefMode.TRACKING) { + if (refMode == RefMode.TRACKING) { return fory.readRef(buffer, fieldInfo.classInfo); } else { - if (fieldInfo.refMode != RefMode.NULL_ONLY || buffer.readByte() != Fory.NULL_FLAG) { + if (refMode != RefMode.NULL_ONLY || buffer.readByte() != Fory.NULL_FLAG) { return fory.readNonRef(buffer, fieldInfo.classInfo); } } } else { - if (fieldInfo.refMode == RefMode.TRACKING) { + if (refMode == RefMode.TRACKING) { return fory.readRef(buffer, fieldInfo.classInfoHolder); } else { - if (fieldInfo.refMode != RefMode.NULL_ONLY || buffer.readByte() != Fory.NULL_FLAG) { + if (refMode != RefMode.NULL_ONLY || buffer.readByte() != Fory.NULL_FLAG) { return fory.readNonRef(buffer, fieldInfo.classInfoHolder); } } @@ -371,33 +381,38 @@ public void writeContainerFieldValue( } @Override - void writeField(SerializationFieldInfo fieldInfo, MemoryBuffer buffer, Object fieldValue) { + void writeField( + SerializationFieldInfo fieldInfo, RefMode refMode, MemoryBuffer buffer, Object fieldValue) { if (fieldInfo.useDeclaredTypeInfo) { Serializer serializer = fieldInfo.classInfo.getSerializer(); - if (fieldInfo.refMode == RefMode.TRACKING) { + if (refMode == RefMode.TRACKING) { if (!refResolver.writeRefOrNull(buffer, fieldValue)) { serializer.write(buffer, fieldValue); } - } else { - if (fieldInfo.refMode == RefMode.NULL_ONLY) { - if (fieldValue == null) { - buffer.writeByte(Fory.NULL_FLAG); - return; - } - serializer.write(buffer, fieldValue); + } else if (refMode == RefMode.NULL_ONLY) { + if (fieldValue == null) { + buffer.writeByte(Fory.NULL_FLAG); + return; } + buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); + serializer.write(buffer, fieldValue); + } else { + // RefMode.NONE - write value directly without null flag + serializer.write(buffer, fieldValue); } } else { - if (fieldInfo.refMode == RefMode.TRACKING) { + if (refMode == RefMode.TRACKING) { fory.writeRef(buffer, fieldValue, fieldInfo.classInfoHolder); - } else { - if (fieldInfo.refMode == RefMode.NULL_ONLY) { - if (fieldValue == null) { - buffer.writeByte(Fory.NULL_FLAG); - return; - } - fory.writeNonRef(buffer, fieldValue, fieldInfo.classInfoHolder); + } else if (refMode == RefMode.NULL_ONLY) { + if (fieldValue == null) { + buffer.writeByte(Fory.NULL_FLAG); + return; } + buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); + fory.writeNonRef(buffer, fieldValue, fieldInfo.classInfoHolder); + } else { + // RefMode.NONE - write value directly without null flag + fory.writeNonRef(buffer, fieldValue, fieldInfo.classInfoHolder); } } } @@ -412,14 +427,15 @@ static final class XlangSerializationBinding extends SerializationBinding { } @Override - void writeField(SerializationFieldInfo fieldInfo, MemoryBuffer buffer, Object fieldValue) { + void writeField( + SerializationFieldInfo fieldInfo, RefMode refMode, MemoryBuffer buffer, Object fieldValue) { if (fieldInfo.useDeclaredTypeInfo) { Serializer serializer = fieldInfo.classInfo.getSerializer(); - if (fieldInfo.refMode == RefMode.TRACKING) { + if (refMode == RefMode.TRACKING) { if (!refResolver.writeRefOrNull(buffer, fieldValue)) { serializer.xwrite(buffer, fieldValue); } - } else if (fieldInfo.refMode == RefMode.NULL_ONLY) { + } else if (refMode == RefMode.NULL_ONLY) { if (fieldValue == null) { buffer.writeByte(Fory.NULL_FLAG); return; @@ -431,9 +447,9 @@ void writeField(SerializationFieldInfo fieldInfo, MemoryBuffer buffer, Object fi serializer.xwrite(buffer, fieldValue); } } else { - if (fieldInfo.refMode == RefMode.TRACKING) { + if (refMode == RefMode.TRACKING) { fory.xwriteRef(buffer, fieldValue, fieldInfo.classInfoHolder); - } else if (fieldInfo.refMode == RefMode.NULL_ONLY) { + } else if (refMode == RefMode.NULL_ONLY) { if (fieldValue == null) { buffer.writeByte(Fory.NULL_FLAG); return; @@ -580,20 +596,20 @@ public void write(MemoryBuffer buffer, Serializer serializer, Object value) { } @Override - Object readField(SerializationFieldInfo fieldInfo, MemoryBuffer buffer) { + Object readField(SerializationFieldInfo fieldInfo, RefMode refMode, MemoryBuffer buffer) { if (fieldInfo.useDeclaredTypeInfo) { - if (fieldInfo.refMode == RefMode.TRACKING) { + if (refMode == RefMode.TRACKING) { return fory.xreadRef(buffer, fieldInfo.classInfo); } else { - if (fieldInfo.refMode != RefMode.NULL_ONLY || buffer.readByte() != Fory.NULL_FLAG) { + if (refMode != RefMode.NULL_ONLY || buffer.readByte() != Fory.NULL_FLAG) { return fory.xreadNonRef(buffer, fieldInfo.classInfo); } } } else { - if (fieldInfo.refMode == RefMode.TRACKING) { + if (refMode == RefMode.TRACKING) { return fory.xreadRef(buffer, fieldInfo.classInfoHolder); } else { - if (fieldInfo.refMode != RefMode.NULL_ONLY || buffer.readByte() != Fory.NULL_FLAG) { + if (refMode != RefMode.NULL_ONLY || buffer.readByte() != Fory.NULL_FLAG) { return fory.xreadNonRef(buffer, fieldInfo.classInfoHolder); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/type/DispatchId.java b/java/fory-core/src/main/java/org/apache/fory/type/DispatchId.java index 6955bb9f95..248e0c54e3 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/DispatchId.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/DispatchId.java @@ -20,192 +20,80 @@ package org.apache.fory.type; import org.apache.fory.Fory; -import org.apache.fory.meta.TypeExtMeta; -import org.apache.fory.reflect.TypeRef; import org.apache.fory.resolver.ClassResolver; +/** + * Dispatch IDs for basic types. These IDs identify the type for serialization dispatch. The + * nullable/primitive handling is determined by SerializationFieldInfo.nullable and + * SerializationFieldInfo.isPrimitiveField at runtime, not encoded in the dispatch ID. + */ public class DispatchId { public static final int UNKNOWN = 0; - public static final int PRIMITIVE_BOOL = 1; - public static final int PRIMITIVE_INT8 = 2; - public static final int PRIMITIVE_INT16 = 3; - public static final int PRIMITIVE_CHAR = 4; - public static final int PRIMITIVE_INT32 = 5; - public static final int PRIMITIVE_VARINT32 = 6; - public static final int PRIMITIVE_INT64 = 7; - public static final int PRIMITIVE_VARINT64 = 8; - public static final int PRIMITIVE_TAGGED_INT64 = 9; - public static final int PRIMITIVE_FLOAT32 = 10; - public static final int PRIMITIVE_FLOAT64 = 11; - public static final int PRIMITIVE_UINT8 = 12; - public static final int PRIMITIVE_UINT16 = 13; - public static final int PRIMITIVE_UINT32 = 14; - public static final int PRIMITIVE_VAR_UINT32 = 15; - public static final int PRIMITIVE_UINT64 = 16; - public static final int PRIMITIVE_VAR_UINT64 = 17; - public static final int PRIMITIVE_TAGGED_UINT64 = 18; - - // Non-nullable boxed types: read value directly (no null flag), box it, use putObject - // Used when remote TypeMeta has nullable=false but local field is boxed (e.g., Integer) - public static final int NOTNULL_BOXED_BOOL = 19; - public static final int NOTNULL_BOXED_INT8 = 20; - public static final int NOTNULL_BOXED_INT16 = 21; - public static final int NOTNULL_BOXED_CHAR = 22; - public static final int NOTNULL_BOXED_INT32 = 23; - public static final int NOTNULL_BOXED_VARINT32 = 24; - public static final int NOTNULL_BOXED_INT64 = 25; - public static final int NOTNULL_BOXED_VARINT64 = 26; - public static final int NOTNULL_BOXED_TAGGED_INT64 = 27; - public static final int NOTNULL_BOXED_FLOAT32 = 28; - public static final int NOTNULL_BOXED_FLOAT64 = 29; - public static final int NOTNULL_BOXED_UINT8 = 30; - public static final int NOTNULL_BOXED_UINT16 = 31; - public static final int NOTNULL_BOXED_UINT32 = 32; - public static final int NOTNULL_BOXED_VAR_UINT32 = 33; - public static final int NOTNULL_BOXED_UINT64 = 34; - public static final int NOTNULL_BOXED_VAR_UINT64 = 35; - public static final int NOTNULL_BOXED_TAGGED_UINT64 = 36; - - // Nullable boxed types: read null flag first, then box if not null - public static final int BOOL = 37; - public static final int INT8 = 38; - public static final int CHAR = 39; - public static final int INT16 = 40; - public static final int INT32 = 41; - public static final int VARINT32 = 42; - public static final int INT64 = 43; - public static final int VARINT64 = 44; - public static final int TAGGED_INT64 = 45; - public static final int FLOAT32 = 46; - public static final int FLOAT64 = 47; - public static final int UINT8 = 48; - public static final int UINT16 = 49; - public static final int UINT32 = 50; - public static final int VAR_UINT32 = 51; - public static final int UINT64 = 52; - public static final int VAR_UINT64 = 53; - public static final int TAGGED_UINT64 = 54; - public static final int STRING = 55; - - // Dispatch mode for determining how to read/write a field - private static final int MODE_PRIMITIVE = 0; // Local field is primitive, use Platform.putInt - private static final int MODE_NOTNULL_BOXED = - 1; // Local is boxed, remote nullable=false, box and putObject - private static final int MODE_NULLABLE_BOXED = - 2; // Local is boxed, remote nullable=true, read null flag + public static final int BOOL = 1; + public static final int INT8 = 2; + public static final int CHAR = 3; + public static final int INT16 = 4; + public static final int INT32 = 5; + public static final int VARINT32 = 6; + public static final int INT64 = 7; + public static final int VARINT64 = 8; + public static final int TAGGED_INT64 = 9; + public static final int FLOAT32 = 10; + public static final int FLOAT64 = 11; + public static final int UINT8 = 12; + public static final int UINT16 = 13; + public static final int UINT32 = 14; + public static final int VAR_UINT32 = 15; + public static final int UINT64 = 16; + public static final int VAR_UINT64 = 17; + public static final int TAGGED_UINT64 = 18; + public static final int STRING = 19; public static int getDispatchId(Fory fory, Descriptor d) { int typeId = Types.getDescriptorTypeId(fory, d); - TypeRef typeRef = d.getTypeRef(); - Class rawType = typeRef.getRawType(); - TypeExtMeta typeExtMeta = typeRef.getTypeExtMeta(); - - // Determine the dispatch mode based on local field type and remote nullable flag - int mode; - boolean localIsPrimitive = rawType.isPrimitive(); - - // Determine remote nullable: if typeExtMeta exists, use remote info; otherwise use local type - // For consistent schema (no typeExtMeta), primitive fields never have null flag - boolean remoteNullable; - if (typeExtMeta != null) { - remoteNullable = typeExtMeta.nullable(); - } else { - // No remote info (consistent schema) - use local field type to determine nullable - // Primitive types are never nullable, boxed/reference types are nullable - remoteNullable = !localIsPrimitive; - } - - if (!remoteNullable) { - // Remote wrote without null flag (or no remote info and local is primitive) - if (localIsPrimitive) { - mode = MODE_PRIMITIVE; - } else if (Types.isPrimitiveType(typeId)) { - mode = MODE_NOTNULL_BOXED; - } else { - mode = MODE_NULLABLE_BOXED; - } - } else { - // Remote wrote with null flag - MUST read null flag regardless of local field type - mode = MODE_NULLABLE_BOXED; - } - if (fory.isCrossLanguage()) { - return xlangTypeIdToDispatchId(typeId, mode); + return xlangTypeIdToDispatchId(typeId); } else { - return nativeIdToDispatchId(typeId, d, mode); + return nativeIdToDispatchId(typeId, d); } } - private static int xlangTypeIdToDispatchId(int typeId, int mode) { + private static int xlangTypeIdToDispatchId(int typeId) { switch (typeId) { case Types.BOOL: - return mode == MODE_PRIMITIVE - ? PRIMITIVE_BOOL - : (mode == MODE_NOTNULL_BOXED ? NOTNULL_BOXED_BOOL : BOOL); + return BOOL; case Types.INT8: - return mode == MODE_PRIMITIVE - ? PRIMITIVE_INT8 - : (mode == MODE_NOTNULL_BOXED ? NOTNULL_BOXED_INT8 : INT8); + return INT8; case Types.INT16: - return mode == MODE_PRIMITIVE - ? PRIMITIVE_INT16 - : (mode == MODE_NOTNULL_BOXED ? NOTNULL_BOXED_INT16 : INT16); + return INT16; case Types.INT32: - return mode == MODE_PRIMITIVE - ? PRIMITIVE_INT32 - : (mode == MODE_NOTNULL_BOXED ? NOTNULL_BOXED_INT32 : INT32); + return INT32; case Types.VARINT32: - return mode == MODE_PRIMITIVE - ? PRIMITIVE_VARINT32 - : (mode == MODE_NOTNULL_BOXED ? NOTNULL_BOXED_VARINT32 : VARINT32); + return VARINT32; case Types.INT64: - return mode == MODE_PRIMITIVE - ? PRIMITIVE_INT64 - : (mode == MODE_NOTNULL_BOXED ? NOTNULL_BOXED_INT64 : INT64); + return INT64; case Types.VARINT64: - return mode == MODE_PRIMITIVE - ? PRIMITIVE_VARINT64 - : (mode == MODE_NOTNULL_BOXED ? NOTNULL_BOXED_VARINT64 : VARINT64); + return VARINT64; case Types.TAGGED_INT64: - return mode == MODE_PRIMITIVE - ? PRIMITIVE_TAGGED_INT64 - : (mode == MODE_NOTNULL_BOXED ? NOTNULL_BOXED_TAGGED_INT64 : TAGGED_INT64); + return TAGGED_INT64; case Types.UINT8: - return mode == MODE_PRIMITIVE - ? PRIMITIVE_UINT8 - : (mode == MODE_NOTNULL_BOXED ? NOTNULL_BOXED_UINT8 : UINT8); + return UINT8; case Types.UINT16: - return mode == MODE_PRIMITIVE - ? PRIMITIVE_UINT16 - : (mode == MODE_NOTNULL_BOXED ? NOTNULL_BOXED_UINT16 : UINT16); + return UINT16; case Types.UINT32: - return mode == MODE_PRIMITIVE - ? PRIMITIVE_UINT32 - : (mode == MODE_NOTNULL_BOXED ? NOTNULL_BOXED_UINT32 : UINT32); + return UINT32; case Types.VAR_UINT32: - return mode == MODE_PRIMITIVE - ? PRIMITIVE_VAR_UINT32 - : (mode == MODE_NOTNULL_BOXED ? NOTNULL_BOXED_VAR_UINT32 : VAR_UINT32); + return VAR_UINT32; case Types.UINT64: - return mode == MODE_PRIMITIVE - ? PRIMITIVE_UINT64 - : (mode == MODE_NOTNULL_BOXED ? NOTNULL_BOXED_UINT64 : UINT64); + return UINT64; case Types.VAR_UINT64: - return mode == MODE_PRIMITIVE - ? PRIMITIVE_VAR_UINT64 - : (mode == MODE_NOTNULL_BOXED ? NOTNULL_BOXED_VAR_UINT64 : VAR_UINT64); + return VAR_UINT64; case Types.TAGGED_UINT64: - return mode == MODE_PRIMITIVE - ? PRIMITIVE_TAGGED_UINT64 - : (mode == MODE_NOTNULL_BOXED ? NOTNULL_BOXED_TAGGED_UINT64 : TAGGED_UINT64); + return TAGGED_UINT64; case Types.FLOAT32: - return mode == MODE_PRIMITIVE - ? PRIMITIVE_FLOAT32 - : (mode == MODE_NOTNULL_BOXED ? NOTNULL_BOXED_FLOAT32 : FLOAT32); + return FLOAT32; case Types.FLOAT64: - return mode == MODE_PRIMITIVE - ? PRIMITIVE_FLOAT64 - : (mode == MODE_NOTNULL_BOXED ? NOTNULL_BOXED_FLOAT64 : FLOAT64); + return FLOAT64; case Types.STRING: return STRING; default: @@ -213,17 +101,12 @@ private static int xlangTypeIdToDispatchId(int typeId, int mode) { } } - private static int nativeIdToDispatchId(int nativeId, Descriptor descriptor, int mode) { + private static int nativeIdToDispatchId(int nativeId, Descriptor descriptor) { if (nativeId >= Types.BOOL && nativeId <= ClassResolver.NATIVE_START_ID) { - return xlangTypeIdToDispatchId(nativeId, mode); - } - if (nativeId == ClassResolver.CHAR_ID) { - return mode == MODE_PRIMITIVE - ? PRIMITIVE_CHAR - : (mode == MODE_NOTNULL_BOXED ? NOTNULL_BOXED_CHAR : CHAR); + return xlangTypeIdToDispatchId(nativeId); } - if (nativeId == ClassResolver.PRIMITIVE_CHAR_ID) { - return PRIMITIVE_CHAR; + if (nativeId == ClassResolver.CHAR_ID || nativeId == ClassResolver.PRIMITIVE_CHAR_ID) { + return CHAR; } if (nativeId >= ClassResolver.PRIMITIVE_VOID_ID && nativeId <= ClassResolver.PRIMITIVE_FLOAT64_ID) { @@ -232,18 +115,11 @@ private static int nativeIdToDispatchId(int nativeId, Descriptor descriptor, int "%s should use `Types.BOOL~Types.FLOAT64` with nullable meta instead, but got %s", descriptor.getField(), nativeId)); } - return xlangTypeIdToDispatchId(nativeId, mode); - } - - public static boolean isPrimitive(int dispatchId) { - return dispatchId >= PRIMITIVE_BOOL && dispatchId <= PRIMITIVE_TAGGED_UINT64; - } - - public static boolean isNotnullBoxed(int dispatchId) { - return dispatchId >= NOTNULL_BOXED_BOOL && dispatchId <= NOTNULL_BOXED_TAGGED_UINT64; + return xlangTypeIdToDispatchId(nativeId); } - public static boolean isNullableBoxed(int dispatchId) { - return dispatchId >= BOOL && dispatchId <= TAGGED_UINT64; + /** Check if the dispatch ID represents a basic type that can be inlined. */ + public static boolean isBasicType(int dispatchId) { + return dispatchId >= BOOL && dispatchId <= STRING; } } diff --git a/java/fory-core/src/test/java/org/apache/fory/builder/ObjectCodecBuilderTest.java b/java/fory-core/src/test/java/org/apache/fory/builder/ObjectCodecBuilderTest.java index c3d6d7f86f..244806a1f7 100644 --- a/java/fory-core/src/test/java/org/apache/fory/builder/ObjectCodecBuilderTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/builder/ObjectCodecBuilderTest.java @@ -108,10 +108,7 @@ public static Object[][] codecConfig() { } @Test(dataProvider = "codecConfig") - public void testSeqCodec( - boolean referenceTracking, - boolean compressNumber, - int fieldsRepeat) { + public void testSeqCodec(boolean referenceTracking, boolean compressNumber, int fieldsRepeat) { Class structClass = Struct.createStructClass("Struct" + fieldsRepeat, fieldsRepeat); Fory fory = Fory.builder() diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java b/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java index 2a6f92d1d6..b2ce5d87ec 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java @@ -1386,7 +1386,7 @@ static class EmptyStruct {} @Data static class OneStringFieldStruct { - @ForyField(id = -1, nullable = true) + @ForyField(nullable = true) String f1; } @@ -1435,6 +1435,8 @@ public void testOneStringFieldCompatible(boolean enableCodegen) throws java.io.I OneStringFieldStruct obj = new OneStringFieldStruct(); obj.f1 = "hello"; + serDeCheck(fory, obj); + MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(64); fory.serialize(buffer, obj); @@ -1461,6 +1463,8 @@ public void testTwoStringFieldCompatible(boolean enableCodegen) throws java.io.I obj.f1 = "first"; obj.f2 = "second"; + serDeCheck(fory, obj); + MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(64); fory.serialize(buffer, obj); From daf1cbe18409e12d20079057174d4c3a1fd5b92e Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 17 Jan 2026 09:15:37 +0800 Subject: [PATCH 10/44] refine field skipper --- .../apache/fory/serializer/FieldSkipper.java | 73 +++++++------------ 1 file changed, 25 insertions(+), 48 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldSkipper.java b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldSkipper.java index e353274dd0..6ab7fe3189 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldSkipper.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldSkipper.java @@ -42,90 +42,67 @@ public class FieldSkipper { */ static void skipField( SerializationBinding binding, SerializationFieldInfo fieldInfo, MemoryBuffer buffer) { - if (!skipFieldValue(binding, fieldInfo, buffer)) { - // Fall back to binding.read for complex types (objects, collections, etc.) - binding.readField(fieldInfo, buffer); - } - } - - /** - * Skip a field value based on its dispatch ID and refMode. The refMode determines whether a null - * flag exists in the stream. - * - * @return true if the field was skipped, false if it needs fallback handling - */ - private static boolean skipFieldValue( - SerializationBinding binding, SerializationFieldInfo fieldInfo, MemoryBuffer buffer) { int dispatchId = fieldInfo.dispatchId; RefMode refMode = fieldInfo.refMode; - // For non-basic types, let the binding handle it + // For non-basic types, fall back to binding.readField if (!DispatchId.isBasicType(dispatchId)) { - return false; + binding.readField(fieldInfo, buffer); + return; } - // If refMode is not NONE, we need to check for null flag first + // For nullable basic types, check null flag first if (refMode != RefMode.NONE) { if (buffer.readByte() == Fory.NULL_FLAG) { - return true; // Field is null, nothing more to skip + return; // Field is null, nothing more to skip } } - // Now skip the actual value bytes based on dispatch ID + // Skip the actual value bytes based on dispatch ID switch (dispatchId) { case DispatchId.BOOL: - buffer.increaseReaderIndex(1); - return true; case DispatchId.INT8: case DispatchId.UINT8: buffer.increaseReaderIndex(1); - return true; + break; case DispatchId.CHAR: - buffer.increaseReaderIndex(2); - return true; case DispatchId.INT16: case DispatchId.UINT16: buffer.increaseReaderIndex(2); - return true; + break; case DispatchId.INT32: case DispatchId.UINT32: + case DispatchId.FLOAT32: buffer.increaseReaderIndex(4); - return true; + break; + case DispatchId.INT64: + case DispatchId.UINT64: + case DispatchId.FLOAT64: + buffer.increaseReaderIndex(8); + break; case DispatchId.VARINT32: buffer.readVarInt32(); - return true; + break; case DispatchId.VAR_UINT32: buffer.readVarUint32(); - return true; - case DispatchId.INT64: - case DispatchId.UINT64: - buffer.increaseReaderIndex(8); - return true; + break; case DispatchId.VARINT64: buffer.readVarInt64(); - return true; - case DispatchId.TAGGED_INT64: - buffer.readTaggedInt64(); - return true; + break; case DispatchId.VAR_UINT64: buffer.readVarUint64(); - return true; + break; + case DispatchId.TAGGED_INT64: + buffer.readTaggedInt64(); + break; case DispatchId.TAGGED_UINT64: buffer.readTaggedUint64(); - return true; - case DispatchId.FLOAT32: - buffer.increaseReaderIndex(4); - return true; - case DispatchId.FLOAT64: - buffer.increaseReaderIndex(8); - return true; + break; case DispatchId.STRING: - // Read and discard the string - no class info in stream for STRING binding.fory.readJavaString(buffer); - return true; + break; default: - // Other types need fallback handling - return false; + throw new IllegalStateException("Unexpected basic dispatchId: " + dispatchId); } } } From a2180c428bc5ad3171bebfc6de590078db7e5e31 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 17 Jan 2026 09:17:29 +0800 Subject: [PATCH 11/44] lint code --- .../java/org/apache/fory/meta/FieldInfo.java | 3 +- .../serializer/AbstractObjectSerializer.java | 64 +++++++++---------- 2 files changed, 33 insertions(+), 34 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java b/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java index ba0f3f82b5..23256d110b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java @@ -135,8 +135,7 @@ Descriptor toDescriptor(TypeResolver resolver, Descriptor descriptor) { Class declaredPrimitive = declared.unwrap().getRawType(); Class remotePrimitive = typeRef.unwrap().getRawType(); boolean bothPrimitives = declaredPrimitive.isPrimitive() && remotePrimitive.isPrimitive(); - boolean samePrimitiveType = - bothPrimitives && declaredPrimitive.equals(remotePrimitive); + boolean samePrimitiveType = bothPrimitives && declaredPrimitive.equals(remotePrimitive); // Set field to null if types are incompatible (not the same primitive type) if (!samePrimitiveType) { builder.field(null); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java index 24079aece4..3f726d47a4 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java @@ -442,6 +442,38 @@ static Object readBuildInFieldValue( return binding.readField(fieldInfo, buffer); } + /** + * Handle all numeric fields read include unsigned and compressed numbers. It also include + * fastpath for common type such as String. + */ + static void readBuildInFieldValue( + SerializationBinding binding, + SerializationFieldInfo fieldInfo, + MemoryBuffer buffer, + Object targetObject) { + FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; + int dispatchId = fieldInfo.dispatchId; + if (fieldInfo.refMode == RefMode.NONE) { + if (fieldInfo.isPrimitiveField) { + readPrimitiveFieldValue(buffer, targetObject, fieldAccessor, dispatchId); + } else { + readNotPrimitiveFieldValue(binding, buffer, targetObject, fieldInfo, dispatchId); + } + } else if (fieldInfo.refMode == RefMode.NULL_ONLY) { + if (buffer.readByte() == Fory.NULL_FLAG) { + return; + } + if (fieldInfo.isPrimitiveField) { + readPrimitiveFieldValue(buffer, targetObject, fieldAccessor, dispatchId); + } else { + readNotPrimitiveFieldValue(binding, buffer, targetObject, fieldInfo, dispatchId); + } + } else { + Object fieldValue = binding.readField(fieldInfo, buffer); + fieldAccessor.putObject(targetObject, fieldValue); + } + } + /** * Read a non-nullable basic object value from buffer and return it. Handles PRIMITIVE_*, and * STRING dispatch IDs with optimized fast paths. @@ -491,38 +523,6 @@ private static Object readNotNullBuildInFieldValue( } } - /** - * Handle all numeric fields read include unsigned and compressed numbers. It also include - * fastpath for common type such as String. - */ - static void readBuildInFieldValue( - SerializationBinding binding, - SerializationFieldInfo fieldInfo, - MemoryBuffer buffer, - Object targetObject) { - FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; - int dispatchId = fieldInfo.dispatchId; - if (fieldInfo.refMode == RefMode.NONE) { - if (fieldInfo.isPrimitiveField) { - readPrimitiveFieldValue(buffer, targetObject, fieldAccessor, dispatchId); - } else { - readNotPrimitiveFieldValue(binding, buffer, targetObject, fieldInfo, dispatchId); - } - } else if (fieldInfo.refMode == RefMode.NULL_ONLY) { - if (buffer.readByte() == Fory.NULL_FLAG) { - return; - } - if (fieldInfo.isPrimitiveField) { - readPrimitiveFieldValue(buffer, targetObject, fieldAccessor, dispatchId); - } else { - readNotPrimitiveFieldValue(binding, buffer, targetObject, fieldInfo, dispatchId); - } - } else { - Object fieldValue = binding.readField(fieldInfo, buffer); - fieldAccessor.putObject(targetObject, fieldValue); - } - } - /** * Read a primitive value from buffer and set it to field referenced by fieldAccessor * of targetObject. From f5e2a95ff30dd8cee577d4c75ea4f12e34bf0959 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 17 Jan 2026 09:24:37 +0800 Subject: [PATCH 12/44] fix object stream serializer --- .../apache/fory/serializer/MetaSharedLayerSerializer.java | 3 +-- .../fory/serializer/MetaSharedLayerSerializerBase.java | 1 - .../org/apache/fory/serializer/ObjectStreamSerializer.java | 7 +++---- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java index 3ecf8580d5..b77bbd0af5 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java @@ -54,7 +54,6 @@ public class MetaSharedLayerSerializer extends MetaSharedLayerSerializerBase< private final SerializationFieldInfo[] otherFields; private final SerializationFieldInfo[] containerFields; private final SerializationBinding binding; - private final TypeResolver typeResolver; /** * Creates a new MetaSharedLayerSerializer. @@ -69,7 +68,7 @@ public MetaSharedLayerSerializer( super(fory, type); this.layerClassDef = layerClassDef; this.layerMarkerClass = layerMarkerClass; - this.typeResolver = fory._getTypeResolver(); + TypeResolver typeResolver = fory._getTypeResolver(); this.binding = SerializationBinding.createBinding(fory); // Build field infos from layerClassDef diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializerBase.java b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializerBase.java index 5568307fea..221c929263 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializerBase.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializerBase.java @@ -30,7 +30,6 @@ * @see MetaSharedLayerSerializer * @see org.apache.fory.builder.MetaSharedLayerCodecBuilder */ -@SuppressWarnings("unchecked") public abstract class MetaSharedLayerSerializerBase extends AbstractObjectSerializer { public MetaSharedLayerSerializerBase(Fory fory, Class type) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java index 9888d3656a..e896555c24 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java @@ -422,9 +422,7 @@ public SlotsInfo(Fory fory, Class type) { Class layerMarkerClass = LayerMarkerClassGenerator.getOrCreate(fory, type, 0); // Create interpreter-mode serializer first - MetaSharedLayerSerializer interpreterSerializer = - new MetaSharedLayerSerializer(fory, type, layerClassDef, layerMarkerClass); - this.slotsSerializer = interpreterSerializer; + this.slotsSerializer = new MetaSharedLayerSerializer(fory, type, layerClassDef, layerMarkerClass); // Register JIT callback to replace with JIT serializer when ready if (fory.getConfig().isCodeGenEnabled()) { @@ -437,7 +435,8 @@ public SlotsInfo(Fory fory, Class type) { type, fory, layerClassDef, layerMarkerClass), c -> thisInfo.slotsSerializer = - (MetaSharedLayerSerializerBase) Serializers.newSerializer(fory, type, c)); + (MetaSharedLayerSerializerBase) Serializers.newSerializer( + fory, type, (Class) c)); } // In GraalVM, ensure serializers are generated for all field types at build time From 23d09bac5a17931f3b0290e65a4fe713827b5b3f Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 17 Jan 2026 09:35:48 +0800 Subject: [PATCH 13/44] fix: add layer class meta reading to MetaSharedLayerCodecBuilder - Override buildDecodeExpression() to include readLayerClassMeta() call to maintain consistency with interpreter-mode MetaSharedLayerSerializer - Fix fallback serializer class from ObjectSerializer to MetaSharedLayerSerializer since ObjectSerializer is not compatible with MetaSharedLayerSerializerBase - Add static helper method readLayerClassMeta() for generated code to call --- .../builder/MetaSharedLayerCodecBuilder.java | 56 ++++++++++++++++++- .../serializer/ObjectStreamSerializer.java | 11 ++-- 2 files changed, 60 insertions(+), 7 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedLayerCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedLayerCodecBuilder.java index 800de0bd33..daa26efcc8 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedLayerCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedLayerCodecBuilder.java @@ -20,6 +20,7 @@ package org.apache.fory.builder; import static org.apache.fory.builder.Generated.GeneratedMetaSharedLayerSerializer.SERIALIZER_FIELD_NAME; +import static org.apache.fory.type.TypeUtils.PRIMITIVE_VOID_TYPE; import java.util.Collection; import java.util.Map; @@ -28,13 +29,15 @@ import org.apache.fory.builder.Generated.GeneratedMetaSharedLayerSerializer; import org.apache.fory.codegen.CodeGenerator; import org.apache.fory.codegen.Expression; +import org.apache.fory.codegen.Expression.ListExpression; +import org.apache.fory.codegen.Expression.Reference; +import org.apache.fory.codegen.Expression.StaticInvoke; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.meta.ClassDef; import org.apache.fory.reflect.TypeRef; import org.apache.fory.serializer.CodegenSerializer; import org.apache.fory.serializer.MetaSharedLayerSerializer; import org.apache.fory.serializer.MetaSharedLayerSerializerBase; -import org.apache.fory.serializer.ObjectSerializer; import org.apache.fory.serializer.Serializers; import org.apache.fory.type.Descriptor; import org.apache.fory.type.DescriptorGrouper; @@ -135,10 +138,12 @@ public static MetaSharedLayerSerializerBase setCodegenSerializer( return (MetaSharedLayerSerializerBase) typeResolver(fory, r -> r.getSerializer(s.getType())); } // This method hold jit lock, so create jit serializer async to avoid block serialization. + // Use MetaSharedLayerSerializer as fallback since it's compatible with + // MetaSharedLayerSerializerBase Class serializerClass = fory.getJITContext() .registerSerializerJITCallback( - () -> ObjectSerializer.class, + () -> MetaSharedLayerSerializer.class, () -> CodegenSerializer.loadCodegenSerializer(fory, s.getType()), c -> s.serializer = @@ -152,6 +157,53 @@ public Expression buildEncodeExpression() { throw new IllegalStateException("unreachable"); } + @Override + public Expression buildDecodeExpression() { + // Build the base decode expression from parent + Expression baseDecodeExpr = super.buildDecodeExpression(); + + // Prepend layer class meta reading if meta share is enabled + if (fory.getConfig().isMetaShareEnabled()) { + ListExpression expressions = new ListExpression(); + Reference buffer = new Reference(BUFFER_NAME, bufferTypeRef, false); + // Call static helper to read layer class meta + Expression readMeta = + new StaticInvoke( + MetaSharedLayerCodecBuilder.class, + "readLayerClassMeta", + PRIMITIVE_VOID_TYPE, + foryRef, + buffer); + expressions.add(readMeta); + expressions.add(baseDecodeExpr); + return expressions; + } + return baseDecodeExpr; + } + + /** + * Static helper method to read layer class meta from buffer. Called by generated code to maintain + * consistency with interpreter-mode MetaSharedLayerSerializer. + * + * @param fory the Fory instance + * @param buffer the memory buffer to read from + */ + public static void readLayerClassMeta(Fory fory, MemoryBuffer buffer) { + org.apache.fory.resolver.MetaContext metaContext = + fory.getSerializationContext().getMetaContext(); + if (metaContext == null) { + return; + } + int indexMarker = buffer.readVarUint32Small14(); + boolean isRef = (indexMarker & 1) == 1; + if (!isRef) { + // New type in stream - read ClassDef inline and skip it + long id = buffer.readInt64(); + ClassDef.skipClassDef(buffer, id); + } + // If isRef, it's a reference to previously read type - nothing to do + } + @Override protected Expression buildComponentsArray() { return buildDefaultComponentsArray(); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java index e896555c24..d65e461b32 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java @@ -422,7 +422,8 @@ public SlotsInfo(Fory fory, Class type) { Class layerMarkerClass = LayerMarkerClassGenerator.getOrCreate(fory, type, 0); // Create interpreter-mode serializer first - this.slotsSerializer = new MetaSharedLayerSerializer(fory, type, layerClassDef, layerMarkerClass); + this.slotsSerializer = + new MetaSharedLayerSerializer(fory, type, layerClassDef, layerMarkerClass); // Register JIT callback to replace with JIT serializer when ready if (fory.getConfig().isCodeGenEnabled()) { @@ -435,8 +436,8 @@ public SlotsInfo(Fory fory, Class type) { type, fory, layerClassDef, layerMarkerClass), c -> thisInfo.slotsSerializer = - (MetaSharedLayerSerializerBase) Serializers.newSerializer( - fory, type, (Class) c)); + (MetaSharedLayerSerializerBase) + Serializers.newSerializer(fory, type, (Class) c)); } // In GraalVM, ensure serializers are generated for all field types at build time @@ -718,7 +719,7 @@ public void writeFields() throws IOException { private void writePutFieldValue(MemoryBuffer buffer, Class fieldType, Object value) { if (fieldType == boolean.class) { - buffer.writeBoolean(value == null ? false : (Boolean) value); + buffer.writeBoolean(value != null && (Boolean) value); } else if (fieldType == byte.class) { buffer.writeByte(value == null ? (byte) 0 : (Byte) value); } else if (fieldType == char.class) { @@ -744,7 +745,7 @@ private void writePutFieldValue(MemoryBuffer buffer, Class fieldType, Object } @Override - public void defaultWriteObject() throws IOException, NotActiveException { + public void defaultWriteObject() throws IOException { if (fieldsWritten) { throw new NotActiveException("not in writeObject invocation or fields already written"); } From 4fbe39bca143f905da5148db901266313622c357 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 17 Jan 2026 09:45:23 +0800 Subject: [PATCH 14/44] fix: handle sender layers not present in receiver for ObjectStreamSerializer The read() method now properly handles schema evolution when: - Sender has class layers that receiver doesn't have (skip sender's data) - Receiver has class layers that sender doesn't have (call readObjectNoData) Previously, the code would cause ArrayIndexOutOfBoundsException or data corruption when sender had parent classes that don't exist in receiver. Changes: - Rewrite read loop to properly match sender and receiver layers - Add skipUnknownLayerData() to skip data for sender-only layers - Handle remaining receiver-only layers at the end --- .../serializer/ObjectStreamSerializer.java | 124 +++++++++++++++--- 1 file changed, 108 insertions(+), 16 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java index d65e461b32..7c2a4f7251 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java @@ -225,32 +225,56 @@ public Object read(MemoryBuffer buffer) { TreeMap callbacks = new TreeMap<>(Collections.reverseOrder()); for (int i = 0; i < numClasses; i++) { Class currentClass = classResolver.readClassInternal(buffer); - SlotInfo slotsInfo = slotsInfos[slotIndex++]; - StreamClassInfo streamClassInfo = slotsInfo.getStreamClassInfo(); - while (currentClass != slotsInfo.getCls()) { - // the receiver's version extends classes that are not extended by the sender's version. - Method readObjectNoData = streamClassInfo.readObjectNoData; - if (readObjectNoData != null) { - if (streamClassInfo.readObjectNoDataFunc != null) { - streamClassInfo.readObjectNoDataFunc.accept(obj); - } else { - readObjectNoData.invoke(obj); + + // Find the matching local slot for sender's class + SlotInfo matchedSlot = null; + while (slotIndex < slotsInfos.length) { + SlotInfo candidateSlot = slotsInfos[slotIndex]; + if (currentClass == candidateSlot.getCls()) { + // Found matching slot + matchedSlot = candidateSlot; + slotIndex++; + break; + } else if (currentClass.isAssignableFrom(candidateSlot.getCls())) { + // Sender's class is an ancestor of candidate's class but they don't match. + // This means sender has a layer (currentClass) that receiver doesn't have. + // We'll skip sender's data for this layer below. + break; + } else { + // Receiver has an extra layer that sender doesn't have - call readObjectNoData + StreamClassInfo streamClassInfo = candidateSlot.getStreamClassInfo(); + Method readObjectNoData = streamClassInfo.readObjectNoData; + if (readObjectNoData != null) { + if (streamClassInfo.readObjectNoDataFunc != null) { + streamClassInfo.readObjectNoDataFunc.accept(obj); + } else { + readObjectNoData.invoke(obj); + } } + slotIndex++; } - slotsInfo = slotsInfos[slotIndex++]; } + + if (matchedSlot == null) { + // Sender has a layer that receiver doesn't have - skip the data + skipUnknownLayerData(buffer, currentClass); + continue; + } + + // Read data for the matched layer + StreamClassInfo streamClassInfo = matchedSlot.getStreamClassInfo(); Method readObjectMethod = streamClassInfo.readObjectMethod; if (readObjectMethod == null) { - slotsInfo.getSlotsSerializer().readAndSetFields(buffer, obj); + matchedSlot.getSlotsSerializer().readAndSetFields(buffer, obj); } else { - ForyObjectInputStream objectInputStream = slotsInfo.getObjectInputStream(); + ForyObjectInputStream objectInputStream = matchedSlot.getObjectInputStream(); MemoryBuffer oldBuffer = objectInputStream.buffer; Object oldObject = objectInputStream.targetObject; ForyObjectInputStream.GetFieldImpl oldGetField = objectInputStream.getField; ForyObjectInputStream.GetFieldImpl getField = - (ForyObjectInputStream.GetFieldImpl) slotsInfo.getFieldPool().popOrNull(); + (ForyObjectInputStream.GetFieldImpl) matchedSlot.getFieldPool().popOrNull(); if (getField == null) { - getField = new ForyObjectInputStream.GetFieldImpl(slotsInfo); + getField = new ForyObjectInputStream.GetFieldImpl(matchedSlot); } boolean fieldsRead = objectInputStream.fieldsRead; try { @@ -269,12 +293,27 @@ public Object read(MemoryBuffer buffer) { objectInputStream.buffer = oldBuffer; objectInputStream.targetObject = oldObject; objectInputStream.getField = oldGetField; - slotsInfo.getFieldPool().add(getField); + matchedSlot.getFieldPool().add(getField); objectInputStream.callbacks = null; Arrays.fill(getField.vals, ForyObjectInputStream.NO_VALUE_STUB); } } } + + // Handle any remaining receiver-only layers at the end + while (slotIndex < slotsInfos.length) { + SlotInfo remainingSlot = slotsInfos[slotIndex++]; + StreamClassInfo streamClassInfo = remainingSlot.getStreamClassInfo(); + Method readObjectNoData = streamClassInfo.readObjectNoData; + if (readObjectNoData != null) { + if (streamClassInfo.readObjectNoDataFunc != null) { + streamClassInfo.readObjectNoDataFunc.accept(obj); + } else { + readObjectNoData.invoke(obj); + } + } + } + for (ObjectInputValidation validation : callbacks.values()) { validation.validateObject(); } @@ -284,6 +323,59 @@ public Object read(MemoryBuffer buffer) { return obj; } + /** + * Skip data for a layer that exists in sender but not in receiver. This is needed for schema + * evolution when sender's class hierarchy has layers that receiver doesn't have. + * + * @param buffer the memory buffer to read from + * @param senderClass the sender's class for this layer (not present in receiver) + */ + private void skipUnknownLayerData(MemoryBuffer buffer, Class senderClass) { + // For layers without custom writeObject, we can skip using a temporary serializer + // created from the ClassDef in the meta context. + // Note: For layers with custom writeObject, the sender would have that class locally, + // and we'd have a matching slot. This method is only called when sender has a layer + // the receiver doesn't have, which means the layer uses standard field serialization. + + // Read and skip the layer class meta + if (fory.getConfig().isMetaShareEnabled()) { + org.apache.fory.resolver.MetaContext metaContext = + fory.getSerializationContext().getMetaContext(); + if (metaContext != null) { + int indexMarker = buffer.readVarUint32Small14(); + boolean isRef = (indexMarker & 1) == 1; + if (!isRef) { + // New type - read ClassDef and use it to skip fields + long id = buffer.readInt64(); + ClassDef classDef = ClassDef.readClassDef(fory, buffer, id); + // Create a temporary serializer to skip the fields + MetaSharedLayerSerializer skipSerializer = + new MetaSharedLayerSerializer<>( + fory, + senderClass, + classDef, + LayerMarkerClassGenerator.getOrCreate(fory, senderClass, 0)); + // Read fields to skip them (result is discarded) + skipSerializer.read(buffer); + } else { + // Reference to previously seen ClassDef - need to look it up + int index = indexMarker >>> 1; + // For referenced types, we need to find the ClassDef and skip accordingly + // This is a simplified approach - in practice we'd need to track layer ClassDefs + throw new UnsupportedOperationException( + "Cannot skip referenced layer class meta for unknown layer: " + + senderClass.getName() + + ". Schema evolution with removed parent classes is not fully supported."); + } + } + } else { + throw new UnsupportedOperationException( + "Cannot skip unknown layer data without meta share enabled for class: " + + senderClass.getName() + + ". Schema evolution with removed parent classes requires meta share."); + } + } + private static void throwUnsupportedEncodingException(Class cls) throws UnsupportedEncodingException { throw new UnsupportedEncodingException( From b132ac0bb79a92ada42e0cab007b12f8180cf0f3 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 17 Jan 2026 09:52:58 +0800 Subject: [PATCH 15/44] feat: comprehensive schema evolution support for ObjectStreamSerializer Add full support for schema evolution when sender has class layers that receiver doesn't have. This includes: 1. LayerReadContext class to track ClassDefs during deserialization: - Maintains list of ClassDefs in stream order - Caches skip serializers for reuse - Supports both new ClassDefs and references to previously seen ones 2. readAndTrackLayerMeta() to read and track layer meta: - Handles new ClassDefs (reads inline and tracks) - Handles references (looks up from tracked list) - Returns the ClassDef for the layer 3. trackLayerMetaFromSerializer() to track ClassDefs from matched layers: - Peeks at buffer to track ClassDefs without consuming - Allows matched layers to contribute to tracking - Enables later unknown layers to reference earlier ClassDefs 4. skipUnknownLayerData() enhanced with context: - Uses tracked ClassDefs for references - Creates skip serializer from ClassDef - Properly advances buffer position 5. MetaSharedLayerSerializer.skipFields() method: - Skips all field types: buildIn, container, other - Uses FieldSkipper for basic types - Uses binding.readField() for complex types --- .../serializer/MetaSharedLayerSerializer.java | 38 +++ .../serializer/ObjectStreamSerializer.java | 224 +++++++++++++++--- 2 files changed, 223 insertions(+), 39 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java index b77bbd0af5..dbfc0a5b60 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java @@ -299,4 +299,42 @@ private void getFieldValuesForPutFields( } } } + + /** + * Skip field data in the buffer without setting it to any object. This is used for schema + * evolution when a class layer exists in the sender but not in the receiver. The layer meta + * should already be consumed before calling this method. + * + * @param buffer the memory buffer to read from + * @param binding the serialization binding for reading field values + */ + public void skipFields(MemoryBuffer buffer, SerializationBinding binding) { + // Skip all fields in order: buildIn, container, other + // We read the values but don't set them anywhere (they're discarded) + skipBuildInFields(buffer, binding); + skipContainerFields(buffer, binding); + skipOtherFields(buffer, binding); + } + + private void skipBuildInFields(MemoryBuffer buffer, SerializationBinding binding) { + for (SerializationFieldInfo fieldInfo : buildInFields) { + // Read the field value (discarding the result) to advance buffer position + FieldSkipper.skipField(binding, fieldInfo, buffer); + } + } + + private void skipContainerFields(MemoryBuffer buffer, SerializationBinding binding) { + Generics generics = fory.getGenerics(); + for (SerializationFieldInfo fieldInfo : containerFields) { + // Read container field value to advance buffer position + AbstractObjectSerializer.readContainerFieldValue(binding, generics, fieldInfo, buffer); + } + } + + private void skipOtherFields(MemoryBuffer buffer, SerializationBinding binding) { + for (SerializationFieldInfo fieldInfo : otherFields) { + // Read field value to advance buffer position + binding.readField(fieldInfo, buffer); + } + } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java index 7c2a4f7251..0ae076ce45 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java @@ -39,7 +39,9 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.TreeMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiConsumer; @@ -221,6 +223,10 @@ public Object read(MemoryBuffer buffer) { fory.getRefResolver().reference(obj); int numClasses = buffer.readInt16(); int slotIndex = 0; + + // Create context to track layer ClassDefs for schema evolution + LayerReadContext layerContext = new LayerReadContext(); + try { TreeMap callbacks = new TreeMap<>(Collections.reverseOrder()); for (int i = 0; i < numClasses; i++) { @@ -257,7 +263,9 @@ public Object read(MemoryBuffer buffer) { if (matchedSlot == null) { // Sender has a layer that receiver doesn't have - skip the data - skipUnknownLayerData(buffer, currentClass); + // First read and track the layer meta, then skip the field data + ClassDef classDef = readAndTrackLayerMeta(buffer, layerContext); + skipUnknownLayerData(buffer, currentClass, layerContext, classDef); continue; } @@ -265,8 +273,12 @@ public Object read(MemoryBuffer buffer) { StreamClassInfo streamClassInfo = matchedSlot.getStreamClassInfo(); Method readObjectMethod = streamClassInfo.readObjectMethod; if (readObjectMethod == null) { + // For standard field serialization, the serializer handles layer meta internally + // We also track the ClassDef for potential future references + trackLayerMetaFromSerializer(buffer, layerContext, matchedSlot.getSlotsSerializer()); matchedSlot.getSlotsSerializer().readAndSetFields(buffer, obj); } else { + // For custom readObject, it handles its own format ForyObjectInputStream objectInputStream = matchedSlot.getObjectInputStream(); MemoryBuffer oldBuffer = objectInputStream.buffer; Object oldObject = objectInputStream.targetObject; @@ -323,57 +335,191 @@ public Object read(MemoryBuffer buffer) { return obj; } + /** + * Track layer meta from a matched serializer. This allows us to track ClassDefs from matched + * layers so they can be referenced by later unknown layers. + * + *

    Note: This is called BEFORE the serializer reads the meta, so we peek at it and track but + * don't consume the bytes. The serializer will read the same bytes again. + */ + private void trackLayerMetaFromSerializer( + MemoryBuffer buffer, LayerReadContext context, MetaSharedLayerSerializerBase serializer) { + if (!fory.getConfig().isMetaShareEnabled()) { + return; + } + org.apache.fory.resolver.MetaContext metaContext = + fory.getSerializationContext().getMetaContext(); + if (metaContext == null) { + return; + } + + // Peek at the buffer to see if this is a new ClassDef or a reference + int readerIndex = buffer.readerIndex(); + int indexMarker = buffer.readVarUint32Small14(); + boolean isRef = (indexMarker & 1) == 1; + int index = indexMarker >>> 1; + + if (isRef) { + // Reference - already tracked, nothing to do + // Reset buffer position so serializer can read it + buffer.readerIndex(readerIndex); + } else { + // New type - read the ClassDef and track it + long id = buffer.readInt64(); + ClassDef classDef = ClassDef.readClassDef(fory, buffer, id); + int storedIndex = context.addClassDef(classDef); + if (storedIndex != index) { + throw new IllegalStateException( + "Layer ClassDef index mismatch: expected " + index + ", got " + storedIndex); + } + // Reset buffer position so serializer can read and skip it + buffer.readerIndex(readerIndex); + } + } + + /** + * Context for tracking layer ClassDefs during deserialization. This is needed for schema + * evolution when sender has layers that receiver doesn't have, and we need to skip data for + * referenced ClassDefs. + */ + private static class LayerReadContext { + /** List of ClassDefs encountered in stream order, indexed by their stream index. */ + private final List layerClassDefs = new ArrayList<>(); + + /** + * Cache of skip serializers for each ClassDef, to avoid recreating them for referenced types. + */ + private final Map> skipSerializerCache = new HashMap<>(); + + /** Add a ClassDef to the tracking list and return its index. */ + int addClassDef(ClassDef classDef) { + int index = layerClassDefs.size(); + layerClassDefs.add(classDef); + return index; + } + + /** Get a ClassDef by its stream index. */ + ClassDef getClassDef(int index) { + if (index < 0 || index >= layerClassDefs.size()) { + throw new IllegalStateException( + "Invalid layer ClassDef index: " + index + ", tracked count: " + layerClassDefs.size()); + } + return layerClassDefs.get(index); + } + + /** Get or create a skip serializer for a ClassDef at the given index. */ + MetaSharedLayerSerializer getOrCreateSkipSerializer( + Fory fory, int index, Class senderClass) { + return skipSerializerCache.computeIfAbsent( + index, + i -> { + ClassDef classDef = getClassDef(i); + return new MetaSharedLayerSerializer<>( + fory, + senderClass, + classDef, + LayerMarkerClassGenerator.getOrCreate(fory, senderClass, 0)); + }); + } + } + + /** + * Read layer class meta from buffer and track in context. This method handles both new ClassDefs + * (read and store) and references (lookup from tracked). + * + * @param buffer the memory buffer to read from + * @param context the layer read context for tracking ClassDefs + * @return the ClassDef for this layer (either newly read or looked up from reference) + */ + private ClassDef readAndTrackLayerMeta(MemoryBuffer buffer, LayerReadContext context) { + if (!fory.getConfig().isMetaShareEnabled()) { + return null; + } + org.apache.fory.resolver.MetaContext metaContext = + fory.getSerializationContext().getMetaContext(); + if (metaContext == null) { + return null; + } + + int indexMarker = buffer.readVarUint32Small14(); + boolean isRef = (indexMarker & 1) == 1; + int index = indexMarker >>> 1; + + if (isRef) { + // Reference to previously seen ClassDef - look it up + return context.getClassDef(index); + } else { + // New type in stream - read ClassDef inline and track it + long id = buffer.readInt64(); + ClassDef classDef = ClassDef.readClassDef(fory, buffer, id); + int storedIndex = context.addClassDef(classDef); + if (storedIndex != index) { + throw new IllegalStateException( + "Layer ClassDef index mismatch: expected " + index + ", got " + storedIndex); + } + return classDef; + } + } + /** * Skip data for a layer that exists in sender but not in receiver. This is needed for schema * evolution when sender's class hierarchy has layers that receiver doesn't have. * * @param buffer the memory buffer to read from * @param senderClass the sender's class for this layer (not present in receiver) + * @param context the layer read context for tracking ClassDefs + * @param classDef the ClassDef for this layer (already read from stream) */ - private void skipUnknownLayerData(MemoryBuffer buffer, Class senderClass) { + private void skipUnknownLayerData( + MemoryBuffer buffer, Class senderClass, LayerReadContext context, ClassDef classDef) { // For layers without custom writeObject, we can skip using a temporary serializer - // created from the ClassDef in the meta context. - // Note: For layers with custom writeObject, the sender would have that class locally, - // and we'd have a matching slot. This method is only called when sender has a layer - // the receiver doesn't have, which means the layer uses standard field serialization. - - // Read and skip the layer class meta - if (fory.getConfig().isMetaShareEnabled()) { - org.apache.fory.resolver.MetaContext metaContext = - fory.getSerializationContext().getMetaContext(); - if (metaContext != null) { - int indexMarker = buffer.readVarUint32Small14(); - boolean isRef = (indexMarker & 1) == 1; - if (!isRef) { - // New type - read ClassDef and use it to skip fields - long id = buffer.readInt64(); - ClassDef classDef = ClassDef.readClassDef(fory, buffer, id); - // Create a temporary serializer to skip the fields - MetaSharedLayerSerializer skipSerializer = - new MetaSharedLayerSerializer<>( - fory, - senderClass, - classDef, - LayerMarkerClassGenerator.getOrCreate(fory, senderClass, 0)); - // Read fields to skip them (result is discarded) - skipSerializer.read(buffer); - } else { - // Reference to previously seen ClassDef - need to look it up - int index = indexMarker >>> 1; - // For referenced types, we need to find the ClassDef and skip accordingly - // This is a simplified approach - in practice we'd need to track layer ClassDefs - throw new UnsupportedOperationException( - "Cannot skip referenced layer class meta for unknown layer: " - + senderClass.getName() - + ". Schema evolution with removed parent classes is not fully supported."); - } - } - } else { + // created from the ClassDef. Note: For layers with custom writeObject, the sender + // would have that class locally, and we'd have a matching slot. This method is only + // called when sender has a layer the receiver doesn't have. + + if (classDef == null) { throw new UnsupportedOperationException( "Cannot skip unknown layer data without meta share enabled for class: " + senderClass.getName() + ". Schema evolution with removed parent classes requires meta share."); } + + // Find the index of this ClassDef in the context + int classDefIndex = -1; + for (int i = 0; i < context.layerClassDefs.size(); i++) { + if (context.layerClassDefs.get(i) == classDef) { + classDefIndex = i; + break; + } + } + + if (classDefIndex < 0) { + // This shouldn't happen - we should have tracked it when reading + throw new IllegalStateException( + "ClassDef not found in context for layer: " + senderClass.getName()); + } + + // Get or create a skip serializer and use it to read (and discard) the fields + MetaSharedLayerSerializer skipSerializer = + context.getOrCreateSkipSerializer(fory, classDefIndex, senderClass); + // Note: We don't call readAndSetFields because we already read the layer meta above. + // We need to just read the field data. The serializer's readAndSetFields would try to + // read meta again, so we directly read the fields instead. + skipLayerFields(buffer, skipSerializer); + } + + /** + * Skip field data using a serializer. This reads the field data without setting it on any object. + * The layer meta should already be consumed before calling this method. + * + * @param buffer the memory buffer to read from + * @param serializer the serializer to use for reading (and discarding) fields + */ + private void skipLayerFields(MemoryBuffer buffer, MetaSharedLayerSerializer serializer) { + // Use SerializationBinding to read fields - the serializer has the field layout + // from the ClassDef, and skipFields will read each field to advance buffer position + SerializationBinding binding = SerializationBinding.createBinding(fory); + serializer.skipFields(buffer, binding); } private static void throwUnsupportedEncodingException(Class cls) From ee22c9e8ce5336acdf5e081d3814414c3478bb6d Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 17 Jan 2026 10:07:01 +0800 Subject: [PATCH 16/44] refactor: read ClassDef in ObjectStreamSerializer before calling serializer The key insight is that different senders may have different ClassDefs for the same class layer (schema evolution). The serializer must be created based on the actual ClassDef from the stream, not the receiver's expected schema. Changes: 1. ObjectStreamSerializer now reads layer ClassDef before calling serializer 2. Creates/caches serializers based on the actual ClassDef from stream 3. Calls readFieldsOnly() instead of readAndSetFields() to avoid double-read 4. MetaSharedLayerSerializer.readFieldsOnly() reads fields without meta 5. Removed trackLayerMetaFromSerializer() - no longer needed 6. Simplified LayerReadContext to use single getOrCreateSerializer() --- .../serializer/MetaSharedLayerSerializer.java | 23 ++- .../serializer/ObjectStreamSerializer.java | 142 +++++++----------- 2 files changed, 70 insertions(+), 95 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java index dbfc0a5b60..0b74a16c27 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java @@ -156,12 +156,28 @@ public T readAndSetFields(MemoryBuffer buffer, T obj) { readLayerClassMeta(buffer); } // Read fields in order: final, container, other - readFinalFields(buffer, obj); - readContainerFields(buffer, obj); - readUserTypeFields(buffer, obj); + readFieldsOnly(buffer, obj); return obj; } + /** + * Read fields only, without reading layer class meta. This is used when the caller has already + * read and processed the layer class meta (e.g., for schema evolution where different senders may + * have different ClassDefs for the same layer). + * + * @param buffer the memory buffer to read from + * @param obj the object to set field values on + * @return the object with fields set + */ + @SuppressWarnings("unchecked") + public T readFieldsOnly(MemoryBuffer buffer, Object obj) { + // Read fields in order: final, container, other + readFinalFields(buffer, (T) obj); + readContainerFields(buffer, (T) obj); + readUserTypeFields(buffer, (T) obj); + return (T) obj; + } + private void readLayerClassMeta(MemoryBuffer buffer) { MetaContext metaContext = fory.getSerializationContext().getMetaContext(); if (metaContext == null) { @@ -169,7 +185,6 @@ private void readLayerClassMeta(MemoryBuffer buffer) { } int indexMarker = buffer.readVarUint32Small14(); boolean isRef = (indexMarker & 1) == 1; - int index = indexMarker >>> 1; if (isRef) { // Reference to previously read type - already in readClassInfos, nothing to do } else { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java index 0ae076ce45..a2a849dde7 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java @@ -261,10 +261,13 @@ public Object read(MemoryBuffer buffer) { } } + // Read and track the layer ClassDef first (for both matched and unmatched) + // This is critical because different senders may have different ClassDefs for the same + // layer + ClassDef classDef = readAndTrackLayerMeta(buffer, layerContext); + if (matchedSlot == null) { // Sender has a layer that receiver doesn't have - skip the data - // First read and track the layer meta, then skip the field data - ClassDef classDef = readAndTrackLayerMeta(buffer, layerContext); skipUnknownLayerData(buffer, currentClass, layerContext, classDef); continue; } @@ -273,10 +276,19 @@ public Object read(MemoryBuffer buffer) { StreamClassInfo streamClassInfo = matchedSlot.getStreamClassInfo(); Method readObjectMethod = streamClassInfo.readObjectMethod; if (readObjectMethod == null) { - // For standard field serialization, the serializer handles layer meta internally - // We also track the ClassDef for potential future references - trackLayerMetaFromSerializer(buffer, layerContext, matchedSlot.getSlotsSerializer()); - matchedSlot.getSlotsSerializer().readAndSetFields(buffer, obj); + // For standard field serialization, create/get a serializer based on the sender's + // ClassDef + // This ensures we read fields according to the sender's schema, not the receiver's + if (classDef != null) { + int classDefIndex = layerContext.getClassDefIndex(classDef); + MetaSharedLayerSerializer layerSerializer = + layerContext.getOrCreateSerializer(fory, classDefIndex, currentClass); + layerSerializer.readFieldsOnly(buffer, obj); + } else { + // Meta share not enabled - use the slot's serializer directly + // Note: readAndSetFields won't try to read meta if meta share is disabled + matchedSlot.getSlotsSerializer().readAndSetFields(buffer, obj); + } } else { // For custom readObject, it handles its own format ForyObjectInputStream objectInputStream = matchedSlot.getObjectInputStream(); @@ -335,61 +347,20 @@ public Object read(MemoryBuffer buffer) { return obj; } - /** - * Track layer meta from a matched serializer. This allows us to track ClassDefs from matched - * layers so they can be referenced by later unknown layers. - * - *

    Note: This is called BEFORE the serializer reads the meta, so we peek at it and track but - * don't consume the bytes. The serializer will read the same bytes again. - */ - private void trackLayerMetaFromSerializer( - MemoryBuffer buffer, LayerReadContext context, MetaSharedLayerSerializerBase serializer) { - if (!fory.getConfig().isMetaShareEnabled()) { - return; - } - org.apache.fory.resolver.MetaContext metaContext = - fory.getSerializationContext().getMetaContext(); - if (metaContext == null) { - return; - } - - // Peek at the buffer to see if this is a new ClassDef or a reference - int readerIndex = buffer.readerIndex(); - int indexMarker = buffer.readVarUint32Small14(); - boolean isRef = (indexMarker & 1) == 1; - int index = indexMarker >>> 1; - - if (isRef) { - // Reference - already tracked, nothing to do - // Reset buffer position so serializer can read it - buffer.readerIndex(readerIndex); - } else { - // New type - read the ClassDef and track it - long id = buffer.readInt64(); - ClassDef classDef = ClassDef.readClassDef(fory, buffer, id); - int storedIndex = context.addClassDef(classDef); - if (storedIndex != index) { - throw new IllegalStateException( - "Layer ClassDef index mismatch: expected " + index + ", got " + storedIndex); - } - // Reset buffer position so serializer can read and skip it - buffer.readerIndex(readerIndex); - } - } - /** * Context for tracking layer ClassDefs during deserialization. This is needed for schema - * evolution when sender has layers that receiver doesn't have, and we need to skip data for - * referenced ClassDefs. + * evolution when sender may have different ClassDefs for the same layer, and we need to create + * appropriate serializers for each unique ClassDef. */ private static class LayerReadContext { /** List of ClassDefs encountered in stream order, indexed by their stream index. */ private final List layerClassDefs = new ArrayList<>(); /** - * Cache of skip serializers for each ClassDef, to avoid recreating them for referenced types. + * Cache of serializers for each ClassDef index. This allows reusing serializers when the same + * ClassDef is referenced multiple times (via the reference mechanism). */ - private final Map> skipSerializerCache = new HashMap<>(); + private final Map> serializerCache = new HashMap<>(); /** Add a ClassDef to the tracking list and return its index. */ int addClassDef(ClassDef classDef) { @@ -407,18 +378,31 @@ ClassDef getClassDef(int index) { return layerClassDefs.get(index); } - /** Get or create a skip serializer for a ClassDef at the given index. */ - MetaSharedLayerSerializer getOrCreateSkipSerializer( - Fory fory, int index, Class senderClass) { - return skipSerializerCache.computeIfAbsent( + /** Get the index of a ClassDef, or -1 if not found. */ + int getClassDefIndex(ClassDef classDef) { + for (int i = 0; i < layerClassDefs.size(); i++) { + if (layerClassDefs.get(i) == classDef) { + return i; + } + } + return -1; + } + + /** + * Get or create a serializer for a ClassDef at the given index. The serializer is created using + * the ClassDef's field layout, allowing it to correctly read data written by a sender with that + * specific schema. + */ + MetaSharedLayerSerializer getOrCreateSerializer(Fory fory, int index, Class targetClass) { + return serializerCache.computeIfAbsent( index, i -> { ClassDef classDef = getClassDef(i); return new MetaSharedLayerSerializer<>( fory, - senderClass, + targetClass, classDef, - LayerMarkerClassGenerator.getOrCreate(fory, senderClass, 0)); + LayerMarkerClassGenerator.getOrCreate(fory, targetClass, 0)); }); } } @@ -472,10 +456,10 @@ private ClassDef readAndTrackLayerMeta(MemoryBuffer buffer, LayerReadContext con */ private void skipUnknownLayerData( MemoryBuffer buffer, Class senderClass, LayerReadContext context, ClassDef classDef) { - // For layers without custom writeObject, we can skip using a temporary serializer - // created from the ClassDef. Note: For layers with custom writeObject, the sender - // would have that class locally, and we'd have a matching slot. This method is only - // called when sender has a layer the receiver doesn't have. + // For layers without custom writeObject, we can skip using a serializer created from the + // ClassDef. Note: For layers with custom writeObject, the sender would have that class + // locally, and we'd have a matching slot. This method is only called when sender has a + // layer the receiver doesn't have. if (classDef == null) { throw new UnsupportedOperationException( @@ -484,42 +468,18 @@ private void skipUnknownLayerData( + ". Schema evolution with removed parent classes requires meta share."); } - // Find the index of this ClassDef in the context - int classDefIndex = -1; - for (int i = 0; i < context.layerClassDefs.size(); i++) { - if (context.layerClassDefs.get(i) == classDef) { - classDefIndex = i; - break; - } - } - + // Get the ClassDef index and create a serializer to skip the fields + int classDefIndex = context.getClassDefIndex(classDef); if (classDefIndex < 0) { - // This shouldn't happen - we should have tracked it when reading throw new IllegalStateException( "ClassDef not found in context for layer: " + senderClass.getName()); } - // Get or create a skip serializer and use it to read (and discard) the fields + // Get or create a serializer and use it to skip (read and discard) the fields MetaSharedLayerSerializer skipSerializer = - context.getOrCreateSkipSerializer(fory, classDefIndex, senderClass); - // Note: We don't call readAndSetFields because we already read the layer meta above. - // We need to just read the field data. The serializer's readAndSetFields would try to - // read meta again, so we directly read the fields instead. - skipLayerFields(buffer, skipSerializer); - } - - /** - * Skip field data using a serializer. This reads the field data without setting it on any object. - * The layer meta should already be consumed before calling this method. - * - * @param buffer the memory buffer to read from - * @param serializer the serializer to use for reading (and discarding) fields - */ - private void skipLayerFields(MemoryBuffer buffer, MetaSharedLayerSerializer serializer) { - // Use SerializationBinding to read fields - the serializer has the field layout - // from the ClassDef, and skipFields will read each field to advance buffer position + context.getOrCreateSerializer(fory, classDefIndex, senderClass); SerializationBinding binding = SerializationBinding.createBinding(fory); - serializer.skipFields(buffer, binding); + skipSerializer.skipFields(buffer, binding); } private static void throwUnsupportedEncodingException(Class cls) From f965519122aa4fd941ea8fca2af4a97059e46cba Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 17 Jan 2026 11:02:18 +0800 Subject: [PATCH 17/44] refactor object stream serializer --- .../builder/MetaSharedLayerCodecBuilder.java | 52 +--- .../org/apache/fory/resolver/ClassInfo.java | 27 ++ .../serializer/MetaSharedLayerSerializer.java | 36 +-- .../MetaSharedLayerSerializerBase.java | 4 +- .../serializer/ObjectStreamSerializer.java | 293 +++++++++--------- 5 files changed, 191 insertions(+), 221 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedLayerCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedLayerCodecBuilder.java index daa26efcc8..463195f576 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedLayerCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedLayerCodecBuilder.java @@ -20,7 +20,6 @@ package org.apache.fory.builder; import static org.apache.fory.builder.Generated.GeneratedMetaSharedLayerSerializer.SERIALIZER_FIELD_NAME; -import static org.apache.fory.type.TypeUtils.PRIMITIVE_VOID_TYPE; import java.util.Collection; import java.util.Map; @@ -29,9 +28,6 @@ import org.apache.fory.builder.Generated.GeneratedMetaSharedLayerSerializer; import org.apache.fory.codegen.CodeGenerator; import org.apache.fory.codegen.Expression; -import org.apache.fory.codegen.Expression.ListExpression; -import org.apache.fory.codegen.Expression.Reference; -import org.apache.fory.codegen.Expression.StaticInvoke; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.meta.ClassDef; import org.apache.fory.reflect.TypeRef; @@ -157,52 +153,8 @@ public Expression buildEncodeExpression() { throw new IllegalStateException("unreachable"); } - @Override - public Expression buildDecodeExpression() { - // Build the base decode expression from parent - Expression baseDecodeExpr = super.buildDecodeExpression(); - - // Prepend layer class meta reading if meta share is enabled - if (fory.getConfig().isMetaShareEnabled()) { - ListExpression expressions = new ListExpression(); - Reference buffer = new Reference(BUFFER_NAME, bufferTypeRef, false); - // Call static helper to read layer class meta - Expression readMeta = - new StaticInvoke( - MetaSharedLayerCodecBuilder.class, - "readLayerClassMeta", - PRIMITIVE_VOID_TYPE, - foryRef, - buffer); - expressions.add(readMeta); - expressions.add(baseDecodeExpr); - return expressions; - } - return baseDecodeExpr; - } - - /** - * Static helper method to read layer class meta from buffer. Called by generated code to maintain - * consistency with interpreter-mode MetaSharedLayerSerializer. - * - * @param fory the Fory instance - * @param buffer the memory buffer to read from - */ - public static void readLayerClassMeta(Fory fory, MemoryBuffer buffer) { - org.apache.fory.resolver.MetaContext metaContext = - fory.getSerializationContext().getMetaContext(); - if (metaContext == null) { - return; - } - int indexMarker = buffer.readVarUint32Small14(); - boolean isRef = (indexMarker & 1) == 1; - if (!isRef) { - // New type in stream - read ClassDef inline and skip it - long id = buffer.readInt64(); - ClassDef.skipClassDef(buffer, id); - } - // If isRef, it's a reference to previously read type - nothing to do - } + // Note: Layer class meta is read by ObjectStreamSerializer before calling this serializer. + // The generated read() method only reads field data, not the layer class meta. @Override protected Expression buildComponentsArray() { diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassInfo.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassInfo.java index 6c0f943ce2..b2dd539247 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassInfo.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassInfo.java @@ -72,6 +72,25 @@ public class ClassInfo { } } + /** + * Creates a ClassInfo for deserialization with a ClassDef. Used when reading class meta from + * stream where the ClassDef specifies the field layout. + * + * @param cls the class + * @param classDef the class definition from stream + */ + public ClassInfo(Class cls, ClassDef classDef) { + this.cls = cls; + this.classDef = classDef; + this.fullNameBytes = null; + this.namespaceBytes = null; + this.typeNameBytes = null; + this.isDynamicGeneratedClass = false; + this.serializer = null; + this.classId = TypeResolver.NO_CLASS_ID; + this.xtypeId = 0; + } + ClassInfo( TypeResolver classResolver, Class cls, @@ -125,6 +144,14 @@ public Class getCls() { return cls; } + public ClassDef getClassDef() { + return classDef; + } + + void setClassDef(ClassDef classDef) { + this.classDef = classDef; + } + public short getClassId() { return classId; } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java index 0b74a16c27..04523fc9d2 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java @@ -136,14 +136,17 @@ private void writeOtherFields(MemoryBuffer buffer, T value) { @Override public T read(MemoryBuffer buffer) { - // This method creates a new object - for layer serialization, use readAndSetFields instead + // Note: Layer class meta is read by ObjectStreamSerializer before calling this method. + // This serializer is designed for use with ObjectStreamSerializer only. T obj = newBean(); refResolver.reference(obj); - return readAndSetFields(buffer, obj); + return readFieldsOnly(buffer, obj); } /** - * Read layer class meta and fields, setting values on the provided object. + * Read fields and set values on the provided object. Note: When meta share is enabled, the caller + * (typically ObjectStreamSerializer) is responsible for reading the layer class meta first. This + * method only reads field data. * * @param buffer the memory buffer to read from * @param obj the object to set field values on @@ -151,13 +154,9 @@ public T read(MemoryBuffer buffer) { */ @Override public T readAndSetFields(MemoryBuffer buffer, T obj) { - // Read and verify layer class meta (only if meta share is enabled) - if (fory.getConfig().isMetaShareEnabled()) { - readLayerClassMeta(buffer); - } - // Read fields in order: final, container, other - readFieldsOnly(buffer, obj); - return obj; + // Note: Layer class meta is read by ObjectStreamSerializer before calling this method + // (when meta share is enabled). This method only reads field values. + return readFieldsOnly(buffer, obj); } /** @@ -178,23 +177,6 @@ public T readFieldsOnly(MemoryBuffer buffer, Object obj) { return (T) obj; } - private void readLayerClassMeta(MemoryBuffer buffer) { - MetaContext metaContext = fory.getSerializationContext().getMetaContext(); - if (metaContext == null) { - return; - } - int indexMarker = buffer.readVarUint32Small14(); - boolean isRef = (indexMarker & 1) == 1; - if (isRef) { - // Reference to previously read type - already in readClassInfos, nothing to do - } else { - // New type in stream - read ClassDef inline - long id = buffer.readInt64(); - ClassDef.skipClassDef(buffer, id); - // Layer class info is managed by this serializer, not stored in readClassInfos - } - } - private void readFinalFields(MemoryBuffer buffer, T targetObject) { for (SerializationFieldInfo fieldInfo : buildInFields) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializerBase.java b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializerBase.java index 221c929263..70dcc27cbc 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializerBase.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializerBase.java @@ -37,7 +37,9 @@ public MetaSharedLayerSerializerBase(Fory fory, Class type) { } /** - * Read layer class meta and fields, setting values on the provided object. + * Read fields and set values on the provided object. Note: When meta share is enabled, the caller + * (typically ObjectStreamSerializer) is responsible for reading the layer class meta first. This + * method only reads field data, not the layer class meta. * * @param buffer the memory buffer to read from * @param obj the object to set field values on diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java index a2a849dde7..fc4d08471c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java @@ -39,9 +39,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.TreeMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiConsumer; @@ -49,6 +47,7 @@ import org.apache.fory.Fory; import org.apache.fory.builder.CodecUtils; import org.apache.fory.builder.LayerMarkerClassGenerator; +import org.apache.fory.collection.LongMap; import org.apache.fory.collection.ObjectArray; import org.apache.fory.collection.ObjectIntMap; import org.apache.fory.logging.Logger; @@ -59,6 +58,8 @@ import org.apache.fory.reflect.ObjectCreator; import org.apache.fory.reflect.ObjectCreators; import org.apache.fory.reflect.ReflectionUtils; +import org.apache.fory.resolver.ClassInfo; +import org.apache.fory.resolver.MetaContext; import org.apache.fory.type.Descriptor; import org.apache.fory.util.ExceptionUtils; import org.apache.fory.util.GraalvmSupport; @@ -82,6 +83,8 @@ public class ObjectStreamSerializer extends AbstractObjectSerializer { private static final Logger LOG = LoggerFactory.getLogger(ObjectStreamSerializer.class); private final SlotInfo[] slotsInfos; + // Instance-level cache: ClassDef ID -> ClassInfo (shared across all slots) + private final LongMap classDefIdToClassInfo = new LongMap<>(4, 0.4f); /** * Interface for slot information used in ObjectStreamSerializer. This allows both full SlotsInfo @@ -94,6 +97,27 @@ private interface SlotInfo { MetaSharedLayerSerializerBase getSlotsSerializer(); + /** + * Read the layer ClassDef from buffer (if meta share enabled) and return the appropriate + * serializer. When meta share is enabled, reads the ClassDef from buffer, caches the serializer + * by ClassDef ID, and returns it. When meta share is disabled, returns the default slots + * serializer. Also stores the returned serializer for later retrieval via {@link + * #getCurrentReadSerializer()}. + * + * @param fory the Fory instance + * @param buffer the memory buffer to read ClassDef from + * @return the serializer to use for reading + */ + MetaSharedLayerSerializerBase getReadSerializer(Fory fory, MemoryBuffer buffer); + + /** + * Get the current read serializer (last returned by {@link #getReadSerializer}). This is used + * by defaultReadObject() to get the serializer without reading from buffer again. + * + * @return the current read serializer + */ + MetaSharedLayerSerializerBase getCurrentReadSerializer(); + ForyObjectOutputStream getObjectOutputStream(); ForyObjectInputStream getObjectInputStream(); @@ -224,9 +248,6 @@ public Object read(MemoryBuffer buffer) { int numClasses = buffer.readInt16(); int slotIndex = 0; - // Create context to track layer ClassDefs for schema evolution - LayerReadContext layerContext = new LayerReadContext(); - try { TreeMap callbacks = new TreeMap<>(Collections.reverseOrder()); for (int i = 0; i < numClasses; i++) { @@ -261,34 +282,22 @@ public Object read(MemoryBuffer buffer) { } } - // Read and track the layer ClassDef first (for both matched and unmatched) - // This is critical because different senders may have different ClassDefs for the same - // layer - ClassDef classDef = readAndTrackLayerMeta(buffer, layerContext); - if (matchedSlot == null) { - // Sender has a layer that receiver doesn't have - skip the data - skipUnknownLayerData(buffer, currentClass, layerContext, classDef); + // Sender has a layer that receiver doesn't have - read ClassDef and skip the data + skipUnknownLayerData(buffer, currentClass); continue; } - // Read data for the matched layer + // Read data for the matched layer - getReadSerializer reads ClassDef from buffer + // This must be called exactly once per layer to read the ClassDef + matchedSlot.getReadSerializer(fory, buffer); + StreamClassInfo streamClassInfo = matchedSlot.getStreamClassInfo(); Method readObjectMethod = streamClassInfo.readObjectMethod; + if (readObjectMethod == null) { - // For standard field serialization, create/get a serializer based on the sender's - // ClassDef - // This ensures we read fields according to the sender's schema, not the receiver's - if (classDef != null) { - int classDefIndex = layerContext.getClassDefIndex(classDef); - MetaSharedLayerSerializer layerSerializer = - layerContext.getOrCreateSerializer(fory, classDefIndex, currentClass); - layerSerializer.readFieldsOnly(buffer, obj); - } else { - // Meta share not enabled - use the slot's serializer directly - // Note: readAndSetFields won't try to read meta if meta share is disabled - matchedSlot.getSlotsSerializer().readAndSetFields(buffer, obj); - } + // For standard field serialization - use getCurrentReadSerializer() + matchedSlot.getCurrentReadSerializer().readAndSetFields(buffer, obj); } else { // For custom readObject, it handles its own format ForyObjectInputStream objectInputStream = matchedSlot.getObjectInputStream(); @@ -347,139 +356,67 @@ public Object read(MemoryBuffer buffer) { return obj; } - /** - * Context for tracking layer ClassDefs during deserialization. This is needed for schema - * evolution when sender may have different ClassDefs for the same layer, and we need to create - * appropriate serializers for each unique ClassDef. - */ - private static class LayerReadContext { - /** List of ClassDefs encountered in stream order, indexed by their stream index. */ - private final List layerClassDefs = new ArrayList<>(); - - /** - * Cache of serializers for each ClassDef index. This allows reusing serializers when the same - * ClassDef is referenced multiple times (via the reference mechanism). - */ - private final Map> serializerCache = new HashMap<>(); - - /** Add a ClassDef to the tracking list and return its index. */ - int addClassDef(ClassDef classDef) { - int index = layerClassDefs.size(); - layerClassDefs.add(classDef); - return index; - } - - /** Get a ClassDef by its stream index. */ - ClassDef getClassDef(int index) { - if (index < 0 || index >= layerClassDefs.size()) { - throw new IllegalStateException( - "Invalid layer ClassDef index: " + index + ", tracked count: " + layerClassDefs.size()); - } - return layerClassDefs.get(index); - } - - /** Get the index of a ClassDef, or -1 if not found. */ - int getClassDefIndex(ClassDef classDef) { - for (int i = 0; i < layerClassDefs.size(); i++) { - if (layerClassDefs.get(i) == classDef) { - return i; - } - } - return -1; - } - - /** - * Get or create a serializer for a ClassDef at the given index. The serializer is created using - * the ClassDef's field layout, allowing it to correctly read data written by a sender with that - * specific schema. - */ - MetaSharedLayerSerializer getOrCreateSerializer(Fory fory, int index, Class targetClass) { - return serializerCache.computeIfAbsent( - index, - i -> { - ClassDef classDef = getClassDef(i); - return new MetaSharedLayerSerializer<>( - fory, - targetClass, - classDef, - LayerMarkerClassGenerator.getOrCreate(fory, targetClass, 0)); - }); - } - } - - /** - * Read layer class meta from buffer and track in context. This method handles both new ClassDefs - * (read and store) and references (lookup from tracked). - * - * @param buffer the memory buffer to read from - * @param context the layer read context for tracking ClassDefs - * @return the ClassDef for this layer (either newly read or looked up from reference) - */ - private ClassDef readAndTrackLayerMeta(MemoryBuffer buffer, LayerReadContext context) { - if (!fory.getConfig().isMetaShareEnabled()) { - return null; - } - org.apache.fory.resolver.MetaContext metaContext = - fory.getSerializationContext().getMetaContext(); - if (metaContext == null) { - return null; - } - - int indexMarker = buffer.readVarUint32Small14(); - boolean isRef = (indexMarker & 1) == 1; - int index = indexMarker >>> 1; - - if (isRef) { - // Reference to previously seen ClassDef - look it up - return context.getClassDef(index); - } else { - // New type in stream - read ClassDef inline and track it - long id = buffer.readInt64(); - ClassDef classDef = ClassDef.readClassDef(fory, buffer, id); - int storedIndex = context.addClassDef(classDef); - if (storedIndex != index) { - throw new IllegalStateException( - "Layer ClassDef index mismatch: expected " + index + ", got " + storedIndex); - } - return classDef; - } - } - /** * Skip data for a layer that exists in sender but not in receiver. This is needed for schema * evolution when sender's class hierarchy has layers that receiver doesn't have. * * @param buffer the memory buffer to read from - * @param senderClass the sender's class for this layer (not present in receiver) - * @param context the layer read context for tracking ClassDefs - * @param classDef the ClassDef for this layer (already read from stream) + * @param senderClass the class from sender that receiver doesn't have */ - private void skipUnknownLayerData( - MemoryBuffer buffer, Class senderClass, LayerReadContext context, ClassDef classDef) { + private void skipUnknownLayerData(MemoryBuffer buffer, Class senderClass) { // For layers without custom writeObject, we can skip using a serializer created from the // ClassDef. Note: For layers with custom writeObject, the sender would have that class // locally, and we'd have a matching slot. This method is only called when sender has a // layer the receiver doesn't have. - if (classDef == null) { + if (!fory.getConfig().isMetaShareEnabled()) { throw new UnsupportedOperationException( "Cannot skip unknown layer data without meta share enabled for class: " + senderClass.getName() + ". Schema evolution with removed parent classes requires meta share."); } - // Get the ClassDef index and create a serializer to skip the fields - int classDefIndex = context.getClassDefIndex(classDef); - if (classDefIndex < 0) { - throw new IllegalStateException( - "ClassDef not found in context for layer: " + senderClass.getName()); + // Read ClassInfo from buffer (maintains index alignment with readClassInfos) + MetaContext metaContext = fory.getSerializationContext().getMetaContext(); + if (metaContext == null) { + throw new IllegalStateException("MetaContext is null but meta share is enabled"); + } + int indexMarker = buffer.readVarUint32Small14(); + boolean isRef = (indexMarker & 1) == 1; + int index = indexMarker >>> 1; + ClassInfo classInfo; + if (isRef) { + // Reference to previously read ClassInfo + classInfo = metaContext.readClassInfos.get(index); + } else { + // New ClassDef in stream - read ID first to check cache + long classDefId = buffer.readInt64(); + classInfo = classDefIdToClassInfo.get(classDefId); + if (classInfo != null) { + // Already cached - skip the ClassDef bytes, reuse existing ClassInfo + ClassDef.skipClassDef(buffer, classDefId); + } else { + // Not cached - read full ClassDef and create ClassInfo + ClassDef classDef = ClassDef.readClassDef(fory, buffer, classDefId); + classInfo = new ClassInfo(senderClass, classDef); + classDefIdToClassInfo.put(classDefId, classInfo); + } + metaContext.readClassInfos.add(classInfo); } - // Get or create a serializer and use it to skip (read and discard) the fields - MetaSharedLayerSerializer skipSerializer = - context.getOrCreateSerializer(fory, classDefIndex, senderClass); + // Get or create serializer from ClassInfo to skip the fields + MetaSharedLayerSerializerBase skipSerializer = + (MetaSharedLayerSerializerBase) classInfo.getSerializer(); + if (skipSerializer == null) { + Class layerMarkerClass = LayerMarkerClassGenerator.getOrCreate(fory, senderClass, 0); + MetaSharedLayerSerializer newSerializer = + new MetaSharedLayerSerializer( + fory, senderClass, classInfo.getClassDef(), layerMarkerClass); + classInfo.setSerializer(newSerializer); + skipSerializer = newSerializer; + } SerializationBinding binding = SerializationBinding.createBinding(fory); - skipSerializer.skipFields(buffer, binding); + ((MetaSharedLayerSerializer) skipSerializer).skipFields(buffer, binding); } private static void throwUnsupportedEncodingException(Class cls) @@ -596,7 +533,7 @@ protected StreamClassInfo computeValue(Class type) { * all the details of serializing and deserializing a single class in the class hierarchy using * Java's ObjectInputStream/ObjectOutputStream protocol. */ - private static class SlotsInfo implements SlotInfo { + private class SlotsInfo implements SlotInfo { private final Class cls; private final StreamClassInfo streamClassInfo; // mark non-final for async-jit to update it to jit-serializer. @@ -607,6 +544,8 @@ private static class SlotsInfo implements SlotInfo { private final ForyObjectOutputStream objectOutputStream; private final ForyObjectInputStream objectInputStream; private final ObjectArray getFieldPool; + // Current read serializer (set by getReadSerializer, used by getCurrentReadSerializer) + private MetaSharedLayerSerializerBase currentReadSerializer; public SlotsInfo(Fory fory, Class type) { this.cls = type; @@ -747,6 +686,72 @@ public Class[] getPutFieldTypes() { return putFieldTypes; } + @Override + @SuppressWarnings("unchecked") + public MetaSharedLayerSerializerBase getReadSerializer(Fory fory, MemoryBuffer buffer) { + MetaSharedLayerSerializerBase result; + if (!fory.getConfig().isMetaShareEnabled()) { + // Meta share not enabled - use the default slots serializer + result = slotsSerializer; + } else { + // Read ClassInfo from buffer (creates new or returns existing) + ClassInfo classInfo = readLayerClassInfo(fory, buffer); + if (classInfo == null) { + result = slotsSerializer; + } else { + // Get or create serializer from ClassInfo + Serializer serializer = classInfo.getSerializer(); + if (serializer != null) { + result = (MetaSharedLayerSerializerBase) serializer; + } else { + // Create a new serializer based on the ClassDef from stream + Class layerMarkerClass = LayerMarkerClassGenerator.getOrCreate(fory, cls, 0); + MetaSharedLayerSerializer newSerializer = + new MetaSharedLayerSerializer(fory, cls, classInfo.getClassDef(), layerMarkerClass); + classInfo.setSerializer(newSerializer); + result = newSerializer; + } + } + } + // Store for getCurrentReadSerializer() + this.currentReadSerializer = result; + return result; + } + + @Override + public MetaSharedLayerSerializerBase getCurrentReadSerializer() { + return currentReadSerializer; + } + + private ClassInfo readLayerClassInfo(Fory fory, MemoryBuffer buffer) { + MetaContext metaContext = fory.getSerializationContext().getMetaContext(); + if (metaContext == null) { + return null; + } + int indexMarker = buffer.readVarUint32Small14(); + boolean isRef = (indexMarker & 1) == 1; + int index = indexMarker >>> 1; + if (isRef) { + // Reference to previously read ClassInfo + return metaContext.readClassInfos.get(index); + } else { + // New ClassDef in stream - read ID first to check cache + long classDefId = buffer.readInt64(); + ClassInfo classInfo = classDefIdToClassInfo.get(classDefId); + if (classInfo != null) { + // Already cached - skip the ClassDef bytes, reuse existing ClassInfo + ClassDef.skipClassDef(buffer, classDefId); + } else { + // Not cached - read full ClassDef and create ClassInfo + ClassDef classDef = ClassDef.readClassDef(fory, buffer, classDefId); + classInfo = new ClassInfo(cls, classDef); + classDefIdToClassInfo.put(classDefId, classInfo); + } + metaContext.readClassInfos.add(classInfo); + return classInfo; + } + } + @Override public String toString() { return "SlotsInfo{" + "cls=" + cls + '}'; @@ -1304,10 +1309,13 @@ private Object readPutFieldValue(MemoryBuffer buffer, Class fieldType) { } @Override + @SuppressWarnings("rawtypes") public void defaultReadObject() throws IOException, ClassNotFoundException { if (fieldsRead) { throw new NotActiveException("not in readObject invocation or fields already read"); } + // Get the serializer that was set by getReadSerializer() - has correct schema from stream + MetaSharedLayerSerializerBase serializer = slotsInfo.getCurrentReadSerializer(); // Read field values in the same format as writeFields, but set to actual class fields Class[] fieldTypes = slotsInfo.getPutFieldTypes(); if (fieldTypes != null && fieldTypes.length > 0) { @@ -1318,11 +1326,10 @@ public void defaultReadObject() throws IOException, ClassNotFoundException { vals[i] = readPutFieldValue(buffer, fieldTypes[i]); } // Now set matching fields on the target object - MetaSharedLayerSerializerBase slotsSerializer = slotsInfo.getSlotsSerializer(); - slotsSerializer.setFieldValuesFromPutFields(targetObject, fieldIndexMap, vals); + serializer.setFieldValuesFromPutFields(targetObject, fieldIndexMap, vals); } else { // No custom writeObject/readObject, use normal serialization - slotsInfo.getSlotsSerializer().readAndSetFields(buffer, targetObject); + serializer.readAndSetFields(buffer, targetObject); } fieldsRead = true; } From 858cd3f471c5b97b270ce9e8f11b21f81057e6ea Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 17 Jan 2026 11:53:33 +0800 Subject: [PATCH 18/44] refactor objectstream serializer --- .../org/apache/fory/builder/Generated.java | 32 +++ .../builder/MetaSharedLayerCodecBuilder.java | 12 ++ .../org/apache/fory/meta/ClassDefEncoder.java | 2 +- .../java/org/apache/fory/meta/FieldInfo.java | 2 +- .../java/org/apache/fory/meta/FieldTypes.java | 2 +- .../serializer/MetaSharedLayerSerializer.java | 92 ++++++++- .../MetaSharedLayerSerializerBase.java | 53 +++++ .../serializer/ObjectStreamSerializer.java | 194 +++++++++--------- 8 files changed, 289 insertions(+), 100 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/Generated.java b/java/fory-core/src/main/java/org/apache/fory/builder/Generated.java index dae5694487..4dadbff23e 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/Generated.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/Generated.java @@ -156,5 +156,37 @@ public Object[] getFieldValuesForPutFields( Object obj, org.apache.fory.collection.ObjectIntMap fieldIndexMap, int arraySize) { return serializer.getFieldValuesForPutFields(obj, fieldIndexMap, arraySize); } + + @Override + public void writeLayerClassMeta(MemoryBuffer buffer) { + serializer.writeLayerClassMeta(buffer); + } + + @Override + public void writeFieldsOnly(MemoryBuffer buffer, Object value) { + serializer.writeFieldsOnly(buffer, value); + } + + @Override + public void writeFieldValues(MemoryBuffer buffer, Object[] vals) { + serializer.writeFieldValues(buffer, vals); + } + + @Override + public Object[] readFieldValues(MemoryBuffer buffer) { + return serializer.readFieldValues(buffer); + } + + @Override + public int getNumFields() { + return serializer.getNumFields(); + } + + @Override + @SuppressWarnings("rawtypes") + public void populateFieldInfo( + org.apache.fory.collection.ObjectIntMap fieldIndexMap, Class[] fieldTypes) { + serializer.populateFieldInfo(fieldIndexMap, fieldTypes); + } } } diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedLayerCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedLayerCodecBuilder.java index 463195f576..d7f6f15c22 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedLayerCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedLayerCodecBuilder.java @@ -28,6 +28,7 @@ import org.apache.fory.builder.Generated.GeneratedMetaSharedLayerSerializer; import org.apache.fory.codegen.CodeGenerator; import org.apache.fory.codegen.Expression; +import org.apache.fory.codegen.Expression.StaticInvoke; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.meta.ClassDef; import org.apache.fory.reflect.TypeRef; @@ -37,6 +38,7 @@ import org.apache.fory.serializer.Serializers; import org.apache.fory.type.Descriptor; import org.apache.fory.type.DescriptorGrouper; +import org.apache.fory.util.ExceptionUtils; import org.apache.fory.util.GraalvmSupport; import org.apache.fory.util.Preconditions; import org.apache.fory.util.StringUtils; @@ -153,6 +155,16 @@ public Expression buildEncodeExpression() { throw new IllegalStateException("unreachable"); } + @Override + protected Expression setFieldValue(Expression bean, Descriptor descriptor, Expression value) { + if (descriptor.getField() == null) { + // Field doesn't exist in current class (e.g., from serialPersistentFields). + // Skip setting this field value but still consume the read value. + return new StaticInvoke(ExceptionUtils.class, "ignore", value); + } + return super.setFieldValue(bean, descriptor, value); + } + // Note: Layer class meta is read by ObjectStreamSerializer before calling this serializer. // The generated read() method only reads field data, not the layer class meta. diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDefEncoder.java b/java/fory-core/src/main/java/org/apache/fory/meta/ClassDefEncoder.java index 62eb5b7a7e..2a896864df 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDefEncoder.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/ClassDefEncoder.java @@ -136,7 +136,7 @@ static ClassDef buildClassDef( classResolver, type, buildFieldsInfo(classResolver, fields), hasFieldsMeta); } - static ClassDef buildClassDefWithFieldInfos( + public static ClassDef buildClassDefWithFieldInfos( ClassResolver classResolver, Class type, List fieldInfos, diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java b/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java index 23256d110b..5c759a1230 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java @@ -47,7 +47,7 @@ public final class FieldInfo implements Serializable { /** Field ID for schema evolution, -1 means no field ID (use field name). */ final short fieldId; - FieldInfo(String definedClass, String fieldName, FieldTypes.FieldType fieldType) { + public FieldInfo(String definedClass, String fieldName, FieldTypes.FieldType fieldType) { this(definedClass, fieldName, fieldType, (short) -1); } diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java index 5f3e3c27f1..bb0104fc44 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java @@ -701,7 +701,7 @@ public String toString() { } public static class EnumFieldType extends FieldType { - private EnumFieldType(boolean nullable, int xtypeId) { + public EnumFieldType(boolean nullable, int xtypeId) { super(xtypeId, nullable, false); } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java index 04523fc9d2..0224692660 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java @@ -87,12 +87,11 @@ public void write(MemoryBuffer buffer, T value) { writeLayerClassMeta(buffer); } // Write fields in order: final, container, other - writeFinalFields(buffer, value); - writeContainerFields(buffer, value); - writeOtherFields(buffer, value); + writeFieldsOnly(buffer, value); } - private void writeLayerClassMeta(MemoryBuffer buffer) { + @Override + public void writeLayerClassMeta(MemoryBuffer buffer) { MetaContext metaContext = fory.getSerializationContext().getMetaContext(); if (metaContext == null) { return; @@ -110,7 +109,15 @@ private void writeLayerClassMeta(MemoryBuffer buffer) { } } - private void writeFinalFields(MemoryBuffer buffer, T value) { + @Override + public void writeFieldsOnly(MemoryBuffer buffer, T value) { + // Write fields in order: buildIn, container, other + writeBuildInFields(buffer, value); + writeContainerFields(buffer, value); + writeOtherFields(buffer, value); + } + + private void writeBuildInFields(MemoryBuffer buffer, T value) { for (SerializationFieldInfo fieldInfo : buildInFields) { AbstractObjectSerializer.writeBuildInField(binding, fieldInfo, buffer, value); } @@ -134,6 +141,81 @@ private void writeOtherFields(MemoryBuffer buffer, T value) { } } + @Override + public void writeFieldValues(MemoryBuffer buffer, Object[] vals) { + // Write fields from array in order: buildIn, container, other + int index = 0; + // Write buildIn fields + for (SerializationFieldInfo fieldInfo : buildInFields) { + AbstractObjectSerializer.writeBuildInFieldValue(binding, fieldInfo, buffer, vals[index++]); + } + // Write container fields + Generics generics = fory.getGenerics(); + for (SerializationFieldInfo fieldInfo : containerFields) { + AbstractObjectSerializer.writeContainerFieldValue( + binding, refResolver, generics, fieldInfo, buffer, vals[index++]); + } + // Write other fields + for (SerializationFieldInfo fieldInfo : otherFields) { + binding.writeField(fieldInfo, buffer, vals[index++]); + } + } + + @Override + public Object[] readFieldValues(MemoryBuffer buffer) { + Object[] vals = new Object[getNumFields()]; + int index = 0; + // Read buildIn fields + for (SerializationFieldInfo fieldInfo : buildInFields) { + vals[index++] = AbstractObjectSerializer.readBuildInFieldValue(binding, fieldInfo, buffer); + } + // Read container fields + Generics generics = fory.getGenerics(); + for (SerializationFieldInfo fieldInfo : containerFields) { + vals[index++] = + AbstractObjectSerializer.readContainerFieldValue(binding, generics, fieldInfo, buffer); + } + // Read other fields + for (SerializationFieldInfo fieldInfo : otherFields) { + vals[index++] = binding.readField(fieldInfo, buffer); + } + return vals; + } + + @Override + public void populateFieldInfo(ObjectIntMap fieldIndexMap, Class[] fieldTypes) { + int index = 0; + // BuildIn fields first + for (SerializationFieldInfo fieldInfo : buildInFields) { + populateSingleFieldInfo(fieldInfo, fieldIndexMap, fieldTypes, index++); + } + // Container fields next + for (SerializationFieldInfo fieldInfo : containerFields) { + populateSingleFieldInfo(fieldInfo, fieldIndexMap, fieldTypes, index++); + } + // Other fields last + for (SerializationFieldInfo fieldInfo : otherFields) { + populateSingleFieldInfo(fieldInfo, fieldIndexMap, fieldTypes, index++); + } + } + + private void populateSingleFieldInfo( + SerializationFieldInfo fieldInfo, + ObjectIntMap fieldIndexMap, + Class[] fieldTypes, + int index) { + FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; + if (fieldAccessor != null) { + fieldIndexMap.put(fieldAccessor.getField().getName(), index); + fieldTypes[index] = fieldAccessor.getField().getType(); + } else { + // Field doesn't exist in actual class (e.g., from serialPersistentFields). + // Use descriptor info instead. + fieldIndexMap.put(fieldInfo.descriptor.getName(), index); + fieldTypes[index] = fieldInfo.descriptor.getRawType(); + } + } + @Override public T read(MemoryBuffer buffer) { // Note: Layer class meta is read by ObjectStreamSerializer before calling this method. diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializerBase.java b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializerBase.java index 70dcc27cbc..f2ae8525f3 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializerBase.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializerBase.java @@ -71,4 +71,57 @@ public abstract void setFieldValuesFromPutFields( @SuppressWarnings("rawtypes") public abstract Object[] getFieldValuesForPutFields( Object obj, ObjectIntMap fieldIndexMap, int arraySize); + + /** + * Write layer class meta to buffer. Called by ObjectStreamSerializer before writing fields. Only + * writes meta if meta share is enabled. + * + * @param buffer the memory buffer to write to + */ + public abstract void writeLayerClassMeta(MemoryBuffer buffer); + + /** + * Write fields only, without layer class meta. The layer class meta should be written by the + * caller (ObjectStreamSerializer) before calling this method. + * + * @param buffer the memory buffer to write to + * @param value the object to write fields from + */ + public abstract void writeFieldsOnly(MemoryBuffer buffer, T value); + + /** + * Write field values from array. Used by writeFields() when values come from PutField. The values + * array should be in the serializer's field order (buildIn, container, other). + * + * @param buffer the memory buffer to write to + * @param vals the values array in field order + */ + public abstract void writeFieldValues(MemoryBuffer buffer, Object[] vals); + + /** + * Read field values into array. Used by readFields() to populate GetField. Returns values in the + * serializer's field order (buildIn, container, other). + * + * @param buffer the memory buffer to read from + * @return array of field values in field order + */ + public abstract Object[] readFieldValues(MemoryBuffer buffer); + + /** + * Get the total number of fields in this layer. + * + * @return number of fields + */ + public abstract int getNumFields(); + + /** + * Populate field index map and field types array in the serializer's field order. This is used by + * ObjectStreamSerializer to build fieldIndexMap and putFieldTypes in the correct order. + * + * @param fieldIndexMap map to populate with field name -> index + * @param fieldTypes array to populate with field types (must be pre-allocated with getNumFields() + * size) + */ + @SuppressWarnings("rawtypes") + public abstract void populateFieldInfo(ObjectIntMap fieldIndexMap, Class[] fieldTypes); } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java index fc4d08471c..b359b3b77f 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java @@ -32,7 +32,6 @@ import java.io.Serializable; import java.io.UnsupportedEncodingException; import java.lang.invoke.MethodHandles; -import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.ArrayList; @@ -41,7 +40,6 @@ import java.util.Collections; import java.util.List; import java.util.TreeMap; -import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiConsumer; import java.util.function.Consumer; import org.apache.fory.Fory; @@ -55,12 +53,17 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.Platform; import org.apache.fory.meta.ClassDef; +import org.apache.fory.meta.ClassDefEncoder; +import org.apache.fory.meta.FieldInfo; +import org.apache.fory.meta.FieldTypes; import org.apache.fory.reflect.ObjectCreator; import org.apache.fory.reflect.ObjectCreators; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.resolver.ClassInfo; import org.apache.fory.resolver.MetaContext; import org.apache.fory.type.Descriptor; +import org.apache.fory.type.TypeUtils; +import org.apache.fory.type.Types; import org.apache.fory.util.ExceptionUtils; import org.apache.fory.util.GraalvmSupport; import org.apache.fory.util.Preconditions; @@ -208,10 +211,16 @@ public void write(MemoryBuffer buffer, Object value) { // create a classinfo to avoid null class bytes when class id is a // replacement id. classResolver.writeClassInternal(buffer, slotsInfo.getCls()); + // Write layer class meta first (if meta share enabled) + MetaSharedLayerSerializerBase serializer = slotsInfo.getSlotsSerializer(); + if (fory.getConfig().isMetaShareEnabled()) { + serializer.writeLayerClassMeta(buffer); + } StreamClassInfo streamClassInfo = slotsInfo.getStreamClassInfo(); Method writeObjectMethod = streamClassInfo.writeObjectMethod; if (writeObjectMethod == null) { - slotsInfo.getSlotsSerializer().write(buffer, value); + // No custom writeObject - write fields directly + serializer.writeFieldsOnly(buffer, value); } else { ForyObjectOutputStream objectOutputStream = slotsInfo.getObjectOutputStream(); Object oldObject = objectOutputStream.targetObject; @@ -465,6 +474,68 @@ private static void ensureFieldSerializersGenerated( } } + /** + * Build a list of FieldInfo from ObjectStreamClass fields (serialPersistentFields). This creates + * FieldInfo objects that represent the serialization contract defined by the class, which may + * differ from the actual class fields when serialPersistentFields is defined. + * + * @param fory the Fory instance + * @param objectStreamClass the ObjectStreamClass for the type + * @param type the class type + * @return list of FieldInfo representing the serialization fields + */ + private static List buildFieldInfoFromObjectStreamClass( + Fory fory, ObjectStreamClass objectStreamClass, Class type) { + ObjectStreamField[] streamFields = objectStreamClass.getFields(); + List fieldInfos = new ArrayList<>(streamFields.length); + String className = type.getName(); + + for (ObjectStreamField streamField : streamFields) { + Class fieldType = streamField.getType(); + String fieldName = streamField.getName(); + + // Build FieldType based on the field's declared type + FieldTypes.FieldType fieldTypeInfo = buildFieldTypeFromClass(fory, fieldType); + + fieldInfos.add(new FieldInfo(className, fieldName, fieldTypeInfo)); + } + return fieldInfos; + } + + /** + * Build a FieldType from a Class. This is a simplified version of FieldTypes.buildFieldType that + * works with Class instead of Field, used for ObjectStreamField which only provides type info. + */ + private static FieldTypes.FieldType buildFieldTypeFromClass(Fory fory, Class fieldType) { + // For primitives, use RegisteredFieldType + if (fieldType.isPrimitive()) { + int typeId = Types.getTypeId(fory, fieldType); + return new FieldTypes.RegisteredFieldType(false, false, typeId); + } + + // For boxed primitives + Class unwrapped = TypeUtils.unwrap(fieldType); + if (unwrapped.isPrimitive()) { + int typeId = Types.getTypeId(fory, unwrapped); + return new FieldTypes.RegisteredFieldType(true, true, typeId); + } + + // For registered types + if (fory.getClassResolver().isRegisteredById(fieldType)) { + Short classId = fory.getClassResolver().getRegisteredClassId(fieldType); + return new FieldTypes.RegisteredFieldType(true, true, classId); + } + + // For enums + if (fieldType.isEnum()) { + return new FieldTypes.EnumFieldType(true, -1); + } + + // For arrays, collections, maps - use ObjectFieldType as a fallback + // The actual type handling will be done during serialization + return new FieldTypes.ObjectFieldType(-1, true, true); + } + /** * Information about a class's stream methods (writeObject, readObject, readObjectNoData) and * their optimized MethodHandle equivalents for fast invocation. @@ -552,8 +623,20 @@ public SlotsInfo(Fory fory, Class type) { ObjectStreamClass objectStreamClass = safeObjectStreamClassLookup(type); streamClassInfo = STREAM_CLASS_INFO_CACHE.get(type); - // Build single-layer ClassDef (resolveParent=false) - ClassDef layerClassDef = fory.getClassResolver().getTypeDef(type, false); + // Build ClassDef from ObjectStreamClass fields (handles serialPersistentFields). + // This ensures the serializer uses the same field layout as defined by + // serialPersistentFields (if present) rather than actual class fields. + ClassDef layerClassDef; + if (objectStreamClass != null) { + List fieldInfos = + buildFieldInfoFromObjectStreamClass(fory, objectStreamClass, type); + layerClassDef = + ClassDefEncoder.buildClassDefWithFieldInfos( + fory.getClassResolver(), type, fieldInfos, true); + } else { + // Fallback when ObjectStreamClass is not available (e.g., GraalVM native image) + layerClassDef = fory.getClassResolver().getTypeDef(type, false); + } // Generate marker class for this layer. Use 0 as layer index since each class // has its own SlotsInfo, and the (class, 0) pair is unique for each class. Class layerMarkerClass = LayerMarkerClassGenerator.getOrCreate(fory, type, 0); @@ -583,36 +666,16 @@ public SlotsInfo(Fory fory, Class type) { ensureFieldSerializersGenerated(fory, layerClassDef, type); } + // Build fieldIndexMap and putFieldTypes from serializer's field order. + // This ensures putFields/writeFields API uses the same order as the serializer + // (buildIn, container, other groups), not ObjectStreamClass order. fieldIndexMap = new ObjectIntMap<>(4, 0.4f); - // Build field list from ObjectStreamClass or class fields - List putFieldInfos = new ArrayList<>(); - if (objectStreamClass != null) { - for (ObjectStreamField serialField : objectStreamClass.getFields()) { - putFieldInfos.add(new PutFieldInfo(serialField.getName(), serialField.getType(), type)); - } - } else { - // Fallback when ObjectStreamClass is not available - Collection descriptors = - layerClassDef.getDescriptors(fory.getClassResolver(), type); - for (Descriptor descriptor : descriptors) { - Field field = descriptor.getField(); - if (field != null) { - putFieldInfos.add(new PutFieldInfo(field.getName(), field.getType(), type)); - } - } - } - if (streamClassInfo != null && (streamClassInfo.writeObjectMethod != null || streamClassInfo.readObjectMethod != null)) { - this.numPutFields = putFieldInfos.size(); + this.numPutFields = slotsSerializer.getNumFields(); this.putFieldTypes = new Class[numPutFields]; - AtomicInteger idx = new AtomicInteger(0); - for (PutFieldInfo fieldInfo : putFieldInfos) { - int index = idx.getAndIncrement(); - fieldIndexMap.put(fieldInfo.name, index); - putFieldTypes[index] = fieldInfo.type; - } + slotsSerializer.populateFieldInfo(fieldIndexMap, putFieldTypes); } else { this.numPutFields = 0; this.putFieldTypes = null; @@ -758,19 +821,6 @@ public String toString() { } } - /** Simple field info for putFields/writeFields support. */ - private static class PutFieldInfo { - final String name; - final Class type; - final Class declaringClass; - - PutFieldInfo(String name, Class type, Class declaringClass) { - this.name = name; - this.type = type; - this.declaringClass = declaringClass; - } - } - /** * Implement serialization for object output with `writeObject/readObject` defined by java * serialization output spec. @@ -906,14 +956,8 @@ public void writeFields() throws IOException { if (curPut == null) { throw new NotActiveException("no current PutField object"); } - // Write field values directly by their types - Class[] fieldTypes = slotsInfo.getPutFieldTypes(); - Object[] vals = curPut.vals; - for (int i = 0; i < vals.length; i++) { - Class fieldType = fieldTypes[i]; - Object value = vals[i]; - writePutFieldValue(buffer, fieldType, value); - } + // Write field values using MetaShare serialization + slotsInfo.getSlotsSerializer().writeFieldValues(buffer, curPut.vals); Arrays.fill(curPut.vals, null); putFieldsCache.add(curPut); this.curPut = null; @@ -952,24 +996,8 @@ public void defaultWriteObject() throws IOException { if (fieldsWritten) { throw new NotActiveException("not in writeObject invocation or fields already written"); } - // Write field values in the same format as writeFields (for compatibility with readFields) - Class[] fieldTypes = slotsInfo.getPutFieldTypes(); - if (fieldTypes != null && fieldTypes.length > 0) { - // Write using putField format (for compatibility with readFields) - ObjectIntMap fieldIndexMap = slotsInfo.getFieldIndexMap(); - MetaSharedLayerSerializerBase slotsSerializer = slotsInfo.getSlotsSerializer(); - Object[] vals = - slotsSerializer.getFieldValuesForPutFields( - targetObject, fieldIndexMap, fieldTypes.length); - for (int i = 0; i < vals.length; i++) { - Class fieldType = fieldTypes[i]; - Object value = vals[i]; - writePutFieldValue(buffer, fieldType, value); - } - } else { - // No custom writeObject/readObject, use normal serialization - slotsInfo.getSlotsSerializer().write(buffer, targetObject); - } + // Write fields using MetaShare serialization (layer meta already written by write()) + slotsInfo.getSlotsSerializer().writeFieldsOnly(buffer, targetObject); fieldsWritten = true; } @@ -1271,12 +1299,9 @@ public GetField readFields() throws IOException { if (fieldsRead) { throw new NotActiveException("not in readObject invocation or fields already read"); } - // Read field values directly by their types - Class[] fieldTypes = slotsInfo.getPutFieldTypes(); - Object[] vals = getField.vals; - for (int i = 0; i < vals.length; i++) { - vals[i] = readPutFieldValue(buffer, fieldTypes[i]); - } + // Read field values using MetaShare serialization + Object[] vals = slotsInfo.getCurrentReadSerializer().readFieldValues(buffer); + System.arraycopy(vals, 0, getField.vals, 0, vals.length); fieldsRead = true; return getField; } @@ -1314,23 +1339,8 @@ public void defaultReadObject() throws IOException, ClassNotFoundException { if (fieldsRead) { throw new NotActiveException("not in readObject invocation or fields already read"); } - // Get the serializer that was set by getReadSerializer() - has correct schema from stream - MetaSharedLayerSerializerBase serializer = slotsInfo.getCurrentReadSerializer(); - // Read field values in the same format as writeFields, but set to actual class fields - Class[] fieldTypes = slotsInfo.getPutFieldTypes(); - if (fieldTypes != null && fieldTypes.length > 0) { - // Read using putField format (for compatibility with writeFields) - ObjectIntMap fieldIndexMap = slotsInfo.getFieldIndexMap(); - Object[] vals = new Object[fieldTypes.length]; - for (int i = 0; i < vals.length; i++) { - vals[i] = readPutFieldValue(buffer, fieldTypes[i]); - } - // Now set matching fields on the target object - serializer.setFieldValuesFromPutFields(targetObject, fieldIndexMap, vals); - } else { - // No custom writeObject/readObject, use normal serialization - serializer.readAndSetFields(buffer, targetObject); - } + // Read fields using MetaShare serialization (layer meta already read by getReadSerializer()) + slotsInfo.getCurrentReadSerializer().readAndSetFields(buffer, targetObject); fieldsRead = true; } From 7b4d341e1f6fdc2ba1aad868870f7093d30bc32c Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 17 Jan 2026 12:01:10 +0800 Subject: [PATCH 19/44] add more tests for object stream serializer --- .../ObjectStreamSerializerTest.java | 834 ++++++++++++++++++ 1 file changed, 834 insertions(+) diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectStreamSerializerTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectStreamSerializerTest.java index ccd545c0c3..c91c824c25 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectStreamSerializerTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/ObjectStreamSerializerTest.java @@ -44,10 +44,12 @@ import lombok.EqualsAndHashCode; import org.apache.fory.Fory; import org.apache.fory.ForyTestBase; +import org.apache.fory.config.CompatibleMode; import org.apache.fory.config.Language; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.util.Preconditions; import org.testng.Assert; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; public class ObjectStreamSerializerTest extends ForyTestBase { @@ -445,4 +447,836 @@ public void testWriteObjectReplaceCopy(Fory fory) throws MalformedURLException { // TODO(chaokunyang) add `readObjectNoData` test for class inheritance change. // @Test public void testReadObjectNoData() {} + + // ==================== Schema Evolution Tests ==================== + + @DataProvider + public static Object[][] compatibleModeProvider() { + return new Object[][] {{CompatibleMode.COMPATIBLE}, {CompatibleMode.SCHEMA_CONSISTENT}}; + } + + // ==================== Layer Count Evolution Tests ==================== + + /** Base class for layer count tests - single layer. */ + @EqualsAndHashCode + public static class SingleLayerClass implements Serializable { + private String name; + private int value; + + public SingleLayerClass() {} + + public SingleLayerClass(String name, int value) { + this.name = name; + this.value = value; + } + + private void writeObject(ObjectOutputStream s) throws IOException { + s.defaultWriteObject(); + } + + private void readObject(ObjectInputStream s) throws IOException, ClassNotFoundException { + s.defaultReadObject(); + } + } + + /** Two-layer class hierarchy - parent. */ + @EqualsAndHashCode + public static class TwoLayerParent implements Serializable { + protected String parentName; + protected int parentValue; + + public TwoLayerParent() {} + + public TwoLayerParent(String parentName, int parentValue) { + this.parentName = parentName; + this.parentValue = parentValue; + } + + private void writeObject(ObjectOutputStream s) throws IOException { + s.defaultWriteObject(); + } + + private void readObject(ObjectInputStream s) throws IOException, ClassNotFoundException { + s.defaultReadObject(); + } + } + + /** Two-layer class hierarchy - child. */ + @EqualsAndHashCode(callSuper = true) + public static class TwoLayerChild extends TwoLayerParent { + private String childName; + private double childValue; + + public TwoLayerChild() {} + + public TwoLayerChild(String parentName, int parentValue, String childName, double childValue) { + super(parentName, parentValue); + this.childName = childName; + this.childValue = childValue; + } + + private void writeObject(ObjectOutputStream s) throws IOException { + s.defaultWriteObject(); + } + + private void readObject(ObjectInputStream s) throws IOException, ClassNotFoundException { + s.defaultReadObject(); + } + } + + /** Three-layer class hierarchy - grandchild. */ + @EqualsAndHashCode(callSuper = true) + public static class ThreeLayerGrandchild extends TwoLayerChild { + private String grandchildName; + private long grandchildValue; + + public ThreeLayerGrandchild() {} + + public ThreeLayerGrandchild( + String parentName, + int parentValue, + String childName, + double childValue, + String grandchildName, + long grandchildValue) { + super(parentName, parentValue, childName, childValue); + this.grandchildName = grandchildName; + this.grandchildValue = grandchildValue; + } + + private void writeObject(ObjectOutputStream s) throws IOException { + s.defaultWriteObject(); + } + + private void readObject(ObjectInputStream s) throws IOException, ClassNotFoundException { + s.defaultReadObject(); + } + } + + @Test(dataProvider = "javaFory") + public void testSingleLayerSerialization(Fory fory) { + fory.registerSerializer( + SingleLayerClass.class, new ObjectStreamSerializer(fory, SingleLayerClass.class)); + SingleLayerClass obj = new SingleLayerClass("test", 42); + serDeCheckSerializer(fory, obj, "ObjectStreamSerializer"); + } + + @Test(dataProvider = "javaFory") + public void testTwoLayerSerialization(Fory fory) { + fory.registerSerializer( + TwoLayerChild.class, new ObjectStreamSerializer(fory, TwoLayerChild.class)); + TwoLayerChild obj = new TwoLayerChild("parent", 10, "child", 3.14); + serDeCheckSerializer(fory, obj, "ObjectStreamSerializer"); + } + + @Test(dataProvider = "javaFory") + public void testThreeLayerSerialization(Fory fory) { + fory.registerSerializer( + ThreeLayerGrandchild.class, new ObjectStreamSerializer(fory, ThreeLayerGrandchild.class)); + ThreeLayerGrandchild obj = + new ThreeLayerGrandchild("parent", 10, "child", 3.14, "grandchild", 100L); + serDeCheckSerializer(fory, obj, "ObjectStreamSerializer"); + } + + // ==================== Field Schema Evolution Tests ==================== + + /** Class with fields that may evolve - version 1. */ + @EqualsAndHashCode + public static class FieldEvolutionV1 implements Serializable { + private String name; + private int oldField; + + public FieldEvolutionV1() {} + + public FieldEvolutionV1(String name, int oldField) { + this.name = name; + this.oldField = oldField; + } + + private void writeObject(ObjectOutputStream s) throws IOException { + s.defaultWriteObject(); + } + + private void readObject(ObjectInputStream s) throws IOException, ClassNotFoundException { + s.defaultReadObject(); + } + } + + /** Class with additional fields - version 2 (reader has more fields). */ + @EqualsAndHashCode + public static class FieldEvolutionV2 implements Serializable { + private String name; + private int oldField; + private String newField; // Added field + private double extraField; // Added field + + public FieldEvolutionV2() {} + + public FieldEvolutionV2(String name, int oldField, String newField, double extraField) { + this.name = name; + this.oldField = oldField; + this.newField = newField; + this.extraField = extraField; + } + + private void writeObject(ObjectOutputStream s) throws IOException { + s.defaultWriteObject(); + } + + private void readObject(ObjectInputStream s) throws IOException, ClassNotFoundException { + s.defaultReadObject(); + } + } + + @Test(dataProvider = "javaFory") + public void testFieldEvolutionSameSchema(Fory fory) { + fory.registerSerializer( + FieldEvolutionV1.class, new ObjectStreamSerializer(fory, FieldEvolutionV1.class)); + FieldEvolutionV1 obj = new FieldEvolutionV1("test", 42); + serDeCheckSerializer(fory, obj, "ObjectStreamSerializer"); + } + + // ==================== PutFields/DefaultReadObject Tests ==================== + + /** Class that uses putFields for writing. */ + @EqualsAndHashCode + public static class PutFieldsWriter implements Serializable { + private String actualField; + private int actualValue; + + // Define serialPersistentFields to control serialization format + private static final ObjectStreamField[] serialPersistentFields = { + new ObjectStreamField("serializedName", String.class), + new ObjectStreamField("serializedValue", Integer.TYPE) + }; + + public PutFieldsWriter() {} + + public PutFieldsWriter(String actualField, int actualValue) { + this.actualField = actualField; + this.actualValue = actualValue; + } + + private void writeObject(ObjectOutputStream s) throws IOException { + ObjectOutputStream.PutField fields = s.putFields(); + fields.put("serializedName", actualField); + fields.put("serializedValue", actualValue); + s.writeFields(); + } + + private void readObject(ObjectInputStream s) throws IOException, ClassNotFoundException { + // Using defaultReadObject - expects fields named in serialPersistentFields + s.defaultReadObject(); + } + } + + /** Class that uses defaultWriteObject and getFields for reading. */ + @EqualsAndHashCode + public static class DefaultWriteGetFieldsReader implements Serializable { + private String name; + private int value; + private transient String computed; + + public DefaultWriteGetFieldsReader() {} + + public DefaultWriteGetFieldsReader(String name, int value) { + this.name = name; + this.value = value; + this.computed = name + ":" + value; + } + + private void writeObject(ObjectOutputStream s) throws IOException { + s.defaultWriteObject(); + } + + private void readObject(ObjectInputStream s) throws IOException, ClassNotFoundException { + ObjectInputStream.GetField fields = s.readFields(); + name = (String) fields.get("name", null); + value = fields.get("value", 0); + computed = name + ":" + value; + } + } + + @Test(dataProvider = "compatibleModeProvider") + public void testPutFieldsWithDefaultReadObject(CompatibleMode compatible) { + Fory fory = + Fory.builder() + .withLanguage(Language.JAVA) + .requireClassRegistration(false) + .withRefTracking(true) + .withCompatibleMode(compatible) + .build(); + fory.registerSerializer( + PutFieldsWriter.class, new ObjectStreamSerializer(fory, PutFieldsWriter.class)); + + PutFieldsWriter obj = new PutFieldsWriter("testName", 123); + MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(256); + fory.serialize(buffer, obj); + + // Deserialize - defaultReadObject should handle the putFields format + buffer.readerIndex(0); + PutFieldsWriter result = (PutFieldsWriter) fory.deserialize(buffer); + // Note: actualField/actualValue won't be populated directly since + // serialPersistentFields maps to different names + } + + @Test(dataProvider = "compatibleModeProvider") + public void testDefaultWriteObjectWithGetFields(CompatibleMode compatible) { + Fory fory = + Fory.builder() + .withLanguage(Language.JAVA) + .requireClassRegistration(false) + .withRefTracking(true) + .withCompatibleMode(compatible) + .build(); + fory.registerSerializer( + DefaultWriteGetFieldsReader.class, + new ObjectStreamSerializer(fory, DefaultWriteGetFieldsReader.class)); + + DefaultWriteGetFieldsReader obj = new DefaultWriteGetFieldsReader("test", 42); + assertEquals(obj.computed, "test:42"); + + MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(256); + fory.serialize(buffer, obj); + + buffer.readerIndex(0); + DefaultWriteGetFieldsReader result = (DefaultWriteGetFieldsReader) fory.deserialize(buffer); + assertEquals(result.name, "test"); + assertEquals(result.value, 42); + assertEquals(result.computed, "test:42"); + } + + // ==================== SerialPersistentFields Inconsistency Tests ==================== + + /** Writer class with specific serialPersistentFields. */ + @EqualsAndHashCode + public static class SerialFieldsWriter implements Serializable { + private String data; + private int number; + + // Define custom serialPersistentFields + private static final ObjectStreamField[] serialPersistentFields = { + new ObjectStreamField("customData", String.class), + new ObjectStreamField("customNumber", Integer.TYPE), + new ObjectStreamField("extraField", Long.TYPE) // Extra field not in actual class + }; + + public SerialFieldsWriter() {} + + public SerialFieldsWriter(String data, int number) { + this.data = data; + this.number = number; + } + + private void writeObject(ObjectOutputStream s) throws IOException { + ObjectOutputStream.PutField fields = s.putFields(); + fields.put("customData", data); + fields.put("customNumber", number); + fields.put("extraField", 999L); + s.writeFields(); + } + + private void readObject(ObjectInputStream s) throws IOException, ClassNotFoundException { + ObjectInputStream.GetField fields = s.readFields(); + data = (String) fields.get("customData", null); + number = fields.get("customNumber", 0); + // extraField is read but not stored + } + } + + @Test(dataProvider = "compatibleModeProvider") + public void testSerialPersistentFieldsInconsistentWithReader(CompatibleMode compatible) { + Fory fory = + Fory.builder() + .withLanguage(Language.JAVA) + .requireClassRegistration(false) + .withRefTracking(true) + .withCompatibleMode(compatible) + .build(); + fory.registerSerializer( + SerialFieldsWriter.class, new ObjectStreamSerializer(fory, SerialFieldsWriter.class)); + + SerialFieldsWriter obj = new SerialFieldsWriter("testData", 42); + MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(256); + fory.serialize(buffer, obj); + + buffer.readerIndex(0); + SerialFieldsWriter result = (SerialFieldsWriter) fory.deserialize(buffer); + assertEquals(result.data, "testData"); + assertEquals(result.number, 42); + } + + /** Class that writes with defaultWriteObject but has serialPersistentFields defined. */ + @EqualsAndHashCode + public static class MixedSerializationClass implements Serializable { + private String name; + private int value; + + // serialPersistentFields different from actual fields + private static final ObjectStreamField[] serialPersistentFields = { + new ObjectStreamField("name", String.class), + new ObjectStreamField("value", Integer.TYPE), + new ObjectStreamField("nonExistentField", Double.TYPE) + }; + + public MixedSerializationClass() {} + + public MixedSerializationClass(String name, int value) { + this.name = name; + this.value = value; + } + + private void writeObject(ObjectOutputStream s) throws IOException { + ObjectOutputStream.PutField fields = s.putFields(); + fields.put("name", name); + fields.put("value", value); + fields.put("nonExistentField", 3.14); + s.writeFields(); + } + + private void readObject(ObjectInputStream s) throws IOException, ClassNotFoundException { + ObjectInputStream.GetField fields = s.readFields(); + name = (String) fields.get("name", null); + value = fields.get("value", 0); + // nonExistentField is ignored + } + } + + @Test(dataProvider = "compatibleModeProvider") + public void testMixedSerialization(CompatibleMode compatible) { + Fory fory = + Fory.builder() + .withLanguage(Language.JAVA) + .requireClassRegistration(false) + .withRefTracking(true) + .withCompatibleMode(compatible) + .build(); + fory.registerSerializer( + MixedSerializationClass.class, + new ObjectStreamSerializer(fory, MixedSerializationClass.class)); + + MixedSerializationClass obj = new MixedSerializationClass("test", 42); + MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(256); + fory.serialize(buffer, obj); + + buffer.readerIndex(0); + MixedSerializationClass result = (MixedSerializationClass) fory.deserialize(buffer); + assertEquals(result.name, "test"); + assertEquals(result.value, 42); + } + + // ==================== Complex Hierarchy Schema Evolution Tests ==================== + + /** Parent class with putFields writer. */ + public static class HierarchyParentPutFields implements Serializable { + protected String parentData; + + private static final ObjectStreamField[] serialPersistentFields = { + new ObjectStreamField("parentData", String.class), + new ObjectStreamField("extraParentField", Integer.TYPE) + }; + + public HierarchyParentPutFields() {} + + public HierarchyParentPutFields(String parentData) { + this.parentData = parentData; + } + + private void writeObject(ObjectOutputStream s) throws IOException { + ObjectOutputStream.PutField fields = s.putFields(); + fields.put("parentData", parentData); + fields.put("extraParentField", 100); + s.writeFields(); + } + + private void readObject(ObjectInputStream s) throws IOException, ClassNotFoundException { + ObjectInputStream.GetField fields = s.readFields(); + parentData = (String) fields.get("parentData", null); + } + } + + /** Child class with defaultWriteObject. */ + @EqualsAndHashCode(callSuper = false) + public static class HierarchyChildDefault extends HierarchyParentPutFields { + private String childData; + private int childValue; + + public HierarchyChildDefault() {} + + public HierarchyChildDefault(String parentData, String childData, int childValue) { + super(parentData); + this.childData = childData; + this.childValue = childValue; + } + + private void writeObject(ObjectOutputStream s) throws IOException { + s.defaultWriteObject(); + } + + private void readObject(ObjectInputStream s) throws IOException, ClassNotFoundException { + s.defaultReadObject(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + HierarchyChildDefault that = (HierarchyChildDefault) o; + return childValue == that.childValue + && java.util.Objects.equals(parentData, that.parentData) + && java.util.Objects.equals(childData, that.childData); + } + + @Override + public int hashCode() { + return java.util.Objects.hash(parentData, childData, childValue); + } + } + + @Test(dataProvider = "compatibleModeProvider") + public void testHierarchyMixedSerialization(CompatibleMode compatible) { + Fory fory = + Fory.builder() + .withLanguage(Language.JAVA) + .requireClassRegistration(false) + .withRefTracking(true) + .withCompatibleMode(compatible) + .build(); + fory.registerSerializer( + HierarchyChildDefault.class, new ObjectStreamSerializer(fory, HierarchyChildDefault.class)); + + HierarchyChildDefault obj = new HierarchyChildDefault("parent", "child", 42); + MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(256); + fory.serialize(buffer, obj); + + buffer.readerIndex(0); + HierarchyChildDefault result = (HierarchyChildDefault) fory.deserialize(buffer); + assertEquals(result.parentData, "parent"); + assertEquals(result.childData, "child"); + assertEquals(result.childValue, 42); + } + + // ==================== Cross-Fory Instance Schema Tests ==================== + + @Test(dataProvider = "compatibleModeProvider") + public void testCrossForyInstanceSerialization(CompatibleMode compatible) { + // Writer Fory instance + Fory writerFory = + Fory.builder() + .withLanguage(Language.JAVA) + .requireClassRegistration(false) + .withRefTracking(true) + .withCompatibleMode(compatible) + .build(); + writerFory.registerSerializer( + MixedSerializationClass.class, + new ObjectStreamSerializer(writerFory, MixedSerializationClass.class)); + + // Reader Fory instance (separate instance) + Fory readerFory = + Fory.builder() + .withLanguage(Language.JAVA) + .requireClassRegistration(false) + .withRefTracking(true) + .withCompatibleMode(compatible) + .build(); + readerFory.registerSerializer( + MixedSerializationClass.class, + new ObjectStreamSerializer(readerFory, MixedSerializationClass.class)); + + // Serialize with writer + MixedSerializationClass obj = new MixedSerializationClass("crossTest", 99); + byte[] bytes = writerFory.serialize(obj); + + // Deserialize with reader + MixedSerializationClass result = (MixedSerializationClass) readerFory.deserialize(bytes); + assertEquals(result.name, "crossTest"); + assertEquals(result.value, 99); + } + + // ==================== Default Value Tests ==================== + + /** Class to test default values when fields are missing. */ + @EqualsAndHashCode + public static class DefaultValueClass implements Serializable { + private String stringField; + private int intField; + private boolean boolField; + private double doubleField; + private Object objectField; + + public DefaultValueClass() {} + + public DefaultValueClass( + String stringField, + int intField, + boolean boolField, + double doubleField, + Object objectField) { + this.stringField = stringField; + this.intField = intField; + this.boolField = boolField; + this.doubleField = doubleField; + this.objectField = objectField; + } + + private void writeObject(ObjectOutputStream s) throws IOException { + s.defaultWriteObject(); + } + + private void readObject(ObjectInputStream s) throws IOException, ClassNotFoundException { + ObjectInputStream.GetField fields = s.readFields(); + stringField = (String) fields.get("stringField", "defaultString"); + intField = fields.get("intField", -1); + boolField = fields.get("boolField", true); + doubleField = fields.get("doubleField", 99.9); + objectField = fields.get("objectField", "defaultObject"); + } + } + + @Test(dataProvider = "compatibleModeProvider") + public void testDefaultValues(CompatibleMode compatible) { + Fory fory = + Fory.builder() + .withLanguage(Language.JAVA) + .requireClassRegistration(false) + .withRefTracking(true) + .withCompatibleMode(compatible) + .build(); + fory.registerSerializer( + DefaultValueClass.class, new ObjectStreamSerializer(fory, DefaultValueClass.class)); + + DefaultValueClass obj = new DefaultValueClass("test", 42, false, 3.14, "objValue"); + MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(256); + fory.serialize(buffer, obj); + + buffer.readerIndex(0); + DefaultValueClass result = (DefaultValueClass) fory.deserialize(buffer); + assertEquals(result.stringField, "test"); + assertEquals(result.intField, 42); + assertEquals(result.boolField, false); + assertEquals(result.doubleField, 3.14, 0.001); + assertEquals(result.objectField, "objValue"); + } + + // ==================== Nested Object Tests ==================== + + /** Nested serializable class. */ + @EqualsAndHashCode + public static class NestedClass implements Serializable { + private String nestedValue; + + public NestedClass() {} + + public NestedClass(String nestedValue) { + this.nestedValue = nestedValue; + } + + private void writeObject(ObjectOutputStream s) throws IOException { + s.defaultWriteObject(); + } + + private void readObject(ObjectInputStream s) throws IOException, ClassNotFoundException { + s.defaultReadObject(); + } + } + + /** Container class with nested objects. */ + @EqualsAndHashCode + public static class ContainerClass implements Serializable { + private String containerName; + private NestedClass nested; + private List nestedList; + + public ContainerClass() {} + + public ContainerClass(String containerName, NestedClass nested, List nestedList) { + this.containerName = containerName; + this.nested = nested; + this.nestedList = nestedList; + } + + private void writeObject(ObjectOutputStream s) throws IOException { + s.defaultWriteObject(); + } + + private void readObject(ObjectInputStream s) throws IOException, ClassNotFoundException { + s.defaultReadObject(); + } + } + + @Test(dataProvider = "compatibleModeProvider") + public void testNestedObjectSerialization(CompatibleMode compatible) { + Fory fory = + Fory.builder() + .withLanguage(Language.JAVA) + .requireClassRegistration(false) + .withRefTracking(true) + .withCompatibleMode(compatible) + .build(); + fory.registerSerializer(NestedClass.class, new ObjectStreamSerializer(fory, NestedClass.class)); + fory.registerSerializer( + ContainerClass.class, new ObjectStreamSerializer(fory, ContainerClass.class)); + + NestedClass nested = new NestedClass("nestedValue"); + List nestedList = new ArrayList<>(); + nestedList.add(new NestedClass("list1")); + nestedList.add(new NestedClass("list2")); + ContainerClass obj = new ContainerClass("container", nested, nestedList); + + MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(512); + fory.serialize(buffer, obj); + + buffer.readerIndex(0); + ContainerClass result = (ContainerClass) fory.deserialize(buffer); + assertEquals(result.containerName, "container"); + assertEquals(result.nested.nestedValue, "nestedValue"); + assertEquals(result.nestedList.size(), 2); + assertEquals(result.nestedList.get(0).nestedValue, "list1"); + assertEquals(result.nestedList.get(1).nestedValue, "list2"); + } + + // ==================== Circular Reference in Custom Serialization ==================== + + /** Class with potential circular reference. */ + public static class CircularRefClass implements Serializable { + private String name; + private CircularRefClass reference; + + public CircularRefClass() {} + + public CircularRefClass(String name) { + this.name = name; + } + + public void setReference(CircularRefClass reference) { + this.reference = reference; + } + + private void writeObject(ObjectOutputStream s) throws IOException { + s.defaultWriteObject(); + } + + private void readObject(ObjectInputStream s) throws IOException, ClassNotFoundException { + s.defaultReadObject(); + } + } + + @Test(dataProvider = "compatibleModeProvider") + public void testCircularReferenceInCustomSerialization(CompatibleMode compatible) { + Fory fory = + Fory.builder() + .withLanguage(Language.JAVA) + .requireClassRegistration(false) + .withRefTracking(true) + .withCompatibleMode(compatible) + .build(); + fory.registerSerializer( + CircularRefClass.class, new ObjectStreamSerializer(fory, CircularRefClass.class)); + + CircularRefClass obj1 = new CircularRefClass("obj1"); + CircularRefClass obj2 = new CircularRefClass("obj2"); + obj1.setReference(obj2); + obj2.setReference(obj1); // Circular reference + + MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(512); + fory.serialize(buffer, obj1); + + buffer.readerIndex(0); + CircularRefClass result = (CircularRefClass) fory.deserialize(buffer); + assertEquals(result.name, "obj1"); + assertEquals(result.reference.name, "obj2"); + assertSame(result.reference.reference, result); // Verify circular reference preserved + } + + // ==================== All Primitive Types Test ==================== + + /** Class with all primitive types using putFields/getFields. */ + @EqualsAndHashCode + public static class AllPrimitivesClass implements Serializable { + private byte byteVal; + private short shortVal; + private int intVal; + private long longVal; + private float floatVal; + private double doubleVal; + private char charVal; + private boolean boolVal; + + public AllPrimitivesClass() {} + + public AllPrimitivesClass( + byte byteVal, + short shortVal, + int intVal, + long longVal, + float floatVal, + double doubleVal, + char charVal, + boolean boolVal) { + this.byteVal = byteVal; + this.shortVal = shortVal; + this.intVal = intVal; + this.longVal = longVal; + this.floatVal = floatVal; + this.doubleVal = doubleVal; + this.charVal = charVal; + this.boolVal = boolVal; + } + + private void writeObject(ObjectOutputStream s) throws IOException { + ObjectOutputStream.PutField fields = s.putFields(); + fields.put("byteVal", byteVal); + fields.put("shortVal", shortVal); + fields.put("intVal", intVal); + fields.put("longVal", longVal); + fields.put("floatVal", floatVal); + fields.put("doubleVal", doubleVal); + fields.put("charVal", charVal); + fields.put("boolVal", boolVal); + s.writeFields(); + } + + private void readObject(ObjectInputStream s) throws IOException, ClassNotFoundException { + ObjectInputStream.GetField fields = s.readFields(); + byteVal = fields.get("byteVal", (byte) 0); + shortVal = fields.get("shortVal", (short) 0); + intVal = fields.get("intVal", 0); + longVal = fields.get("longVal", 0L); + floatVal = fields.get("floatVal", 0.0f); + doubleVal = fields.get("doubleVal", 0.0); + charVal = fields.get("charVal", '\0'); + boolVal = fields.get("boolVal", false); + } + } + + @Test(dataProvider = "compatibleModeProvider") + public void testAllPrimitiveTypes(CompatibleMode compatible) { + Fory fory = + Fory.builder() + .withLanguage(Language.JAVA) + .requireClassRegistration(false) + .withRefTracking(true) + .withCompatibleMode(compatible) + .build(); + fory.registerSerializer( + AllPrimitivesClass.class, new ObjectStreamSerializer(fory, AllPrimitivesClass.class)); + + AllPrimitivesClass obj = + new AllPrimitivesClass((byte) 1, (short) 2, 3, 4L, 5.5f, 6.6, 'A', true); + + MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(256); + fory.serialize(buffer, obj); + + buffer.readerIndex(0); + AllPrimitivesClass result = (AllPrimitivesClass) fory.deserialize(buffer); + assertEquals(result.byteVal, (byte) 1); + assertEquals(result.shortVal, (short) 2); + assertEquals(result.intVal, 3); + assertEquals(result.longVal, 4L); + assertEquals(result.floatVal, 5.5f, 0.001f); + assertEquals(result.doubleVal, 6.6, 0.001); + assertEquals(result.charVal, 'A'); + assertEquals(result.boolVal, true); + } } From 4ec6af487decb8c32c056aa88f825a8a082f6421 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 17 Jan 2026 12:07:17 +0800 Subject: [PATCH 20/44] moke latest jdk tests as a module of fory --- AGENTS.md | 4 ---- ci/format.sh | 2 +- ci/release.py | 3 +-- ci/run_ci.sh | 10 --------- ci/tasks/java.py | 17 -------------- integration_tests/README.md | 1 - .../fory-latest-jdk-tests}/pom.xml | 22 ++++++------------- .../ImmutableCollectionSerializersTest.java | 0 .../fory/integration_tests/RecordRowTest.java | 0 .../RecordSerializersTest.java | 0 .../fory/integration_tests/Records.java | 0 .../fory/integration_tests/TestUtils.java | 0 java/pom.xml | 1 + 13 files changed, 10 insertions(+), 50 deletions(-) rename {integration_tests/latest_jdk_tests => java/fory-latest-jdk-tests}/pom.xml (78%) rename {integration_tests/latest_jdk_tests => java/fory-latest-jdk-tests}/src/test/java/org/apache/fory/integration_tests/ImmutableCollectionSerializersTest.java (100%) rename {integration_tests/latest_jdk_tests => java/fory-latest-jdk-tests}/src/test/java/org/apache/fory/integration_tests/RecordRowTest.java (100%) rename {integration_tests/latest_jdk_tests => java/fory-latest-jdk-tests}/src/test/java/org/apache/fory/integration_tests/RecordSerializersTest.java (100%) rename {integration_tests/latest_jdk_tests => java/fory-latest-jdk-tests}/src/test/java/org/apache/fory/integration_tests/Records.java (100%) rename {integration_tests/latest_jdk_tests => java/fory-latest-jdk-tests}/src/test/java/org/apache/fory/integration_tests/TestUtils.java (100%) diff --git a/AGENTS.md b/AGENTS.md index 1ab1a72655..f93c714021 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -296,9 +296,6 @@ it_dir=$(pwd) # Run graalvm tests cd $it_dir/graalvm_tests && mvn -T16 -DskipTests=true -Pnative package && target/main -# Run latest_jdk_tests -cd $it_dir/latest_jdk_tests && mvn -T16 test - # Run JDK compatibility tests cd $it_dir/jdk_compatibility_tests && mvn -T16 test @@ -499,7 +496,6 @@ Fory rust provides macro-based serialization and deserialization. Fory rust cons - Note that fory use codegen to support graalvm instead of reflection, fory don't use `reflect-config.json` for serialization, this is the core advantage of compared to graalvm JDK serialization. - **jdk_compatibility_tests**: test suite for fory serialization compatibility between multiple JDK versions -- **latest_jdk_tests**: test suite for `jdk17+` versions ## Key Development Guidelines diff --git a/ci/format.sh b/ci/format.sh index 46e806cc3b..2ded19e0b3 100755 --- a/ci/format.sh +++ b/ci/format.sh @@ -143,7 +143,7 @@ format_java() { cd "$ROOT/benchmarks/java_benchmark" mvn -T10 --no-transfer-progress spotless:apply cd "$ROOT/integration_tests" - dirs=("graalvm_tests" "jdk_compatibility_tests" "latest_jdk_tests") + dirs=("graalvm_tests" "jdk_compatibility_tests") for d in "${dirs[@]}" ; do pushd "$d" mvn -T10 --no-transfer-progress spotless:apply diff --git a/ci/release.py b/ci/release.py index e050fa9253..a19bffe793 100644 --- a/ci/release.py +++ b/ci/release.py @@ -186,8 +186,6 @@ def bump_java_version(new_version): "integration_tests/graalvm_tests", "integration_tests/jdk_compatibility_tests", "integration_tests/jpms_tests", - "integration_tests/latest_jdk_tests", - "integration_tests/latest_jdk_tests", "benchmarks/java_benchmark", "java/fory-core", "java/fory-format", @@ -195,6 +193,7 @@ def bump_java_version(new_version): "java/fory-extensions", "java/fory-test-core", "java/fory-testsuite", + "java/fory-latest-jdk-tests", ]: _bump_version(p, "pom.xml", new_version, _update_pom_parent_version) # mvn versions:set too slow diff --git a/ci/run_ci.sh b/ci/run_ci.sh index a408e7a8cd..9a70b49ce1 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -182,12 +182,6 @@ integration_tests() { echo "benchmark tests" cd "$ROOT"/benchmarks/java_benchmark mvn -T10 -B --no-transfer-progress clean test install -Pjmh - echo "Start latest jdk tests" - cd "$ROOT"/integration_tests/latest_jdk_tests - echo "latest_jdk_tests: JDK 21" - export JAVA_HOME="$ROOT/zulu21.28.85-ca-jdk21.0.0-linux_x64" - export PATH=$JAVA_HOME/bin:$PATH - mvn -T10 -B --no-transfer-progress clean test echo "Start JPMS tests" cd "$ROOT"/integration_tests/jpms_tests mvn -T10 -B --no-transfer-progress clean compile @@ -220,10 +214,6 @@ jdk17_plus_tests() { exit $testcode fi echo "Executing fory java tests succeeds" - echo "Executing latest_jdk_tests" - cd "$ROOT"/integration_tests/latest_jdk_tests - mvn -T10 -B --no-transfer-progress clean test - echo "Executing latest_jdk_tests succeeds" } kotlin_tests() { diff --git a/ci/tasks/java.py b/ci/tasks/java.py index d6abbb1d26..4a740fd22d 100644 --- a/ci/tasks/java.py +++ b/ci/tasks/java.py @@ -175,12 +175,6 @@ def run_jdk17_plus(java_version="17"): common.exec_cmd("mvn -T10 --batch-mode --no-transfer-progress install") logging.info("Executing fory java tests succeeds") - logging.info("Executing latest_jdk_tests") - - common.cd_project_subdir("integration_tests/latest_jdk_tests") - common.exec_cmd("mvn -T10 -B --no-transfer-progress clean test") - - logging.info("Executing latest_jdk_tests succeeds") def run_windows_java21(): @@ -218,17 +212,6 @@ def run_integration_tests(): common.cd_project_subdir("benchmarks/java_benchmark") common.exec_cmd("mvn -T10 -B --no-transfer-progress clean test install -Pjmh") - logging.info("Start latest jdk tests") - common.cd_project_subdir("integration_tests/latest_jdk_tests") - logging.info("latest_jdk_tests: JDK 21") - - # Set Java 21 as the current JDK - java_home = os.path.join(common.PROJECT_ROOT_DIR, JDKS["21"]) - os.environ["JAVA_HOME"] = java_home - os.environ["PATH"] = f"{java_home}/bin:{os.environ.get('PATH', '')}" - - common.exec_cmd("mvn -T10 -B --no-transfer-progress clean test") - logging.info("Start JPMS tests") common.cd_project_subdir("integration_tests/jpms_tests") common.exec_cmd("mvn -T10 -B --no-transfer-progress clean compile") diff --git a/integration_tests/README.md b/integration_tests/README.md index e764c76315..845c07740c 100644 --- a/integration_tests/README.md +++ b/integration_tests/README.md @@ -1,7 +1,6 @@ # Integration tests for fory: - [jdk_compatibility_tests](jdk_compatibility_tests): test fory compatibility across multiple jdk versions. -- [latest_jdk_tests](latest_jdk_tests): test latest jdk. - [graalvm_tests](graalvm_tests): test graalvm native image support. - [jpms_tests](jpms_tests): test JPMS module names. - [cpython_benchmark](cpython_benchmark): fory CPython microbenchmark. diff --git a/integration_tests/latest_jdk_tests/pom.xml b/java/fory-latest-jdk-tests/pom.xml similarity index 78% rename from integration_tests/latest_jdk_tests/pom.xml rename to java/fory-latest-jdk-tests/pom.xml index 29d78b0d6e..000b1adcfe 100644 --- a/integration_tests/latest_jdk_tests/pom.xml +++ b/java/fory-latest-jdk-tests/pom.xml @@ -26,15 +26,16 @@ org.apache.fory fory-parent 0.15.0-SNAPSHOT - ../../java 4.0.0 - latest_jdk_tests + fory-latest-jdk-tests 17 17 - UTF-8 + true + true + ${basedir}/.. @@ -58,19 +59,10 @@ - com.diffplug.spotless - spotless-maven-plugin - 2.5.0 + org.apache.maven.plugins + maven-deploy-plugin - - - 1.7 - - - - **/generated/*.java - - + true diff --git a/integration_tests/latest_jdk_tests/src/test/java/org/apache/fory/integration_tests/ImmutableCollectionSerializersTest.java b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ImmutableCollectionSerializersTest.java similarity index 100% rename from integration_tests/latest_jdk_tests/src/test/java/org/apache/fory/integration_tests/ImmutableCollectionSerializersTest.java rename to java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/ImmutableCollectionSerializersTest.java diff --git a/integration_tests/latest_jdk_tests/src/test/java/org/apache/fory/integration_tests/RecordRowTest.java b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/RecordRowTest.java similarity index 100% rename from integration_tests/latest_jdk_tests/src/test/java/org/apache/fory/integration_tests/RecordRowTest.java rename to java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/RecordRowTest.java diff --git a/integration_tests/latest_jdk_tests/src/test/java/org/apache/fory/integration_tests/RecordSerializersTest.java b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/RecordSerializersTest.java similarity index 100% rename from integration_tests/latest_jdk_tests/src/test/java/org/apache/fory/integration_tests/RecordSerializersTest.java rename to java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/RecordSerializersTest.java diff --git a/integration_tests/latest_jdk_tests/src/test/java/org/apache/fory/integration_tests/Records.java b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/Records.java similarity index 100% rename from integration_tests/latest_jdk_tests/src/test/java/org/apache/fory/integration_tests/Records.java rename to java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/Records.java diff --git a/integration_tests/latest_jdk_tests/src/test/java/org/apache/fory/integration_tests/TestUtils.java b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/TestUtils.java similarity index 100% rename from integration_tests/latest_jdk_tests/src/test/java/org/apache/fory/integration_tests/TestUtils.java rename to java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/TestUtils.java diff --git a/java/pom.xml b/java/pom.xml index 52c826f223..e573fbe969 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -96,6 +96,7 @@ fory-simd fory-graalvm-feature + fory-latest-jdk-tests From 6b883755be078386d90db46bb5ffc43fd18f3c89 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 17 Jan 2026 14:48:29 +0800 Subject: [PATCH 21/44] fix record serializer --- .../fory/builder/BaseObjectCodecBuilder.java | 3 +- .../fory/builder/MetaSharedCodecBuilder.java | 5 +- .../fory/builder/ObjectCodecBuilder.java | 6 +- .../apache/fory/codegen/ExpressionUtils.java | 3 + .../java/org/apache/fory/meta/ClassDef.java | 4 + .../fory/serializer/MetaSharedSerializer.java | 5 +- .../NonexistentClassSerializers.java | 16 +- .../fory/serializer/ObjectSerializer.java | 6 +- .../main/java/org/apache/fory/util/Utils.java | 2 +- .../integration_tests/RecordXlangTest.java | 696 ++++++++++++++++++ 10 files changed, 732 insertions(+), 14 deletions(-) create mode 100644 java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/RecordXlangTest.java diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java index ce887481e0..3c7411b34c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java @@ -1740,7 +1740,8 @@ protected Expression deserializeForNullable( if (typeResolver(r -> r.needToWriteRef(typeRef))) { return readRef(buffer, callback, () -> deserializeForNotNull(buffer, typeRef, null)); } else { - if (typeRef.isPrimitive()) { + if (typeRef.isPrimitive() && !nullable) { + // Only skip null check if BOTH: local type is primitive AND sender didn't write null flag Expression value = deserializeForNotNull(buffer, typeRef, null); // Should put value expr ahead to avoid generated code in wrong scope. return new ListExpression(value, callback.apply(value)); diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java index 235b2fd7b2..832dba2b8a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java @@ -95,11 +95,12 @@ public MetaSharedCodecBuilder(TypeRef beanType, Fory fory, ClassDef classDef) DescriptorGrouper grouper = typeResolver(r -> r.createDescriptorGrouper(descriptors, false)); List sortedDescriptors = grouper.getSortedDescriptors(); if (org.apache.fory.util.Utils.DEBUG_OUTPUT_ENABLED) { - LOG.info("========== sorted descriptors for {} ==========", classDef.getClassName()); + LOG.info("========== {} sorted descriptors for {} ==========", + classDef.getFieldCount(), classDef.getClassName()); for (Descriptor d : sortedDescriptors) { LOG.info( " {} -> {}, ref {}, nullable {}", - d.getName(), + StringUtils.toSnakeCase(d.getName()), d.getTypeName(), d.isTrackingRef(), d.isNullable()); diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java index 67e5851a7e..49c0158a80 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java @@ -64,6 +64,7 @@ import org.apache.fory.type.DescriptorGrouper; import org.apache.fory.type.DispatchId; import org.apache.fory.type.TypeUtils; +import org.apache.fory.util.StringUtils; import org.apache.fory.util.function.SerializableSupplier; import org.apache.fory.util.record.RecordUtils; @@ -105,12 +106,13 @@ public ObjectCodecBuilder(Class beanClass, Fory fory) { Collection p = descriptors; DescriptorGrouper grouper = typeResolver(r -> r.createDescriptorGrouper(p, false)); if (org.apache.fory.util.Utils.DEBUG_OUTPUT_ENABLED) { - LOG.info("========== sorted descriptors for {} ==========", beanClass.getSimpleName()); + LOG.info("========== {} sorted descriptors for {} ==========", + descriptors.size(), beanClass.getSimpleName()); List sortedDescriptors = grouper.getSortedDescriptors(); for (Descriptor d : sortedDescriptors) { LOG.info( " {} -> {}, ref {}, nullable {}", - d.getName(), + StringUtils.toSnakeCase(d.getName()), d.getTypeName(), d.isTrackingRef(), d.isNullable()); diff --git a/java/fory-core/src/main/java/org/apache/fory/codegen/ExpressionUtils.java b/java/fory-core/src/main/java/org/apache/fory/codegen/ExpressionUtils.java index 112bdd11c4..75536b3476 100644 --- a/java/fory-core/src/main/java/org/apache/fory/codegen/ExpressionUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/codegen/ExpressionUtils.java @@ -152,6 +152,9 @@ public static Expression not(Expression target) { } public static Literal nullValue(TypeRef type) { + if (type.isPrimitive()) { + return defaultValue(type.getRawType()); + } return new Literal(null, type); } diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java b/java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java index aaa2444c6f..7d2427448c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java @@ -162,6 +162,10 @@ public byte[] getEncoded() { return encoded; } + public int getFieldCount() { + return fieldsInfo.size(); + } + @Override public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java index 6f8123e2e0..3662411caf 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java @@ -43,6 +43,7 @@ import org.apache.fory.util.DefaultValueUtils; import org.apache.fory.util.GraalvmSupport; import org.apache.fory.util.Preconditions; +import org.apache.fory.util.StringUtils; import org.apache.fory.util.Utils; import org.apache.fory.util.record.RecordInfo; import org.apache.fory.util.record.RecordUtils; @@ -84,7 +85,7 @@ public MetaSharedSerializer(Fory fory, Class type, ClassDef classDef) { fory.getConfig().isMetaShareEnabled(), "Meta share must be enabled."); if (Utils.DEBUG_OUTPUT_ENABLED) { LOG.info("========== MetaSharedSerializer ClassDef for {} ==========", type.getName()); - LOG.info("ClassDef fieldsInfo count: {}", classDef.getFieldsInfo().size()); + LOG.info("ClassDef fieldsInfo count: {}", classDef.getFieldCount()); for (int i = 0; i < classDef.getFieldsInfo().size(); i++) { LOG.info(" [{}] {}", i, classDef.getFieldsInfo().get(i)); } @@ -98,7 +99,7 @@ public MetaSharedSerializer(Fory fory, Class type, ClassDef classDef) { for (Descriptor d : descriptorGrouper.getSortedDescriptors()) { LOG.info( " {} -> {}, ref {}, nullable {}, type id {}", - d.getName(), + StringUtils.toSnakeCase(d.getName()), d.getTypeName(), d.isTrackingRef(), d.isNullable(), diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java index b4ea521061..362c071c2f 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java @@ -43,6 +43,7 @@ import org.apache.fory.type.DescriptorGrouper; import org.apache.fory.type.Generics; import org.apache.fory.util.Preconditions; +import org.apache.fory.util.Utils; @SuppressWarnings({"rawtypes", "unchecked"}) public final class NonexistentClassSerializers { @@ -73,6 +74,13 @@ public NonexistentClassSerializer(Fory fory, ClassDef classDef) { fieldsInfoMap = new LongMap<>(); binding = SerializationBinding.createBinding(fory); Preconditions.checkArgument(fory.getConfig().isMetaShareEnabled()); + if (Utils.DEBUG_OUTPUT_ENABLED && classDef != null) { + LOG.info("========== NonexistentClassSerializer ClassDef for {} ==========", type.getName()); + LOG.info("ClassDef fieldsInfo count: {}", classDef.getFieldCount()); + for (int i = 0; i < classDef.getFieldsInfo().size(); i++) { + LOG.info(" [{}] {}", i, classDef.getFieldsInfo().get(i)); + } + } } /** @@ -158,19 +166,19 @@ public Object read(MemoryBuffer buffer) { refResolver.reference(obj); List entries = new ArrayList<>(); // read order: primitive,boxed,final,other,collection,map - ClassFieldsInfo fieldsInfo = getClassFieldsInfo(classDef); - for (SerializationFieldInfo fieldInfo : fieldsInfo.buildInFields) { + ClassFieldsInfo allFieldsInfo = getClassFieldsInfo(classDef); + for (SerializationFieldInfo fieldInfo : allFieldsInfo.buildInFields) { Object fieldValue = AbstractObjectSerializer.readBuildInFieldValue(binding, fieldInfo, buffer); entries.add(new MapEntry(fieldInfo.qualifiedFieldName, fieldValue)); } Generics generics = fory.getGenerics(); - for (SerializationFieldInfo fieldInfo : fieldsInfo.containerFields) { + for (SerializationFieldInfo fieldInfo : allFieldsInfo.containerFields) { Object fieldValue = AbstractObjectSerializer.readContainerFieldValue(binding, generics, fieldInfo, buffer); entries.add(new MapEntry(fieldInfo.qualifiedFieldName, fieldValue)); } - for (SerializationFieldInfo fieldInfo : fieldsInfo.otherFields) { + for (SerializationFieldInfo fieldInfo : allFieldsInfo.otherFields) { Object fieldValue = binding.readField(fieldInfo, buffer); entries.add(new MapEntry(fieldInfo.qualifiedFieldName, fieldValue)); } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java index e12d870ffb..12006ffd9a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java @@ -41,6 +41,7 @@ import org.apache.fory.type.DescriptorGrouper; import org.apache.fory.type.Generics; import org.apache.fory.util.MurmurHash3; +import org.apache.fory.util.StringUtils; import org.apache.fory.util.Utils; import org.apache.fory.util.record.RecordInfo; import org.apache.fory.util.record.RecordUtils; @@ -103,11 +104,12 @@ public ObjectSerializer(Fory fory, Class cls, boolean resolveParent) { DescriptorGrouper grouper = typeResolver.createDescriptorGrouper(descriptors, false); descriptors = grouper.getSortedDescriptors(); if (Utils.DEBUG_OUTPUT_ENABLED) { - LOG.info("========== ObjectSerializer sorted descriptors for {} ==========", cls.getName()); + LOG.info("========== ObjectSerializer {} sorted descriptors for {} ==========", + descriptors.size(), cls.getName()); for (Descriptor d : descriptors) { LOG.info( " {} -> {}, ref {}, nullable {}", - d.getName(), + StringUtils.toSnakeCase(d.getName()), d.getTypeName(), d.isTrackingRef(), d.isNullable()); diff --git a/java/fory-core/src/main/java/org/apache/fory/util/Utils.java b/java/fory-core/src/main/java/org/apache/fory/util/Utils.java index 6687619f77..c3dd9aa8d2 100644 --- a/java/fory-core/src/main/java/org/apache/fory/util/Utils.java +++ b/java/fory-core/src/main/java/org/apache/fory/util/Utils.java @@ -25,6 +25,6 @@ public class Utils { public static final boolean DEBUG_OUTPUT_ENABLED; static { - DEBUG_OUTPUT_ENABLED = "1".equals(System.getenv("ENABLE_FORY_DEBUG_OUTPUT")); + DEBUG_OUTPUT_ENABLED = true; } } diff --git a/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/RecordXlangTest.java b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/RecordXlangTest.java new file mode 100644 index 0000000000..6aac6c2895 --- /dev/null +++ b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/RecordXlangTest.java @@ -0,0 +1,696 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.integration_tests; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import lombok.Data; +import org.apache.fory.Fory; +import org.apache.fory.annotation.ForyField; +import org.apache.fory.config.CompatibleMode; +import org.apache.fory.config.Language; +import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.meta.DeflaterMetaCompressor; +import org.testng.Assert; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +/** + * Tests for Java Records with cross-language serialization nullable field handling. + * + *

    This test verifies that Java Records work correctly with Fory's xlang serialization, testing + * nullable field handling in both SCHEMA_CONSISTENT and COMPATIBLE modes. + * + *

    For each mode, we test: + * + *

      + *
    • Record as Java data type with annotations on constructor parameters + *
    • POJO as xlang type representation (what other languages would use) + *
    • Serialization from Record, deserialization to POJO and vice versa + *
    + */ +public class RecordXlangTest { + + @DataProvider(name = "enableCodegen") + public static Object[][] enableCodegen() { + return new Object[][] {{true}, {false}}; + } + + // ==================== SCHEMA_CONSISTENT Mode Records and POJOs ==================== + + /** + * Record for SCHEMA_CONSISTENT mode testing. + * + *

    Fields: + * + *

      + *
    • Base non-nullable fields: primitives and reference types + *
    • Nullable fields: boxed types and reference types with @ForyField(nullable=true) + *
    + */ + public record NullableRecordSchemaConsistent( + // Base non-nullable primitive fields + byte byteField, + short shortField, + int intField, + long longField, + float floatField, + double doubleField, + boolean boolField, + // Base non-nullable reference fields + String stringField, + List listField, + Set setField, + Map mapField, + // Nullable fields - boxed types + @ForyField(nullable = true) Integer nullableInt, + @ForyField(nullable = true) Long nullableLong, + @ForyField(nullable = true) Float nullableFloat, + // Nullable fields - reference types + @ForyField(nullable = true) Double nullableDouble, + @ForyField(nullable = true) Boolean nullableBool, + @ForyField(nullable = true) String nullableString, + @ForyField(nullable = true) List nullableList, + @ForyField(nullable = true) Set nullableSet, + @ForyField(nullable = true) Map nullableMap) {} + + /** + * POJO for SCHEMA_CONSISTENT mode xlang type. Same structure as the Record - both Java and other + * languages use the same field nullability. + */ + @Data + public static class NullablePojoSchemaConsistentXlang { + // Base non-nullable primitive fields + byte byteField; + short shortField; + int intField; + long longField; + float floatField; + double doubleField; + boolean boolField; + + // Base non-nullable reference fields + String stringField; + List listField; + Set setField; + Map mapField; + + // Nullable fields - boxed types + @ForyField(nullable = true) + Integer nullableInt; + + @ForyField(nullable = true) + Long nullableLong; + + @ForyField(nullable = true) + Float nullableFloat; + + // Nullable fields - reference types + @ForyField(nullable = true) + Double nullableDouble; + + @ForyField(nullable = true) + Boolean nullableBool; + + @ForyField(nullable = true) + String nullableString; + + @ForyField(nullable = true) + List nullableList; + + @ForyField(nullable = true) + Set nullableSet; + + @ForyField(nullable = true) + Map nullableMap; + } + + // ==================== COMPATIBLE Mode Records and POJOs ==================== + + /** + * Record for COMPATIBLE mode testing. Matches XlangTestBase.NullableComprehensiveCompatible. + * + *

    Fields are organized as: + * + *

      + *
    • Group 1 (non-nullable in Java): primitives, boxed types, and reference types + *
    • Group 2 (nullable in Java): boxed types and reference types with + * @ForyField(nullable=true) + *
    + * + *

    In Go, Group 1 fields are nullable (*int8, *int16, etc.) and Group 2 fields are + * non-nullable (int32, int64, etc.) - this tests schema evolution with inverted nullability. + */ + public record NullableRecordCompatible( + // Group 1: Non-nullable in Java (nullable in Go with pointer types) + // Primitive fields + byte byteField, + short shortField, + int intField, + long longField, + float floatField, + double doubleField, + boolean boolField, + // Boxed fields (non-nullable in Java) + Integer boxedInt, + Long boxedLong, + Float boxedFloat, + Double boxedDouble, + Boolean boxedBool, + // Reference fields (non-nullable in Java) + String stringField, + List listField, + Set setField, + Map mapField, + // Group 2: Nullable in Java (non-nullable in Go with value types) + // Boxed types with @ForyField(nullable=true) + @ForyField(nullable = true) Integer nullableInt1, + @ForyField(nullable = true) Long nullableLong1, + @ForyField(nullable = true) Float nullableFloat1, + @ForyField(nullable = true) Double nullableDouble1, + @ForyField(nullable = true) Boolean nullableBool1, + // Reference types with @ForyField(nullable=true) + @ForyField(nullable = true) String nullableString2, + @ForyField(nullable = true) List nullableList2, + @ForyField(nullable = true) Set nullableSet2, + @ForyField(nullable = true) Map nullableMap2) {} + + /** + * POJO for COMPATIBLE mode xlang type with INVERTED nullability. + * + *

    This matches Go's NullableComprehensiveCompatible struct where: + * + *

      + *
    • Group 1 fields are nullable (pointer types in Go: *int8, *int16, etc.) + *
    • Group 2 fields are non-nullable (value types in Go: int32, int64, etc.) + *
    + */ + @Data + public static class NullablePojoCompatibleXlang { + // Group 1: NULLABLE in xlang (non-nullable in Java Record) + // These match Go's pointer types (*int8, *int16, etc.) + @ForyField(nullable = true) + Byte byteField; + + @ForyField(nullable = true) + Short shortField; + + @ForyField(nullable = true) + Integer intField; + + @ForyField(nullable = true) + Long longField; + + @ForyField(nullable = true) + Float floatField; + + @ForyField(nullable = true) + Double doubleField; + + @ForyField(nullable = true) + Boolean boolField; + + @ForyField(nullable = true) + Integer boxedInt; + + @ForyField(nullable = true) + Long boxedLong; + + @ForyField(nullable = true) + Float boxedFloat; + + @ForyField(nullable = true) + Double boxedDouble; + + @ForyField(nullable = true) + Boolean boxedBool; + + @ForyField(nullable = true) + String stringField; + + @ForyField(nullable = true) + List listField; + + @ForyField(nullable = true) + Set setField; + + @ForyField(nullable = true) + Map mapField; + + // Group 2: NON-NULLABLE in xlang (nullable in Java Record) + // These match Go's value types (int32, int64, etc.) + int nullableInt1; + long nullableLong1; + float nullableFloat1; + double nullableDouble1; + boolean nullableBool1; + String nullableString2; + List nullableList2; + Set nullableSet2; + Map nullableMap2; + } + + // ==================== SCHEMA_CONSISTENT Mode Tests ==================== + + @Test(dataProvider = "enableCodegen") + public void testRecordNullableFieldSchemaConsistentNotNull(boolean enableCodegen) { + Fory fory = + Fory.builder() + .withLanguage(Language.XLANG) + .withCompatibleMode(CompatibleMode.SCHEMA_CONSISTENT) + .withCodegen(enableCodegen) + .build(); + fory.register(NullableRecordSchemaConsistent.class, 0); + + Fory foryXlang = + Fory.builder() + .withLanguage(Language.XLANG) + .withCompatibleMode(CompatibleMode.SCHEMA_CONSISTENT) + .withCodegen(enableCodegen) + .build(); + foryXlang.register(NullablePojoSchemaConsistentXlang.class, 0); + + // Create record with all nullable fields having values + NullableRecordSchemaConsistent record = + new NullableRecordSchemaConsistent( + // Base non-nullable primitive fields + (byte) 1, + (short) 2, + 42, + 123456789L, + 1.5f, + 2.5, + true, + // Base non-nullable reference fields + "hello", + Arrays.asList("a", "b", "c"), + new HashSet<>(Arrays.asList("x", "y")), + createMap("key1", "value1", "key2", "value2"), + // Nullable fields - all have values + 100, + 200L, + 1.5f, + 2.5, + false, + "nullable_value", + Arrays.asList("p", "q"), + new HashSet<>(Arrays.asList("m", "n")), + createMap("nk1", "nv1")); + + // Serialize record + MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(512); + fory.serialize(buffer, record); + + // Deserialize as POJO (simulating xlang) + buffer.readerIndex(0); + NullablePojoSchemaConsistentXlang xlangObj = + (NullablePojoSchemaConsistentXlang) foryXlang.deserialize(buffer); + + // Verify all fields + Assert.assertEquals(xlangObj.byteField, record.byteField()); + Assert.assertEquals(xlangObj.shortField, record.shortField()); + Assert.assertEquals(xlangObj.intField, record.intField()); + Assert.assertEquals(xlangObj.longField, record.longField()); + Assert.assertEquals(xlangObj.floatField, record.floatField(), 0.001f); + Assert.assertEquals(xlangObj.doubleField, record.doubleField(), 0.001); + Assert.assertEquals(xlangObj.boolField, record.boolField()); + Assert.assertEquals(xlangObj.stringField, record.stringField()); + Assert.assertEquals(xlangObj.listField, record.listField()); + Assert.assertEquals(xlangObj.setField, record.setField()); + Assert.assertEquals(xlangObj.mapField, record.mapField()); + Assert.assertEquals(xlangObj.nullableInt, record.nullableInt()); + Assert.assertEquals(xlangObj.nullableLong, record.nullableLong()); + Assert.assertEquals(xlangObj.nullableFloat, record.nullableFloat()); + Assert.assertEquals(xlangObj.nullableDouble, record.nullableDouble()); + Assert.assertEquals(xlangObj.nullableBool, record.nullableBool()); + Assert.assertEquals(xlangObj.nullableString, record.nullableString()); + Assert.assertEquals(xlangObj.nullableList, record.nullableList()); + Assert.assertEquals(xlangObj.nullableSet, record.nullableSet()); + Assert.assertEquals(xlangObj.nullableMap, record.nullableMap()); + + // Serialize POJO back and deserialize as Record + MemoryBuffer buffer2 = MemoryBuffer.newHeapBuffer(512); + foryXlang.serialize(buffer2, xlangObj); + buffer2.readerIndex(0); + NullableRecordSchemaConsistent result = + (NullableRecordSchemaConsistent) fory.deserialize(buffer2); + Assert.assertEquals(result, record); + } + + @Test(dataProvider = "enableCodegen") + public void testRecordNullableFieldSchemaConsistentNull(boolean enableCodegen) { + Fory fory = + Fory.builder() + .withLanguage(Language.XLANG) + .withCompatibleMode(CompatibleMode.SCHEMA_CONSISTENT) + .withCodegen(enableCodegen) + .build(); + fory.register(NullableRecordSchemaConsistent.class, 0); + + Fory foryXlang = + Fory.builder() + .withLanguage(Language.XLANG) + .withCompatibleMode(CompatibleMode.SCHEMA_CONSISTENT) + .withCodegen(enableCodegen) + .build(); + foryXlang.register(NullablePojoSchemaConsistentXlang.class, 0); + + // Create record with all nullable fields as null + NullableRecordSchemaConsistent record = + new NullableRecordSchemaConsistent( + // Base non-nullable primitive fields + (byte) 1, + (short) 2, + 42, + 123456789L, + 1.5f, + 2.5, + true, + // Base non-nullable reference fields + "hello", + Arrays.asList("a", "b", "c"), + new HashSet<>(Arrays.asList("x", "y")), + createMap("key1", "value1", "key2", "value2"), + // Nullable fields - all null + null, + null, + null, + null, + null, + null, + null, + null, + null); + + // Serialize record + MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(512); + fory.serialize(buffer, record); + + // Deserialize as POJO (simulating xlang) + buffer.readerIndex(0); + NullablePojoSchemaConsistentXlang xlangObj = + (NullablePojoSchemaConsistentXlang) foryXlang.deserialize(buffer); + + // Verify base fields + Assert.assertEquals(xlangObj.byteField, record.byteField()); + Assert.assertEquals(xlangObj.shortField, record.shortField()); + Assert.assertEquals(xlangObj.intField, record.intField()); + Assert.assertEquals(xlangObj.longField, record.longField()); + Assert.assertEquals(xlangObj.floatField, record.floatField(), 0.001f); + Assert.assertEquals(xlangObj.doubleField, record.doubleField(), 0.001); + Assert.assertEquals(xlangObj.boolField, record.boolField()); + Assert.assertEquals(xlangObj.stringField, record.stringField()); + Assert.assertEquals(xlangObj.listField, record.listField()); + Assert.assertEquals(xlangObj.setField, record.setField()); + Assert.assertEquals(xlangObj.mapField, record.mapField()); + + // Verify nullable fields are null + Assert.assertNull(xlangObj.nullableInt); + Assert.assertNull(xlangObj.nullableLong); + Assert.assertNull(xlangObj.nullableFloat); + Assert.assertNull(xlangObj.nullableDouble); + Assert.assertNull(xlangObj.nullableBool); + Assert.assertNull(xlangObj.nullableString); + Assert.assertNull(xlangObj.nullableList); + Assert.assertNull(xlangObj.nullableSet); + Assert.assertNull(xlangObj.nullableMap); + + // Serialize POJO back and deserialize as Record + MemoryBuffer buffer2 = MemoryBuffer.newHeapBuffer(512); + foryXlang.serialize(buffer2, xlangObj); + buffer2.readerIndex(0); + NullableRecordSchemaConsistent result = + (NullableRecordSchemaConsistent) fory.deserialize(buffer2); + Assert.assertEquals(result, record); + } + + // ==================== COMPATIBLE Mode Tests ==================== + + @Test(dataProvider = "enableCodegen") + public void testRecordNullableFieldCompatibleNotNull(boolean enableCodegen) { + Fory fory = + Fory.builder() + .withLanguage(Language.XLANG) + .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withCodegen(enableCodegen) + .withMetaCompressor(new DeflaterMetaCompressor()) + .build(); + fory.register(NullableRecordCompatible.class, 1); + + Fory foryXlang = + Fory.builder() + .withLanguage(Language.XLANG) + .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withCodegen(enableCodegen) + .withMetaCompressor(new DeflaterMetaCompressor()) + .build(); + foryXlang.register(NullablePojoCompatibleXlang.class, 1); + + // Create record with all fields having values + NullableRecordCompatible record = + new NullableRecordCompatible( + // Group 1: Non-nullable in Java (nullable in xlang) + (byte) 1, + (short) 2, + 42, + 123456789L, + 1.5f, + 2.5, + true, + 10, + 20L, + 1.1f, + 2.2, + true, + "hello", + Arrays.asList("a", "b", "c"), + new HashSet<>(Arrays.asList("x", "y")), + createMap("key1", "value1", "key2", "value2"), + // Group 2: Nullable in Java (non-nullable in xlang) + 100, + 200L, + 1.5f, + 2.5, + false, + "nullable_value", + Arrays.asList("p", "q"), + new HashSet<>(Arrays.asList("m", "n")), + createMap("nk1", "nv1")); + + // Serialize record + MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(1024); + fory.serialize(buffer, record); + + // Deserialize as POJO with inverted nullability (simulating xlang) + buffer.readerIndex(0); + NullablePojoCompatibleXlang xlangObj = + (NullablePojoCompatibleXlang) foryXlang.deserialize(buffer); + + // Verify Group 1 fields (non-nullable in Record -> nullable in xlang POJO) + Assert.assertEquals((byte) xlangObj.byteField, record.byteField()); + Assert.assertEquals((short) xlangObj.shortField, record.shortField()); + Assert.assertEquals((int) xlangObj.intField, record.intField()); + Assert.assertEquals((long) xlangObj.longField, record.longField()); + Assert.assertEquals(xlangObj.floatField, record.floatField(), 0.001f); + Assert.assertEquals(xlangObj.doubleField, record.doubleField(), 0.001); + Assert.assertEquals(xlangObj.boolField, record.boolField()); + Assert.assertEquals(xlangObj.boxedInt, record.boxedInt()); + Assert.assertEquals(xlangObj.boxedLong, record.boxedLong()); + Assert.assertEquals(xlangObj.boxedFloat, record.boxedFloat()); + Assert.assertEquals(xlangObj.boxedDouble, record.boxedDouble()); + Assert.assertEquals(xlangObj.boxedBool, record.boxedBool()); + Assert.assertEquals(xlangObj.stringField, record.stringField()); + Assert.assertEquals(xlangObj.listField, record.listField()); + Assert.assertEquals(xlangObj.setField, record.setField()); + Assert.assertEquals(xlangObj.mapField, record.mapField()); + + // Verify Group 2 fields (nullable in Record -> non-nullable in xlang POJO) + Assert.assertEquals(xlangObj.nullableInt1, (int) record.nullableInt1()); + Assert.assertEquals(xlangObj.nullableLong1, (long) record.nullableLong1()); + Assert.assertEquals(xlangObj.nullableFloat1, record.nullableFloat1(), 0.001f); + Assert.assertEquals(xlangObj.nullableDouble1, record.nullableDouble1(), 0.001); + Assert.assertEquals(xlangObj.nullableBool1, record.nullableBool1()); + Assert.assertEquals(xlangObj.nullableString2, record.nullableString2()); + Assert.assertEquals(xlangObj.nullableList2, record.nullableList2()); + Assert.assertEquals(xlangObj.nullableSet2, record.nullableSet2()); + Assert.assertEquals(xlangObj.nullableMap2, record.nullableMap2()); + + // Serialize POJO back and deserialize as Record + MemoryBuffer buffer2 = MemoryBuffer.newHeapBuffer(1024); + foryXlang.serialize(buffer2, xlangObj); + buffer2.readerIndex(0); + NullableRecordCompatible result = (NullableRecordCompatible) fory.deserialize(buffer2); + Assert.assertEquals(result, record); + } + + @Test(dataProvider = "enableCodegen") + public void testRecordNullableFieldCompatibleNull(boolean enableCodegen) { + Fory fory = + Fory.builder() + .withLanguage(Language.XLANG) + .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withCodegen(enableCodegen) + .withMetaCompressor(new DeflaterMetaCompressor()) + .build(); + fory.register(NullableRecordCompatible.class, 1); + + Fory foryXlang = + Fory.builder() + .withLanguage(Language.XLANG) + .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withCodegen(enableCodegen) + .withMetaCompressor(new DeflaterMetaCompressor()) + .build(); + foryXlang.register(NullablePojoCompatibleXlang.class, 1); + + // Create record with Group 1 having values, Group 2 all null + NullableRecordCompatible record = + new NullableRecordCompatible( + // Group 1: Non-nullable in Java (nullable in xlang) - must have values + (byte) 1, + (short) 2, + 42, + 123456789L, + 1.5f, + 2.5, + true, + 10, + 20L, + 1.1f, + 2.2, + true, + "hello", + Arrays.asList("a", "b", "c"), + new HashSet<>(Arrays.asList("x", "y")), + createMap("key1", "value1", "key2", "value2"), + // Group 2: Nullable in Java (non-nullable in xlang) - all null + null, + null, + null, + null, + null, + null, + null, + null, + null); + + // Serialize record + MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(1024); + fory.serialize(buffer, record); + + // Deserialize as POJO with inverted nullability (simulating xlang) + buffer.readerIndex(0); + NullablePojoCompatibleXlang xlangObj = + (NullablePojoCompatibleXlang) foryXlang.deserialize(buffer); + + // Verify Group 1 fields + Assert.assertEquals((byte) xlangObj.byteField, record.byteField()); + Assert.assertEquals((short) xlangObj.shortField, record.shortField()); + Assert.assertEquals((int) xlangObj.intField, record.intField()); + Assert.assertEquals((long) xlangObj.longField, record.longField()); + Assert.assertEquals(xlangObj.floatField, record.floatField(), 0.001f); + Assert.assertEquals(xlangObj.doubleField, record.doubleField(), 0.001); + Assert.assertEquals(xlangObj.boolField, record.boolField()); + Assert.assertEquals(xlangObj.boxedInt, record.boxedInt()); + Assert.assertEquals(xlangObj.boxedLong, record.boxedLong()); + Assert.assertEquals(xlangObj.boxedFloat, record.boxedFloat()); + Assert.assertEquals(xlangObj.boxedDouble, record.boxedDouble()); + Assert.assertEquals(xlangObj.boxedBool, record.boxedBool()); + Assert.assertEquals(xlangObj.stringField, record.stringField()); + Assert.assertEquals(xlangObj.listField, record.listField()); + Assert.assertEquals(xlangObj.setField, record.setField()); + Assert.assertEquals(xlangObj.mapField, record.mapField()); + + // Verify Group 2 fields - xlang POJO has non-nullable fields, so nulls become defaults + // Primitive types get default values + Assert.assertEquals(xlangObj.nullableInt1, 0); + Assert.assertEquals(xlangObj.nullableLong1, 0L); + Assert.assertEquals(xlangObj.nullableFloat1, 0.0f, 0.001f); + Assert.assertEquals(xlangObj.nullableDouble1, 0.0, 0.001); + Assert.assertEquals(xlangObj.nullableBool1, false); + // Reference types become null initially + Assert.assertNull(xlangObj.nullableString2); + Assert.assertNull(xlangObj.nullableList2); + Assert.assertNull(xlangObj.nullableSet2); + Assert.assertNull(xlangObj.nullableMap2); + + // Fill null reference fields with default values to simulate Go/Rust behavior + // In Go/Rust, non-nullable reference fields get empty values, not null + xlangObj.nullableString2 = ""; + xlangObj.nullableList2 = new ArrayList<>(); + xlangObj.nullableSet2 = new HashSet<>(); + xlangObj.nullableMap2 = new HashMap<>(); + + // Serialize POJO back and deserialize as Record + MemoryBuffer buffer2 = MemoryBuffer.newHeapBuffer(1024); + foryXlang.serialize(buffer2, xlangObj); + buffer2.readerIndex(0); + NullableRecordCompatible result = (NullableRecordCompatible) fory.deserialize(buffer2); + + // Build expected: Group 2 fields will have default values instead of null + NullableRecordCompatible expected = + new NullableRecordCompatible( + // Group 1: unchanged + record.byteField(), + record.shortField(), + record.intField(), + record.longField(), + record.floatField(), + record.doubleField(), + record.boolField(), + record.boxedInt(), + record.boxedLong(), + record.boxedFloat(), + record.boxedDouble(), + record.boxedBool(), + record.stringField(), + record.listField(), + record.setField(), + record.mapField(), + // Group 2: xlang's non-nullable fields send default values + 0, // nullableInt1 + 0L, // nullableLong1 + 0.0f, // nullableFloat1 + 0.0, // nullableDouble1 + false, // nullableBool1 + "", // nullableString2 - empty string + new ArrayList<>(), // nullableList2 - empty list + new HashSet<>(), // nullableSet2 - empty set + new HashMap<>() // nullableMap2 - empty map + ); + + Assert.assertEquals(result, expected); + } + + // ==================== Helper Methods ==================== + + private static Map createMap(String... keyValues) { + Map map = new HashMap<>(); + for (int i = 0; i < keyValues.length; i += 2) { + map.put(keyValues[i], keyValues[i + 1]); + } + return map; + } +} From d00d245d55a405eea055221436ecb8617b38b904 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 17 Jan 2026 15:42:02 +0800 Subject: [PATCH 22/44] add comprehensive tests --- .../java/org/apache/fory/meta/FieldTypes.java | 86 +++--- .../apache/fory/resolver/XtypeResolver.java | 6 +- .../NonexistentClassSerializers.java | 7 +- .../test/java/org/apache/fory/TestUtils.java | 28 ++ .../NonexistentClassSerializersTest.java | 259 ++++++++++++++++++ .../fory/xlang/MetaSharedXlangTest.java | 1 + .../integration_tests/RecordXlangTest.java | 96 ++++++- 7 files changed, 433 insertions(+), 50 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java index bb0104fc44..5376cd8826 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java @@ -163,25 +163,22 @@ private static FieldType buildFieldType( } else { if (rawType.isEnum()) { return new EnumFieldType(nullable, xtypeId); - } else if (isXlang - && !Types.isUserDefinedType((byte) xtypeId) - && resolver.isRegisteredById(rawType)) { - return new RegisteredFieldType(nullable, trackingRef, xtypeId); - } else if (!isXlang && resolver.isRegisteredById(rawType)) { - Short classId = ((ClassResolver) resolver).getRegisteredClassId(rawType); - return new RegisteredFieldType(nullable, trackingRef, classId); - } else { - if (rawType.isArray()) { - Class elemType = rawType.getComponentType(); - while (elemType.isArray()) { - elemType = elemType.getComponentType(); - } - if (isXlang && !elemType.isPrimitive()) { - return new CollectionFieldType( - xtypeId, - nullable, - trackingRef, - buildFieldType(resolver, null, GenericType.build(elemType))); + } + if (rawType.isArray()) { + Class elemType = rawType.getComponentType(); + if (elemType.isPrimitive()) { + return new RegisteredFieldType(nullable, trackingRef, xtypeId); + } + if (isXlang) { + // primitive array has bee handled before + return new CollectionFieldType( + xtypeId, + nullable, + trackingRef, + buildFieldType(resolver, null, GenericType.build(elemType))); + } else { + if (((ClassResolver)resolver).isInternalRegistered(rawType)) { + return new RegisteredFieldType(nullable, trackingRef, xtypeId); } Tuple2, Integer> arrayComponentInfo = getArrayComponentInfo(rawType); return new ArrayFieldType( @@ -191,6 +188,13 @@ private static FieldType buildFieldType( buildFieldType(resolver, null, GenericType.build(arrayComponentInfo.f0)), arrayComponentInfo.f1); } + } + if (isXlang && !Types.isUserDefinedType((byte) xtypeId) && resolver.isRegisteredById(rawType)) { + return new RegisteredFieldType(nullable, trackingRef, xtypeId); + } else if (!isXlang && resolver.isRegisteredById(rawType)) { + Short classId = ((ClassResolver) resolver).getRegisteredClassId(rawType); + return new RegisteredFieldType(nullable, trackingRef, classId); + } else { return new ObjectFieldType(xtypeId, nullable, trackingRef); } } @@ -523,7 +527,7 @@ public FieldType getElementType() { } @Override - public TypeRef toTypeToken(TypeResolver classResolver, TypeRef declared) { + public TypeRef toTypeToken(TypeResolver resolver, TypeRef declared) { // TODO support preserve element TypeExtMeta Class declaredClass; TypeRef declElementType; @@ -542,34 +546,34 @@ public TypeRef toTypeToken(TypeResolver classResolver, TypeRef declared) { declElementType = declElementType.resolveAllWildcards(); } } - TypeRef elementType = this.elementType.toTypeToken(classResolver, declElementType); + TypeRef elementType = this.elementType.toTypeToken(resolver, declElementType); if (declared == null) { return collectionOf(elementType, new TypeExtMeta(xtypeId, nullable, trackingRef)); } - TypeRef> collectionTypeRef = - collectionOf(declaredClass, elementType, new TypeExtMeta(xtypeId, nullable, trackingRef)); if (!declaredClass.isArray()) { if (declElementType.equals(elementType)) { return declared; } - return collectionTypeRef; - } - Tuple2, Integer> info = TypeUtils.getArrayComponentInfo(declaredClass); - List> typeRefs = new ArrayList<>(info.f1 + 1); - typeRefs.add(collectionTypeRef); - for (int i = 0; i < info.f1; i++) { - typeRefs.add(TypeUtils.getElementType(typeRefs.get(i))); - } - Collections.reverse(typeRefs); - for (int i = 1; i < typeRefs.size(); i++) { - TypeRef arrayType = typeRefs.get(i - 1); - TypeRef typeRef = - TypeRef.of( - Array.newInstance(arrayType.getRawType(), 1).getClass(), - typeRefs.get(i).getTypeExtMeta()); - typeRefs.set(i, typeRef); - } - return typeRefs.get(typeRefs.size() - 1); + return collectionOf(declaredClass, elementType, new TypeExtMeta(xtypeId, nullable, trackingRef)); + } + // Build array type from element type + // elementType could be base type (int) or intermediate array (int[]) + // Calculate how many dimensions to add + int declaredDimensions = getArrayDimensions(declaredClass); + Class elemRawType = elementType.getRawType(); + int elementDimensions = elemRawType.isArray() ? getArrayDimensions(elemRawType) : 0; + int dimensionsToAdd = declaredDimensions - elementDimensions; + TypeRef currentType = elementType; + for (int i = 0; i < dimensionsToAdd; i++) { + Class arrayClass = Array.newInstance(currentType.getRawType(), 0).getClass(); + // Apply field metadata (nullable, trackingRef) to outermost array only + TypeExtMeta meta = + (i == dimensionsToAdd - 1) + ? new TypeExtMeta(xtypeId, nullable, trackingRef) + : currentType.getTypeExtMeta(); + currentType = TypeRef.of(arrayClass, meta); + } + return currentType; } @Override diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java index e76748004b..dd445b7bee 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java @@ -735,7 +735,11 @@ public void writeClassInfo(MemoryBuffer buffer, ClassInfo classInfo) { case Types.NAMED_COMPATIBLE_STRUCT: case Types.COMPATIBLE_STRUCT: assert shareMeta : "Meta share must be enabled for compatible mode"; - writeSharedClassMeta(buffer, classInfo); + // Skip writeSharedClassMeta for NonexistentMetaShared since + // NonexistentClassSerializer.writeClassDef will handle writing the correct classDef. + if (classInfo.cls != NonexistentMetaShared.class) { + writeSharedClassMeta(buffer, classInfo); + } break; default: break; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java index 362c071c2f..89bbf8eb0f 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java @@ -87,12 +87,11 @@ public NonexistentClassSerializer(Fory fory, ClassDef classDef) { * Multiple un existed class will correspond to this `NonexistentMetaSharedClass`. When querying * classinfo by `class`, it may dispatch to same `NonexistentClassSerializer`, so we can't use * `classDef` in this serializer, but use `classDef` in `NonexistentMetaSharedClass` instead. + * + *

    Note: XtypeResolver.writeClassInfo skips writeSharedClassMeta for NonexistentMetaShared, + * so this method writes the classDef directly without reverting any buffer bytes. */ private void writeClassDef(MemoryBuffer buffer, NonexistentClass.NonexistentMetaShared value) { - // Register NotFoundClass ahead to skip write meta shared info, - // then revert written class id to write class info here, - // since it's the only place to hold class def for not found class. - buffer.increaseWriterIndex(-2); MetaContext metaContext = fory.getSerializationContext().getMetaContext(); IdentityObjectIntMap classMap = metaContext.classMap; int newId = classMap.size; diff --git a/java/fory-core/src/test/java/org/apache/fory/TestUtils.java b/java/fory-core/src/test/java/org/apache/fory/TestUtils.java index f7e52f0fef..afa0d72501 100644 --- a/java/fory-core/src/test/java/org/apache/fory/TestUtils.java +++ b/java/fory-core/src/test/java/org/apache/fory/TestUtils.java @@ -31,8 +31,10 @@ import java.util.stream.Collectors; import org.apache.fory.collection.Tuple3; import org.apache.fory.memory.Platform; +import org.apache.fory.meta.ClassDef; import org.apache.fory.reflect.FieldAccessor; import org.apache.fory.reflect.ReflectionUtils; +import org.apache.fory.type.Descriptor; import org.apache.fory.util.unsafe._JDKAccess; import org.testng.SkipException; @@ -331,4 +333,30 @@ public static Tuple3, Map, Map> getCom commonFields.retainAll(fieldMap2.keySet()); return Tuple3.of(commonFields, fieldMap1, fieldMap2); } + + /** + * Convert an object to a Map using Fory's field descriptors. The map uses qualified field names + * (className.fieldName) as keys to match NonexistentClass format. + * + * @param fory the Fory instance + * @param obj the object to convert + * @return a map of qualified field names to field values + */ + public static Map objectToMap(Fory fory, Object obj) { + Class cls = obj.getClass(); + ClassDef classDef = fory.getClassResolver().getTypeDef(cls, true); + List descriptors = classDef.getDescriptors(fory._getTypeResolver(), cls); + Map result = new LinkedHashMap<>(); + for (Descriptor descriptor : descriptors) { + Field field = descriptor.getField(); + if (field != null) { + FieldAccessor accessor = FieldAccessor.createAccessor(field); + Object value = accessor.get(obj); + // Use qualified field name format: className.fieldName + String qualifiedName = descriptor.getDeclaringClass() + "." + descriptor.getName(); + result.put(qualifiedName, value); + } + } + return result; + } } diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/NonexistentClassSerializersTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/NonexistentClassSerializersTest.java index 08f6763162..5ce58dca30 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/NonexistentClassSerializersTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/NonexistentClassSerializersTest.java @@ -25,14 +25,21 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; import java.lang.reflect.Array; +import java.util.HashMap; import java.util.List; +import java.util.Map; + +import lombok.Data; import org.apache.fory.Fory; import org.apache.fory.ForyTestBase; +import org.apache.fory.TestUtils; +import org.apache.fory.annotation.ForyField; import org.apache.fory.codegen.CompileUnit; import org.apache.fory.codegen.JaninoUtils; import org.apache.fory.config.CompatibleMode; import org.apache.fory.config.ForyBuilder; import org.apache.fory.config.Language; +import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.resolver.MetaContext; import org.apache.fory.test.bean.Struct; @@ -362,4 +369,256 @@ public void testThrowExceptionIfClassNotExist() { byte[] bytes = fory.serialize(pojo); assertThrowsCause(RuntimeException.class, () -> fory2.deserialize(bytes)); } + + + /** + * Simple test class with primitive types for NonexistentClass serialization testing. Avoids + * collection types which require complex type registration in xlang mode. + */ + @Data + static class SimpleTestClass { + // Primitive fields + byte byteField; + short shortField; + int intField; + long longField; + float floatField; + double doubleField; + boolean boolField; + + // Boxed fields + Integer boxedInt; + Long boxedLong; + Float boxedFloat; + Double boxedDouble; + Boolean boxedBool; + + // String field + String stringField; + + // Nullable fields + @ForyField(nullable = true) + Integer nullableInt; + + @ForyField(nullable = true) + String nullableString; + } + + /** Create a populated test object with all fields set. */ + private static SimpleTestClass createTestObject() { + SimpleTestClass obj = new SimpleTestClass(); + // Primitive fields + obj.byteField = 1; + obj.shortField = 2; + obj.intField = 42; + obj.longField = 123456789L; + obj.floatField = 1.5f; + obj.doubleField = 2.5; + obj.boolField = true; + + // Boxed fields + obj.boxedInt = 10; + obj.boxedLong = 20L; + obj.boxedFloat = 1.1f; + obj.boxedDouble = 2.2; + obj.boxedBool = true; + + // String field + obj.stringField = "hello"; + + // Nullable fields - set values + obj.nullableInt = 100; + obj.nullableString = "nullable_value"; + + return obj; + } + + /** + * Test that NonexistentClass correctly preserves field values when deserializing an unknown + * class. This simulates the scenario where fory2 doesn't have the class registered, so it + * deserializes to NonexistentMetaShared. + */ + @Test(dataProvider = "language") + public void testNonexistentClassDeserializationPreservesValues(Language language) { + // Fory1: serializer with class registered + Fory fory1 = + Fory.builder() + .withLanguage(language) + .withCodegen(false) + .withCompatibleMode(CompatibleMode.COMPATIBLE) + .build(); + fory1.register(SimpleTestClass.class, "test.SimpleTestClass"); + + // Fory2: deserializer without class registered - will use NonexistentClassSerializer + Fory fory2 = + Fory.builder() + .withLanguage(language) + .withCodegen(false) + .withCompatibleMode(CompatibleMode.COMPATIBLE) + .build(); + // Don't register SimpleTestClass - fory2 doesn't know this class + + // Create and serialize object with fory1 + SimpleTestClass obj = createTestObject(); + MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(1024); + fory1.serialize(buffer, obj); + + // Convert original object to map for comparison + Map expectedMap = TestUtils.objectToMap(fory1, obj); + + // Deserialize with fory2 - should return NonexistentMetaShared + buffer.readerIndex(0); + Object result = fory2.deserialize(buffer); + + // Verify result is NonexistentMetaShared + assertEquals(result.getClass(), NonexistentClass.NonexistentMetaShared.class); + + NonexistentClass.NonexistentMetaShared nonexistent = + (NonexistentClass.NonexistentMetaShared) result; + + // Convert NonexistentMetaShared to a map keyed by simple field name + Map actualMap = new HashMap<>(); + for (Object key : nonexistent.keySet()) { + String qualifiedKey = (String) key; + // Extract simple field name from "className.fieldName" + String simpleFieldName = qualifiedKey.substring(qualifiedKey.lastIndexOf('.') + 1); + actualMap.put(simpleFieldName, nonexistent.get(key)); + } + + // Verify all field values are preserved (compare by simple field name) + for (Map.Entry entry : expectedMap.entrySet()) { + String qualifiedFieldName = entry.getKey(); + String simpleFieldName = + qualifiedFieldName.substring(qualifiedFieldName.lastIndexOf('.') + 1); + Object expectedValue = entry.getValue(); + Object actualValue = actualMap.get(simpleFieldName); + + assertEquals( + actualValue, expectedValue, String.format("Field '%s' value mismatch", simpleFieldName)); + } + } + + /** Test NonexistentClass with null values in nullable fields. */ + @Test(dataProvider = "language") + public void testNonexistentClassDeserializationWithNulls(Language language) { + // Fory1: serializer with class registered + Fory fory1 = + Fory.builder() + .withLanguage(language) + .withCodegen(false) + .withCompatibleMode(CompatibleMode.COMPATIBLE) + .build(); + fory1.register(SimpleTestClass.class, "test.SimpleTestClass"); + + // Fory2: deserializer without class registered + Fory fory2 = + Fory.builder() + .withLanguage(language) + .withCodegen(false) + .withCompatibleMode(CompatibleMode.COMPATIBLE) + .build(); + + // Create object with null nullable fields + SimpleTestClass obj = createTestObject(); + obj.nullableInt = null; + obj.nullableString = null; + + MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(1024); + fory1.serialize(buffer, obj); + + Map expectedMap = TestUtils.objectToMap(fory1, obj); + + buffer.readerIndex(0); + Object result = fory2.deserialize(buffer); + + assertEquals(result.getClass(), NonexistentClass.NonexistentMetaShared.class); + NonexistentClass.NonexistentMetaShared nonexistent = + (NonexistentClass.NonexistentMetaShared) result; + + // Convert NonexistentMetaShared to a map keyed by simple field name + Map actualMap = new HashMap<>(); + for (Object key : nonexistent.keySet()) { + String qualifiedKey = (String) key; + String simpleFieldName = qualifiedKey.substring(qualifiedKey.lastIndexOf('.') + 1); + actualMap.put(simpleFieldName, nonexistent.get(key)); + } + + // Verify values including nulls (compare by simple field name) + for (Map.Entry entry : expectedMap.entrySet()) { + String qualifiedFieldName = entry.getKey(); + String simpleFieldName = + qualifiedFieldName.substring(qualifiedFieldName.lastIndexOf('.') + 1); + Object expectedValue = entry.getValue(); + Object actualValue = actualMap.get(simpleFieldName); + + assertEquals( + actualValue, expectedValue, String.format("Field '%s' value mismatch", simpleFieldName)); + } + } + + /** + * Test that NonexistentMetaShared can be serialized and deserialized again by the same Fory + * instance that doesn't know the class. This verifies that unknown class data is preserved across + * serialization cycles. + */ + @Test(dataProvider = "language") + public void testNonexistentClassRoundTripWithinSameFory(Language language) { + // Fory1: knows the class + Fory fory1 = + Fory.builder() + .withLanguage(language) + .withCodegen(false) + .withCompatibleMode(CompatibleMode.COMPATIBLE) + .build(); + fory1.register(SimpleTestClass.class, "test.SimpleTestClass"); + + // Fory2: doesn't know the class + Fory fory2 = + Fory.builder() + .withLanguage(language) + .withCodegen(false) + .withCompatibleMode(CompatibleMode.COMPATIBLE) + .build(); + + // Step 1: Serialize with fory1 + SimpleTestClass original = createTestObject(); + Map expectedMap = TestUtils.objectToMap(fory1, original); + MemoryBuffer buffer1 = MemoryBuffer.newHeapBuffer(1024); + fory1.serialize(buffer1, original); + + // Step 2: Deserialize with fory2 (gets NonexistentMetaShared) + buffer1.readerIndex(0); + Object nonexistent1 = fory2.deserialize(buffer1); + assertEquals(nonexistent1.getClass(), NonexistentClass.NonexistentMetaShared.class); + + // Step 3: Serialize NonexistentMetaShared with fory2 + byte[] bytes = fory2.serialize(nonexistent1); + + // Step 4: Deserialize again with fory2 (should get NonexistentMetaShared with same values) + Object nonexistent2 = fory2.deserialize(bytes); + assertEquals(nonexistent2.getClass(), NonexistentClass.NonexistentMetaShared.class); + + // Verify values are preserved across the round-trip + NonexistentClass.NonexistentMetaShared result = + (NonexistentClass.NonexistentMetaShared) nonexistent2; + Map actualMap = new HashMap<>(); + for (Object key : result.keySet()) { + String qualifiedKey = (String) key; + String simpleFieldName = qualifiedKey.substring(qualifiedKey.lastIndexOf('.') + 1); + actualMap.put(simpleFieldName, result.get(key)); + } + + for (Map.Entry entry : expectedMap.entrySet()) { + String qualifiedFieldName = entry.getKey(); + String simpleFieldName = + qualifiedFieldName.substring(qualifiedFieldName.lastIndexOf('.') + 1); + Object expectedValue = entry.getValue(); + Object actualValue = actualMap.get(simpleFieldName); + + assertEquals( + actualValue, + expectedValue, + String.format("Field '%s' value mismatch after round-trip", simpleFieldName)); + } + } } diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java index e03353093b..34eb2c1c1a 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java @@ -75,4 +75,5 @@ public void testMDArrayField() { s.arr = new int[][] {{1, 2}, {3, 4}}; serDeCheck(fory, s); } + } diff --git a/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/RecordXlangTest.java b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/RecordXlangTest.java index 6aac6c2895..b707aac97e 100644 --- a/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/RecordXlangTest.java +++ b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/RecordXlangTest.java @@ -156,12 +156,12 @@ public static class NullablePojoSchemaConsistentXlang { * *

      *
    • Group 1 (non-nullable in Java): primitives, boxed types, and reference types - *
    • Group 2 (nullable in Java): boxed types and reference types with - * @ForyField(nullable=true) + *
    • Group 2 (nullable in Java): boxed types and reference types + * with @ForyField(nullable=true) *
    * - *

    In Go, Group 1 fields are nullable (*int8, *int16, etc.) and Group 2 fields are - * non-nullable (int32, int64, etc.) - this tests schema evolution with inverted nullability. + *

    In Go, Group 1 fields are nullable (*int8, *int16, etc.) and Group 2 fields are non-nullable + * (int32, int64, etc.) - this tests schema evolution with inverted nullability. */ public record NullableRecordCompatible( // Group 1: Non-nullable in Java (nullable in Go with pointer types) @@ -684,6 +684,94 @@ public void testRecordNullableFieldCompatibleNull(boolean enableCodegen) { Assert.assertEquals(result, expected); } + // ==================== Reference Tracking Tests ==================== + + /** Record for inner struct in reference tracking tests. */ + public record RefInnerRecord(int id, String name) {} + + /** + * Record for outer struct in reference tracking tests. Contains two fields that can point to the + * same RefInnerRecord instance. + */ + public record RefOuterRecord( + @ForyField(ref = true, nullable = true, dynamic = ForyField.Dynamic.FALSE) + RefInnerRecord inner1, + @ForyField(ref = true, nullable = true, dynamic = ForyField.Dynamic.FALSE) + RefInnerRecord inner2) {} + + /** + * Test reference tracking with Record in SCHEMA_CONSISTENT mode. Creates an outer struct with two + * fields pointing to the same inner struct instance. Verifies that reference identity is + * preserved after serialization/deserialization. + */ + @Test(dataProvider = "enableCodegen") + public void testRecordRefSchemaConsistent(boolean enableCodegen) { + Fory fory = + Fory.builder() + .withLanguage(Language.XLANG) + .withCompatibleMode(CompatibleMode.SCHEMA_CONSISTENT) + .withRefTracking(true) + .withCodegen(enableCodegen) + .build(); + fory.register(RefInnerRecord.class, 501); + fory.register(RefOuterRecord.class, 502); + + // Create inner record + RefInnerRecord inner = new RefInnerRecord(42, "shared_inner"); + + // Create outer record with both fields pointing to the same inner record + RefOuterRecord outer = new RefOuterRecord(inner, inner); + + // Serialize and deserialize + MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(256); + fory.serialize(buffer, outer); + buffer.readerIndex(0); + RefOuterRecord result = (RefOuterRecord) fory.deserialize(buffer); + + // Verify reference identity is preserved + Assert.assertSame( + result.inner1(), result.inner2(), "inner1 and inner2 should be same object"); + Assert.assertEquals(result.inner1().id(), 42); + Assert.assertEquals(result.inner1().name(), "shared_inner"); + } + + /** + * Test reference tracking with Record in COMPATIBLE mode. Creates an outer struct with two fields + * pointing to the same inner struct instance. Verifies that reference identity is preserved after + * serialization/deserialization with schema evolution support. + */ + @Test(dataProvider = "enableCodegen") + public void testRecordRefCompatible(boolean enableCodegen) { + Fory fory = + Fory.builder() + .withLanguage(Language.XLANG) + .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withRefTracking(true) + .withCodegen(enableCodegen) + .withMetaCompressor(new DeflaterMetaCompressor()) + .build(); + fory.register(RefInnerRecord.class, 503); + fory.register(RefOuterRecord.class, 504); + + // Create inner record + RefInnerRecord inner = new RefInnerRecord(99, "compatible_shared"); + + // Create outer record with both fields pointing to the same inner record + RefOuterRecord outer = new RefOuterRecord(inner, inner); + + // Serialize and deserialize + MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(512); + fory.serialize(buffer, outer); + buffer.readerIndex(0); + RefOuterRecord result = (RefOuterRecord) fory.deserialize(buffer); + + // Verify reference identity is preserved + Assert.assertSame( + result.inner1(), result.inner2(), "inner1 and inner2 should be same object"); + Assert.assertEquals(result.inner1().id(), 99); + Assert.assertEquals(result.inner1().name(), "compatible_shared"); + } + // ==================== Helper Methods ==================== private static Map createMap(String... keyValues) { From 4304f86d745e1e445d2d21138cb82c60ab932737 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 17 Jan 2026 17:48:39 +0800 Subject: [PATCH 23/44] fix failed tests --- .../fory/builder/BaseObjectCodecBuilder.java | 59 ++++++++----------- .../fory/builder/ObjectCodecBuilder.java | 2 +- .../apache/fory/codegen/ExpressionUtils.java | 7 +++ .../java/org/apache/fory/meta/FieldTypes.java | 14 +++-- .../apache/fory/resolver/ClassResolver.java | 13 +--- .../apache/fory/resolver/XtypeResolver.java | 15 +++-- .../NonexistentClassSerializers.java | 4 +- .../fory/serializer/SerializationBinding.java | 6 ++ .../collection/ChildContainerSerializers.java | 38 ++++++++++-- .../main/java/org/apache/fory/type/Types.java | 2 + 10 files changed, 97 insertions(+), 63 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java index 3c7411b34c..00588baca8 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java @@ -19,12 +19,6 @@ package org.apache.fory.builder; -import static org.apache.fory.builder.CodecBuilder.readFloat32Func; -import static org.apache.fory.builder.CodecBuilder.readFloat64Func; -import static org.apache.fory.builder.CodecBuilder.readInt16Func; -import static org.apache.fory.builder.CodecBuilder.readIntFunc; -import static org.apache.fory.builder.CodecBuilder.readLongFunc; -import static org.apache.fory.builder.CodecBuilder.readVarInt32Func; import static org.apache.fory.codegen.CodeGenerator.getPackage; import static org.apache.fory.codegen.Expression.Invoke.inlineInvoke; import static org.apache.fory.codegen.Expression.Literal.ofInt; @@ -35,7 +29,6 @@ import static org.apache.fory.codegen.ExpressionUtils.bitand; import static org.apache.fory.codegen.ExpressionUtils.bitor; import static org.apache.fory.codegen.ExpressionUtils.cast; -import static org.apache.fory.codegen.ExpressionUtils.defaultValue; import static org.apache.fory.codegen.ExpressionUtils.eq; import static org.apache.fory.codegen.ExpressionUtils.eqNull; import static org.apache.fory.codegen.ExpressionUtils.gt; @@ -1727,16 +1720,17 @@ protected Expression deserializeFor( // Should put value expr ahead to avoid generated code in wrong scope. return new ListExpression(value, callback.apply(value)); } - return readNullable( + return readNullableField( buffer, typeRef, callback, () -> deserializeForNotNull(buffer, typeRef, invokeHint)); } } - protected Expression deserializeForNullable( + protected Expression deserializeForNullableField( Expression buffer, - TypeRef typeRef, + Descriptor descriptor, Function callback, boolean nullable) { + TypeRef typeRef = descriptor.getTypeRef(); if (typeResolver(r -> r.needToWriteRef(typeRef))) { return readRef(buffer, callback, () -> deserializeForNotNull(buffer, typeRef, null)); } else { @@ -1746,8 +1740,15 @@ protected Expression deserializeForNullable( // Should put value expr ahead to avoid generated code in wrong scope. return new ListExpression(value, callback.apply(value)); } - return readNullable( - buffer, typeRef, callback, () -> deserializeForNotNull(buffer, typeRef, null), nullable); + // Pass local field type so readNullable can use default value for primitives when null + Class localFieldType = typeRef.isPrimitive() ? typeRef.getRawType() : null; + return readNullableField( + buffer, + descriptor, + callback, + () -> deserializeForNotNull(buffer, typeRef, null), + nullable + ); } } @@ -1772,7 +1773,7 @@ private Expression readRef( false); } - private Expression readNullable( + private Expression readNullableField( Expression buffer, TypeRef typeRef, Function callback, @@ -1786,22 +1787,12 @@ private Expression readNullable( return new If(notNull, callback.apply(value), callback.apply(nullValue(typeRef)), false); } - private Expression readNullable( + private Expression readNullableField( Expression buffer, - TypeRef typeRef, + Descriptor descriptor, Function callback, Supplier deserializeForNotNull, boolean nullable) { - return readNullable(buffer, typeRef, callback, deserializeForNotNull, nullable, null); - } - - private Expression readNullable( - Expression buffer, - TypeRef typeRef, - Function callback, - Supplier deserializeForNotNull, - boolean nullable, - Class localFieldType) { if (nullable) { Expression notNull = neq( @@ -1810,12 +1801,8 @@ private Expression readNullable( Expression value = deserializeForNotNull.get(); // When local field is primitive but remote was nullable (boxed), use default value // instead of null. This handles compatibility between boxed/primitive field types. - Expression nullExpr; - if (localFieldType != null && isPrimitive(localFieldType)) { - nullExpr = defaultValue(localFieldType); - } else { - nullExpr = nullValue(typeRef); - } + Expression nullExpr = nullValue(descriptor.getField() != null + ? descriptor.getField().getType() : descriptor.getRawType()); // use false to ignore null. return new If(notNull, callback.apply(value), callback.apply(nullExpr), false); } else { @@ -1902,13 +1889,13 @@ protected Expression deserializeField( java.lang.reflect.Field field = descriptor.getField(); Class localFieldType = field != null ? field.getType() : null; Expression readNullableExpr = - readNullable( + readNullableField( buffer, - typeRef, + descriptor, callback, () -> deserializeForNotNullForField(buffer, descriptor, null), - true, - localFieldType); + true + ); if (serializerCallsReference) { Expression preserveStubRefId = @@ -2251,7 +2238,7 @@ private Expression readContainerElement( read = new If( hasNull, - readNullable( + readNullableField( buffer, elementType, callback, diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java index 49c0158a80..25b7d72acb 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java @@ -613,7 +613,7 @@ protected Expression deserializeGroupForRecord( // use Reference to cut-off expr dependency. for (Descriptor d : group) { boolean nullable = d.isNullable(); - Expression v = deserializeForNullable(buffer, d.getTypeRef(), expr -> expr, nullable); + Expression v = deserializeForNullableField(buffer, d, expr -> expr, nullable); Expression action = setFieldValue(bean, d, tryInlineCast(v, d.getTypeRef())); groupExpressions.add(action); } diff --git a/java/fory-core/src/main/java/org/apache/fory/codegen/ExpressionUtils.java b/java/fory-core/src/main/java/org/apache/fory/codegen/ExpressionUtils.java index 75536b3476..9a80d1e5dc 100644 --- a/java/fory-core/src/main/java/org/apache/fory/codegen/ExpressionUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/codegen/ExpressionUtils.java @@ -151,6 +151,13 @@ public static Expression not(Expression target) { return new Not(target); } + public static Literal nullValue(Class type) { + if (type.isPrimitive()) { + return defaultValue(type); + } + return new Literal(null, TypeRef.of(type)); + } + public static Literal nullValue(TypeRef type) { if (type.isPrimitive()) { return defaultValue(type.getRawType()); diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java index 5376cd8826..5956b2aa8f 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java @@ -166,19 +166,23 @@ private static FieldType buildFieldType( } if (rawType.isArray()) { Class elemType = rawType.getComponentType(); - if (elemType.isPrimitive()) { - return new RegisteredFieldType(nullable, trackingRef, xtypeId); - } if (isXlang) { - // primitive array has bee handled before + if (elemType.isPrimitive()) { + // For xlang mode, use xlang primitive array type IDs + int elemTypeId = Types.getTypeId(resolver.getFory(), elemType); + int arrayTypeId = Types.getPrimitiveArrayTypeId(elemTypeId); + return new RegisteredFieldType(nullable, trackingRef, arrayTypeId); + } return new CollectionFieldType( xtypeId, nullable, trackingRef, buildFieldType(resolver, null, GenericType.build(elemType))); } else { + // For native mode, use Java class IDs for arrays if (((ClassResolver)resolver).isInternalRegistered(rawType)) { - return new RegisteredFieldType(nullable, trackingRef, xtypeId); + Short classId = ((ClassResolver) resolver).getRegisteredClassId(rawType); + return new RegisteredFieldType(nullable, trackingRef, classId); } Tuple2, Integer> arrayComponentInfo = getArrayComponentInfo(rawType); return new ArrayFieldType( diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index c74e1a5791..e49217ab8d 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -704,6 +704,9 @@ public boolean isMonomorphic(Descriptor descriptor) { */ @Override public boolean isMonomorphic(Class clz) { + if (clz == NonexistentMetaShared.class) { + return true; + } if (fory.getConfig().isMetaShareEnabled()) { // can't create final map/collection type using TypeUtils.mapOf(TypeToken, // TypeToken) @@ -1513,16 +1516,6 @@ public void writeClassInfo(MemoryBuffer buffer, ClassInfo classInfo) { } public void writeClassInfoWithMetaShare(MemoryBuffer buffer, ClassInfo classInfo) { - // For NonexistentClassSerializer, the serializer handles ClassDef writing itself - // because the ClassDef comes from the object being serialized, not from the class. - // We just write a placeholder that the serializer will overwrite. - if (classInfo.serializer != null - && classInfo.serializer.getClass() - == NonexistentClassSerializers.NonexistentClassSerializer.class) { - // Write a 2-byte placeholder that NonexistentClassSerializer.writeClassDef will overwrite - buffer.writeInt16((short) 0); - return; - } // For dynamically generated classes (lambdas, JDK proxies), use their stub class // for the ClassDef since the dynamic class names cannot be loaded by name. // The serializer will handle the actual serialization correctly. diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java index dd445b7bee..46cfe47e9a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java @@ -449,6 +449,9 @@ public boolean isMonomorphic(Descriptor descriptor) { if (rawType.isEnum()) { return true; } + if (rawType == NonexistentMetaShared.class) { + return true; + } byte xtypeId = getXtypeId(rawType); if (fory.isCompatible()) { return !Types.isUserDefinedType(xtypeId) && xtypeId != Types.UNKNOWN; @@ -465,6 +468,9 @@ public boolean isMonomorphic(Class clz) { if (clz.isArray()) { return true; } + if (clz == NonexistentMetaShared.class) { + return true; + } ClassInfo classInfo = getClassInfo(clz, false); if (classInfo != null) { Serializer s = classInfo.serializer; @@ -486,6 +492,9 @@ public boolean isMonomorphic(Class clz) { public boolean isBuildIn(Descriptor descriptor) { Class rawType = descriptor.getRawType(); byte xtypeId = getXtypeId(rawType); + if (rawType == NonexistentMetaShared.class) { + return true; + } return !Types.isUserDefinedType(xtypeId) && xtypeId != Types.UNKNOWN; } @@ -735,11 +744,7 @@ public void writeClassInfo(MemoryBuffer buffer, ClassInfo classInfo) { case Types.NAMED_COMPATIBLE_STRUCT: case Types.COMPATIBLE_STRUCT: assert shareMeta : "Meta share must be enabled for compatible mode"; - // Skip writeSharedClassMeta for NonexistentMetaShared since - // NonexistentClassSerializer.writeClassDef will handle writing the correct classDef. - if (classInfo.cls != NonexistentMetaShared.class) { - writeSharedClassMeta(buffer, classInfo); - } + writeSharedClassMeta(buffer, classInfo); break; default: break; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java index 89bbf8eb0f..2c8256482a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java @@ -88,8 +88,8 @@ public NonexistentClassSerializer(Fory fory, ClassDef classDef) { * classinfo by `class`, it may dispatch to same `NonexistentClassSerializer`, so we can't use * `classDef` in this serializer, but use `classDef` in `NonexistentMetaSharedClass` instead. * - *

    Note: XtypeResolver.writeClassInfo skips writeSharedClassMeta for NonexistentMetaShared, - * so this method writes the classDef directly without reverting any buffer bytes. + *

    XtypeResolver.writeSharedClassMeta writes a stub (1-byte ref marker + stub bytes) for + * NonexistentMetaShared. This method rewinds past that stub and writes the actual classDef. */ private void writeClassDef(MemoryBuffer buffer, NonexistentClass.NonexistentMetaShared value) { MetaContext metaContext = fory.getSerializationContext().getMetaContext(); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java index 6101e560ab..80f56010a9 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java @@ -273,6 +273,9 @@ Object readField(SerializationFieldInfo fieldInfo, RefMode refMode, MemoryBuffer return fory.readRef(buffer, fieldInfo.classInfo); } else { if (refMode != RefMode.NULL_ONLY || buffer.readByte() != Fory.NULL_FLAG) { + // Preserve a dummy ref ID so ObjectSerializer.read() can pop it. + // This is needed when global ref tracking is enabled but field ref tracking is disabled. + refResolver.preserveRefId(-1); return fory.readNonRef(buffer, fieldInfo.classInfo); } } @@ -281,6 +284,9 @@ Object readField(SerializationFieldInfo fieldInfo, RefMode refMode, MemoryBuffer return fory.readRef(buffer, fieldInfo.classInfoHolder); } else { if (refMode != RefMode.NULL_ONLY || buffer.readByte() != Fory.NULL_FLAG) { + // Preserve a dummy ref ID so ObjectSerializer.read() can pop it. + // This is needed when global ref tracking is enabled but field ref tracking is disabled. + refResolver.preserveRefId(-1); return fory.readNonRef(buffer, fieldInfo.classInfoHolder); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java index 98247cff92..c831a5ad15 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java @@ -42,6 +42,7 @@ import org.apache.fory.meta.ClassDef; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.resolver.ClassResolver; +import org.apache.fory.resolver.MetaContext; import org.apache.fory.serializer.AbstractObjectSerializer; import org.apache.fory.serializer.FieldGroups; import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo; @@ -151,7 +152,7 @@ public Collection onCollectionWrite(MemoryBuffer buffer, T value) { public Collection newCollection(MemoryBuffer buffer) { Collection collection = super.newCollection(buffer); - readAndSetFields(buffer, collection, slotsSerializers); + readAndSetFields(fory, buffer, collection, slotsSerializers); return collection; } @@ -213,7 +214,7 @@ public Map onMapWrite(MemoryBuffer buffer, T value) { @Override public Map newMap(MemoryBuffer buffer) { Map map = super.newMap(buffer); - readAndSetFields(buffer, map, slotsSerializers); + readAndSetFields(fory, buffer, map, slotsSerializers); return map; } @@ -254,13 +255,42 @@ private static Serializer[] buildSlotsSerializers( } private static void readAndSetFields( - MemoryBuffer buffer, Object collection, Serializer[] slotsSerializers) { + Fory fory, MemoryBuffer buffer, Object collection, Serializer[] slotsSerializers) { for (Serializer slotsSerializer : slotsSerializers) { if (slotsSerializer instanceof MetaSharedLayerSerializer) { - ((MetaSharedLayerSerializer) slotsSerializer).readAndSetFields(buffer, collection); + MetaSharedLayerSerializer metaSerializer = (MetaSharedLayerSerializer) slotsSerializer; + // Read layer class meta first if meta share is enabled + // This corresponds to writeLayerClassMeta() in MetaSharedLayerSerializer.write() + if (fory.getConfig().isMetaShareEnabled()) { + readAndSkipLayerClassMeta(fory, buffer); + } + metaSerializer.readAndSetFields(buffer, collection); } else { ((ObjectSerializer) slotsSerializer).readAndSetFields(buffer, collection); } } } + + /** + * Read and skip the layer class meta from buffer. This is used to skip over the class definition + * that was written by MetaSharedLayerSerializer.writeLayerClassMeta(). For ChildContainerSerializers, + * we use the same serializer on both write and read sides, so we just need to skip the meta + * without actually parsing it. + */ + private static void readAndSkipLayerClassMeta(Fory fory, MemoryBuffer buffer) { + MetaContext metaContext = fory.getSerializationContext().getMetaContext(); + if (metaContext == null) { + return; + } + int indexMarker = buffer.readVarUint32Small14(); + boolean isRef = (indexMarker & 1) == 1; + int index = indexMarker >>> 1; + if (isRef) { + // Reference to previously read type - nothing more to read + return; + } + // New type - need to read and skip the ClassDef bytes + long id = buffer.readInt64(); + ClassDef.skipClassDef(buffer, id); + } } diff --git a/java/fory-core/src/main/java/org/apache/fory/type/Types.java b/java/fory-core/src/main/java/org/apache/fory/type/Types.java index b20d03a3ed..c6337a6ab3 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/Types.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/Types.java @@ -275,8 +275,10 @@ public static int getPrimitiveArrayTypeId(int typeId) { case INT16: return INT16_ARRAY; case INT32: + case VARINT32: return INT32_ARRAY; case INT64: + case VARINT64: return INT64_ARRAY; case UINT8: return UINT8_ARRAY; From 152b0d7e2f7c08ec88045d0620f52bd9e5e319f1 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 18 Jan 2026 13:03:07 +0800 Subject: [PATCH 24/44] refactor type system for java --- .../src/main/java/org/apache/fory/Fory.java | 8 +- .../org/apache/fory/meta/ClassDefEncoder.java | 7 +- .../java/org/apache/fory/meta/FieldInfo.java | 2 +- .../java/org/apache/fory/meta/FieldTypes.java | 6 +- .../org/apache/fory/meta/TypeDefEncoder.java | 6 +- .../org/apache/fory/resolver/ClassInfo.java | 81 +-- .../apache/fory/resolver/ClassResolver.java | 370 +++--------- .../org/apache/fory/resolver/MetaContext.java | 13 + .../apache/fory/resolver/TypeResolver.java | 549 +++++++++++++++++- .../apache/fory/resolver/XtypeResolver.java | 521 ++++++++--------- .../fory/serializer/ArraySerializers.java | 3 +- .../fory/serializer/OptionalSerializers.java | 3 +- .../fory/serializer/PrimitiveSerializers.java | 3 +- .../apache/fory/serializer/Serializers.java | 3 +- .../fory/serializer/TimeSerializers.java | 3 +- .../collection/CollectionLikeSerializer.java | 2 +- .../collection/CollectionSerializers.java | 3 +- .../GuavaCollectionSerializers.java | 3 +- .../ImmutableCollectionSerializers.java | 3 +- .../collection/MapLikeSerializer.java | 12 +- .../serializer/collection/MapSerializers.java | 8 +- .../collection/SubListSerializers.java | 3 +- .../collection/SynchronizedSerializers.java | 3 +- .../collection/UnmodifiableSerializers.java | 3 +- .../main/java/org/apache/fory/type/Types.java | 2 +- 25 files changed, 985 insertions(+), 635 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java index c702cb4859..e04c351017 100644 --- a/java/fory-core/src/main/java/org/apache/fory/Fory.java +++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java @@ -585,7 +585,7 @@ public void xwriteData(MemoryBuffer buffer, ClassInfo classInfo, Object obj) { /** Write not null data to buffer. */ private void writeData(MemoryBuffer buffer, ClassInfo classInfo, Object obj) { - switch (classInfo.getClassId()) { + switch (classInfo.getTypeId()) { case Types.BOOL: buffer.writeBoolean((Boolean) obj); break; @@ -989,7 +989,7 @@ public Object readData(MemoryBuffer buffer, ClassInfo classInfo) { } private Object readDataInternal(MemoryBuffer buffer, ClassInfo classInfo) { - switch (classInfo.getClassId()) { + switch (classInfo.getTypeId()) { case Types.BOOL: return buffer.readBoolean(); case Types.INT8: @@ -1021,7 +1021,7 @@ private Object readDataInternal(MemoryBuffer buffer, ClassInfo classInfo) { } private Object xreadDataInternal(MemoryBuffer buffer, ClassInfo classInfo) { - switch (classInfo.getClassId()) { + switch (classInfo.getTypeId()) { case Types.BOOL: return buffer.readBoolean(); case Types.INT8: @@ -1406,7 +1406,7 @@ public T copyObject(T obj) { } Object copy; ClassInfo classInfo = classResolver.getOrUpdateClassInfo(obj.getClass()); - switch (classInfo.getClassId()) { + switch (classInfo.getTypeId()) { case Types.BOOL: case Types.INT8: case ClassResolver.CHAR_ID: diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDefEncoder.java b/java/fory-core/src/main/java/org/apache/fory/meta/ClassDefEncoder.java index 2a896864df..c6bf209a3d 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDefEncoder.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/ClassDefEncoder.java @@ -76,13 +76,14 @@ static List buildFields(Fory fory, Class cls, boolean resolveParent) { descriptorGrouper .getBuildInDescriptors() .forEach(descriptor -> fields.add(descriptor.getField())); - descriptorGrouper - .getOtherDescriptors() - .forEach(descriptor -> fields.add(descriptor.getField())); + // Order must match ObjectSerializer serialization order: buildIn, container, other descriptorGrouper .getCollectionDescriptors() .forEach(descriptor -> fields.add(descriptor.getField())); descriptorGrouper.getMapDescriptors().forEach(descriptor -> fields.add(descriptor.getField())); + descriptorGrouper + .getOtherDescriptors() + .forEach(descriptor -> fields.add(descriptor.getField())); return fields; } diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java b/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java index 5c759a1230..ccb2cdc4a3 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java @@ -94,7 +94,7 @@ Descriptor toDescriptor(TypeResolver resolver, Descriptor descriptor) { String typeName = fieldType.getTypeName(resolver, typeRef); if (fieldType instanceof FieldTypes.RegisteredFieldType) { if (!Types.isPrimitiveType(fieldType.xtypeId)) { - typeName = String.valueOf(((FieldTypes.RegisteredFieldType) fieldType).getClassId()); + typeName = String.valueOf(((FieldTypes.RegisteredFieldType) fieldType).getTypeId()); } } // Get nullable and trackingRef from remote FieldType - these are what the remote peer diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java index 5956b2aa8f..e3aaddc9c8 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java @@ -97,7 +97,7 @@ private static FieldType buildFieldType( } else { ClassInfo info = resolver.getClassInfo(genericType.getCls(), false); if (info != null) { - xtypeId = info.getXtypeId(); + xtypeId = info.getTypeId(); } else { xtypeId = Types.UNKNOWN; } @@ -259,7 +259,7 @@ public void write(MemoryBuffer buffer, boolean writeHeader) { // - bits 2+: typeId byte header = (byte) ((nullable ? 0b10 : 0) | (trackingRef ? 0b1 : 0)); if (this instanceof RegisteredFieldType) { - short classId = ((RegisteredFieldType) this).getClassId(); + short classId = ((RegisteredFieldType) this).getTypeId(); buffer.writeVarUint32Small7(writeHeader ? ((5 + classId) << 2) | header : 5 + classId); } else if (this instanceof EnumFieldType) { buffer.writeVarUint32Small7(writeHeader ? ((4) << 2) | header : 4); @@ -416,7 +416,7 @@ public RegisteredFieldType(boolean nullable, boolean trackingRef, int classId) { this.classId = (short) classId; } - public short getClassId() { + public short getTypeId() { return classId; } diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefEncoder.java b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefEncoder.java index 854b41c018..420f75b51f 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefEncoder.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefEncoder.java @@ -68,8 +68,8 @@ static ClassDef buildTypeDef(Fory fory, Class type) { Function.identity()); ClassInfo classInfo = fory._getTypeResolver().getClassInfo(type); List fields; - int xtypeId = classInfo.getXtypeId(); - if (Types.isStructType(xtypeId & 0xff)) { + int typeId = classInfo.getTypeId(); + if (Types.isStructType(typeId & 0xff)) { fields = descriptorGrouper.getSortedDescriptors().stream() .map(Descriptor::getField) @@ -144,7 +144,7 @@ static MemoryBuffer encodeClassDef( buffer.writeVarUint32(fields.size() - SMALL_NUM_FIELDS_THRESHOLD); } if (resolver.isRegisteredById(type)) { - buffer.writeVarUint32(classInfo.getXtypeId()); + buffer.writeVarUint32(classInfo.getTypeId()); } else { Preconditions.checkArgument(resolver.isRegisteredByName(type)); currentClassHeader |= REGISTER_BY_NAME_FLAG; diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassInfo.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassInfo.java index b2dd539247..a9d38b2b13 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassInfo.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassInfo.java @@ -29,6 +29,7 @@ import org.apache.fory.meta.MetaString.Encoding; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.serializer.Serializer; +import org.apache.fory.type.Types; import org.apache.fory.util.Preconditions; import org.apache.fory.util.function.Functions; @@ -42,11 +43,12 @@ public class ClassInfo { final MetaStringBytes namespaceBytes; final MetaStringBytes typeNameBytes; final boolean isDynamicGeneratedClass; - int xtypeId; + // Unified type ID for both native and xlang modes. + // - Types 0-30: Shared internal types (Types.BOOL, Types.STRING, etc.) + // - Types 31-255: Native-only internal types (VOID_ID, CHAR_ID, etc.) + // - Types 256+: User-registered types encoded as (userTypeId << 8) | internalTypeId + int typeId; Serializer serializer; - // use primitive to avoid boxing - // class id must be less than Integer.MAX_VALUE/2 since we use bit 0 as class id flag. - short classId; ClassDef classDef; boolean needToWriteClassDef; @@ -57,17 +59,15 @@ public class ClassInfo { MetaStringBytes typeNameBytes, boolean isDynamicGeneratedClass, Serializer serializer, - short classId, - int xtypeId) { + int typeId) { this.cls = cls; this.fullNameBytes = fullNameBytes; this.namespaceBytes = namespaceBytes; this.typeNameBytes = typeNameBytes; this.isDynamicGeneratedClass = isDynamicGeneratedClass; - this.xtypeId = xtypeId; + this.typeId = typeId; this.serializer = serializer; - this.classId = classId; - if (cls != null && classId == TypeResolver.NO_CLASS_ID) { + if (cls != null && typeId == TypeResolver.NO_CLASS_ID) { Preconditions.checkArgument(typeNameBytes != null); } } @@ -87,16 +87,10 @@ public ClassInfo(Class cls, ClassDef classDef) { this.typeNameBytes = null; this.isDynamicGeneratedClass = false; this.serializer = null; - this.classId = TypeResolver.NO_CLASS_ID; - this.xtypeId = 0; + this.typeId = TypeResolver.NO_CLASS_ID; } - ClassInfo( - TypeResolver classResolver, - Class cls, - Serializer serializer, - short classId, - short xtypeId) { + ClassInfo(TypeResolver classResolver, Class cls, Serializer serializer, int typeId) { this.cls = cls; this.serializer = serializer; needToWriteClassDef = serializer != null && classResolver.needToWriteClassDef(serializer); @@ -108,12 +102,19 @@ public ClassInfo(Class cls, ClassDef classDef) { } else { this.fullNameBytes = null; } - // When `classId == ClassResolver.REPLACE_STUB_ID` was established, - // means only classes are serialized, not the instance. If we - // serialize such class only, we need to write classname bytes. - if (cls != null - && (classId == ClassResolver.NO_CLASS_ID || classId == ClassResolver.REPLACE_STUB_ID)) { - // REPLACE_STUB_ID for write replace class in `ClassSerializer`. + // When typeId indicates a named type, we need to create classname bytes for serialization. + // - NAMED_STRUCT: unregistered struct classes + // - NAMED_COMPATIBLE_STRUCT: unregistered classes in compatible mode + // - NAMED_ENUM, NAMED_EXT: other named types + // - REPLACE_STUB_ID: for write replace class in `ClassSerializer` + // - NO_CLASS_ID: legacy support (should use NAMED_STRUCT instead) + boolean isNamedType = typeId == Types.NAMED_STRUCT + || typeId == Types.NAMED_COMPATIBLE_STRUCT + || typeId == Types.NAMED_ENUM + || typeId == Types.NAMED_EXT + || typeId == ClassResolver.REPLACE_STUB_ID + || typeId == TypeResolver.NO_CLASS_ID; + if (cls != null && isNamedType) { Tuple2 tuple2 = Encoders.encodePkgAndClass(cls); this.namespaceBytes = metaStringResolver.getOrCreateMetaStringBytes(Encoders.encodePackage(tuple2.f0)); @@ -123,17 +124,16 @@ public ClassInfo(Class cls, ClassDef classDef) { this.namespaceBytes = null; this.typeNameBytes = null; } - this.xtypeId = xtypeId; - this.classId = classId; + this.typeId = typeId; if (cls != null) { boolean isLambda = Functions.isLambda(cls); - boolean isProxy = classId != ClassResolver.REPLACE_STUB_ID && ReflectionUtils.isJdkProxy(cls); + boolean isProxy = typeId != ClassResolver.REPLACE_STUB_ID && ReflectionUtils.isJdkProxy(cls); this.isDynamicGeneratedClass = isLambda || isProxy; if (isLambda) { - this.classId = ClassResolver.LAMBDA_STUB_ID; + this.typeId = ClassResolver.LAMBDA_STUB_ID; } if (isProxy) { - this.classId = ClassResolver.JDK_PROXY_STUB_ID; + this.typeId = ClassResolver.JDK_PROXY_STUB_ID; } } else { this.isDynamicGeneratedClass = false; @@ -152,12 +152,21 @@ void setClassDef(ClassDef classDef) { this.classDef = classDef; } - public short getClassId() { - return classId; - } - - public int getXtypeId() { - return xtypeId; + /** + * Returns the unified type ID for this class. + * + *

    Type ID encoding: + * + *

      + *
    • 0-30: Shared internal types (Types.BOOL, Types.STRING, etc.) + *
    • 31-255: Native-only internal types (VOID_ID, CHAR_ID, etc.) + *
    • 256+: User-registered types encoded as (userTypeId << 8) | internalTypeId + *
    + * + * @return the unified type ID + */ + public int getTypeId() { + return typeId; } @SuppressWarnings("unchecked") @@ -193,8 +202,8 @@ public String toString() { + isDynamicGeneratedClass + ", serializer=" + serializer - + ", classId=" - + classId + + ", typeId=" + + typeId + '}'; } } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index e49217ab8d..649f17a6fe 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -199,7 +199,7 @@ public class ClassResolver extends TypeResolver { */ public static final short USER_ID_BASE = 256; - public static final int NATIVE_START_ID = Types.STRING + 1; + public static final int NATIVE_START_ID = Types.NAMED_EXT + 1; public static final int VOID_ID = NATIVE_START_ID; public static final int CHAR_ID = NATIVE_START_ID + 1; // Note: following pre-defined class id should be continuous, since they may be used based range. @@ -232,13 +232,12 @@ public class ClassResolver extends TypeResolver { public static final short REPLACE_STUB_ID = NATIVE_START_ID + 28; private final Fory fory; - XtypeResolver xtypeResolver; private ClassInfo[] registeredId2ClassInfo = new ClassInfo[] {}; private ClassInfo classInfoCache; // Every deserialization for unregistered class will query it, performance is important. private final ObjectMap compositeNameBytes2ClassInfo = new ObjectMap<>(16, foryMapLoadFactor); - private final Map, ClassDef> classDefMap = new HashMap<>(); + // classDefMap is inherited from TypeResolver private Class currentReadClass; // class id of last default registered class. private short innerEndClassId; @@ -343,8 +342,8 @@ private void addDefaultSerializers() { // Those class id must be known in advance, here is two bytes, so // `NonexistentClassSerializer.writeClassDef` // can overwrite written classinfo and replace with real classinfo. - short classId = - Objects.requireNonNull(classInfoMap.get(NonexistentMetaShared.class)).classId; + int classId = + Objects.requireNonNull(classInfoMap.get(NonexistentMetaShared.class)).typeId; Preconditions.checkArgument(classId > 63 && classId < 8192, classId); } else { registerInternal(NonexistentSkip.class); @@ -500,7 +499,7 @@ public void register(Class cls, String namespace, String name) { metaStringResolver.getOrCreateMetaStringBytes(encodePackage(namespace)); MetaStringBytes nameBytes = metaStringResolver.getOrCreateMetaStringBytes(encodeTypeName(name)); ClassInfo classInfo = - new ClassInfo(cls, fullNameBytes, nsBytes, nameBytes, false, null, NO_CLASS_ID, (short) -1); + new ClassInfo(cls, fullNameBytes, nsBytes, nameBytes, false, null, NO_CLASS_ID); classInfoMap.put(cls, classInfo); compositeNameBytes2ClassInfo.put( new TypeNameBytes(nsBytes.hashCode, nameBytes.hashCode), classInfo); @@ -572,9 +571,9 @@ private void registerImpl(Class cls, int classId) { } ClassInfo classInfo = classInfoMap.get(cls); if (classInfo != null) { - classInfo.classId = id; + classInfo.typeId = id; } else { - classInfo = new ClassInfo(this, cls, null, id, NOT_SUPPORT_XLANG); + classInfo = new ClassInfo(this, cls, null, id); // make `extRegistry.registeredClassIdMap` and `classInfoMap` share same classInfo // instances. classInfoMap.put(cls, classInfo); @@ -704,9 +703,6 @@ public boolean isMonomorphic(Descriptor descriptor) { */ @Override public boolean isMonomorphic(Class clz) { - if (clz == NonexistentMetaShared.class) { - return true; - } if (fory.getConfig().isMetaShareEnabled()) { // can't create final map/collection type using TypeUtils.mapOf(TypeToken, // TypeToken) @@ -742,7 +738,7 @@ public boolean isInternalRegistered(Class cls) { if (classId == null) { ClassInfo classInfo = getClassInfo(cls, false); if (classInfo != null) { - classId = classInfo.getClassId(); + classId = (short) classInfo.getTypeId(); } } return classId != null && classId != NO_CLASS_ID && classId < innerEndClassId; @@ -827,6 +823,7 @@ public void registerSerializer(Class type, Serializer serializer) { * @param type class needed to be serialized/deserialized * @param serializer serializer for object of {@code type} */ + @Override public void registerInternalSerializer(Class type, Serializer serializer) { registerSerializerImpl(type, serializer); } @@ -956,17 +953,28 @@ public void addSerializer(Class type, Serializer serializer) { && !(serializer instanceof FinalFieldReplaceResolveSerializer)) { classId = REPLACE_STUB_ID; } else { - classId = NO_CLASS_ID; + // For unregistered classes, use NAMED_STRUCT or NAMED_COMPATIBLE_STRUCT + // so that writeClassInfo writes the namespace and typename bytes (or meta-share) + classId = (short) (metaContextShareEnabled + ? Types.NAMED_COMPATIBLE_STRUCT : Types.NAMED_STRUCT); } classInfo = classInfoMap.get(type); } - if (classInfo == null || classId != classInfo.classId) { - classInfo = new ClassInfo(this, type, null, classId, (short) 0); + if (classInfo == null || classId != classInfo.typeId) { + classInfo = new ClassInfo(this, type, null, classId); classInfoMap.put(type, classInfo); if (registered) { registeredId2ClassInfo[classId] = classInfo; } + // Add to compositeNameBytes2ClassInfo for unregistered classes so that + // readClassInfo can find the ClassInfo by name bytes during deserialization. + // This is important for dynamically created classes that can't be loaded by name. + if (classInfo.namespaceBytes != null && classInfo.typeNameBytes != null) { + TypeNameBytes typeNameBytes = + new TypeNameBytes(classInfo.namespaceBytes.hashCode, classInfo.typeNameBytes.hashCode); + compositeNameBytes2ClassInfo.put(typeNameBytes, classInfo); + } } // 2. Set `Serializer` for `ClassInfo`. @@ -1306,7 +1314,7 @@ public ClassInfo getOrUpdateClassInfo(Class cls) { private ClassInfo getOrUpdateClassInfo(short classId) { ClassInfo classInfo = classInfoCache; - if (classInfo.classId != classId) { + if (classInfo.typeId != classId) { classInfo = registeredId2ClassInfo[classId]; if (classInfo.serializer == null) { addSerializer(classInfo.cls, createSerializer(classInfo.cls)); @@ -1465,6 +1473,18 @@ private boolean isSecure(Class cls) { // } } + /** + * Check if a typeId corresponds to a valid registered class in this resolver. + * For ClassResolver, this checks if the typeId is in range and has an entry + * in registeredId2ClassInfo. + */ + @Override + protected boolean isValidRegisteredTypeId(int typeId) { + return typeId > 0 + && typeId < registeredId2ClassInfo.length + && registeredId2ClassInfo[typeId] != null; + } + /** * Write class info to buffer. TODO(chaokunyang): The method should try to write * aligned data to reduce cpu instruction overhead. `writeClassInfo` is the last step before @@ -1493,62 +1513,8 @@ public void writeClassAndUpdateCache(MemoryBuffer buffer, Class cls) { // return classInfo; // } - /** Write classname for java serialization. */ @Override - public void writeClassInfo(MemoryBuffer buffer, ClassInfo classInfo) { - if (metaContextShareEnabled) { - // FIXME(chaokunyang) Register class but not register serializer can't be used with - // meta share mode, because no class def are sent to peer. - writeClassInfoWithMetaShare(buffer, classInfo); - } else { - if (classInfo.classId == NO_CLASS_ID) { // no class id provided. - // use classname - // if it's null, it's a bug. - assert classInfo.namespaceBytes != null; - metaStringResolver.writeMetaStringBytesWithFlag(buffer, classInfo.namespaceBytes); - assert classInfo.typeNameBytes != null; - metaStringResolver.writeMetaStringBytes(buffer, classInfo.typeNameBytes); - } else { - // use classId - buffer.writeVarUint32(classInfo.classId << 1); - } - } - } - - public void writeClassInfoWithMetaShare(MemoryBuffer buffer, ClassInfo classInfo) { - // For dynamically generated classes (lambdas, JDK proxies), use their stub class - // for the ClassDef since the dynamic class names cannot be loaded by name. - // The serializer will handle the actual serialization correctly. - Class classForMeta = classInfo.cls; - if (classInfo.isDynamicGeneratedClass) { - if (classInfo.classId == LAMBDA_STUB_ID) { - classForMeta = LambdaSerializer.ReplaceStub.class; - } else if (classInfo.classId == JDK_PROXY_STUB_ID) { - classForMeta = JdkProxySerializer.ReplaceStub.class; - } - } - MetaContext metaContext = fory.getSerializationContext().getMetaContext(); - assert metaContext != null : SET_META__CONTEXT_MSG; - IdentityObjectIntMap> classMap = metaContext.classMap; - int newId = classMap.size; - int id = classMap.putOrGet(classForMeta, newId); - if (id >= 0) { - // Reference to previously written type: (index << 1) | 1, LSB=1 - buffer.writeVarUint32((id << 1) | 1); - } else { - // New type: index << 1, LSB=0, followed by ClassDef bytes inline - buffer.writeVarUint32(newId << 1); - ClassInfo stubClassInfo = - classForMeta == classInfo.cls ? classInfo : getClassInfo(classForMeta); - ClassDef classDef = stubClassInfo.classDef; - if (classDef == null) { - classDef = buildClassDef(stubClassInfo); - } - buffer.writeBytes(classDef.getEncoded()); - } - } - - private ClassDef buildClassDef(ClassInfo classInfo) { + protected ClassDef buildClassDef(ClassInfo classInfo) { ClassDef classDef; Serializer serializer = classInfo.serializer; Preconditions.checkArgument(serializer.getClass() != NonexistentClassSerializer.class); @@ -1565,53 +1531,7 @@ private ClassDef buildClassDef(ClassInfo classInfo) { return classDef; } - @Override - public ClassInfo readSharedClassMeta(MemoryBuffer buffer, MetaContext metaContext) { - assert metaContext != null : SET_META__CONTEXT_MSG; - int indexMarker = buffer.readVarUint32Small14(); - boolean isRef = (indexMarker & 1) == 1; - int index = indexMarker >>> 1; - ClassInfo classInfo; - if (isRef) { - // Reference to previously read type in this stream - classInfo = metaContext.readClassInfos.get(index); - } else { - // New type in stream - but may already be known from registry - long id = buffer.readInt64(); - Tuple2 tuple2 = extRegistry.classIdToDef.get(id); - if (tuple2 != null) { - // Already known - skip the ClassDef bytes, reuse existing ClassInfo - ClassDef.skipClassDef(buffer, id); - classInfo = tuple2.f1; - if (classInfo == null) { - classInfo = buildMetaSharedClassInfo(tuple2, tuple2.f0); - } - } else { - // Unknown - read ClassDef and create ClassInfo - tuple2 = readClassDef(buffer, id); - classInfo = tuple2.f1; - if (classInfo == null) { - classInfo = buildMetaSharedClassInfo(tuple2, tuple2.f0); - } - } - // index == readClassInfos.size() since types are written sequentially - metaContext.readClassInfos.add(classInfo); - } - return classInfo; - } - - @Override - public ClassDef getTypeDef(Class cls, boolean resolveParent) { - if (resolveParent) { - return classDefMap.computeIfAbsent(cls, k -> ClassDef.buildClassDef(fory, cls)); - } - ClassDef classDef = extRegistry.currentLayerClassDef.get(cls); - if (classDef == null) { - classDef = ClassDef.buildClassDef(fory, cls, false); - extRegistry.currentLayerClassDef.put(cls, classDef); - } - return classDef; - } + // getTypeDef is inherited from TypeResolver // Note: Thread safe for jit thread to call. public Expression writeClassExpr(Expression buffer, short classId) { @@ -1640,30 +1560,28 @@ public void writeClassInternal(MemoryBuffer buffer, Class cls) { Short classId = extRegistry.registeredClassIdMap.get(cls); // Don't create serializer in case the object for class is non-serializable, // Or class is abstract or interface. - classInfo = - new ClassInfo( - this, cls, null, classId == null ? NO_CLASS_ID : classId, NOT_SUPPORT_XLANG); + classInfo = new ClassInfo(this, cls, null, classId == null ? NO_CLASS_ID : classId); classInfoMap.put(cls, classInfo); } writeClassInternal(buffer, classInfo); } public void writeClassInternal(MemoryBuffer buffer, ClassInfo classInfo) { - short classId = classInfo.classId; - if (classId == REPLACE_STUB_ID) { - // clear class id to avoid replaced class written as + int typeId = classInfo.typeId; + if (typeId == REPLACE_STUB_ID) { + // clear type id to avoid replaced class written as // ReplaceResolveSerializer.ReplaceStub - classInfo.classId = NO_CLASS_ID; + classInfo.typeId = NO_CLASS_ID; } - if (classInfo.classId != NO_CLASS_ID) { - buffer.writeVarUint32(classInfo.classId << 1); + if (classInfo.typeId != NO_CLASS_ID) { + buffer.writeVarUint32(classInfo.typeId << 1); } else { // let the lowermost bit of next byte be set, so the deserialization can know // whether need to read class by name in advance metaStringResolver.writeMetaStringBytesWithFlag(buffer, classInfo.namespaceBytes); metaStringResolver.writeMetaStringBytes(buffer, classInfo.typeNameBytes); } - classInfo.classId = classId; + classInfo.typeId = typeId; } /** @@ -1681,111 +1599,39 @@ public Class readClassInternal(MemoryBuffer buffer) { MetaStringBytes simpleClassNameBytes = metaStringResolver.readMetaStringBytes(buffer); classInfo = loadBytesToClassInfo(packageBytes, simpleClassNameBytes); } else { - classInfo = registeredId2ClassInfo[(short) (header >> 1)]; + classInfo = registeredId2ClassInfo[header >> 1]; } final Class cls = classInfo.cls; currentReadClass = cls; return cls; } - /** - * Read class info from java data buffer. {@link #readClassInfo(MemoryBuffer, - * ClassInfo)} is faster since it use a non-global class info cache. - */ - public ClassInfo readClassInfo(MemoryBuffer buffer) { - if (metaContextShareEnabled) { - return readSharedClassMeta(buffer, fory.getSerializationContext().getMetaContext()); - } - int header = buffer.readVarUint32Small14(); - ClassInfo classInfo; - if ((header & 0b1) != 0) { - classInfo = readClassInfoFromBytes(buffer, classInfoCache, header); - classInfoCache = classInfo; - } else { - classInfo = getOrUpdateClassInfo((short) (header >> 1)); - } - currentReadClass = classInfo.cls; - return classInfo; - } - - /** - * Read class info from java data buffer. `classInfoCache` is used as a cache to - * reduce map lookup to load class from binary. - */ - @CodegenInvoke - @Override - public ClassInfo readClassInfo(MemoryBuffer buffer, ClassInfo classInfoCache) { - if (metaContextShareEnabled) { - return readSharedClassMeta(buffer, fory.getSerializationContext().getMetaContext()); - } - int header = buffer.readVarUint32Small14(); - if ((header & 0b1) != 0) { - return readClassInfoByCache(buffer, classInfoCache, header); - } else { - return getClassInfo((short) (header >> 1)); - } - } - - /** Read class info, update classInfoHolder if cache not hit. */ @Override - @CodegenInvoke - public ClassInfo readClassInfo(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { - if (metaContextShareEnabled) { - return readSharedClassMeta(buffer, fory.getSerializationContext().getMetaContext()); - } - int header = buffer.readVarUint32Small14(); - if ((header & 0b1) != 0) { - return readClassInfoFromBytes(buffer, classInfoHolder, header); - } else { - return getClassInfo((short) (header >> 1)); - } - } - - private ClassInfo readClassInfoByCache( - MemoryBuffer buffer, ClassInfo classInfoCache, int header) { - if (metaContextShareEnabled) { - return readSharedClassMeta(buffer, fory.getSerializationContext().getMetaContext()); - } - return readClassInfoFromBytes(buffer, classInfoCache, header); - } - - private ClassInfo readClassInfoFromBytes( - MemoryBuffer buffer, ClassInfoHolder classInfoHolder, int header) { - if (metaContextShareEnabled) { - return readSharedClassMeta(buffer, fory.getSerializationContext().getMetaContext()); - } - ClassInfo classInfo = readClassInfoFromBytes(buffer, classInfoHolder.classInfo, header); - classInfoHolder.classInfo = classInfo; - return classInfo; - } - - private ClassInfo readClassInfoFromBytes( - MemoryBuffer buffer, ClassInfo classInfoCache, int header) { - MetaStringBytes typeNameBytesCache = classInfoCache.typeNameBytes; - MetaStringBytes namespaceBytes; - MetaStringBytes simpleClassNameBytes; - if (typeNameBytesCache != null) { - MetaStringBytes packageNameBytesCache = classInfoCache.namespaceBytes; - namespaceBytes = - metaStringResolver.readMetaStringBytesWithFlag(buffer, packageNameBytesCache, header); - assert packageNameBytesCache != null; - simpleClassNameBytes = metaStringResolver.readMetaStringBytes(buffer, typeNameBytesCache); - if (typeNameBytesCache.hashCode == simpleClassNameBytes.hashCode - && packageNameBytesCache.hashCode == namespaceBytes.hashCode) { - return classInfoCache; - } - } else { - namespaceBytes = metaStringResolver.readMetaStringBytesWithFlag(buffer, header); - simpleClassNameBytes = metaStringResolver.readMetaStringBytes(buffer); - } - ClassInfo classInfo = loadBytesToClassInfo(namespaceBytes, simpleClassNameBytes); - if (classInfo.serializer == null) { - return getClassInfo(classInfo.cls); + protected ClassInfo getClassInfoByTypeId(int typeId) { + // For native mode: typeId is the direct index into registeredId2ClassInfo + // - Internal types (0-255) are stored at their type ID index + // - User types (256+) are stored at (userId + USER_ID_BASE) index + if (typeId < 0 || typeId >= registeredId2ClassInfo.length) { + throw new IllegalStateException( + String.format( + "Invalid typeId %d in meta share mode. This usually indicates a protocol mismatch " + + "or buffer corruption. Expected typeId in range [0, %d). " + + "Check that the serializer and deserializer use the same protocol version.", + typeId, registeredId2ClassInfo.length)); + } + ClassInfo classInfo = registeredId2ClassInfo[typeId]; + // Ensure serializer is set for registered classes (they may have been registered + // without a serializer via registerInternal) + if (classInfo != null && classInfo.serializer == null) { + addSerializer(classInfo.cls, createSerializer(classInfo.cls)); + // Re-read from registeredId2ClassInfo since addSerializer updates it for registered classes + classInfo = registeredId2ClassInfo[typeId]; } return classInfo; } - ClassInfo loadBytesToClassInfo( + @Override + protected ClassInfo loadBytesToClassInfo( MetaStringBytes packageBytes, MetaStringBytes simpleClassNameBytes) { TypeNameBytes typeNameBytes = new TypeNameBytes(packageBytes.hashCode, simpleClassNameBytes.hashCode); @@ -1793,6 +1639,25 @@ ClassInfo loadBytesToClassInfo( if (classInfo == null) { classInfo = populateBytesToClassInfo(typeNameBytes, packageBytes, simpleClassNameBytes); } + // Note: Don't create serializer here - this method is used by both readClassInfo + // (which needs serializer) and readClassInternal (which doesn't need serializer). + // Serializer creation is handled by ensureSerializerForClassInfo in TypeResolver. + return classInfo; + } + + @Override + protected ClassInfo ensureSerializerForClassInfo(ClassInfo classInfo) { + if (classInfo.serializer == null) { + // Get or create ClassInfo with serializer + ClassInfo newClassInfo = getClassInfo(classInfo.cls); + // Update the cache with the correct ClassInfo that has a serializer + if (classInfo.typeNameBytes != null) { + TypeNameBytes typeNameBytes = + new TypeNameBytes(classInfo.namespaceBytes.hashCode, classInfo.typeNameBytes.hashCode); + compositeNameBytes2ClassInfo.put(typeNameBytes, newClassInfo); + } + return newClassInfo; + } return classInfo; } @@ -1809,14 +1674,7 @@ private ClassInfo populateBytesToClassInfo( Class cls = loadClass(classSpec.entireClassName, classSpec.isEnum, classSpec.dimension); ClassInfo classInfo = new ClassInfo( - cls, - fullClassNameBytes, - packageBytes, - simpleClassNameBytes, - false, - null, - NO_CLASS_ID, - NOT_SUPPORT_XLANG); + cls, fullClassNameBytes, packageBytes, simpleClassNameBytes, false, null, NO_CLASS_ID); if (NonexistentClass.class.isAssignableFrom(TypeUtils.getComponentIfArray(cls))) { classInfo.serializer = NonexistentClassSerializers.getSerializer(fory, classSpec.entireClassName, cls); @@ -1844,60 +1702,14 @@ public void resetRead() {} public void resetWrite() {} - @Override - public GenericType buildGenericType(TypeRef typeRef) { - return GenericType.build( - typeRef, - t -> { - if (t.getClass() == Class.class) { - return isMonomorphic((Class) t); - } else { - return isMonomorphic(getRawType(t)); - } - }); - } - - @Override - public GenericType buildGenericType(Type type) { - GenericType genericType = extRegistry.genericTypes.get(type); - if (genericType != null) { - return genericType; - } - return populateGenericType(type); - } - - private GenericType populateGenericType(Type type) { - GenericType genericType = - GenericType.build( - type, - t -> { - if (t.getClass() == Class.class) { - return isMonomorphic((Class) t); - } else { - return isMonomorphic(getRawType(t)); - } - }); - extRegistry.genericTypes.put(type, genericType); - return genericType; - } + // buildGenericType, nilClassInfo, nilClassInfoHolder are inherited from TypeResolver public GenericType getObjectGenericType() { return extRegistry.objectGenericType; } - public ClassInfo newClassInfo(Class cls, Serializer serializer, short classId) { - return new ClassInfo(this, cls, serializer, classId, NOT_SUPPORT_XLANG); - } - - // Invoked by fory JIT. - @Override - public ClassInfo nilClassInfo() { - return new ClassInfo(this, null, null, NO_CLASS_ID, NOT_SUPPORT_XLANG); - } - - @Override - public ClassInfoHolder nilClassInfoHolder() { - return new ClassInfoHolder(nilClassInfo()); + public ClassInfo newClassInfo(Class cls, Serializer serializer, int typeId) { + return new ClassInfo(this, cls, serializer, typeId); } public boolean isPrimitive(short classId) { diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/MetaContext.java b/java/fory-core/src/main/java/org/apache/fory/resolver/MetaContext.java index 83f95f49c4..8282f9b261 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/MetaContext.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/MetaContext.java @@ -21,6 +21,8 @@ import org.apache.fory.collection.IdentityObjectIntMap; import org.apache.fory.collection.ObjectArray; +import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.meta.ClassDef; /** * Context for sharing class meta across multiple serialization. Class name, field name and field @@ -30,6 +32,17 @@ public class MetaContext { /** Classes which has sent definitions to peer. */ public final IdentityObjectIntMap> classMap = new IdentityObjectIntMap<>(1, 0.5f); + /** Class definitions read from peer. */ + public final ObjectArray readClassDefs = new ObjectArray<>(); + /** ClassInfos read from peer for reference lookup during deserialization. */ public final ObjectArray readClassInfos = new ObjectArray<>(); + + /** + * New class definition which needs sending to peer. This will be filled up when there are new + * class definition need sending, and will be cleared after writing to buffer. + * + * @see ClassResolver#writeClassDefs(MemoryBuffer) + */ + public final ObjectArray writingClassDefs = new ObjectArray<>(); } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index 852d88f3b6..c01874685c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -24,6 +24,7 @@ import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; +import org.apache.fory.collection.ObjectArray; import java.lang.reflect.Field; import java.lang.reflect.Member; import java.lang.reflect.Type; @@ -45,10 +46,12 @@ import org.apache.fory.builder.CodecUtils; import org.apache.fory.builder.Generated.GeneratedMetaSharedSerializer; import org.apache.fory.builder.Generated.GeneratedObjectSerializer; +import org.apache.fory.builder.JITContext; import org.apache.fory.codegen.CodeGenerator; import org.apache.fory.codegen.Expression; import org.apache.fory.codegen.Expression.Invoke; import org.apache.fory.collection.IdentityMap; +import org.apache.fory.collection.IdentityObjectIntMap; import org.apache.fory.collection.LongMap; import org.apache.fory.collection.Tuple2; import org.apache.fory.config.CompatibleMode; @@ -61,6 +64,7 @@ import org.apache.fory.meta.TypeExtMeta; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.reflect.TypeRef; +import org.apache.fory.serializer.CodegenSerializer; import org.apache.fory.serializer.CodegenSerializer.LazyInitBeanSerializer; import org.apache.fory.serializer.MetaSharedSerializer; import org.apache.fory.serializer.NonexistentClass; @@ -91,9 +95,9 @@ public abstract class TypeResolver { private static final Logger LOG = LoggerFactory.getLogger(ClassResolver.class); - public static final short NO_CLASS_ID = (short) 0; + public static final int NO_CLASS_ID = 0; static final ClassInfo NIL_CLASS_INFO = - new ClassInfo(null, null, null, null, false, null, NO_CLASS_ID, NOT_SUPPORT_XLANG); + new ClassInfo(null, null, null, null, false, null, NO_CLASS_ID); // use a lower load factor to minimize hash collision static final float foryMapLoadFactor = 0.25f; static final int estimatedNumRegistered = 150; @@ -108,6 +112,10 @@ public abstract class TypeResolver { // IdentityMap has better lookup performance, when loadFactor is 0.05f, performance is better final IdentityMap, ClassInfo> classInfoMap = new IdentityMap<>(64, foryMapLoadFactor); final ExtRegistry extRegistry; + final Map, ClassDef> classDefMap = new HashMap<>(); + // Cache for readClassInfo(MemoryBuffer) - persists between calls to avoid reloading + // dynamically created classes that can't be found by Class.forName + private ClassInfo classInfoCache; protected TypeResolver(Fory fory) { this.fory = fory; @@ -180,6 +188,15 @@ public void register(String className, String namespace, String typeName) { public abstract void registerSerializer( Class type, Class serializerClass); + /** + * Registers a serializer for internal types (those with fixed IDs in the type system). This + * method is used for built-in types like ArrayList, HashMap, etc. + * + * @param type the class to register + * @param serializer the serializer to use + */ + public abstract void registerInternalSerializer(Class type, Serializer serializer); + /** * Whether to track reference for this type. If false, reference tracing of subclasses may be * ignored too. @@ -238,7 +255,79 @@ public final boolean needToWriteClassDef(Serializer serializer) { public abstract ClassInfo getClassInfo(Class cls, ClassInfoHolder classInfoHolder); - public abstract void writeClassInfo(MemoryBuffer buffer, ClassInfo classInfo); + /** + * Writes class info to buffer using the unified type system. This is the single implementation + * shared by both ClassResolver and XtypeResolver. + * + *

    Encoding: + *

      + *
    • NAMED_ENUM/NAMED_STRUCT/NAMED_EXT: namespace + typename bytes (or meta-share if enabled) + *
    • NAMED_COMPATIBLE_STRUCT/COMPATIBLE_STRUCT: always meta-share + *
    • Other types: just the type ID + *
    + */ + public final void writeClassInfo(MemoryBuffer buffer, ClassInfo classInfo) { + // In meta share mode, use the meta share protocol directly + // Protocol: LSB=0 means registered type by ID, LSB=1 means meta share data + if (metaContextShareEnabled) { + writeClassInfoWithMetaShare(buffer, classInfo); + return; + } + + // Non-meta share mode: write typeId followed by optional class name bytes + int typeId = classInfo.getTypeId(); + int internalTypeId = typeId & 0xff; + buffer.writeVarUint32Small7(typeId); + + switch (internalTypeId) { + case Types.NAMED_ENUM: + case Types.NAMED_STRUCT: + case Types.NAMED_EXT: + assert classInfo.namespaceBytes != null; + metaStringResolver.writeMetaStringBytes(buffer, classInfo.namespaceBytes); + assert classInfo.typeNameBytes != null; + metaStringResolver.writeMetaStringBytes(buffer, classInfo.typeNameBytes); + break; + default: + // No additional data needed - type ID already written + break; + } + } + + /** + * Writes class info using meta share protocol. + * Protocol: LSB=0 means registered type by ID, LSB=1 means meta share reference/definition. + */ + private void writeClassInfoWithMetaShare(MemoryBuffer buffer, ClassInfo classInfo) { + int typeId = classInfo.getTypeId(); + // For registered types that don't need ClassDef, just write the type ID. + // The isValidRegisteredTypeId check ensures we only use the fast path for + // classes that are actually registered in this resolver's registry. + // Named types (NAMED_STRUCT, NAMED_COMPATIBLE_STRUCT, etc.) will not pass + // this check because they don't have entries in the registry. + if (isValidRegisteredTypeId(typeId) && !classInfo.needToWriteClassDef) { + buffer.writeVarUint32(typeId << 1); + return; + } + // For types that need ClassDef or are not registered, use the meta share protocol + MetaContext metaContext = fory.getSerializationContext().getMetaContext(); + assert metaContext != null : SET_META__CONTEXT_MSG; + IdentityObjectIntMap> classMap = metaContext.classMap; + int newId = classMap.size; + int id = classMap.putOrGet(classInfo.cls, newId); + if (id >= 0) { + // Reference to previously written type: (index << 1) | 1 + buffer.writeVarUint32((id << 1) | 1); + } else { + // New type: (index << 1) | 1, followed by ClassDef bytes + buffer.writeVarUint32((newId << 1) | 1); + ClassDef classDef = classInfo.classDef; + if (classDef == null) { + classDef = buildClassDef(classInfo); + } + metaContext.writingClassDefs.add(classDef); + } + } /** * Native code for ClassResolver.writeClassInfo is too big to inline, so inline it manually. @@ -252,15 +341,356 @@ public Expression writeClassExpr( return new Invoke(classResolverRef, "writeClassInfo", buffer, classInfo); } - public abstract ClassInfo readClassInfo(MemoryBuffer buffer, ClassInfoHolder classInfoHolder); + /** + * Writes shared class metadata using the meta-share protocol. Protocol: If class already written, + * writes (index << 1) | 1 (reference). If new class, writes (index << 1) followed by ClassDef + * bytes. + * + *

    This method is shared between XtypeResolver and ClassResolver. + */ + protected final void writeSharedClassMeta(MemoryBuffer buffer, ClassInfo classInfo) { + MetaContext metaContext = fory.getSerializationContext().getMetaContext(); + assert metaContext != null : SET_META__CONTEXT_MSG; + IdentityObjectIntMap> classMap = metaContext.classMap; + int newId = classMap.size; + int id = classMap.putOrGet(classInfo.cls, newId); + if (id >= 0) { + // Reference to previously written type: (index << 1) | 1, LSB=1 + buffer.writeVarUint32((id << 1) | 1); + } else { + // New type: index << 1, LSB=0, followed by ClassDef bytes inline + buffer.writeVarUint32(newId << 1); + ClassDef classDef = classInfo.classDef; + if (classDef == null) { + classDef = buildClassDef(classInfo); + } + buffer.writeBytes(classDef.getEncoded()); + } + } - public abstract ClassInfo readClassInfo(MemoryBuffer buffer, ClassInfo classInfoCache); + /** + * Build ClassDef for the given ClassInfo. Used by writeSharedClassMeta when the classDef is not + * yet created. + */ + protected abstract ClassDef buildClassDef(ClassInfo classInfo); - abstract ClassInfo readSharedClassMeta(MemoryBuffer buffer, MetaContext metaContext); + /** + * Reads class info from buffer using the unified type system. This is the single implementation + * shared by both ClassResolver and XtypeResolver. + * + *

    Note: {@link #readClassInfo(MemoryBuffer, ClassInfo)} is faster since it uses a non-global + * class info cache. + */ + public final ClassInfo readClassInfo(MemoryBuffer buffer) { + // In meta share mode, use the meta share protocol directly + // Protocol: LSB=0 means registered type by ID, LSB=1 means meta share data + if (metaContextShareEnabled) { + return readClassInfoWithMetaShare(buffer); + } + + // Non-meta share mode: read typeId followed by optional class name bytes + int header = buffer.readVarUint32Small14(); + int internalTypeId = header & 0xff; + + // Check if this is a named type (needs class name bytes) + if (isNamedType(internalTypeId)) { + // Use cache to avoid reloading dynamically created classes + // ensureSerializerForClassInfo is called within readClassInfoFromBytes + ClassInfo classInfo = readClassInfoFromBytes(buffer, classInfoCache, header); + classInfoCache = classInfo; + return classInfo; + } else { + // Lookup by type ID from registry + return getClassInfoByTypeId(header); + } + } + + /** + * Reads class info using meta share protocol. + * Protocol: LSB=0 means registered type by ID, LSB=1 means meta share reference/definition. + */ + private ClassInfo readClassInfoWithMetaShare(MemoryBuffer buffer) { + MetaContext metaContext = fory.getSerializationContext().getMetaContext(); + assert metaContext != null : SET_META__CONTEXT_MSG; + int header = buffer.readVarUint32Small14(); + int id = header >>> 1; + if ((header & 0b1) == 0) { + // Registered type by ID + return getClassInfoByTypeId(id); + } + // Meta share reference or definition + ClassInfo classInfo = metaContext.readClassInfos.get(id); + if (classInfo == null) { + classInfo = readSharedClassMetaById(metaContext, id); + } + return classInfo; + } + + /** + * Read a ClassDef from meta share context and create ClassInfo. + * The ClassDef is looked up from metaContext.readClassDefs by index. + */ + protected final ClassInfo readSharedClassMetaById(MetaContext metaContext, int index) { + ClassDef classDef = metaContext.readClassDefs.get(index); + Tuple2 classDefTuple = extRegistry.classIdToDef.get(classDef.getId()); + ClassInfo classInfo; + if (classDefTuple == null || classDefTuple.f1 == null || classDefTuple.f1.serializer == null) { + classInfo = buildMetaSharedClassInfo(classDefTuple, classDef); + } else { + classInfo = classDefTuple.f1; + } + metaContext.readClassInfos.set(index, classInfo); + return classInfo; + } + + /** + * Write collected ClassDefs to buffer. ClassDefs are collected during serialization + * and written at the end of the serialization process. + */ + public final void writeClassDefs(MemoryBuffer buffer) { + MetaContext metaContext = fory.getSerializationContext().getMetaContext(); + ObjectArray writingClassDefs = metaContext.writingClassDefs; + final int size = writingClassDefs.size; + buffer.writeVarUint32Small7(size); + if (buffer.isHeapFullyWriteable()) { + for (int i = 0; i < size; i++) { + buffer.writeBytes(writingClassDefs.get(i).getEncoded()); + } + } else { + for (int i = 0; i < size; i++) { + writingClassDefs.get(i).writeClassDef(buffer); + } + } + metaContext.writingClassDefs.size = 0; + } + + /** + * Read ClassDefs from buffer and populate metaContext.readClassDefs. + * Must be called before deserializing objects in meta share mode. + */ + public final void readClassDefs(MemoryBuffer buffer) { + MetaContext metaContext = fory.getSerializationContext().getMetaContext(); + assert metaContext != null : SET_META__CONTEXT_MSG; + int numClassDefs = buffer.readVarUint32Small7(); + for (int i = 0; i < numClassDefs; i++) { + long id = buffer.readInt64(); + Tuple2 tuple2 = extRegistry.classIdToDef.get(id); + if (tuple2 != null) { + ClassDef.skipClassDef(buffer, id); + } else { + tuple2 = readClassDefFromBuffer(buffer, id); + } + metaContext.readClassDefs.add(tuple2.f0); + metaContext.readClassInfos.add(tuple2.f1); + } + } + + private Tuple2 readClassDefFromBuffer(MemoryBuffer buffer, long header) { + ClassDef readClassDef = ClassDef.readClassDef(fory, buffer, header); + Tuple2 tuple2 = extRegistry.classIdToDef.get(readClassDef.getId()); + if (tuple2 == null) { + tuple2 = Tuple2.of(readClassDef, null); + extRegistry.classIdToDef.put(readClassDef.getId(), tuple2); + } + return tuple2; + } + + /** + * Read class info from buffer with ClassInfo cache. This version is faster than {@link + * #readClassInfo(MemoryBuffer)} because it uses the provided classInfoCache to reduce map lookups + * when reading class from binary. + * + * @param buffer the buffer to read from + * @param classInfoCache cache for class info to speed up repeated reads + * @return the ClassInfo read from buffer + */ + @CodegenInvoke + public final ClassInfo readClassInfo(MemoryBuffer buffer, ClassInfo classInfoCache) { + // In meta share mode, use the meta share protocol directly + if (metaContextShareEnabled) { + return readClassInfoWithMetaShare(buffer); + } + + // Non-meta share mode: read typeId followed by optional class name bytes + int header = buffer.readVarUint32Small14(); + int internalTypeId = header & 0xff; + + // Check if this is a named type (needs class name bytes) + if (isNamedType(internalTypeId)) { + return readClassInfoByCache(buffer, classInfoCache, header); + } else { + // Lookup by type ID from registry + return getClassInfoByTypeId(header); + } + } + + /** + * Read class info from buffer with ClassInfoHolder cache. This version updates the + * classInfoHolder if the cache doesn't hit, allowing callers to maintain the cache across calls. + * + * @param buffer the buffer to read from + * @param classInfoHolder holder containing cache, will be updated on cache miss + * @return the ClassInfo read from buffer + */ + @CodegenInvoke + public final ClassInfo readClassInfo(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { + // In meta share mode, use the meta share protocol directly + if (metaContextShareEnabled) { + return readClassInfoWithMetaShare(buffer); + } + + // Non-meta share mode: read typeId followed by optional class name bytes + int header = buffer.readVarUint32Small14(); + int internalTypeId = header & 0xff; + + // Check if this is a named type (needs class name bytes) + if (isNamedType(internalTypeId)) { + return readClassInfoFromBytes(buffer, classInfoHolder, header); + } else { + // Lookup by type ID from registry + return getClassInfoByTypeId(header); + } + } + + /** Helper to check if a type ID represents a named type that needs class name bytes. */ + private boolean isNamedType(int internalTypeId) { + return internalTypeId == Types.NAMED_ENUM + || internalTypeId == Types.NAMED_STRUCT + || internalTypeId == Types.NAMED_EXT + || internalTypeId == Types.NAMED_COMPATIBLE_STRUCT; + } + + /** + * Read class info using the provided cache. Returns cached ClassInfo if the namespace and type + * name bytes match. + */ + protected final ClassInfo readClassInfoByCache( + MemoryBuffer buffer, ClassInfo classInfoCache, int header) { + return readClassInfoFromBytes(buffer, classInfoCache, header); + } + + /** Read class info and update the ClassInfoHolder's cache. */ + protected final ClassInfo readClassInfoFromBytes( + MemoryBuffer buffer, ClassInfoHolder classInfoHolder, int header) { + ClassInfo classInfo = readClassInfoFromBytes(buffer, classInfoHolder.classInfo, header); + classInfoHolder.classInfo = classInfo; + return classInfo; + } + + /** + * Read class info from bytes with cache optimization. Uses the cached namespace and type name + * bytes to avoid map lookups when the class is the same as the cached one (hash comparison). + */ + protected final ClassInfo readClassInfoFromBytes( + MemoryBuffer buffer, ClassInfo classInfoCache, int header) { + MetaStringBytes typeNameBytesCache = + classInfoCache != null ? classInfoCache.typeNameBytes : null; + MetaStringBytes namespaceBytes; + MetaStringBytes simpleClassNameBytes; + + if (typeNameBytesCache != null) { + // Use cache for faster comparison + MetaStringBytes packageNameBytesCache = classInfoCache.namespaceBytes; + namespaceBytes = metaStringResolver.readMetaStringBytes(buffer, packageNameBytesCache); + assert packageNameBytesCache != null; + simpleClassNameBytes = metaStringResolver.readMetaStringBytes(buffer, typeNameBytesCache); + + // Fast path: if hashes match, return cached ClassInfo (already has serializer) + if (typeNameBytesCache.hashCode == simpleClassNameBytes.hashCode + && packageNameBytesCache.hashCode == namespaceBytes.hashCode) { + return classInfoCache; + } + } else { + // No cache available, read fresh + namespaceBytes = metaStringResolver.readMetaStringBytes(buffer); + simpleClassNameBytes = metaStringResolver.readMetaStringBytes(buffer); + } + + // Load class info from bytes (subclass-specific) and ensure serializer is set + ClassInfo classInfo = loadBytesToClassInfo(namespaceBytes, simpleClassNameBytes); + return ensureSerializerForClassInfo(classInfo); + } + + /** + * Reads shared class metadata from buffer. This is the shared implementation used by both + * ClassResolver and XtypeResolver. + */ + protected final ClassInfo readSharedClassMeta(MemoryBuffer buffer) { + MetaContext metaContext = fory.getSerializationContext().getMetaContext(); + assert metaContext != null : SET_META__CONTEXT_MSG; + int indexMarker = buffer.readVarUint32Small14(); + boolean isRef = (indexMarker & 1) == 1; + int index = indexMarker >>> 1; + ClassInfo classInfo; + if (isRef) { + // Reference to previously read type in this stream + classInfo = metaContext.readClassInfos.get(index); + } else { + // New type in stream - but may already be known from registry + long id = buffer.readInt64(); + Tuple2 tuple2 = extRegistry.classIdToDef.get(id); + if (tuple2 != null) { + // Already known - skip the ClassDef bytes, reuse existing ClassInfo + ClassDef.skipClassDef(buffer, id); + classInfo = tuple2.f1; + if (classInfo == null) { + classInfo = buildMetaSharedClassInfo(tuple2, tuple2.f0); + } + } else { + // Unknown - read ClassDef and create ClassInfo + tuple2 = readClassDef(buffer, id); + classInfo = tuple2.f1; + if (classInfo == null) { + classInfo = buildMetaSharedClassInfo(tuple2, tuple2.f0); + } + } + // index == readClassInfos.size() since types are written sequentially + metaContext.readClassInfos.add(classInfo); + } + return classInfo; + } + + /** + * Load class info from namespace and type name bytes. Subclasses implement this to resolve the + * class and create/lookup ClassInfo. + * + *

    Note: This method should NOT create serializers. It's used by both readClassInfo + * (which needs serializers) and readClassInternal (which doesn't need serializers). + * Use {@link #ensureSerializerForClassInfo} after calling this if a serializer is needed. + */ + protected abstract ClassInfo loadBytesToClassInfo( + MetaStringBytes namespaceBytes, MetaStringBytes simpleClassNameBytes); + + /** + * Ensure the ClassInfo has a serializer set. Called after loading class info for deserialization. + * If the class is abstract/interface or can't be serialized, this may throw an exception. + * + * @param classInfo the class info to ensure has a serializer + * @return the ClassInfo with serializer set (may be the same instance or a different one) + */ + protected abstract ClassInfo ensureSerializerForClassInfo(ClassInfo classInfo); + + /** + * Get ClassInfo by type ID from the registry. For internal types (0-255), returns the + * pre-registered ClassInfo. For user types, decodes the type ID and looks up in the appropriate + * registry. + */ + protected abstract ClassInfo getClassInfoByTypeId(int typeId); + + /** + * Check if a typeId corresponds to a valid registered class that can be written + * using the fast path (typeId << 1) in meta share mode. + * + *

    For ClassResolver, this checks if the typeId has an entry in registeredId2ClassInfo. + * For XtypeResolver, this checks if the typeId is a valid internal type or has a registered entry. + * + * @param typeId the type ID to check + * @return true if this typeId can use the fast path, false if meta share protocol is needed + */ + protected abstract boolean isValidRegisteredTypeId(int typeId); public final ClassInfo readSharedClassMeta(MemoryBuffer buffer, Class targetClass) { - ClassInfo classInfo = - readSharedClassMeta(buffer, fory.getSerializationContext().getMetaContext()); + ClassInfo classInfo = readSharedClassMeta(buffer); Class readClass = classInfo.getCls(); // replace target class if needed if (targetClass != readClass) { @@ -314,7 +744,7 @@ private ClassInfo getMetaSharedClassInfo(ClassDef classDef, Class clz) { Class cls = clz; Short classId = extRegistry.registeredClassIdMap.get(cls); ClassInfo classInfo = - new ClassInfo(this, cls, null, classId == null ? NO_CLASS_ID : classId, NOT_SUPPORT_XLANG); + new ClassInfo(this, cls, null, classId == null ? NO_CLASS_ID : classId); classInfo.classDef = classDef; if (NonexistentClass.class.isAssignableFrom(TypeUtils.getComponentIfArray(cls))) { if (cls == NonexistentMetaShared.class) { @@ -424,13 +854,44 @@ final Class loadClass( public abstract void setSerializerIfAbsent(Class cls, Serializer serializer); - public abstract ClassInfo nilClassInfo(); + public final ClassInfo nilClassInfo() { + return NIL_CLASS_INFO; + } - public abstract ClassInfoHolder nilClassInfoHolder(); + public final ClassInfoHolder nilClassInfoHolder() { + return new ClassInfoHolder(NIL_CLASS_INFO); + } - public abstract GenericType buildGenericType(TypeRef typeRef); + public final GenericType buildGenericType(TypeRef typeRef) { + return GenericType.build( + typeRef, + t -> { + if (t.getClass() == Class.class) { + return isMonomorphic((Class) t); + } else { + return isMonomorphic(TypeUtils.getRawType(t)); + } + }); + } - public abstract GenericType buildGenericType(Type type); + public final GenericType buildGenericType(Type type) { + GenericType genericType = extRegistry.genericTypes.get(type); + if (genericType != null) { + return genericType; + } + GenericType newGenericType = + GenericType.build( + type, + t -> { + if (t.getClass() == Class.class) { + return isMonomorphic((Class) t); + } else { + return isMonomorphic(TypeUtils.getRawType(t)); + } + }); + extRegistry.genericTypes.put(type, newGenericType); + return newGenericType; + } @CodegenInvoke public GenericType getGenericTypeInStruct(Class cls, String genericTypeStr) { @@ -443,7 +904,17 @@ public GenericType getGenericTypeInStruct(Class cls, String genericTypeStr) { public abstract void ensureSerializersCompiled(); - public abstract ClassDef getTypeDef(Class cls, boolean resolveParent); + public final ClassDef getTypeDef(Class cls, boolean resolveParent) { + if (resolveParent) { + return classDefMap.computeIfAbsent(cls, k -> ClassDef.buildClassDef(fory, cls)); + } + ClassDef classDef = extRegistry.currentLayerClassDef.get(cls); + if (classDef == null) { + classDef = ClassDef.buildClassDef(fory, cls, false); + extRegistry.currentLayerClassDef.put(cls, classDef); + } + return classDef; + } public final boolean isSerializable(Class cls) { // Enums are always serializable, even if abstract (enums with abstract methods) @@ -473,6 +944,56 @@ public final boolean isSerializable(Class cls) { public abstract Class getSerializerClass(Class cls, boolean codegen); + /** + * Get the serializer class for object serialization with JIT support. This is used by both + * ClassResolver and XtypeResolver for creating object serializers. + */ + public Class getObjectSerializerClass( + Class cls, + boolean shareMeta, + boolean codegen, + JITContext.SerializerJITCallback> callback) { + if (codegen) { + if (extRegistry.getClassCtx.contains(cls)) { + // avoid potential recursive call for seq codec generation. + return CodegenSerializer.LazyInitBeanSerializer.class; + } else { + try { + extRegistry.getClassCtx.add(cls); + Class sc; + switch (fory.getCompatibleMode()) { + case SCHEMA_CONSISTENT: + sc = + fory.getJITContext() + .registerSerializerJITCallback( + () -> ObjectSerializer.class, + () -> CodegenSerializer.loadCodegenSerializer(fory, cls), + callback); + return sc; + case COMPATIBLE: + // Always use ObjectSerializer for compatible mode. + // Class definition will be sent to peer to create serializer for deserialization. + sc = + fory.getJITContext() + .registerSerializerJITCallback( + () -> ObjectSerializer.class, + () -> CodegenSerializer.loadCodegenSerializer(fory, cls), + callback); + return sc; + default: + throw new UnsupportedOperationException( + String.format("Unsupported mode %s", fory.getCompatibleMode())); + } + } finally { + extRegistry.getClassCtx.remove(cls); + } + } + } else { + // Always use ObjectSerializer for both modes + return ObjectSerializer.class; + } + } + public final boolean isCollection(Class cls) { if (Collection.class.isAssignableFrom(cls)) { return true; diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java index 46cfe47e9a..a44fed244e 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java @@ -80,9 +80,11 @@ import org.apache.fory.serializer.NonexistentClassSerializers; import org.apache.fory.serializer.NonexistentClassSerializers.NonexistentClassSerializer; import org.apache.fory.serializer.ObjectSerializer; +import org.apache.fory.serializer.PrimitiveSerializers; import org.apache.fory.serializer.SerializationUtils; import org.apache.fory.serializer.Serializer; import org.apache.fory.serializer.Serializers; +import org.apache.fory.serializer.StringSerializer; import org.apache.fory.serializer.TimeSerializers; import org.apache.fory.serializer.UnionSerializer; import org.apache.fory.serializer.collection.CollectionLikeSerializer; @@ -114,7 +116,6 @@ public class XtypeResolver extends TypeResolver { private final Config config; private final Fory fory; - private final ClassResolver classResolver; private final ClassInfoHolder classInfoCache = new ClassInfoHolder(NIL_CLASS_INFO); private final MetaStringResolver metaStringResolver; @@ -123,7 +124,7 @@ public class XtypeResolver extends TypeResolver { new ObjectMap<>(16, loadFactor); private final ObjectMap qualifiedType2ClassInfo = new ObjectMap<>(16, loadFactor); - private final Map, ClassDef> classDefMap = new HashMap<>(); + // classDefMap is inherited from TypeResolver private final boolean shareMeta; private int xtypeIdGenerator = 64; @@ -137,8 +138,6 @@ public XtypeResolver(Fory fory) { super(fory); this.config = fory.getConfig(); this.fory = fory; - this.classResolver = fory.getClassResolver(); - classResolver.xtypeResolver = this; shareMeta = fory.getConfig().isMetaShareEnabled(); this.generics = fory.getGenerics(); this.metaStringResolver = fory.getMetaStringResolver(); @@ -177,9 +176,9 @@ public void register(Class type, int userTypeId) { Serializer serializer = null; if (classInfo != null) { serializer = classInfo.serializer; - if (classInfo.xtypeId != 0) { + if (classInfo.typeId != 0) { throw new IllegalArgumentException( - String.format("Type %s has been registered with id %s", type, classInfo.xtypeId)); + String.format("Type %s has been registered with id %s", type, classInfo.typeId)); } String prevNamespace = classInfo.decodeNamespace(); String prevTypeName = classInfo.decodeTypeName(); @@ -276,7 +275,7 @@ private void register( () -> { if (ref.get() == null) { Class c = - classResolver.getObjectSerializerClass( + getObjectSerializerClass( type, shareMeta, fory.getConfig().isCodeGenEnabled(), @@ -324,17 +323,17 @@ private boolean isStructType(Serializer serializer) { return serializer instanceof DeferredLazyObjectSerializer; } - private ClassInfo newClassInfo(Class type, Serializer serializer, int xtypeId) { + private ClassInfo newClassInfo(Class type, Serializer serializer, int typeId) { return newClassInfo( type, serializer, ReflectionUtils.getPackage(type), ReflectionUtils.getClassNameWithoutPackage(type), - xtypeId); + typeId); } private ClassInfo newClassInfo( - Class type, Serializer serializer, String namespace, String typeName, int xtypeId) { + Class type, Serializer serializer, String namespace, String typeName, int typeId) { MetaStringBytes fullClassNameBytes = metaStringResolver.getOrCreateMetaStringBytes( GENERIC_ENCODER.encode(type.getName(), MetaString.Encoding.UTF_8)); @@ -343,7 +342,7 @@ private ClassInfo newClassInfo( MetaStringBytes classNameBytes = metaStringResolver.getOrCreateMetaStringBytes(Encoders.encodeTypeName(typeName)); return new ClassInfo( - type, fullClassNameBytes, nsBytes, classNameBytes, false, serializer, NO_CLASS_ID, xtypeId); + type, fullClassNameBytes, nsBytes, classNameBytes, false, serializer, typeId); } public void registerSerializer(Class type, Class serializerClass) { @@ -357,30 +356,62 @@ public void registerSerializer(Class type, Serializer serializer) { if (!serializer.getClass().getPackage().getName().startsWith("org.apache.fory")) { SerializationUtils.validate(type, serializer.getClass()); } - int oldXtypeId = classInfo.xtypeId; - int foryId = oldXtypeId & 0xff; + int oldTypeId = classInfo.typeId; + int foryId = oldTypeId & 0xff; - if (oldXtypeId != 0 && xtypeIdToClassMap.get(oldXtypeId) == classInfo) { - xtypeIdToClassMap.remove(oldXtypeId); - registeredTypeIds.remove(oldXtypeId); + if (oldTypeId != 0 && xtypeIdToClassMap.get(oldTypeId) == classInfo) { + xtypeIdToClassMap.remove(oldTypeId); + registeredTypeIds.remove(oldTypeId); } if (foryId != Types.EXT && foryId != Types.NAMED_EXT) { if (foryId == Types.STRUCT || foryId == Types.COMPATIBLE_STRUCT) { - classInfo.xtypeId = (oldXtypeId & 0xffffff00) | Types.EXT; + classInfo.typeId = (oldTypeId & 0xffffff00) | Types.EXT; } else if (foryId == Types.NAMED_STRUCT || foryId == Types.NAMED_COMPATIBLE_STRUCT) { - classInfo.xtypeId = (oldXtypeId & 0xffffff00) | Types.NAMED_EXT; + classInfo.typeId = (oldTypeId & 0xffffff00) | Types.NAMED_EXT; } else { throw new IllegalArgumentException( - String.format("Can't register serializer for type %s with id %s", type, oldXtypeId)); + String.format("Can't register serializer for type %s with id %s", type, oldTypeId)); } } classInfo.serializer = serializer; - int newXtypeId = classInfo.xtypeId; - if (newXtypeId != 0) { - xtypeIdToClassMap.put(newXtypeId, classInfo); - registeredTypeIds.add(newXtypeId); + int newTypeId = classInfo.typeId; + if (newTypeId != 0) { + xtypeIdToClassMap.put(newTypeId, classInfo); + registeredTypeIds.add(newTypeId); + } + } + + @Override + public void registerInternalSerializer(Class type, Serializer serializer) { + checkRegisterAllowed(); + ClassInfo classInfo = classInfoMap.get(type); + if (classInfo != null) { + classInfo.serializer = serializer; + } else { + // Determine appropriate type ID based on the type + int typeId = determineTypeIdForClass(type); + classInfo = newClassInfo(type, serializer, typeId); + classInfoMap.put(type, classInfo); + } + } + + /** + * Determine the appropriate xlang type ID for a class. + * For collection types, use the collection-specific type IDs. + * For other types, use NAMED_STRUCT which writes namespace and typename bytes. + */ + private int determineTypeIdForClass(Class type) { + if (List.class.isAssignableFrom(type)) { + return Types.LIST; + } else if (Set.class.isAssignableFrom(type)) { + return Types.SET; + } else if (Map.class.isAssignableFrom(type)) { + return Types.MAP; + } else { + // For unregistered classes, use NAMED_STRUCT so that class name is written + return Types.NAMED_STRUCT; } } @@ -388,7 +419,7 @@ private ClassInfo checkClassRegistration(Class type) { ClassInfo classInfo = classInfoMap.get(type); Preconditions.checkArgument( classInfo != null - && (classInfo.xtypeId != 0 || !type.getSimpleName().equals(classInfo.decodeTypeName())), + && (classInfo.typeId != 0 || !type.getSimpleName().equals(classInfo.decodeTypeName())), "Type %s should be registered with id or namespace+typename before register serializer", type); return classInfo; @@ -405,8 +436,8 @@ public boolean isRegisteredById(Class cls) { if (classInfo == null) { return false; } - int xtypeId = classInfo.xtypeId & 0xff; - switch (xtypeId) { + int typeId = classInfo.typeId & 0xff; + switch (typeId) { case Types.NAMED_COMPATIBLE_STRUCT: case Types.NAMED_ENUM: case Types.NAMED_STRUCT: @@ -423,8 +454,8 @@ public boolean isRegisteredByName(Class cls) { if (classInfo == null) { return false; } - int xtypeId = classInfo.xtypeId & 0xff; - switch (xtypeId) { + int typeId = classInfo.typeId & 0xff; + switch (typeId) { case Types.NAMED_COMPATIBLE_STRUCT: case Types.NAMED_ENUM: case Types.NAMED_STRUCT: @@ -452,11 +483,11 @@ public boolean isMonomorphic(Descriptor descriptor) { if (rawType == NonexistentMetaShared.class) { return true; } - byte xtypeId = getXtypeId(rawType); + byte typeIdByte = getInternalTypeId(rawType); if (fory.isCompatible()) { - return !Types.isUserDefinedType(xtypeId) && xtypeId != Types.UNKNOWN; + return !Types.isUserDefinedType(typeIdByte) && typeIdByte != Types.UNKNOWN; } - return xtypeId != Types.UNKNOWN; + return typeIdByte != Types.UNKNOWN; } } @@ -491,11 +522,11 @@ public boolean isMonomorphic(Class clz) { public boolean isBuildIn(Descriptor descriptor) { Class rawType = descriptor.getRawType(); - byte xtypeId = getXtypeId(rawType); + byte typeIdByte = getInternalTypeId(rawType); if (rawType == NonexistentMetaShared.class) { return true; } - return !Types.isUserDefinedType(xtypeId) && xtypeId != Types.UNKNOWN; + return !Types.isUserDefinedType(typeIdByte) && typeIdByte != Types.UNKNOWN; } @Override @@ -542,157 +573,195 @@ public ClassInfo getUserTypeInfo(int userTypeId) { return xtypeIdToClassMap.get(userTypeId); } - @Override - public GenericType buildGenericType(TypeRef typeRef) { - return classResolver.buildGenericType(typeRef); - } - - @Override - public GenericType buildGenericType(Type type) { - return classResolver.buildGenericType(type); - } + // buildGenericType methods are inherited from TypeResolver private ClassInfo buildClassInfo(Class cls) { Serializer serializer; - int xtypeId; - if (classResolver.isSet(cls)) { + int typeId; + if (isSet(cls)) { if (cls.isAssignableFrom(HashSet.class)) { cls = HashSet.class; serializer = new HashSetSerializer(fory); } else { serializer = getCollectionSerializer(cls); } - xtypeId = Types.SET; - } else if (classResolver.isCollection(cls)) { + typeId = Types.SET; + } else if (isCollection(cls)) { if (cls.isAssignableFrom(ArrayList.class)) { cls = ArrayList.class; serializer = new ArrayListSerializer(fory); } else { serializer = getCollectionSerializer(cls); } - xtypeId = Types.LIST; + typeId = Types.LIST; } else if (cls.isArray() && !cls.getComponentType().isPrimitive()) { serializer = new ArraySerializers.ObjectArraySerializer(fory, cls); - xtypeId = Types.LIST; - } else if (classResolver.isMap(cls)) { + typeId = Types.LIST; + } else if (isMap(cls)) { if (cls.isAssignableFrom(HashMap.class)) { cls = HashMap.class; serializer = new HashMapSerializer(fory); } else { - ClassInfo classInfo = classResolver.getClassInfo(cls, false); - if (classInfo != null && classInfo.serializer != null) { - if (classInfo.serializer instanceof MapLikeSerializer - && ((MapLikeSerializer) classInfo.serializer).supportCodegenHook()) { - serializer = classInfo.serializer; - } else { - serializer = new MapSerializer(fory, cls); - } + ClassInfo classInfo = classInfoMap.get(cls); + if (classInfo != null && classInfo.serializer != null + && classInfo.serializer instanceof MapLikeSerializer + && ((MapLikeSerializer) classInfo.serializer).supportCodegenHook()) { + serializer = classInfo.serializer; } else { serializer = new MapSerializer(fory, cls); } } - xtypeId = Types.MAP; + typeId = Types.MAP; } else if (NonexistentClass.class.isAssignableFrom(cls)) { serializer = NonexistentClassSerializers.getSerializer(fory, "Unknown", cls); if (cls.isEnum()) { - xtypeId = Types.ENUM; + typeId = Types.ENUM; } else { - xtypeId = shareMeta ? Types.COMPATIBLE_STRUCT : Types.STRUCT; + typeId = shareMeta ? Types.COMPATIBLE_STRUCT : Types.STRUCT; } } else if (cls == Object.class) { - return classResolver.getClassInfo(cls); + // Object.class is handled as unknown type in xlang + return getClassInfo(cls); } else { Class enclosingClass = (Class) cls.getEnclosingClass(); if (enclosingClass != null && enclosingClass.isEnum()) { serializer = new EnumSerializer(fory, (Class) cls); - xtypeId = getClassInfo(enclosingClass).xtypeId; + typeId = getClassInfo(enclosingClass).typeId; } else { throw new ClassUnregisteredException(cls); } } - ClassInfo info = newClassInfo(cls, serializer, (short) xtypeId); + ClassInfo info = newClassInfo(cls, serializer, typeId); classInfoMap.put(cls, info); return info; } private Serializer getCollectionSerializer(Class cls) { - ClassInfo classInfo = classResolver.getClassInfo(cls, false); - if (classInfo != null && classInfo.serializer != null) { - if (classInfo.serializer instanceof CollectionLikeSerializer - && ((CollectionLikeSerializer) (classInfo.serializer)).supportCodegenHook()) { - return classInfo.serializer; - } + ClassInfo classInfo = classInfoMap.get(cls); + if (classInfo != null && classInfo.serializer != null + && classInfo.serializer instanceof CollectionLikeSerializer + && ((CollectionLikeSerializer) (classInfo.serializer)).supportCodegenHook()) { + return classInfo.serializer; } return new CollectionSerializer(fory, cls); } private void registerDefaultTypes() { - registerDefaultTypes(Types.BOOL, Boolean.class, boolean.class, AtomicBoolean.class); - registerDefaultTypes(Types.UINT8, Byte.class, byte.class); - registerDefaultTypes(Types.UINT16, Short.class, short.class); - registerDefaultTypes(Types.UINT32, Integer.class, int.class, AtomicInteger.class); - registerDefaultTypes(Types.UINT64, Long.class, long.class, AtomicLong.class); - registerDefaultTypes(Types.TAGGED_UINT64, Long.class, long.class, AtomicLong.class); - registerDefaultTypes(Types.INT32, Integer.class, int.class, AtomicInteger.class); - registerDefaultTypes(Types.INT64, Long.class, long.class, AtomicLong.class); - registerDefaultTypes(Types.TAGGED_INT64, Long.class, long.class, AtomicLong.class); - - registerDefaultTypes(Types.INT8, Byte.class, byte.class); - registerDefaultTypes(Types.INT16, Short.class, short.class); - registerDefaultTypes(Types.VARINT32, Integer.class, int.class, AtomicInteger.class); - registerDefaultTypes(Types.VARINT64, Long.class, long.class, AtomicLong.class); - registerDefaultTypes(Types.FLOAT32, Float.class, float.class); - registerDefaultTypes(Types.FLOAT64, Double.class, double.class); - registerDefaultTypes(Types.STRING, String.class, StringBuilder.class, StringBuffer.class); - registerDefaultTypes(Types.DURATION, Duration.class); - registerDefaultTypes( - Types.TIMESTAMP, - Instant.class, - Date.class, - java.sql.Date.class, - Timestamp.class, - LocalDateTime.class); - registerDefaultTypes(Types.DECIMAL, BigDecimal.class, BigInteger.class); - registerDefaultTypes( - Types.BINARY, - byte[].class, - Platform.HEAP_BYTE_BUFFER_CLASS, - Platform.DIRECT_BYTE_BUFFER_CLASS); - registerDefaultTypes(Types.BOOL_ARRAY, boolean[].class); - registerDefaultTypes(Types.INT16_ARRAY, short[].class); - registerDefaultTypes(Types.INT32_ARRAY, int[].class); - registerDefaultTypes(Types.INT64_ARRAY, long[].class); - registerDefaultTypes(Types.FLOAT32_ARRAY, float[].class); - registerDefaultTypes(Types.FLOAT64_ARRAY, double[].class); - registerDefaultTypes(Types.LIST, ArrayList.class, Object[].class, List.class, Collection.class); - registerDefaultTypes(Types.SET, HashSet.class, LinkedHashSet.class, Set.class); - registerDefaultTypes(Types.MAP, HashMap.class, LinkedHashMap.class, Map.class); - registerDefaultTypes(Types.LOCAL_DATE, LocalDate.class); + // Boolean types + registerType(Types.BOOL, Boolean.class, new PrimitiveSerializers.BooleanSerializer(fory, Boolean.class)); + registerType(Types.BOOL, boolean.class, new PrimitiveSerializers.BooleanSerializer(fory, boolean.class)); + registerType(Types.BOOL, AtomicBoolean.class, new Serializers.AtomicBooleanSerializer(fory)); + + // Byte types + registerType(Types.UINT8, Byte.class, new PrimitiveSerializers.ByteSerializer(fory, Byte.class)); + registerType(Types.UINT8, byte.class, new PrimitiveSerializers.ByteSerializer(fory, byte.class)); + registerType(Types.INT8, Byte.class, new PrimitiveSerializers.ByteSerializer(fory, Byte.class)); + registerType(Types.INT8, byte.class, new PrimitiveSerializers.ByteSerializer(fory, byte.class)); + + // Short types + registerType(Types.UINT16, Short.class, new PrimitiveSerializers.ShortSerializer(fory, Short.class)); + registerType(Types.UINT16, short.class, new PrimitiveSerializers.ShortSerializer(fory, short.class)); + registerType(Types.INT16, Short.class, new PrimitiveSerializers.ShortSerializer(fory, Short.class)); + registerType(Types.INT16, short.class, new PrimitiveSerializers.ShortSerializer(fory, short.class)); + + // Integer types + registerType(Types.UINT32, Integer.class, new PrimitiveSerializers.IntSerializer(fory, Integer.class)); + registerType(Types.UINT32, int.class, new PrimitiveSerializers.IntSerializer(fory, int.class)); + registerType(Types.UINT32, AtomicInteger.class, new Serializers.AtomicIntegerSerializer(fory)); + registerType(Types.INT32, Integer.class, new PrimitiveSerializers.IntSerializer(fory, Integer.class)); + registerType(Types.INT32, int.class, new PrimitiveSerializers.IntSerializer(fory, int.class)); + registerType(Types.INT32, AtomicInteger.class, new Serializers.AtomicIntegerSerializer(fory)); + registerType(Types.VARINT32, Integer.class, new PrimitiveSerializers.IntSerializer(fory, Integer.class)); + registerType(Types.VARINT32, int.class, new PrimitiveSerializers.IntSerializer(fory, int.class)); + registerType(Types.VARINT32, AtomicInteger.class, new Serializers.AtomicIntegerSerializer(fory)); + + // Long types + registerType(Types.UINT64, Long.class, new PrimitiveSerializers.LongSerializer(fory, Long.class)); + registerType(Types.UINT64, long.class, new PrimitiveSerializers.LongSerializer(fory, long.class)); + registerType(Types.UINT64, AtomicLong.class, new Serializers.AtomicLongSerializer(fory)); + registerType(Types.TAGGED_UINT64, Long.class, new PrimitiveSerializers.LongSerializer(fory, Long.class)); + registerType(Types.TAGGED_UINT64, long.class, new PrimitiveSerializers.LongSerializer(fory, long.class)); + registerType(Types.TAGGED_UINT64, AtomicLong.class, new Serializers.AtomicLongSerializer(fory)); + registerType(Types.INT64, Long.class, new PrimitiveSerializers.LongSerializer(fory, Long.class)); + registerType(Types.INT64, long.class, new PrimitiveSerializers.LongSerializer(fory, long.class)); + registerType(Types.INT64, AtomicLong.class, new Serializers.AtomicLongSerializer(fory)); + registerType(Types.TAGGED_INT64, Long.class, new PrimitiveSerializers.LongSerializer(fory, Long.class)); + registerType(Types.TAGGED_INT64, long.class, new PrimitiveSerializers.LongSerializer(fory, long.class)); + registerType(Types.TAGGED_INT64, AtomicLong.class, new Serializers.AtomicLongSerializer(fory)); + registerType(Types.VARINT64, Long.class, new PrimitiveSerializers.LongSerializer(fory, Long.class)); + registerType(Types.VARINT64, long.class, new PrimitiveSerializers.LongSerializer(fory, long.class)); + registerType(Types.VARINT64, AtomicLong.class, new Serializers.AtomicLongSerializer(fory)); + + // Float types + registerType(Types.FLOAT32, Float.class, new PrimitiveSerializers.FloatSerializer(fory, Float.class)); + registerType(Types.FLOAT32, float.class, new PrimitiveSerializers.FloatSerializer(fory, float.class)); + registerType(Types.FLOAT64, Double.class, new PrimitiveSerializers.DoubleSerializer(fory, Double.class)); + registerType(Types.FLOAT64, double.class, new PrimitiveSerializers.DoubleSerializer(fory, double.class)); + + // String types + registerType(Types.STRING, String.class, new StringSerializer(fory)); + registerType(Types.STRING, StringBuilder.class, new Serializers.StringBuilderSerializer(fory)); + registerType(Types.STRING, StringBuffer.class, new Serializers.StringBufferSerializer(fory)); + + // Time types + registerType(Types.DURATION, Duration.class, new TimeSerializers.DurationSerializer(fory)); + registerType(Types.TIMESTAMP, Instant.class, new TimeSerializers.InstantSerializer(fory)); + registerType(Types.TIMESTAMP, Date.class, new TimeSerializers.DateSerializer(fory)); + registerType(Types.TIMESTAMP, java.sql.Date.class, new TimeSerializers.SqlDateSerializer(fory)); + registerType(Types.TIMESTAMP, Timestamp.class, new TimeSerializers.TimestampSerializer(fory)); + registerType(Types.TIMESTAMP, LocalDateTime.class, new TimeSerializers.LocalDateTimeSerializer(fory)); + registerType(Types.LOCAL_DATE, LocalDate.class, new TimeSerializers.LocalDateSerializer(fory)); + + // Decimal types + registerType(Types.DECIMAL, BigDecimal.class, new Serializers.BigDecimalSerializer(fory)); + registerType(Types.DECIMAL, BigInteger.class, new Serializers.BigIntegerSerializer(fory)); + + // Binary types + registerType(Types.BINARY, byte[].class, new ArraySerializers.ByteArraySerializer(fory)); + @SuppressWarnings("unchecked") + Class heapByteBufferClass = (Class) Platform.HEAP_BYTE_BUFFER_CLASS; + registerType(Types.BINARY, Platform.HEAP_BYTE_BUFFER_CLASS, + new org.apache.fory.serializer.BufferSerializers.ByteBufferSerializer(fory, heapByteBufferClass)); + @SuppressWarnings("unchecked") + Class directByteBufferClass = (Class) Platform.DIRECT_BYTE_BUFFER_CLASS; + registerType(Types.BINARY, Platform.DIRECT_BYTE_BUFFER_CLASS, + new org.apache.fory.serializer.BufferSerializers.ByteBufferSerializer(fory, directByteBufferClass)); + + // Primitive arrays + registerType(Types.BOOL_ARRAY, boolean[].class, new ArraySerializers.BooleanArraySerializer(fory)); + registerType(Types.INT16_ARRAY, short[].class, new ArraySerializers.ShortArraySerializer(fory)); + registerType(Types.INT32_ARRAY, int[].class, new ArraySerializers.IntArraySerializer(fory)); + registerType(Types.INT64_ARRAY, long[].class, new ArraySerializers.LongArraySerializer(fory)); + registerType(Types.FLOAT32_ARRAY, float[].class, new ArraySerializers.FloatArraySerializer(fory)); + registerType(Types.FLOAT64_ARRAY, double[].class, new ArraySerializers.DoubleArraySerializer(fory)); + + // Collections + registerType(Types.LIST, ArrayList.class, new ArrayListSerializer(fory)); + registerType(Types.LIST, Object[].class, new ArraySerializers.ObjectArraySerializer(fory, Object[].class)); + registerType(Types.LIST, List.class, new XlangListDefaultSerializer(fory, List.class)); + registerType(Types.LIST, Collection.class, new XlangListDefaultSerializer(fory, Collection.class)); + + // Sets + registerType(Types.SET, HashSet.class, new HashSetSerializer(fory)); + registerType(Types.SET, LinkedHashSet.class, + new org.apache.fory.serializer.collection.CollectionSerializers.LinkedHashSetSerializer(fory)); + registerType(Types.SET, Set.class, new XlangSetDefaultSerializer(fory, Set.class)); + + // Maps + registerType(Types.MAP, HashMap.class, + new org.apache.fory.serializer.collection.MapSerializers.HashMapSerializer(fory)); + registerType(Types.MAP, LinkedHashMap.class, + new org.apache.fory.serializer.collection.MapSerializers.LinkedHashMapSerializer(fory)); + registerType(Types.MAP, Map.class, new XlangMapSerializer(fory, Map.class)); + registerUnionTypes(); } - private void registerDefaultTypes(int xtypeId, Class defaultType, Class... otherTypes) { - ClassInfo classInfo = - newClassInfo(defaultType, classResolver.getSerializer(defaultType), (short) xtypeId); - classInfoMap.put(defaultType, classInfo); - xtypeIdToClassMap.put(xtypeId, classInfo); - for (Class otherType : otherTypes) { - Serializer serializer; - if (ReflectionUtils.isAbstract(otherType)) { - if (isMap(otherType)) { - serializer = new XlangMapSerializer(fory, otherType); - } else if (isSet(otherType)) { - serializer = new XlangSetDefaultSerializer(fory, otherType); - } else if (isCollection(otherType)) { - serializer = new XlangListDefaultSerializer(fory, otherType); - } else { - serializer = classInfo.serializer; - } - } else { - serializer = classResolver.getSerializer(otherType); - } - ClassInfo info = newClassInfo(otherType, serializer, (short) xtypeId); - classInfoMap.put(otherType, info); + private void registerType(int xtypeId, Class type, Serializer serializer) { + ClassInfo classInfo = newClassInfo(type, serializer, (short) xtypeId); + classInfoMap.put(type, classInfo); + if (!xtypeIdToClassMap.containsKey(xtypeId)) { + xtypeIdToClassMap.put(xtypeId, classInfo); } } @@ -724,54 +793,7 @@ public ClassInfo writeClassInfo(MemoryBuffer buffer, Object obj) { } @Override - public void writeClassInfo(MemoryBuffer buffer, ClassInfo classInfo) { - int xtypeId = classInfo.getXtypeId(); - int internalTypeId = xtypeId & 0xff; - buffer.writeVarUint32Small7(xtypeId); - switch (internalTypeId) { - case Types.NAMED_ENUM: - case Types.NAMED_STRUCT: - case Types.NAMED_EXT: - if (shareMeta) { - writeSharedClassMeta(buffer, classInfo); - return; - } - assert classInfo.namespaceBytes != null; - metaStringResolver.writeMetaStringBytes(buffer, classInfo.namespaceBytes); - assert classInfo.typeNameBytes != null; - metaStringResolver.writeMetaStringBytes(buffer, classInfo.typeNameBytes); - break; - case Types.NAMED_COMPATIBLE_STRUCT: - case Types.COMPATIBLE_STRUCT: - assert shareMeta : "Meta share must be enabled for compatible mode"; - writeSharedClassMeta(buffer, classInfo); - break; - default: - break; - } - } - - public void writeSharedClassMeta(MemoryBuffer buffer, ClassInfo classInfo) { - MetaContext metaContext = fory.getSerializationContext().getMetaContext(); - assert metaContext != null : SET_META__CONTEXT_MSG; - IdentityObjectIntMap> classMap = metaContext.classMap; - int newId = classMap.size; - int id = classMap.putOrGet(classInfo.cls, newId); - if (id >= 0) { - // Reference to previously written type: (index << 1) | 1, LSB=1 - buffer.writeVarUint32((id << 1) | 1); - } else { - // New type: index << 1, LSB=0, followed by ClassDef bytes inline - buffer.writeVarUint32(newId << 1); - ClassDef classDef = classInfo.classDef; - if (classDef == null) { - classDef = buildClassDef(classInfo); - } - buffer.writeBytes(classDef.getEncoded()); - } - } - - private ClassDef buildClassDef(ClassInfo classInfo) { + protected ClassDef buildClassDef(ClassInfo classInfo) { ClassDef classDef = classDefMap.computeIfAbsent(classInfo.cls, cls -> ClassDef.buildClassDef(fory, cls)); classInfo.classDef = classDef; @@ -800,95 +822,40 @@ public void setSerializerIfAbsent(Class cls, Serializer serializer) { Preconditions.checkNotNull(classInfo.serializer); } - @Override - public ClassInfo nilClassInfo() { - return classResolver.nilClassInfo(); - } - - @Override - public ClassInfoHolder nilClassInfoHolder() { - return classResolver.nilClassInfoHolder(); - } - - @Override - public ClassInfo readClassInfo(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { - return readClassInfo(buffer); - } + // nilClassInfo and nilClassInfoHolder are inherited from TypeResolver @Override - public ClassInfo readClassInfo(MemoryBuffer buffer, ClassInfo classInfoCache) { - // TODO support type cache to speed up lookup - return readClassInfo(buffer); - } - - public ClassInfo readClassInfo(MemoryBuffer buffer) { - int xtypeId = buffer.readVarUint32Small14(); - int internalTypeId = xtypeId & 0xff; + protected ClassInfo getClassInfoByTypeId(int typeId) { + int internalTypeId = typeId & 0xff; switch (internalTypeId) { - case Types.NAMED_ENUM: - case Types.NAMED_STRUCT: - case Types.NAMED_EXT: - if (shareMeta) { - return readSharedClassMeta(buffer); - } - MetaStringBytes packageBytes = metaStringResolver.readMetaStringBytes(buffer); - MetaStringBytes simpleClassNameBytes = metaStringResolver.readMetaStringBytes(buffer); - return loadBytesToClassInfo(internalTypeId, packageBytes, simpleClassNameBytes); - case Types.NAMED_COMPATIBLE_STRUCT: - case Types.COMPATIBLE_STRUCT: - assert shareMeta : "Meta share must be enabled for compatible mode"; - return readSharedClassMeta(buffer); case Types.LIST: return getListClassInfo(); case Types.TIMESTAMP: return getGenericClassInfo(); default: - ClassInfo classInfo = xtypeIdToClassMap.get(xtypeId); + ClassInfo classInfo = xtypeIdToClassMap.get(typeId); if (classInfo == null) { - throwUnexpectTypeIdException(xtypeId); + throwUnexpectTypeIdException(typeId); } return classInfo; } } @Override - public ClassInfo readSharedClassMeta(MemoryBuffer buffer, MetaContext metaContext) { - return readClassInfo(buffer); - } - - private ClassInfo readSharedClassMeta(MemoryBuffer buffer) { - MetaContext metaContext = fory.getSerializationContext().getMetaContext(); - assert metaContext != null : SET_META__CONTEXT_MSG; - int indexMarker = buffer.readVarUint32Small14(); - boolean isRef = (indexMarker & 1) == 1; - int index = indexMarker >>> 1; - ClassInfo classInfo; - if (isRef) { - // Reference to previously read type in this stream - classInfo = metaContext.readClassInfos.get(index); - } else { - // New type in stream - but may already be known from registry - long id = buffer.readInt64(); - Tuple2 tuple2 = extRegistry.classIdToDef.get(id); - if (tuple2 != null) { - // Already known - skip the ClassDef bytes, reuse existing ClassInfo - ClassDef.skipClassDef(buffer, id); - classInfo = tuple2.f1; - if (classInfo == null) { - classInfo = buildMetaSharedClassInfo(tuple2, tuple2.f0); - } - } else { - // Unknown - read ClassDef and create ClassInfo - tuple2 = readClassDef(buffer, id); - classInfo = tuple2.f1; - if (classInfo == null) { - classInfo = buildMetaSharedClassInfo(tuple2, tuple2.f0); - } - } - // index == readClassInfos.size() since types are written sequentially - metaContext.readClassInfos.add(classInfo); + protected boolean isValidRegisteredTypeId(int typeId) { + // For XtypeResolver, a valid registered type ID means: + // 1. It's a built-in internal type (not UNKNOWN, and not a named type) + // 2. Or it has an entry in xtypeIdToClassMap + if (typeId == NO_CLASS_ID) { + return false; } - return classInfo; + int internalTypeId = typeId & 0xff; + // Named types are not "registered" - they use meta share + if (Types.isNamedType(internalTypeId)) { + return false; + } + // Check if it's in the registry + return xtypeIdToClassMap.containsKey(typeId); } private void throwUnexpectTypeIdException(long xtypeId) { @@ -924,7 +891,30 @@ private ClassInfo getOrBuildClassInfo(Class cls) { return classInfo; } - private ClassInfo loadBytesToClassInfo( + @Override + protected ClassInfo loadBytesToClassInfo( + MetaStringBytes packageBytes, MetaStringBytes simpleClassNameBytes) { + // Default to NAMED_STRUCT when called without internalTypeId + return loadBytesToClassInfoWithTypeId(Types.NAMED_STRUCT, packageBytes, simpleClassNameBytes); + } + + @Override + protected ClassInfo ensureSerializerForClassInfo(ClassInfo classInfo) { + if (classInfo.serializer == null) { + // Get or create ClassInfo with serializer + ClassInfo newClassInfo = getClassInfo(classInfo.cls); + // Update the cache with the correct ClassInfo that has a serializer + if (classInfo.typeNameBytes != null) { + TypeNameBytes typeNameBytes = + new TypeNameBytes(classInfo.namespaceBytes.hashCode, classInfo.typeNameBytes.hashCode); + compositeClassNameBytes2ClassInfo.put(typeNameBytes, newClassInfo); + } + return newClassInfo; + } + return classInfo; + } + + private ClassInfo loadBytesToClassInfoWithTypeId( int internalTypeId, MetaStringBytes packageBytes, MetaStringBytes simpleClassNameBytes) { TypeNameBytes typeNameBytes = new TypeNameBytes(packageBytes.hashCode, simpleClassNameBytes.hashCode); @@ -978,7 +968,6 @@ private ClassInfo populateBytesToClassInfo( simpleClassNameBytes, false, null, - NO_CLASS_ID, NOT_SUPPORT_XLANG); if (NonexistentClass.class.isAssignableFrom(TypeUtils.getComponentIfArray(type))) { classInfo.serializer = NonexistentClassSerializers.getSerializer(fory, qualifiedName, type); @@ -1000,19 +989,19 @@ public DescriptorGrouper createDescriptorGrouper( descriptorUpdator, getPrimitiveComparator(), (o1, o2) -> { - int xtypeId = getXtypeId(o1.getRawType()); - int xtypeId2 = getXtypeId(o2.getRawType()); - if (xtypeId == xtypeId2) { + int typeId1 = getInternalTypeId(o1.getRawType()); + int typeId2 = getInternalTypeId(o2.getRawType()); + if (typeId1 == typeId2) { return getFieldSortKey(o1).compareTo(getFieldSortKey(o2)); } else { - return xtypeId - xtypeId2; + return typeId1 - typeId2; } }) .setOtherDescriptorComparator(Comparator.comparing(TypeResolver::getFieldSortKey)) .sort(); } - private byte getXtypeId(Class cls) { + private byte getInternalTypeId(Class cls) { if (isSet(cls)) { return Types.SET; } @@ -1026,7 +1015,7 @@ private byte getXtypeId(Class cls) { return Types.MAP; } if (isRegistered(cls)) { - return (byte) (getClassInfo(cls).getXtypeId() & 0xff); + return (byte) (getClassInfo(cls).getTypeId() & 0xff); } else { if (cls.isEnum()) { return Types.ENUM; @@ -1041,15 +1030,9 @@ private byte getXtypeId(Class cls) { } } - @Override - public List getFieldDescriptors(Class clz, boolean searchParent) { - return classResolver.getFieldDescriptors(clz, searchParent); - } + // getFieldDescriptors is inherited from TypeResolver - @Override - public ClassDef getTypeDef(Class cls, boolean resolveParent) { - return classResolver.getTypeDef(cls, resolveParent); - } + // getTypeDef is inherited from TypeResolver @Override public Class getSerializerClass(Class cls) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ArraySerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ArraySerializers.java index fe3ffc77b5..0c16356bf3 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ArraySerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ArraySerializers.java @@ -29,6 +29,7 @@ import org.apache.fory.resolver.ClassInfo; import org.apache.fory.resolver.ClassInfoHolder; import org.apache.fory.resolver.ClassResolver; +import org.apache.fory.resolver.TypeResolver; import org.apache.fory.resolver.RefResolver; import org.apache.fory.serializer.collection.CollectionFlags; import org.apache.fory.serializer.collection.ForyArrayAsListSerializer; @@ -981,7 +982,7 @@ public String[] xread(MemoryBuffer buffer) { } public static void registerDefaultSerializers(Fory fory) { - ClassResolver resolver = fory.getClassResolver(); + TypeResolver resolver = fory._getTypeResolver(); resolver.registerInternalSerializer( Object[].class, new ObjectArraySerializer<>(fory, Object[].class)); resolver.registerInternalSerializer( diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/OptionalSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/OptionalSerializers.java index a19fde5ef0..ad51fdf50d 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/OptionalSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/OptionalSerializers.java @@ -26,6 +26,7 @@ import org.apache.fory.Fory; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.resolver.ClassResolver; +import org.apache.fory.resolver.TypeResolver; /** * Serializers for {@link Optional}, {@link OptionalInt}, {@link OptionalLong} and {@link @@ -131,7 +132,7 @@ public OptionalDouble read(MemoryBuffer buffer) { } public static void registerDefaultSerializers(Fory fory) { - ClassResolver resolver = fory.getClassResolver(); + TypeResolver resolver = fory._getTypeResolver(); resolver.registerInternalSerializer(Optional.class, new OptionalSerializer(fory)); resolver.registerInternalSerializer(OptionalInt.class, new OptionalIntSerializer(fory)); resolver.registerInternalSerializer(OptionalLong.class, new OptionalLongSerializer(fory)); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/PrimitiveSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/PrimitiveSerializers.java index 34ff88938e..475656f3c5 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/PrimitiveSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/PrimitiveSerializers.java @@ -28,6 +28,7 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.Platform; import org.apache.fory.resolver.ClassResolver; +import org.apache.fory.resolver.TypeResolver; import org.apache.fory.util.Preconditions; /** Serializers for java primitive types. */ @@ -294,7 +295,7 @@ public Double read(MemoryBuffer buffer) { public static void registerDefaultSerializers(Fory fory) { // primitive types will be boxed. - ClassResolver resolver = fory.getClassResolver(); + TypeResolver resolver = fory._getTypeResolver(); resolver.registerInternalSerializer(boolean.class, new BooleanSerializer(fory, boolean.class)); resolver.registerInternalSerializer(byte.class, new ByteSerializer(fory, byte.class)); resolver.registerInternalSerializer(short.class, new ShortSerializer(fory, short.class)); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java index 020ac2efc4..7bf15c9c30 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java @@ -50,6 +50,7 @@ import org.apache.fory.meta.ClassDef; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.resolver.ClassResolver; +import org.apache.fory.resolver.TypeResolver; import org.apache.fory.util.ExceptionUtils; import org.apache.fory.util.GraalvmSupport; import org.apache.fory.util.GraalvmSupport.GraalvmSerializerHolder; @@ -561,7 +562,7 @@ public Object read(MemoryBuffer buffer) { } public static void registerDefaultSerializers(Fory fory) { - ClassResolver resolver = fory.getClassResolver(); + TypeResolver resolver = fory._getTypeResolver(); resolver.registerInternalSerializer(Class.class, new ClassSerializer(fory)); resolver.registerInternalSerializer(StringBuilder.class, new StringBuilderSerializer(fory)); resolver.registerInternalSerializer(StringBuffer.class, new StringBufferSerializer(fory)); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/TimeSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/TimeSerializers.java index a328b65c51..83f6797d86 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/TimeSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/TimeSerializers.java @@ -42,6 +42,7 @@ import org.apache.fory.Fory; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.resolver.ClassResolver; +import org.apache.fory.resolver.TypeResolver; import org.apache.fory.util.DateTimeUtils; /** Serializers for all time related types. */ @@ -708,7 +709,7 @@ public OffsetDateTime read(MemoryBuffer buffer) { } public static void registerDefaultSerializers(Fory fory) { - ClassResolver resolver = fory.getClassResolver(); + TypeResolver resolver = fory._getTypeResolver(); resolver.registerInternalSerializer(Date.class, new DateSerializer(fory)); resolver.registerInternalSerializer(java.sql.Date.class, new SqlDateSerializer(fory)); resolver.registerInternalSerializer(Time.class, new SqlTimeSerializer(fory)); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionLikeSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionLikeSerializer.java index 4589065265..319bb5f0d2 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionLikeSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionLikeSerializer.java @@ -476,7 +476,7 @@ public void copyElements(Collection originCollection, Collection newCollection) if (element != null) { ClassInfo classInfo = typeResolver.getClassInfo(element.getClass(), elementClassInfoHolder); if (!classInfo.getSerializer().isImmutable()) { - element = fory.copyObject(element, classInfo.getClassId()); + element = fory.copyObject(element, classInfo.getTypeId()); } } newCollection.add(element); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java index a7a7225bb3..99aa91309c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java @@ -57,6 +57,7 @@ import org.apache.fory.resolver.ClassInfo; import org.apache.fory.resolver.ClassInfoHolder; import org.apache.fory.resolver.ClassResolver; +import org.apache.fory.resolver.TypeResolver; import org.apache.fory.resolver.RefResolver; import org.apache.fory.serializer.ReplaceResolveSerializer; import org.apache.fory.serializer.Serializer; @@ -1001,7 +1002,7 @@ public Set newCollection(MemoryBuffer buffer) { // TODO Support ArraySubListSerializer, SubListSerializer public static void registerDefaultSerializers(Fory fory) { - ClassResolver resolver = fory.getClassResolver(); + TypeResolver resolver = fory._getTypeResolver(); resolver.registerInternalSerializer(ArrayList.class, new ArrayListSerializer(fory)); Class arrayAsListClass = Arrays.asList(1, 2).getClass(); resolver.registerInternalSerializer( diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/GuavaCollectionSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/GuavaCollectionSerializers.java index 535178886a..5256d8c24d 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/GuavaCollectionSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/GuavaCollectionSerializers.java @@ -38,6 +38,7 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.Platform; import org.apache.fory.resolver.ClassResolver; +import org.apache.fory.resolver.TypeResolver; import org.apache.fory.util.unsafe._JDKAccess; /** Serializers for common guava types. */ @@ -401,7 +402,7 @@ public static void registerDefaultSerializers(Fory fory) { // inconsistent if peers load different version of guava. // For example: guava 20 return ImmutableBiMap for ImmutableMap.of(), but guava 27 return // ImmutableMap. - ClassResolver resolver = fory.getClassResolver(); + TypeResolver resolver = fory._getTypeResolver(); Class cls = loadClass(pkg + ".RegularImmutableBiMap", ImmutableBiMap.of("k1", 1, "k2", 4).getClass()); resolver.registerInternalSerializer(cls, new ImmutableBiMapSerializer(fory, cls)); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ImmutableCollectionSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ImmutableCollectionSerializers.java index 32e051cb37..29de60b9e1 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ImmutableCollectionSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ImmutableCollectionSerializers.java @@ -33,6 +33,7 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.Platform; import org.apache.fory.resolver.ClassResolver; +import org.apache.fory.resolver.TypeResolver; import org.apache.fory.util.unsafe._JDKAccess; /** Serializers for jdk9+ java.util.ImmutableCollections. */ @@ -260,7 +261,7 @@ public Map onMapRead(Map map) { } public static void registerSerializers(Fory fory) { - ClassResolver resolver = fory.getClassResolver(); + TypeResolver resolver = fory._getTypeResolver(); resolver.registerInternalSerializer(List12, new ImmutableListSerializer(fory, List12)); resolver.registerInternalSerializer(ListN, new ImmutableListSerializer(fory, ListN)); resolver.registerInternalSerializer(SubList, new ImmutableListSerializer(fory, SubList)); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapLikeSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapLikeSerializer.java index 79b984e375..9973743fca 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapLikeSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapLikeSerializer.java @@ -531,7 +531,7 @@ protected void copyEntry(Map originMap, Map newMap) { if (key != null) { ClassInfo classInfo = classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache); if (!classInfo.getSerializer().isImmutable()) { - key = fory.copyObject(key, classInfo.getClassId()); + key = fory.copyObject(key, classInfo.getTypeId()); } } V value = entry.getValue(); @@ -539,7 +539,7 @@ protected void copyEntry(Map originMap, Map newMap) { ClassInfo classInfo = classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache); if (!classInfo.getSerializer().isImmutable()) { - value = fory.copyObject(value, classInfo.getClassId()); + value = fory.copyObject(value, classInfo.getTypeId()); } } newMap.put(key, value); @@ -553,7 +553,7 @@ protected void copyEntry(Map originMap, Builder builder) { if (key != null) { ClassInfo classInfo = classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache); if (!classInfo.getSerializer().isImmutable()) { - key = fory.copyObject(key, classInfo.getClassId()); + key = fory.copyObject(key, classInfo.getTypeId()); } } V value = entry.getValue(); @@ -561,7 +561,7 @@ protected void copyEntry(Map originMap, Builder builder) { ClassInfo classInfo = classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache); if (!classInfo.getSerializer().isImmutable()) { - value = fory.copyObject(value, classInfo.getClassId()); + value = fory.copyObject(value, classInfo.getTypeId()); } } builder.put(key, value); @@ -576,7 +576,7 @@ protected void copyEntry(Map originMap, Object[] elements) { if (key != null) { ClassInfo classInfo = classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache); if (!classInfo.getSerializer().isImmutable()) { - key = fory.copyObject(key, classInfo.getClassId()); + key = fory.copyObject(key, classInfo.getTypeId()); } } V value = entry.getValue(); @@ -584,7 +584,7 @@ protected void copyEntry(Map originMap, Object[] elements) { ClassInfo classInfo = classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache); if (!classInfo.getSerializer().isImmutable()) { - value = fory.copyObject(value, classInfo.getClassId()); + value = fory.copyObject(value, classInfo.getTypeId()); } } elements[index++] = key; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapSerializers.java index 1a05377dce..346a56938a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapSerializers.java @@ -39,6 +39,7 @@ import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.resolver.ClassInfo; import org.apache.fory.resolver.ClassResolver; +import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.ReplaceResolveSerializer; import org.apache.fory.serializer.Serializer; import org.apache.fory.serializer.Serializers; @@ -351,7 +352,7 @@ protected void copyEntry(Map originMap, Map newMap) { ClassInfo classInfo = classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache); if (!classInfo.getSerializer().isImmutable()) { - value = fory.copyObject(value, classInfo.getClassId()); + value = fory.copyObject(value, classInfo.getTypeId()); } } newMap.put(entry.getKey(), value); @@ -494,10 +495,9 @@ public Object onMapRead(Map map) { // TODO(chaokunyang) support ConcurrentSkipListMap.SubMap mo efficiently. public static void registerDefaultSerializers(Fory fory) { - ClassResolver resolver = fory.getClassResolver(); + TypeResolver resolver = fory._getTypeResolver(); resolver.registerInternalSerializer(HashMap.class, new HashMapSerializer(fory)); - fory.getClassResolver() - .registerInternalSerializer(LinkedHashMap.class, new LinkedHashMapSerializer(fory)); + resolver.registerInternalSerializer(LinkedHashMap.class, new LinkedHashMapSerializer(fory)); resolver.registerInternalSerializer( TreeMap.class, new SortedMapSerializer<>(fory, TreeMap.class)); resolver.registerInternalSerializer( diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SubListSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SubListSerializers.java index 27da62670b..7319e22745 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SubListSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SubListSerializers.java @@ -28,6 +28,7 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.resolver.ClassResolver; +import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.ObjectSerializer; @SuppressWarnings({"rawtypes", "unchecked"}) @@ -64,7 +65,7 @@ private interface Stub {} } public static void registerSerializers(Fory fory, boolean preserveView) { - ClassResolver classResolver = fory.getClassResolver(); + TypeResolver classResolver = fory._getTypeResolver(); // java.util.ImmutableCollections$SubList is already registered in // ImmutableCollectionSerializers for (Class cls : diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SynchronizedSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SynchronizedSerializers.java index 9c8bf2c66e..f1ee86c5a2 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SynchronizedSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SynchronizedSerializers.java @@ -41,6 +41,7 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.Platform; import org.apache.fory.resolver.ClassResolver; +import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.Serializer; import org.apache.fory.util.ExceptionUtils; @@ -212,7 +213,7 @@ static Tuple2, Function>[] synchronizedFactories() { */ public static void registerSerializers(Fory fory) { try { - ClassResolver resolver = fory.getClassResolver(); + TypeResolver resolver = fory._getTypeResolver(); for (Tuple2, Function> factory : synchronizedFactories()) { resolver.registerInternalSerializer(factory.f0, createSerializer(fory, factory)); } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/UnmodifiableSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/UnmodifiableSerializers.java index cc6797158d..feb270f514 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/UnmodifiableSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/UnmodifiableSerializers.java @@ -40,6 +40,7 @@ import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.Platform; import org.apache.fory.resolver.ClassResolver; +import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.Serializer; import org.apache.fory.util.ExceptionUtils; import org.apache.fory.util.Preconditions; @@ -210,7 +211,7 @@ static Tuple2, Function>[] unmodifiableFactories() { */ public static void registerSerializers(Fory fory) { try { - ClassResolver resolver = fory.getClassResolver(); + TypeResolver resolver = fory._getTypeResolver(); for (Tuple2, Function> factory : unmodifiableFactories()) { resolver.registerInternalSerializer(factory.f0, createSerializer(fory, factory)); } diff --git a/java/fory-core/src/main/java/org/apache/fory/type/Types.java b/java/fory-core/src/main/java/org/apache/fory/type/Types.java index c6337a6ab3..aacfe39167 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/Types.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/Types.java @@ -351,7 +351,7 @@ public static int getTypeId(Fory fory, Class clz) { } ClassInfo classInfo = fory._getTypeResolver().getClassInfo(clz, false); if (classInfo != null) { - return fory.isCrossLanguage() ? classInfo.getXtypeId() : classInfo.getClassId(); + return classInfo.getTypeId(); } return Types.UNKNOWN; } From 10e369c5d535ca3ffc4a4abe88dd40b3ed437d7e Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 18 Jan 2026 19:34:05 +0800 Subject: [PATCH 25/44] refactor type system --- docs/specification/java_serialization_spec.md | 560 ------------------ .../src/main/java/org/apache/fory/Fory.java | 20 +- .../fory/builder/BaseObjectCodecBuilder.java | 13 +- .../fory/builder/MetaSharedCodecBuilder.java | 6 +- .../fory/builder/ObjectCodecBuilder.java | 6 +- .../java/org/apache/fory/meta/ClassDef.java | 20 + .../org/apache/fory/meta/ClassDefDecoder.java | 26 +- .../org/apache/fory/meta/ClassDefEncoder.java | 8 +- .../java/org/apache/fory/meta/FieldInfo.java | 2 +- .../java/org/apache/fory/meta/FieldTypes.java | 217 +++---- .../org/apache/fory/meta/TypeDefDecoder.java | 8 +- .../org/apache/fory/resolver/ClassInfo.java | 13 +- .../apache/fory/resolver/ClassResolver.java | 196 ++++-- .../org/apache/fory/resolver/MetaContext.java | 13 - .../apache/fory/resolver/TypeResolver.java | 346 ++++------- .../apache/fory/resolver/XtypeResolver.java | 186 ++++-- .../fory/serializer/ArraySerializers.java | 2 +- .../NonexistentClassSerializers.java | 54 +- .../fory/serializer/ObjectSerializer.java | 6 +- .../serializer/ObjectStreamSerializer.java | 4 +- .../fory/serializer/OptionalSerializers.java | 1 - .../fory/serializer/PrimitiveSerializers.java | 1 - .../fory/serializer/SerializationBinding.java | 6 +- .../apache/fory/serializer/Serializers.java | 1 - .../fory/serializer/TimeSerializers.java | 1 - .../collection/ChildContainerSerializers.java | 6 +- .../collection/CollectionSerializers.java | 2 +- .../GuavaCollectionSerializers.java | 1 - .../ImmutableCollectionSerializers.java | 1 - .../collection/SubListSerializers.java | 1 - .../collection/SynchronizedSerializers.java | 1 - .../collection/UnmodifiableSerializers.java | 1 - ...inalFieldReplaceResolveSerializerTest.java | 2 +- .../NonexistentClassSerializersTest.java | 2 - .../fory/xlang/MetaSharedXlangTest.java | 1 - .../integration_tests/RecordXlangTest.java | 6 +- 36 files changed, 662 insertions(+), 1078 deletions(-) diff --git a/docs/specification/java_serialization_spec.md b/docs/specification/java_serialization_spec.md index c7b8d22d80..e69de29bb2 100644 --- a/docs/specification/java_serialization_spec.md +++ b/docs/specification/java_serialization_spec.md @@ -1,560 +0,0 @@ ---- -title: Java Serialization Format -sidebar_position: 1 -id: java_serialization_spec -license: | - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---- - -## Spec overview - -Apache Fory™ Java Serialization is an automatic object serialization framework that supports reference and polymorphism. Apache Fory™ -will -convert an object from/to fory java serialization binary format. Apache Fory™ has two core concepts for java serialization: - -- **Apache Fory™ Java Binary format** -- **Framework to convert object to/from Apache Fory™ Java Binary format** - -The serialization format is a dynamic binary format. The dynamics and reference/polymorphism support make Apache Fory™ flexible, -much more easy to use, but -also introduce more complexities compared to static serialization frameworks. So the format will be more complex. - -Here is the overall format: - -``` -| fory header | object ref meta | object class meta | object value data | -``` - -The data are serialized using little endian byte order overall. If bytes swap is costly for some object, -Fory will write the byte order for that object into the data instead of converting it to little endian. - -## Fory header - -Fory header consists starts one byte: - -``` -| 4 bits | 1 bit | 1 bit | 1 bit | 1 bit | optional 4 bytes | -+---------------+-------+-------+--------+-------+------------------------------------+ -| reserved bits | oob | xlang | endian | null | unsigned int for meta start offset | -``` - -- null flag: 1 when object is null, 0 otherwise. If an object is null, other bits won't be set. -- endian flag: 1 when data is encoded by little endian, 0 for big endian. -- xlang flag: 1 when serialization uses xlang format, 0 when serialization uses Fory java format. -- oob flag: 1 when passed `BufferCallback` is not null, 0 otherwise. - -If meta share mode is enabled, an uncompressed unsigned int is appended to indicate the start offset of metadata. - -## Reference Meta - -Reference tracking handles whether the object is null, and whether to track reference for the object by writing -corresponding flags and maintaining internal state. - -Reference flags: - -| Flag | Byte Value | Description | -| ------------------- | ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | -| NULL FLAG | `-3` | This flag indicates the object is a null value. We don't use another byte to indicate REF, so that we can save one byte. | -| REF FLAG | `-2` | This flag indicates the object is already serialized previously, and fory will write a ref id with unsigned varint format instead of serialize it again | -| NOT_NULL VALUE FLAG | `-1` | This flag indicates the object is a non-null value and fory doesn't track ref for this type of object. | -| REF VALUE FLAG | `0` | This flag indicates the object is referencable and the first time to serialize. | - -When reference tracking is disabled globally or for specific types, or for certain types within a particular -context(e.g., a field of a class), only the `NULL` and `NOT_NULL VALUE` flags will be used for reference meta. - -## Class Meta - -Fory supports to register class by an optional id, the registration can be used for security check and class -identification. -If a class is registered, it will have a user-provided or an auto-growing unsigned int i.e. `class_id`. - -Depending on whether meta share mode and registration is enabled for current class, Fory will write class meta -differently. - -### Schema consistent - -If schema consistent mode is enabled globally or enabled for current class, class meta will be written as follows: - -- If class is registered, it will be written as a fory unsigned varint: `class_id << 1`. -- If class is not registered: - - If class is not an array, fory will write one byte `0bxxxxxxx1` first, then write class name. - - The first little bit is `1`, which is different from first bit `0` of - encoded class id. Fory can use this information to determine whether to read class by class id for - deserialization. - - If class is not registered and class is an array, fory will write one byte `dimensions << 1 | 1` first, then write - component - class subsequently. This can reduce array class name cost if component class is or will be serialized. - - Class will be written as two enumerated fory unsigned by default: `package name` and `class name`. If meta share - mode is - enabled, - class will be written as an unsigned varint which points to index in `MetaContext`. - -### Schema evolution - -If schema evolution mode is enabled globally or enabled for current class, class meta will be written as follows: - -- If meta share mode is not enabled, class meta will be written as schema consistent mode. Additionally, field meta such - as field type - and name will be written with the field value using a key-value like layout. -- If meta share mode is enabled, class meta will be written as a meta-share encoded binary if class hasn't been written - before, otherwise an unsigned varint id which references to previous written class meta will be written. - -## Meta share - -> This mode will forbid streaming writing since it needs to look back for update the start offset after the whole object -> graph -> writing and meta collecting is finished. Only in this way we can ensure deserialization failure doesn't lost shared -> meta. -> Meta streamline will be supported in the future for enclosed meta sharing which doesn't cross multiple serializations -> of different objects. - -For Schema consistent mode, class will be encoded as an enumerated string by full class name. Here we mainly describe -the meta layout for schema evolution mode: - -``` -| 8 bytes global meta header | 1~2 bytes | variable bytes | variable bytes | variable bytes | -+-------------------------------+-------------|--------------------+-------------------+----------------+ -| 50 bits hash + 14 bits header | type header | current class meta | parent class meta | ... | -``` - -Class meta are encoded from parent class to leaf class, only class with serializable fields will be encoded. - -### Global meta header - -Meta header is a 64 bits number value encoded in little endian order. - -- lower 12 bits are used to encode meta size. If meta size `>= 0b1111_1111_1111`, then write - `meta_ size - 0b1111_1111_1111` next. -- 13rd bit is used to indicate whether to write fields meta. When this class is schema-consistent or use registered - serializer, fields meta will be skipped. Class Meta will be used for share namespace + type name only. -- 14rd bit is used to indicate whether meta is compressed. -- Other 50 bits is used to store the unique hash of `flags + all layers class meta`. - -### Type header - -- Lowest 4 digits `0b0000~0b1110` are used to record num classes. `0b1111` is preserved to indicate that Fory need to - read more bytes for length using Fory unsigned int encoding. If current class doesn't has parent class, or parent - class doesn't have fields to serialize, or we're in a context which serialize fields of current class - only(`ObjectStreamSerializer#SlotInfo` is an example), num classes will be 1. -- Other 4 bits are preserved to future extensions. -- If num classes are greater than or equal to `0b1111`, write `num_classes - 0b1111` as varuint next. - -### Single layer class meta - -``` -| unsigned varint | meta string | meta string | field info: variable bytes | variable bytes | ... | -+----------------------------+-----------------------+---------------------+-------------------------------+-----------------+-----+ -| num fields + register flag | header + package name | header + class name | header + type id + field name | next field info | ... | -``` - -- num fields: encode `num fields << 1 | register flag(1 when class registered)` as unsigned varint. - - If class is registered, then an unsigned varint class id will be written next, package and class name will be - omitted. - - If current class is schema consistent, then num field will be `0` to flag it. - - If current class isn't schema consistent, then num field will be the number of compatible fields. For example, - users - can use tag id to mark some field as compatible field in schema consistent context. In such cases, schema - consistent - fields will be serialized first, then compatible fields will be serialized next. At deserialization, Fory will use - fields info of those fields which aren't annotated by tag id for deserializing schema consistent fields, then use - fields info in meta for deserializing compatible fields. -- Package name encoding(omitted when class is registered): - - encoding algorithm: `UTF8/ALL_TO_LOWER_SPECIAL/LOWER_UPPER_DIGIT_SPECIAL` - - Header: `6 bits size | 2 bits encoding flags`. The `6 bits size: 0~63` will be used to indicate size `0~63`, - the value `63` the size need more byte to read, the encoding will encode `size - 63` as a varint next. -- Class name encoding(omitted when class is registered): - - encoding algorithm: `UTF8/LOWER_UPPER_DIGIT_SPECIAL/FIRST_TO_LOWER_SPECIAL/ALL_TO_LOWER_SPECIAL` - - header: `6 bits size | 2 bits encoding flags`. The `6 bits size: 0~63` will be used to indicate size `0~63`, - the value `63` the size need more byte to read, the encoding will encode `size - 63` as a varint next. -- Field info: - - header(8 - bits): `3 bits size + 2 bits field name encoding + polymorphism flag + nullability flag + ref tracking flag`. - Users can use annotation to provide those info. - - 2 bits field name encoding: - - encoding: `UTF8/ALL_TO_LOWER_SPECIAL/LOWER_UPPER_DIGIT_SPECIAL/TAG_ID` - - If tag id is used, i.e. field name is written by an unsigned varint tag id. 2 bits encoding will be `11`. - - size of field name: - - The `3 bits size: 0~7` will be used to indicate length `1~7`, the value `6` the size read more bytes, - the encoding will encode `size - 7` as a varint next. - - If encoding is `TAG_ID`, then num_bytes of field name will be used to store tag id. - - ref tracking: when set to 1, ref tracking will be enabled for this field. - - nullability: when set to 1, this field can be null. - - polymorphism: when set to 1, the actual type of field will be the declared field type even the type if - not `final`. - - type id: - - For registered type-consistent classes, it will be the registered class id. - - Otherwise it will be encoded as `OBJECT_ID` if it isn't `final` and `FINAL_OBJECT_ID` if it's `final`. The - meta for such types is written separately instead of inlining here is to reduce meta space cost if object of - this type is serialized in current object graph multiple times, and the field value may be null too. - - Field name: If type id is set, type id will be used instead. Otherwise meta string encoding length and data will - be written instead. - -Field order are left as implementation details, which is not exposed to specification, the deserialization need to -resort fields based on Fory field comparator. In this way, fory can compute statistics for field names or types and -using a more compact encoding. - -### Other layers class meta - -Same encoding algorithm as the previous layer except: - -- header + package name: - - Header: - - If package name has been written before: `varint index + sharing flag(set)` will be written - - If package name hasn't been written before: - - If meta string encoding is `LOWER_SPECIAL` and the length of encoded string `<=` 64, then header will be - `6 bits size + encoding flag(set) + sharing flag(unset)`. - - Otherwise, header will - be `3 bits unset + 3 bits encoding flags + encoding flag(unset) + sharing flag(unset)` - -## Meta String - -Meta string is mainly used to encode meta strings such as class name and field names. - -### Encoding Algorithms - -String binary encoding algorithm: - -| Algorithm | Pattern | Description | -| ------------------------- | ------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| LOWER_SPECIAL | `a-z._$\|` | every char is written using 5 bits, `a-z`: `0b00000~0b11001`, `._$\|`: `0b11010~0b11101`, prepend one bit at the start to indicate whether strip last char since last byte may have 7 redundant bits(1 indicates strip last char) | -| LOWER_UPPER_DIGIT_SPECIAL | `a-zA-Z0~9._` | every char is written using 6 bits, `a-z`: `0b00000~0b11001`, `A-Z`: `0b11010~0b110011`, `0~9`: `0b110100~0b111101`, `._`: `0b111110~0b111111`, prepend one bit at the start to indicate whether strip last char since last byte may have 7 redundant bits(1 indicates strip last char) | -| UTF-8 | any chars | UTF-8 encoding | - -Encoding flags: - -| Encoding Flag | Pattern | Encoding Algorithm | -| ------------------------- | ------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | -| LOWER_SPECIAL | every char is in `a-z._$\|` | `LOWER_SPECIAL` | -| FIRST_TO_LOWER_SPECIAL | every char is in `a-z[c1,c2]` except first char is upper case | replace first upper case char to lower case, then use `LOWER_SPECIAL` | -| ALL_TO_LOWER_SPECIAL | every char is in `a-zA-Z[c1,c2]` | replace every upper case char by `\|` + `lower case`, then use `LOWER_SPECIAL`, use this encoding if it's smaller than Encoding `LOWER_UPPER_DIGIT_SPECIAL` | -| LOWER_UPPER_DIGIT_SPECIAL | every char is in `a-zA-Z[c1,c2]` | use `LOWER_UPPER_DIGIT_SPECIAL` encoding if it's smaller than Encoding `FIRST_TO_LOWER_SPECIAL` | -| UTF8 | any utf-8 char | use `UTF-8` encoding | -| Compression | any utf-8 char | lossless compression | - -Notes: - -- For package name encoding, `c1,c2` should be `._`; For field/type name encoding, `c1,c2` should be `_$`; -- Depending on cases, one can choose encoding `flags + data` jointly, uses 3 bits of first byte for flags and other - bytes - for data. - -### Shared meta string - -The shared meta string format consists of header and encoded string binary. Header of encoded string binary will be -inlined -in shared meta header. - -Header is written using little endian order, Fory can read this flag first to determine how to deserialize the data. - -#### Write by data - -If string hasn't been written before, the data will be written as follows: - -``` -| unsigned varint: string binary size + 1 bit: not written before | 56 bits: unique hash | 3 bits encoding flags + string binary | -``` - -If string binary size is less than `16` bytes, the hash will be omitted to save spaces. Unique hash can be omitted too -if caller pass a flag to disable it. In such cases, the format will be: - -``` -| unsigned varint: string binary size + 1 bit: not written before | 3 bits encoding flags + string binary | -``` - -#### Write by ref - -If string has been written before, the data will be written as follows: - -``` -| unsigned varint: written string id + 1 bit: written before | -``` - -## Value Format - -### Basic types - -#### Bool - -- size: 1 byte -- format: 0 for `false`, 1 for `true` - -#### Byte - -- size: 1 byte -- format: write as pure byte. - -#### Short - -- size: 2 byte -- byte order: little endian order - -#### Char - -- size: 2 byte -- byte order: little endian order - -#### Unsigned int - -- size: 1~5 byte -- Format: The most significant bit (MSB) in every byte indicates whether to have the next byte. If first bit is set - i.e. `b & 0x80 == 0x80`, then - the next byte should be read until the first bit of the next byte is unset. - -#### Signed int - -- size: 1~5 byte -- Format: First convert the number into positive unsigned int by `(v << 1) ^ (v >> 31)` ZigZag algorithm, then encoding - it as an unsigned int. - -#### Unsigned long - -- size: 1~9 byte -- Fory PVL(Progressive Variable-length Long) Encoding: - - positive long format: first bit in every byte indicates whether to have the next byte. If first bit is set - i.e. `b & 0x80 == 0x80`, then the next byte should be read until the first bit is unset. - -#### Signed long - -- size: 1~9 byte -- Fory SLI(Small long as int) Encoding: - - If long is in [-1073741824, 1073741823], encode as 4 bytes int: `| little-endian: ((int) value) << 1 |` - - Otherwise write as 9 bytes: `| 0b1 | little-endian 8 bytes long |` -- Fory PVL(Progressive Variable-length Long) Encoding: - - First convert the number into positive unsigned long by `(v << 1) ^ (v >> 63)` ZigZag algorithm to reduce cost of - small negative numbers, then encoding it as an unsigned long. - -#### Float - -- size: 4 byte -- format: convert float to 4 bytes int by `Float.floatToRawIntBits`, then write as binary by little endian order. - -#### Double - -- size: 8 byte -- format: convert double to 8 bytes int by `Double.doubleToRawLongBits`, then write as binary by little endian order. - -### String - -Format: - -``` -| header: size << 2 | 2 bits encoding flags | binary data | -``` - -- `size + encoding` will be concat as a long and encoded as an unsigned var long. The little 2 bits is used for - encoding: - 0 for `latin`, 1 for `utf-16`, 2 for `utf-8`. -- encoded string binary data based on encoding: `latin/utf-16/utf-8`. - -Which encoding to choose: - -- For JDK8: fory detect `latin` at runtime, if string is `latin` string, then use `latin` encoding, otherwise - use `utf-16`. -- For JDK9+: fory use `coder` in `String` object for encoding, `latin`/`utf-16` will be used for encoding. -- If the string is encoded by `utf-8`, then fory will use `utf-8` to decode the data. But currently fory doesn't enable - utf-8 encoding by default for java. Cross-language string serialization of fory uses `utf-8` by default. - -### Collection - -> All collection serializers must extend `CollectionLikeSerializer`. - -Format: - -``` -length(unsigned varint) | collection header | elements header | elements data -``` - -#### Collection header - -- For `ArrayList/LinkedArrayList/HashSet/LinkedHashSet`, this will be empty. -- For `TreeSet`, this will be `Comparator` -- For subclass of `ArrayList`, this may be extra object field info. - -#### Elements header - -In most cases, all collection elements are same type and not null, elements header will encode those homogeneous -information to avoid the cost of writing it for every element. Specifically, there are four kinds of information -which will be encoded by elements header, each use one bit: - -- If track elements ref, use the first bit `0b1` of the header to flag it. -- If the collection has null, use the second bit `0b10` of the header to flag it. If ref tracking is enabled for this - element type, this flag is invalid. -- If the collection element types are the declared type, use the 3rd bit `0b100` of the header to flag it. -- If the collection element types are same, use the 4th bit `0b1000` header to flag it. - -By default, all bits are unset, which means all elements won't track ref, all elements are same type, not null and -the actual element is the declared type in the custom class field. - -The implementation can generate different deserialization code based read header, and look up the generated code from a -linear map/list. - -#### Elements data - -Based on the elements header, the serialization of elements data may skip `ref flag`/`null flag`/`element class info`. - -`CollectionSerializer#write/read` can be taken as an example. - -### Array - -#### Primitive array - -Primitive array are taken as a binary buffer, serialization will just write the length of array size as an unsigned int, -then copy the whole buffer into the stream. - -Such serialization won't compress the array. If users want to compress primitive array, users need to register custom -serializers for such types. - -#### Object array - -Object array is serialized using the collection format. Object component type will be taken as collection element -generic -type. - -### Map - -> All Map serializers must extend `MapLikeSerializer`. - -Format: - -``` -| length(unsigned varint) | map header | key value pairs data | -``` - -#### Map header - -- For `HashMap/LinkedHashMap`, this will be empty. -- For `TreeMap`, this will be `Comparator` -- For other `Map`, this may be extra object field info. - -#### Map Key-Value data - -Map iteration is too expensive, Fory won't compute the header like for collection before since it introduce -[considerable overhead](https://github.com/apache/fory/issues/925). -Users can use `MapFieldInfo` annotation to provide header in advance. Otherwise Fory will use first key-value pair to -predict header optimistically, and update the chunk header if the prediction failed at some pair. - -Fory will serialize map chunk by chunk, every chunk has 127 pairs at most. - -``` -| 1 byte | 1 byte | variable bytes | -+----------------+----------------+-----------------+ -| KV header | chunk size: N | N*2 objects | -``` - -KV header: - -- If track key ref, use the first bit `0b1` of the header to flag it. -- If the key has null, use the second bit `0b10` of the header to flag it. If ref tracking is enabled for this - key type, this flag is invalid. -- If the actual key type of map is the declared key type, use the 3rd bit `0b100` of the header to flag it. -- If track value ref, use the 4th bit `0b1000` of the header to flag it. -- If the value has null, use the 5th bit `0b10000` of the header to flag it. If ref tracking is enabled for this - value type, this flag is invalid. -- If the value type of map is the declared value type, use the 6rd bit `0b100000` of the header to flag it. -- If key or value is null, that key and value will be written as a separate chunk, and chunk size writing will be - skipped too. - -If streaming write is enabled, which means Fory can't update written `chunk size`. In such cases, map key-value data -format will be: - -``` -| 1 byte | variable bytes | -+----------------+-----------------+ -| KV header | N*2 objects | -``` - -`KV header` will be a header marked by `MapFieldInfo` in java. The implementation can generate different deserialization -code based read header, and look up the generated code from a linear map/list. - -### Enum - -Enums are serialized as an unsigned var int. If the order of enum values change, the deserialized enum value may not be -the value users expect. In such cases, users must register enum serializer by make it write enum value as an enumerated -string with unique hash disabled. - -### Object - -Object means object of `pojo/struct/bean/record` type. -Object will be serialized by writing its fields data in fory order. - -Depending on schema compatibility, objects will have different formats. - -#### Field order - -Field will be ordered as following, every group of fields will have its own order: - -- primitive fields: larger size type first, smaller later, variable size type last. -- boxed primitive fields: same order as primitive fields -- final fields: same type together, then sorted by field name lexicographically. -- collection fields: same order as final fields -- map fields: same order as final fields -- other fields: same order as final fields - -#### Schema consistent - -Object fields will be serialized one by one using following format: - -``` -Primitive field value: -| var bytes | -+----------------+ -| value data | -+----------------+ -Boxed field value: -| one byte | var bytes | -+-----------+---------------+ -| null flag | field value | -+-----------+---------------+ -field value of final type with ref tracking: -| var bytes | var objects | -+-----------+-------------+ -| ref meta | value data | -+-----------+-------------+ -field value of final type without ref tracking: -| one byte | var objects | -+-----------+-------------+ -| null flag | field value | -+-----------+-------------+ -field value of non-final type with ref tracking: -| one byte | var bytes | var objects | -+-----------+-------------+-------------+ -| ref meta | class meta | value data | -+-----------+-------------+-------------+ -field value of non-final type without ref tracking: -| one byte | var bytes | var objects | -+-----------+------------+------------+ -| null flag | class meta | value data | -+-----------+------------+------------+ -``` - -#### Schema evolution - -Schema evolution have similar format as schema consistent mode for object except: - -- For this object type itself, `schema consistent` mode will write class by id/name, but `schema evolution` mode will - write class field names, types and other meta too, see [Class meta](#class-meta). -- Class meta of `final custom type` needs to be written too, because peers may not have this class defined. - -### Class - -Class will be serialized using class meta format. - -## Implementation guidelines - -- Try to merge multiple bytes into an int/long write before writing to reduce memory IO and bound check cost. -- Read multiple bytes as an int/long, then split into multiple bytes to reduce memory IO and bound check cost. -- Try to use one varint/long to write flags and length together to save one byte cost and reduce memory io. -- Condition branches are less expensive compared to memory IO cost unless there are too many branches. diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java index e04c351017..431ae9142e 100644 --- a/java/fory-core/src/main/java/org/apache/fory/Fory.java +++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java @@ -550,7 +550,8 @@ public void xwriteNonRef(MemoryBuffer buffer, Object obj, Serializer serializer) } public void xwriteData(MemoryBuffer buffer, ClassInfo classInfo, Object obj) { - switch (classInfo.getXtypeId()) { + int internalTypeId = classInfo.getTypeId() & 0xff; + switch (internalTypeId) { case Types.BOOL: buffer.writeBoolean((Boolean) obj); break; @@ -585,7 +586,8 @@ public void xwriteData(MemoryBuffer buffer, ClassInfo classInfo, Object obj) { /** Write not null data to buffer. */ private void writeData(MemoryBuffer buffer, ClassInfo classInfo, Object obj) { - switch (classInfo.getTypeId()) { + int internalTypeId = classInfo.getTypeId() & 0xff; + switch (internalTypeId) { case Types.BOOL: buffer.writeBoolean((Boolean) obj); break; @@ -989,7 +991,8 @@ public Object readData(MemoryBuffer buffer, ClassInfo classInfo) { } private Object readDataInternal(MemoryBuffer buffer, ClassInfo classInfo) { - switch (classInfo.getTypeId()) { + int internalTypeId = classInfo.getTypeId() & 0xff; + switch (internalTypeId) { case Types.BOOL: return buffer.readBoolean(); case Types.INT8: @@ -1021,7 +1024,8 @@ private Object readDataInternal(MemoryBuffer buffer, ClassInfo classInfo) { } private Object xreadDataInternal(MemoryBuffer buffer, ClassInfo classInfo) { - switch (classInfo.getTypeId()) { + int internalTypeId = classInfo.getTypeId() & 0xff; + switch (internalTypeId) { case Types.BOOL: return buffer.readBoolean(); case Types.INT8: @@ -1124,7 +1128,8 @@ public Object xreadNonRef(MemoryBuffer buffer, Serializer serializer) { public Object xreadNonRef(MemoryBuffer buffer, ClassInfo classInfo) { assert classInfo != null; - switch (classInfo.getXtypeId()) { + int internalTypeId = classInfo.getTypeId() & 0xff; + switch (internalTypeId) { case Types.BOOL: return buffer.readBoolean(); case Types.INT8: @@ -1236,7 +1241,7 @@ public T deserializeJavaObject(MemoryBuffer buffer, Class cls) { if (nextReadRefId >= NOT_NULL_VALUE_FLAG) { ClassInfo classInfo; if (shareMeta) { - classInfo = classResolver.readSharedClassMeta(buffer, cls); + classInfo = classResolver.readClassInfo(buffer, cls); } else { classInfo = classResolver.getClassInfo(cls); } @@ -1406,7 +1411,8 @@ public T copyObject(T obj) { } Object copy; ClassInfo classInfo = classResolver.getOrUpdateClassInfo(obj.getClass()); - switch (classInfo.getTypeId()) { + int internalTypeId = classInfo.getTypeId() & 0xff; + switch (internalTypeId) { case Types.BOOL: case Types.INT8: case ClassResolver.CHAR_ID: diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java index 00588baca8..8dd67b5c82 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java @@ -1747,8 +1747,7 @@ protected Expression deserializeForNullableField( descriptor, callback, () -> deserializeForNotNull(buffer, typeRef, null), - nullable - ); + nullable); } } @@ -1801,8 +1800,11 @@ private Expression readNullableField( Expression value = deserializeForNotNull.get(); // When local field is primitive but remote was nullable (boxed), use default value // instead of null. This handles compatibility between boxed/primitive field types. - Expression nullExpr = nullValue(descriptor.getField() != null - ? descriptor.getField().getType() : descriptor.getRawType()); + Expression nullExpr = + nullValue( + descriptor.getField() != null + ? descriptor.getField().getType() + : descriptor.getRawType()); // use false to ignore null. return new If(notNull, callback.apply(value), callback.apply(nullExpr), false); } else { @@ -1894,8 +1896,7 @@ protected Expression deserializeField( descriptor, callback, () -> deserializeForNotNullForField(buffer, descriptor, null), - true - ); + true); if (serializerCallsReference) { Expression preserveStubRefId = diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java index 832dba2b8a..38e0091053 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java @@ -95,8 +95,10 @@ public MetaSharedCodecBuilder(TypeRef beanType, Fory fory, ClassDef classDef) DescriptorGrouper grouper = typeResolver(r -> r.createDescriptorGrouper(descriptors, false)); List sortedDescriptors = grouper.getSortedDescriptors(); if (org.apache.fory.util.Utils.DEBUG_OUTPUT_ENABLED) { - LOG.info("========== {} sorted descriptors for {} ==========", - classDef.getFieldCount(), classDef.getClassName()); + LOG.info( + "========== {} sorted descriptors for {} ==========", + classDef.getFieldCount(), + classDef.getClassName()); for (Descriptor d : sortedDescriptors) { LOG.info( " {} -> {}, ref {}, nullable {}", diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java index 25b7d72acb..f511cb87ae 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java @@ -106,8 +106,10 @@ public ObjectCodecBuilder(Class beanClass, Fory fory) { Collection p = descriptors; DescriptorGrouper grouper = typeResolver(r -> r.createDescriptorGrouper(p, false)); if (org.apache.fory.util.Utils.DEBUG_OUTPUT_ENABLED) { - LOG.info("========== {} sorted descriptors for {} ==========", - descriptors.size(), beanClass.getSimpleName()); + LOG.info( + "========== {} sorted descriptors for {} ==========", + descriptors.size(), + beanClass.getSimpleName()); List sortedDescriptors = grouper.getSortedDescriptors(); for (Descriptor d : sortedDescriptors) { LOG.info( diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java b/java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java index 7d2427448c..1587c16848 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java @@ -45,6 +45,8 @@ import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.MetaSharedSerializer; import org.apache.fory.type.Descriptor; +import org.apache.fory.type.Types; +import org.apache.fory.util.Preconditions; import org.apache.fory.util.StringUtils; /** @@ -166,6 +168,24 @@ public int getFieldCount() { return fieldsInfo.size(); } + public boolean isNamed() { + return classSpec.typeId < 0 || Types.isNamedType(classSpec.typeId & 0xff); + } + + public boolean isCompatible() { + if (classSpec.typeId < 0) { + return false; + } + int internalTypeId = classSpec.typeId & 0xff; + return internalTypeId == Types.COMPATIBLE_STRUCT + || internalTypeId == Types.NAMED_COMPATIBLE_STRUCT; + } + + public int getUserTypeId() { + Preconditions.checkArgument(!isNamed(), "Named types don't have user type id"); + return classSpec.typeId >>> 8; + } + @Override public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) { diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDefDecoder.java b/java/fory-core/src/main/java/org/apache/fory/meta/ClassDefDecoder.java index f912017f1d..bda9ca420e 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDefDecoder.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/ClassDefDecoder.java @@ -37,6 +37,7 @@ import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.NonexistentClass; +import org.apache.fory.type.Types; import org.apache.fory.util.Preconditions; /** @@ -83,13 +84,18 @@ public static ClassDef decodeClassDef(ClassResolver resolver, MemoryBuffer buffe int numFields = currentClassHeader >>> 1; if (isRegistered) { short registeredId = (short) classDefBuf.readVarUint32Small7(); + int internalTypeId = + resolver.getFory().getConfig().isMetaShareEnabled() + ? Types.COMPATIBLE_STRUCT + : Types.STRUCT; + int typeId = (registeredId << 8) | internalTypeId; if (resolver.getRegisteredClass(registeredId) == null) { - classSpec = new ClassSpec(NonexistentClass.NonexistentMetaShared.class); + classSpec = new ClassSpec(NonexistentClass.NonexistentMetaShared.class, typeId); className = classSpec.entireClassName; } else { Class cls = resolver.getRegisteredClass(registeredId); className = cls.getName(); - classSpec = new ClassSpec(cls); + classSpec = new ClassSpec(cls, resolver.getTypeIdForClassDef(cls)); } } else { String pkg = readPkgName(classDefBuf); @@ -99,7 +105,21 @@ public static ClassDef decodeClassDef(ClassResolver resolver, MemoryBuffer buffe if (resolver.isRegisteredByName(className)) { Class cls = resolver.getRegisteredClass(className); className = cls.getName(); - classSpec = new ClassSpec(cls); + classSpec = new ClassSpec(cls, resolver.getTypeIdForClassDef(cls)); + } else { + int typeId = + classSpec.isEnum + ? Types.NAMED_ENUM + : (resolver.getFory().getConfig().isMetaShareEnabled() + ? Types.NAMED_COMPATIBLE_STRUCT + : Types.NAMED_STRUCT); + classSpec = + new ClassSpec( + classSpec.entireClassName, + classSpec.isEnum, + classSpec.isArray, + classSpec.dimension, + typeId); } } List fieldInfos = readFieldsInfo(classDefBuf, resolver, className, numFields); diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDefEncoder.java b/java/fory-core/src/main/java/org/apache/fory/meta/ClassDefEncoder.java index c6bf209a3d..e40119bc3a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDefEncoder.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/ClassDefEncoder.java @@ -147,12 +147,10 @@ public static ClassDef buildClassDefWithFieldInfos( classLayers.values().forEach(fieldInfos::addAll); MemoryBuffer encodeClassDef = encodeClassDef(classResolver, type, classLayers, hasFieldsMeta); byte[] classDefBytes = encodeClassDef.getBytes(0, encodeClassDef.writerIndex()); + int typeId = classResolver.getTypeIdForClassDef(type); + ClassSpec classSpec = new ClassSpec(type, typeId); return new ClassDef( - Encoders.buildClassSpec(type), - fieldInfos, - hasFieldsMeta, - encodeClassDef.getInt64(0), - classDefBytes); + classSpec, fieldInfos, hasFieldsMeta, encodeClassDef.getInt64(0), classDefBytes); } // see spec documentation: docs/specification/java_serialization_spec.md diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java b/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java index ccb2cdc4a3..980321b324 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/FieldInfo.java @@ -93,7 +93,7 @@ Descriptor toDescriptor(TypeResolver resolver, Descriptor descriptor) { TypeRef typeRef = fieldType.toTypeToken(resolver, declared); String typeName = fieldType.getTypeName(resolver, typeRef); if (fieldType instanceof FieldTypes.RegisteredFieldType) { - if (!Types.isPrimitiveType(fieldType.xtypeId)) { + if (!Types.isPrimitiveType(fieldType.typeId)) { typeName = String.valueOf(((FieldTypes.RegisteredFieldType) fieldType).getTypeId()); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java index e3aaddc9c8..b65ffa99cf 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java @@ -30,10 +30,6 @@ import java.io.Serializable; import java.lang.reflect.Array; import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; import java.util.Objects; import org.apache.fory.annotation.ForyField; import org.apache.fory.collection.Tuple2; @@ -87,19 +83,41 @@ private static FieldType buildFieldType( boolean isXlang = resolver.getFory().isCrossLanguage(); // Get type ID for both xlang and native mode // This supports unsigned types and field-configurable compression in both modes - int xtypeId; + int typeId; if (TypeUtils.unwrap(rawType).isPrimitive()) { if (field != null) { - xtypeId = Types.getDescriptorTypeId(resolver.getFory(), field); + typeId = Types.getDescriptorTypeId(resolver.getFory(), field); } else { - xtypeId = Types.getTypeId(resolver.getFory(), rawType); + typeId = Types.getTypeId(resolver.getFory(), rawType); } } else { - ClassInfo info = resolver.getClassInfo(genericType.getCls(), false); + ClassInfo info = resolver.getClassInfo(rawType, false); if (info != null) { - xtypeId = info.getTypeId(); + typeId = info.getTypeId(); + } else if (isXlang) { + if (rawType.isArray()) { + Class componentType = rawType.getComponentType(); + if (componentType.isPrimitive()) { + int elemTypeId = Types.getTypeId(resolver.getFory(), componentType); + typeId = Types.getPrimitiveArrayTypeId(elemTypeId); + } else { + typeId = Types.LIST; + } + } else if (rawType.isEnum()) { + typeId = Types.ENUM; + } else if (resolver.isSet(rawType)) { + typeId = Types.SET; + } else if (resolver.isCollection(rawType)) { + typeId = Types.LIST; + } else if (resolver.isMap(rawType)) { + typeId = Types.MAP; + } else { + typeId = Types.UNKNOWN; + } + } else if (resolver instanceof ClassResolver) { + typeId = ((ClassResolver) resolver).getTypeIdForClassDef(rawType); } else { - xtypeId = Types.UNKNOWN; + typeId = Types.UNKNOWN; } } // For xlang: ref tracking is false by default (no shared ownership like Rust's Rc/Arc) @@ -129,7 +147,7 @@ private static FieldType buildFieldType( if (COLLECTION_TYPE.isSupertypeOf(genericType.getTypeRef())) { return new CollectionFieldType( - xtypeId, + typeId, nullable, trackingRef, buildFieldType( @@ -140,7 +158,7 @@ private static FieldType buildFieldType( : genericType.getTypeParameter0())); } else if (MAP_TYPE.isSupertypeOf(genericType.getTypeRef())) { return new MapFieldType( - xtypeId, + typeId, nullable, trackingRef, buildFieldType( @@ -159,10 +177,10 @@ private static FieldType buildFieldType( return new UnionFieldType(nullable, trackingRef); } else if (TypeUtils.unwrap(rawType).isPrimitive()) { // unified basic types for xlang and native mode - return new RegisteredFieldType(nullable, trackingRef, xtypeId); + return new RegisteredFieldType(nullable, trackingRef, typeId); } else { if (rawType.isEnum()) { - return new EnumFieldType(nullable, xtypeId); + return new EnumFieldType(nullable, typeId); } if (rawType.isArray()) { Class elemType = rawType.getComponentType(); @@ -174,45 +192,45 @@ private static FieldType buildFieldType( return new RegisteredFieldType(nullable, trackingRef, arrayTypeId); } return new CollectionFieldType( - xtypeId, + typeId, nullable, trackingRef, buildFieldType(resolver, null, GenericType.build(elemType))); } else { // For native mode, use Java class IDs for arrays - if (((ClassResolver)resolver).isInternalRegistered(rawType)) { - Short classId = ((ClassResolver) resolver).getRegisteredClassId(rawType); - return new RegisteredFieldType(nullable, trackingRef, classId); + if (resolver.isRegisteredById(rawType)) { + return new RegisteredFieldType(nullable, trackingRef, typeId); } Tuple2, Integer> arrayComponentInfo = getArrayComponentInfo(rawType); return new ArrayFieldType( - xtypeId, + typeId, nullable, trackingRef, buildFieldType(resolver, null, GenericType.build(arrayComponentInfo.f0)), arrayComponentInfo.f1); } } - if (isXlang && !Types.isUserDefinedType((byte) xtypeId) && resolver.isRegisteredById(rawType)) { - return new RegisteredFieldType(nullable, trackingRef, xtypeId); + if (isXlang + && !Types.isUserDefinedType((byte) typeId) + && resolver.isRegisteredById(rawType)) { + return new RegisteredFieldType(nullable, trackingRef, typeId); } else if (!isXlang && resolver.isRegisteredById(rawType)) { - Short classId = ((ClassResolver) resolver).getRegisteredClassId(rawType); - return new RegisteredFieldType(nullable, trackingRef, classId); + return new RegisteredFieldType(nullable, trackingRef, typeId); } else { - return new ObjectFieldType(xtypeId, nullable, trackingRef); + return new ObjectFieldType(typeId, nullable, trackingRef); } } } public abstract static class FieldType implements Serializable { - protected final int xtypeId; + protected final int typeId; protected final boolean nullable; protected final boolean trackingRef; - public FieldType(int xtypeId, boolean nullable, boolean trackingRef) { + public FieldType(int typeId, boolean nullable, boolean trackingRef) { this.trackingRef = trackingRef; this.nullable = nullable; - this.xtypeId = xtypeId; + this.typeId = typeId; } public boolean trackingRef() { @@ -259,8 +277,8 @@ public void write(MemoryBuffer buffer, boolean writeHeader) { // - bits 2+: typeId byte header = (byte) ((nullable ? 0b10 : 0) | (trackingRef ? 0b1 : 0)); if (this instanceof RegisteredFieldType) { - short classId = ((RegisteredFieldType) this).getTypeId(); - buffer.writeVarUint32Small7(writeHeader ? ((5 + classId) << 2) | header : 5 + classId); + int typeId = ((RegisteredFieldType) this).getTypeId(); + buffer.writeVarUint32Small7(writeHeader ? ((5 + typeId) << 2) | header : 5 + typeId); } else if (this instanceof EnumFieldType) { buffer.writeVarUint32Small7(writeHeader ? ((4) << 2) | header : 4); } else if (this instanceof ArrayFieldType) { @@ -325,19 +343,19 @@ public static FieldType read( } public final void xwrite(MemoryBuffer buffer, boolean writeFlags) { - int xtypeId = this.xtypeId; + int typeId = this.typeId; if (writeFlags) { - xtypeId = (xtypeId << 2); + typeId = (typeId << 2); if (nullable) { - xtypeId |= 0b10; + typeId |= 0b10; } if (trackingRef) { - xtypeId |= 0b1; + typeId |= 0b1; } } - buffer.writeVarUint32Small7(xtypeId); - // Use the original xtypeId for the switch (not the one with flags) - switch (this.xtypeId & 0xff) { + buffer.writeVarUint32Small7(typeId); + // Use the original typeId for the switch (not the one with flags) + switch (this.typeId & 0xff) { case Types.LIST: case Types.SET: ((CollectionFieldType) this).getElementType().xwrite(buffer, true); @@ -354,52 +372,52 @@ public final void xwrite(MemoryBuffer buffer, boolean writeFlags) { } public static FieldType xread(MemoryBuffer buffer, XtypeResolver resolver) { - int xtypeId = buffer.readVarUint32Small7(); - boolean trackingRef = (xtypeId & 0b1) != 0; - boolean nullable = (xtypeId & 0b10) != 0; - xtypeId = xtypeId >>> 2; - return xread(buffer, resolver, xtypeId, nullable, trackingRef); + int typeId = buffer.readVarUint32Small7(); + boolean trackingRef = (typeId & 0b1) != 0; + boolean nullable = (typeId & 0b10) != 0; + typeId = typeId >>> 2; + return xread(buffer, resolver, typeId, nullable, trackingRef); } public static FieldType xread( MemoryBuffer buffer, XtypeResolver resolver, - int xtypeId, + int typeId, boolean nullable, boolean trackingRef) { - switch (xtypeId & 0xff) { + switch (typeId & 0xff) { case Types.LIST: case Types.SET: - return new CollectionFieldType(xtypeId, nullable, trackingRef, xread(buffer, resolver)); + return new CollectionFieldType(typeId, nullable, trackingRef, xread(buffer, resolver)); case Types.MAP: return new MapFieldType( - xtypeId, nullable, trackingRef, xread(buffer, resolver), xread(buffer, resolver)); + typeId, nullable, trackingRef, xread(buffer, resolver), xread(buffer, resolver)); case Types.ENUM: case Types.NAMED_ENUM: - return new EnumFieldType(nullable, xtypeId); + return new EnumFieldType(nullable, typeId); case Types.UNION: return new UnionFieldType(nullable, trackingRef); case Types.UNKNOWN: - return new ObjectFieldType(xtypeId, nullable, trackingRef); + return new ObjectFieldType(typeId, nullable, trackingRef); default: { - if (Types.isPrimitiveType(xtypeId)) { + if (Types.isPrimitiveType(typeId)) { // unsigned types share same class with signed numeric types, so unsigned types are // not registered. - return new RegisteredFieldType(nullable, trackingRef, xtypeId); + return new RegisteredFieldType(nullable, trackingRef, typeId); } - if (!Types.isUserDefinedType((byte) xtypeId)) { - ClassInfo classInfo = resolver.getXtypeInfo(xtypeId); + if (!Types.isUserDefinedType((byte) typeId)) { + ClassInfo classInfo = resolver.getXtypeInfo(typeId); if (classInfo == null) { // Type not registered locally - this can happen in compatible mode // when remote sends a type ID that's not registered here. // Fall back to ObjectFieldType to handle gracefully. - LOG.warn("Type {} not registered locally, treating as ObjectFieldType", xtypeId); - return new ObjectFieldType(xtypeId, nullable, trackingRef); + LOG.warn("Type {} not registered locally, treating as ObjectFieldType", typeId); + return new ObjectFieldType(typeId, nullable, trackingRef); } - return new RegisteredFieldType(nullable, trackingRef, xtypeId); + return new RegisteredFieldType(nullable, trackingRef, typeId); } else { - return new ObjectFieldType(xtypeId, nullable, trackingRef); + return new ObjectFieldType(typeId, nullable, trackingRef); } } } @@ -408,23 +426,21 @@ public static FieldType xread( /** Class for field type which is registered. */ public static class RegisteredFieldType extends FieldType { - private final short classId; - - public RegisteredFieldType(boolean nullable, boolean trackingRef, int classId) { - super(classId, nullable, trackingRef); - Preconditions.checkArgument(classId > 0); - this.classId = (short) classId; + public RegisteredFieldType(boolean nullable, boolean trackingRef, int typeId) { + super(typeId, nullable, trackingRef); + Preconditions.checkArgument(typeId > 0); } - public short getTypeId() { - return classId; + public int getTypeId() { + return typeId; } @Override public TypeRef toTypeToken(TypeResolver resolver, TypeRef declared) { Class cls; - if (Types.isPrimitiveType(classId)) { - cls = Types.getClassForTypeId(classId); + int internalTypeId = typeId & 0xff; + if (Types.isPrimitiveType(internalTypeId)) { + cls = Types.getClassForTypeId(internalTypeId); if (declared == null) { // For primitive types, ensure we use the correct primitive/boxed form // based on the nullable flag, not the declared type @@ -442,20 +458,21 @@ public TypeRef toTypeToken(TypeResolver resolver, TypeRef declared) { cls = declared.getRawType(); } } - return TypeRef.of(cls, new TypeExtMeta(classId, nullable, trackingRef)); + return TypeRef.of(cls, new TypeExtMeta(typeId, nullable, trackingRef)); } if (resolver instanceof XtypeResolver) { - ClassInfo xtypeInfo = ((XtypeResolver) resolver).getXtypeInfo(classId); + ClassInfo xtypeInfo = ((XtypeResolver) resolver).getXtypeInfo(typeId); Preconditions.checkNotNull(xtypeInfo); cls = xtypeInfo.getCls(); } else { - cls = ((ClassResolver) resolver).getRegisteredClass(classId); + int classId = typeId < ClassResolver.USER_ID_BASE ? typeId : (typeId >>> 8); + cls = ((ClassResolver) resolver).getRegisteredClass((short) classId); } if (cls == null) { - LOG.warn("Class {} not registered, take it as Struct type for deserialization.", classId); + LOG.warn("Class {} not registered, take it as Struct type for deserialization.", typeId); cls = NonexistentClass.NonexistentMetaShared.class; } - return TypeRef.of(cls, new TypeExtMeta(classId, nullable, trackingRef)); + return TypeRef.of(cls, new TypeExtMeta(typeId, nullable, trackingRef)); } @Override @@ -467,13 +484,14 @@ public String getTypeName(TypeResolver resolver, TypeRef typeRef) { if (resolver instanceof ClassResolver) { ClassResolver classResolver = (ClassResolver) resolver; // Peer class may not register this class id, which will introduce inconsistent field order + int classId = typeId < ClassResolver.USER_ID_BASE ? typeId : (typeId >>> 8); if (classResolver.isInternalRegistered(classId)) { return String.valueOf(classId); } else { return "Registered"; } } - return String.valueOf(classId); + return String.valueOf(typeId); } @Override @@ -488,12 +506,12 @@ public boolean equals(Object o) { return false; } RegisteredFieldType that = (RegisteredFieldType) o; - return classId == that.classId; + return typeId == that.typeId; } @Override public int hashCode() { - return Objects.hash(super.hashCode(), classId); + return Objects.hash(super.hashCode(), typeId); } @Override @@ -503,8 +521,8 @@ public String toString() { + nullable() + ", trackingRef=" + trackingRef() - + ", classId=" - + classId + + ", typeId=" + + typeId + '}'; } } @@ -521,8 +539,8 @@ public static class CollectionFieldType extends FieldType { private final FieldType elementType; public CollectionFieldType( - int xtypeId, boolean nullable, boolean trackingRef, FieldType elementType) { - super(xtypeId, nullable, trackingRef); + int typeId, boolean nullable, boolean trackingRef, FieldType elementType) { + super(typeId, nullable, trackingRef); this.elementType = elementType; } @@ -552,13 +570,14 @@ public TypeRef toTypeToken(TypeResolver resolver, TypeRef declared) { } TypeRef elementType = this.elementType.toTypeToken(resolver, declElementType); if (declared == null) { - return collectionOf(elementType, new TypeExtMeta(xtypeId, nullable, trackingRef)); + return collectionOf(elementType, new TypeExtMeta(typeId, nullable, trackingRef)); } if (!declaredClass.isArray()) { if (declElementType.equals(elementType)) { return declared; } - return collectionOf(declaredClass, elementType, new TypeExtMeta(xtypeId, nullable, trackingRef)); + return collectionOf( + declaredClass, elementType, new TypeExtMeta(typeId, nullable, trackingRef)); } // Build array type from element type // elementType could be base type (int) or intermediate array (int[]) @@ -573,7 +592,7 @@ public TypeRef toTypeToken(TypeResolver resolver, TypeRef declared) { // Apply field metadata (nullable, trackingRef) to outermost array only TypeExtMeta meta = (i == dimensionsToAdd - 1) - ? new TypeExtMeta(xtypeId, nullable, trackingRef) + ? new TypeExtMeta(typeId, nullable, trackingRef) : currentType.getTypeExtMeta(); currentType = TypeRef.of(arrayClass, meta); } @@ -626,12 +645,8 @@ public static class MapFieldType extends FieldType { private final FieldType valueType; public MapFieldType( - int xtypeId, - boolean nullable, - boolean trackingRef, - FieldType keyType, - FieldType valueType) { - super(xtypeId, nullable, trackingRef); + int typeId, boolean nullable, boolean trackingRef, FieldType keyType, FieldType valueType) { + super(typeId, nullable, trackingRef); this.keyType = keyType; this.valueType = valueType; } @@ -665,12 +680,12 @@ public TypeRef toTypeToken(TypeResolver classResolver, TypeRef declared) { declared.getRawType(), keyType.toTypeToken(classResolver, keyDecl), valueType.toTypeToken(classResolver, valueDecl), - new TypeExtMeta(xtypeId, nullable, trackingRef)); + new TypeExtMeta(typeId, nullable, trackingRef)); } return mapOf( keyType.toTypeToken(classResolver, keyDecl), valueType.toTypeToken(classResolver, valueDecl), - new TypeExtMeta(xtypeId, nullable, trackingRef)); + new TypeExtMeta(typeId, nullable, trackingRef)); } @Override @@ -709,8 +724,8 @@ public String toString() { } public static class EnumFieldType extends FieldType { - public EnumFieldType(boolean nullable, int xtypeId) { - super(xtypeId, nullable, false); + public EnumFieldType(boolean nullable, int typeId) { + super(typeId, nullable, false); } @Override @@ -728,7 +743,7 @@ public String getTypeName(TypeResolver resolver, TypeRef typeRef) { @Override public String toString() { - return "EnumFieldType{" + "xtypeId=" + xtypeId + ", nullable=" + nullable + '}'; + return "EnumFieldType{" + "typeId=" + typeId + ", nullable=" + nullable + '}'; } } @@ -737,12 +752,12 @@ public static class ArrayFieldType extends FieldType { private final int dimensions; public ArrayFieldType( - int xtypeId, + int typeId, boolean nullable, boolean trackingRef, FieldType componentType, int dimensions) { - super(xtypeId, nullable, trackingRef); + super(typeId, nullable, trackingRef); this.componentType = componentType; this.dimensions = dimensions; } @@ -758,11 +773,11 @@ public TypeRef toTypeToken(TypeResolver classResolver, TypeRef declared) { return TypeRef.of( NonexistentClass.getNonexistentClass( componentType instanceof EnumFieldType, dimensions, true), - new TypeExtMeta(xtypeId, nullable, trackingRef)); + new TypeExtMeta(typeId, nullable, trackingRef)); } else { return TypeRef.of( Array.newInstance(componentRawType, new int[dimensions]).getClass(), - new TypeExtMeta(xtypeId, nullable, trackingRef)); + new TypeExtMeta(typeId, nullable, trackingRef)); } } @@ -821,14 +836,14 @@ public String toString() { /** Class for field type which isn't registered and not collection/map type too. */ public static class ObjectFieldType extends FieldType { - public ObjectFieldType(int xtypeId, boolean nullable, boolean trackingRef) { - super(xtypeId, nullable, trackingRef); + public ObjectFieldType(int typeId, boolean nullable, boolean trackingRef) { + super(typeId, nullable, trackingRef); } @Override public TypeRef toTypeToken(TypeResolver classResolver, TypeRef declared) { Class clz = declared == null ? Object.class : declared.getRawType(); - return TypeRef.of(clz, new TypeExtMeta(xtypeId, nullable, trackingRef)); + return TypeRef.of(clz, new TypeExtMeta(typeId, nullable, trackingRef)); } @Override @@ -851,8 +866,8 @@ public int hashCode() { @Override public String toString() { return "ObjectFieldType{" - + "xtypeId=" - + xtypeId + + "typeId=" + + typeId + ", nullable=" + nullable + ", trackingRef=" @@ -876,7 +891,7 @@ public TypeRef toTypeToken(TypeResolver classResolver, TypeRef declared) { } // Fallback to base Union class if no declared type return TypeRef.of( - org.apache.fory.type.union.Union.class, new TypeExtMeta(xtypeId, nullable, trackingRef)); + org.apache.fory.type.union.Union.class, new TypeExtMeta(typeId, nullable, trackingRef)); } @Override diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefDecoder.java b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefDecoder.java index 3f02c62dfa..9b6e851ac2 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefDecoder.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefDecoder.java @@ -72,12 +72,12 @@ public static ClassDef decodeClassDef(XtypeResolver resolver, MemoryBuffer input classSpec = new ClassSpec(userTypeInfo.getCls()); } } else { - int xtypeId = buffer.readVarUint32Small7(); - ClassInfo userTypeInfo = resolver.getUserTypeInfo(xtypeId); + int typeId = buffer.readVarUint32Small7(); + ClassInfo userTypeInfo = resolver.getUserTypeInfo(typeId); if (userTypeInfo == null) { - classSpec = new ClassSpec(NonexistentClass.NonexistentMetaShared.class, xtypeId); + classSpec = new ClassSpec(NonexistentClass.NonexistentMetaShared.class, typeId); } else { - classSpec = new ClassSpec(userTypeInfo.getCls(), xtypeId); + classSpec = new ClassSpec(userTypeInfo.getCls(), typeId); } } List classFields = diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassInfo.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassInfo.java index a9d38b2b13..9d3455409e 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassInfo.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassInfo.java @@ -108,12 +108,13 @@ public ClassInfo(Class cls, ClassDef classDef) { // - NAMED_ENUM, NAMED_EXT: other named types // - REPLACE_STUB_ID: for write replace class in `ClassSerializer` // - NO_CLASS_ID: legacy support (should use NAMED_STRUCT instead) - boolean isNamedType = typeId == Types.NAMED_STRUCT - || typeId == Types.NAMED_COMPATIBLE_STRUCT - || typeId == Types.NAMED_ENUM - || typeId == Types.NAMED_EXT - || typeId == ClassResolver.REPLACE_STUB_ID - || typeId == TypeResolver.NO_CLASS_ID; + boolean isNamedType = + typeId == Types.NAMED_STRUCT + || typeId == Types.NAMED_COMPATIBLE_STRUCT + || typeId == Types.NAMED_ENUM + || typeId == Types.NAMED_EXT + || typeId == ClassResolver.REPLACE_STUB_ID + || typeId == TypeResolver.NO_CLASS_ID; if (cls != null && isNamedType) { Tuple2 tuple2 = Encoders.encodePkgAndClass(cls); this.namespaceBytes = diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 649f17a6fe..e104a6bda3 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -19,7 +19,6 @@ package org.apache.fory.resolver; -import static org.apache.fory.Fory.NOT_SUPPORT_XLANG; import static org.apache.fory.meta.Encoders.GENERIC_ENCODER; import static org.apache.fory.meta.Encoders.PACKAGE_DECODER; import static org.apache.fory.meta.Encoders.PACKAGE_ENCODER; @@ -29,13 +28,11 @@ import static org.apache.fory.serializer.CodegenSerializer.loadCodegenSerializer; import static org.apache.fory.serializer.CodegenSerializer.supportCodegenForJavaSerialization; import static org.apache.fory.type.TypeUtils.OBJECT_TYPE; -import static org.apache.fory.type.TypeUtils.getRawType; import java.io.Externalizable; import java.io.IOException; import java.io.Serializable; import java.lang.invoke.SerializedLambda; -import java.lang.reflect.Type; import java.math.BigDecimal; import java.math.BigInteger; import java.nio.ByteBuffer; @@ -80,12 +77,12 @@ import org.apache.fory.annotation.CodegenInvoke; import org.apache.fory.annotation.ForyField; import org.apache.fory.annotation.Internal; +import org.apache.fory.builder.Generated; import org.apache.fory.builder.JITContext; import org.apache.fory.codegen.CodeGenerator; import org.apache.fory.codegen.Expression; import org.apache.fory.codegen.Expression.Invoke; import org.apache.fory.codegen.Expression.Literal; -import org.apache.fory.collection.IdentityObjectIntMap; import org.apache.fory.collection.ObjectMap; import org.apache.fory.collection.Tuple2; import org.apache.fory.config.Language; @@ -100,10 +97,10 @@ import org.apache.fory.meta.MetaString; import org.apache.fory.reflect.ObjectCreators; import org.apache.fory.reflect.ReflectionUtils; -import org.apache.fory.reflect.TypeRef; import org.apache.fory.serializer.ArraySerializers; import org.apache.fory.serializer.BufferSerializers; import org.apache.fory.serializer.CodegenSerializer.LazyInitBeanSerializer; +import org.apache.fory.serializer.DeferedLazySerializer; import org.apache.fory.serializer.EnumSerializer; import org.apache.fory.serializer.ExternalizableSerializer; import org.apache.fory.serializer.FinalFieldReplaceResolveSerializer; @@ -230,6 +227,7 @@ public class ClassResolver extends TypeResolver { public static final short LAMBDA_STUB_ID = NATIVE_START_ID + 26; public static final short JDK_PROXY_STUB_ID = NATIVE_START_ID + 27; public static final short REPLACE_STUB_ID = NATIVE_START_ID + 28; + public static final int NONEXISTENT_META_SHARED_ID = REPLACE_STUB_ID + 1; private final Fory fory; private ClassInfo[] registeredId2ClassInfo = new ClassInfo[] {}; @@ -247,7 +245,7 @@ public ClassResolver(Fory fory) { super(fory); this.fory = fory; classInfoCache = NIL_CLASS_INFO; - extRegistry.classIdGenerator = REPLACE_STUB_ID + 1; + extRegistry.classIdGenerator = NONEXISTENT_META_SHARED_ID + 1; shimDispatcher = new ShimDispatcher(fory); _addGraalvmClassRegistry(fory.getConfig().getConfigHash(), this); } @@ -337,14 +335,9 @@ private void addDefaultSerializers() { } if (fory.getConfig().deserializeNonexistentClass()) { if (metaContextShareEnabled) { - addDefaultSerializer( + registerInternal(NonexistentMetaShared.class, NONEXISTENT_META_SHARED_ID); + registerInternalSerializer( NonexistentMetaShared.class, new NonexistentClassSerializer(fory, null)); - // Those class id must be known in advance, here is two bytes, so - // `NonexistentClassSerializer.writeClassDef` - // can overwrite written classinfo and replace with real classinfo. - int classId = - Objects.requireNonNull(classInfoMap.get(NonexistentMetaShared.class)).typeId; - Preconditions.checkArgument(classId > 63 && classId < 8192, classId); } else { registerInternal(NonexistentSkip.class); } @@ -498,8 +491,12 @@ public void register(Class cls, String namespace, String name) { MetaStringBytes nsBytes = metaStringResolver.getOrCreateMetaStringBytes(encodePackage(namespace)); MetaStringBytes nameBytes = metaStringResolver.getOrCreateMetaStringBytes(encodeTypeName(name)); + int typeId = + cls.isEnum() + ? Types.NAMED_ENUM + : (metaContextShareEnabled ? Types.NAMED_COMPATIBLE_STRUCT : Types.NAMED_STRUCT); ClassInfo classInfo = - new ClassInfo(cls, fullNameBytes, nsBytes, nameBytes, false, null, NO_CLASS_ID); + new ClassInfo(cls, fullNameBytes, nsBytes, nameBytes, false, null, typeId); classInfoMap.put(cls, classInfo); compositeNameBytes2ClassInfo.put( new TypeNameBytes(nsBytes.hashCode, nameBytes.hashCode), classInfo); @@ -569,11 +566,12 @@ private void registerImpl(Class cls, int classId) { System.arraycopy(registeredId2ClassInfo, 0, tmp, 0, registeredId2ClassInfo.length); registeredId2ClassInfo = tmp; } + int typeId = buildRegisteredTypeId(cls, classId, null); ClassInfo classInfo = classInfoMap.get(cls); if (classInfo != null) { - classInfo.typeId = id; + classInfo.typeId = typeId; } else { - classInfo = new ClassInfo(this, cls, null, id); + classInfo = new ClassInfo(this, cls, null, typeId); // make `extRegistry.registeredClassIdMap` and `classInfoMap` share same classInfo // instances. classInfoMap.put(cls, classInfo); @@ -584,6 +582,41 @@ private void registerImpl(Class cls, int classId) { GraalvmSupport.registerClass(cls, fory.getConfig().getConfigHash()); } + private int buildRegisteredTypeId(Class cls, int classId, Serializer serializer) { + if (classId < USER_ID_BASE) { + return classId; + } + int internalTypeId; + if (cls.isEnum()) { + internalTypeId = Types.ENUM; + } else if (serializer != null && !isStructSerializer(serializer)) { + internalTypeId = Types.EXT; + } else { + internalTypeId = metaContextShareEnabled ? Types.COMPATIBLE_STRUCT : Types.STRUCT; + } + return (classId << 8) | internalTypeId; + } + + private int buildUnregisteredTypeId(Class cls, Serializer serializer) { + if (serializer instanceof ReplaceResolveSerializer + && !(serializer instanceof FinalFieldReplaceResolveSerializer)) { + return REPLACE_STUB_ID; + } + if (cls.isEnum()) { + return Types.NAMED_ENUM; + } + if (serializer != null && !isStructSerializer(serializer)) { + return Types.NAMED_EXT; + } + return metaContextShareEnabled ? Types.NAMED_COMPATIBLE_STRUCT : Types.NAMED_STRUCT; + } + + private boolean isStructSerializer(Serializer serializer) { + return serializer instanceof ObjectSerializer + || serializer instanceof Generated.GeneratedSerializer + || serializer instanceof DeferedLazySerializer.DeferredLazyObjectSerializer; + } + private void checkRegistration(Class cls, short classId, String name) { if (extRegistry.registeredClassIdMap.containsKey(cls)) { throw new IllegalArgumentException( @@ -680,6 +713,35 @@ public String getTypeAlias(Class cls) { return cls.getName(); } + /** + * Compute the typeId used in ClassDef without forcing serializer creation. This avoids recursive + * serializer construction while building class metadata. + */ + public int getTypeIdForClassDef(Class cls) { + ClassInfo classInfo = classInfoMap.get(cls); + if (classInfo != null) { + return classInfo.typeId; + } + Short classId = extRegistry.registeredClassIdMap.get(cls); + if (classId != null) { + if (classId < USER_ID_BASE) { + return classId; + } + int internalTypeId = + cls.isEnum() + ? Types.ENUM + : (metaContextShareEnabled ? Types.COMPATIBLE_STRUCT : Types.STRUCT); + return (classId << 8) | internalTypeId; + } + if (useReplaceResolveSerializer(cls)) { + return REPLACE_STUB_ID; + } + if (cls.isEnum()) { + return Types.NAMED_ENUM; + } + return metaContextShareEnabled ? Types.NAMED_COMPATIBLE_STRUCT : Types.NAMED_STRUCT; + } + @Override public boolean isMonomorphic(Descriptor descriptor) { ForyField foryField = descriptor.getForyField(); @@ -735,12 +797,6 @@ public boolean isInternalRegistered(int classId) { /** Returns true if cls is fory inner registered class. */ public boolean isInternalRegistered(Class cls) { Short classId = extRegistry.registeredClassIdMap.get(cls); - if (classId == null) { - ClassInfo classInfo = getClassInfo(cls, false); - if (classInfo != null) { - classId = (short) classInfo.getTypeId(); - } - } return classId != null && classId != NO_CLASS_ID && classId < innerEndClassId; } @@ -814,6 +870,21 @@ public void registerSerializer(Class type, Serializer serializer) { if (!extRegistry.registeredClassIdMap.containsKey(type) && !fory.isCrossLanguage()) { register(type); } + ClassInfo classInfo = classInfoMap.get(type); + if (classInfo == null) { + classInfo = getClassInfo(type); + } + if (!isStructSerializer(serializer)) { + int typeId = classInfo.typeId; + int internalTypeId = typeId & 0xff; + int userPart = typeId & 0xffffff00; + if (internalTypeId == Types.STRUCT || internalTypeId == Types.COMPATIBLE_STRUCT) { + classInfo.typeId = userPart | Types.EXT; + } else if (internalTypeId == Types.NAMED_STRUCT + || internalTypeId == Types.NAMED_COMPATIBLE_STRUCT) { + classInfo.typeId = Types.NAMED_EXT; + } + } registerSerializerImpl(type, serializer); } @@ -940,32 +1011,27 @@ public void clearSerializer(Class cls) { /** Add serializer for specified class. */ public void addSerializer(Class type, Serializer serializer) { Preconditions.checkNotNull(serializer); - // 1. Try to get ClassInfo from `registeredId2ClassInfo` and - // `classInfoMap` or create a new `ClassInfo`. ClassInfo classInfo; Short classId = extRegistry.registeredClassIdMap.get(type); boolean registered = classId != null; - // set serializer for class if it's registered by now. if (registered) { classInfo = registeredId2ClassInfo[classId]; - } else { - if (serializer instanceof ReplaceResolveSerializer - && !(serializer instanceof FinalFieldReplaceResolveSerializer)) { - classId = REPLACE_STUB_ID; + int typeId = buildRegisteredTypeId(type, classId, serializer); + if (classInfo == null) { + classInfo = new ClassInfo(this, type, null, typeId); + classInfoMap.put(type, classInfo); + registeredId2ClassInfo[classId] = classInfo; } else { - // For unregistered classes, use NAMED_STRUCT or NAMED_COMPATIBLE_STRUCT - // so that writeClassInfo writes the namespace and typename bytes (or meta-share) - classId = (short) (metaContextShareEnabled - ? Types.NAMED_COMPATIBLE_STRUCT : Types.NAMED_STRUCT); + classInfo.typeId = typeId; } + } else { + int typeId = buildUnregisteredTypeId(type, serializer); classInfo = classInfoMap.get(type); - } - - if (classInfo == null || classId != classInfo.typeId) { - classInfo = new ClassInfo(this, type, null, classId); - classInfoMap.put(type, classInfo); - if (registered) { - registeredId2ClassInfo[classId] = classInfo; + if (classInfo == null) { + classInfo = new ClassInfo(this, type, null, typeId); + classInfoMap.put(type, classInfo); + } else { + classInfo.typeId = typeId; } // Add to compositeNameBytes2ClassInfo for unregistered classes so that // readClassInfo can find the ClassInfo by name bytes during deserialization. @@ -1314,7 +1380,8 @@ public ClassInfo getOrUpdateClassInfo(Class cls) { private ClassInfo getOrUpdateClassInfo(short classId) { ClassInfo classInfo = classInfoCache; - if (classInfo.typeId != classId) { + Short cachedId = extRegistry.registeredClassIdMap.get(classInfo.cls); + if (cachedId == null || cachedId != classId) { classInfo = registeredId2ClassInfo[classId]; if (classInfo.serializer == null) { addSerializer(classInfo.cls, createSerializer(classInfo.cls)); @@ -1474,15 +1541,15 @@ private boolean isSecure(Class cls) { } /** - * Check if a typeId corresponds to a valid registered class in this resolver. - * For ClassResolver, this checks if the typeId is in range and has an entry - * in registeredId2ClassInfo. + * Check if a typeId corresponds to a valid registered class in this resolver. For ClassResolver, + * this checks if the typeId is in range and has an entry in registeredId2ClassInfo. */ @Override protected boolean isValidRegisteredTypeId(int typeId) { - return typeId > 0 - && typeId < registeredId2ClassInfo.length - && registeredId2ClassInfo[typeId] != null; + int classId = typeId < USER_ID_BASE ? typeId : (typeId >>> 8); + return classId > 0 + && classId < registeredId2ClassInfo.length + && registeredId2ClassInfo[classId] != null; } /** @@ -1494,9 +1561,9 @@ protected boolean isValidRegisteredTypeId(int typeId) { public void writeClassAndUpdateCache(MemoryBuffer buffer, Class cls) { // fast path for common type if (cls == Integer.class) { - buffer.writeVarUint32Small7(Types.INT32 << 1); + buffer.writeVarUint32Small7(Types.INT32); } else if (cls == Long.class) { - buffer.writeVarUint32Small7(Types.INT64 << 1); + buffer.writeVarUint32Small7(Types.INT64); } else { writeClassInfo(buffer, getOrUpdateClassInfo(cls)); } @@ -1568,20 +1635,21 @@ public void writeClassInternal(MemoryBuffer buffer, Class cls) { public void writeClassInternal(MemoryBuffer buffer, ClassInfo classInfo) { int typeId = classInfo.typeId; - if (typeId == REPLACE_STUB_ID) { - // clear type id to avoid replaced class written as - // ReplaceResolveSerializer.ReplaceStub - classInfo.typeId = NO_CLASS_ID; - } - if (classInfo.typeId != NO_CLASS_ID) { - buffer.writeVarUint32(classInfo.typeId << 1); + int internalTypeId = typeId & 0xff; + Short classId = extRegistry.registeredClassIdMap.get(classInfo.cls); + boolean writeById = + classId != null + && typeId != NO_CLASS_ID + && typeId != REPLACE_STUB_ID + && !Types.isNamedType(internalTypeId); + if (writeById) { + buffer.writeVarUint32(classId << 1); } else { // let the lowermost bit of next byte be set, so the deserialization can know // whether need to read class by name in advance metaStringResolver.writeMetaStringBytesWithFlag(buffer, classInfo.namespaceBytes); metaStringResolver.writeMetaStringBytes(buffer, classInfo.typeNameBytes); } - classInfo.typeId = typeId; } /** @@ -1608,10 +1676,8 @@ public Class readClassInternal(MemoryBuffer buffer) { @Override protected ClassInfo getClassInfoByTypeId(int typeId) { - // For native mode: typeId is the direct index into registeredId2ClassInfo - // - Internal types (0-255) are stored at their type ID index - // - User types (256+) are stored at (userId + USER_ID_BASE) index - if (typeId < 0 || typeId >= registeredId2ClassInfo.length) { + int classId = typeId < USER_ID_BASE ? typeId : (typeId >>> 8); + if (classId < 0 || classId >= registeredId2ClassInfo.length) { throw new IllegalStateException( String.format( "Invalid typeId %d in meta share mode. This usually indicates a protocol mismatch " @@ -1619,13 +1685,13 @@ protected ClassInfo getClassInfoByTypeId(int typeId) { + "Check that the serializer and deserializer use the same protocol version.", typeId, registeredId2ClassInfo.length)); } - ClassInfo classInfo = registeredId2ClassInfo[typeId]; + ClassInfo classInfo = registeredId2ClassInfo[classId]; // Ensure serializer is set for registered classes (they may have been registered // without a serializer via registerInternal) if (classInfo != null && classInfo.serializer == null) { addSerializer(classInfo.cls, createSerializer(classInfo.cls)); // Re-read from registeredId2ClassInfo since addSerializer updates it for registered classes - classInfo = registeredId2ClassInfo[typeId]; + classInfo = registeredId2ClassInfo[classId]; } return classInfo; } @@ -1648,6 +1714,10 @@ protected ClassInfo loadBytesToClassInfo( @Override protected ClassInfo ensureSerializerForClassInfo(ClassInfo classInfo) { if (classInfo.serializer == null) { + Class cls = classInfo.cls; + if (cls != null && (ReflectionUtils.isAbstract(cls) || cls.isInterface())) { + return classInfo; + } // Get or create ClassInfo with serializer ClassInfo newClassInfo = getClassInfo(classInfo.cls); // Update the cache with the correct ClassInfo that has a serializer diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/MetaContext.java b/java/fory-core/src/main/java/org/apache/fory/resolver/MetaContext.java index 8282f9b261..83f95f49c4 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/MetaContext.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/MetaContext.java @@ -21,8 +21,6 @@ import org.apache.fory.collection.IdentityObjectIntMap; import org.apache.fory.collection.ObjectArray; -import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.meta.ClassDef; /** * Context for sharing class meta across multiple serialization. Class name, field name and field @@ -32,17 +30,6 @@ public class MetaContext { /** Classes which has sent definitions to peer. */ public final IdentityObjectIntMap> classMap = new IdentityObjectIntMap<>(1, 0.5f); - /** Class definitions read from peer. */ - public final ObjectArray readClassDefs = new ObjectArray<>(); - /** ClassInfos read from peer for reference lookup during deserialization. */ public final ObjectArray readClassInfos = new ObjectArray<>(); - - /** - * New class definition which needs sending to peer. This will be filled up when there are new - * class definition need sending, and will be cleared after writing to buffer. - * - * @see ClassResolver#writeClassDefs(MemoryBuffer) - */ - public final ObjectArray writingClassDefs = new ObjectArray<>(); } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index c01874685c..2be6acb12b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -19,12 +19,10 @@ package org.apache.fory.resolver; -import static org.apache.fory.Fory.NOT_SUPPORT_XLANG; import static org.apache.fory.type.TypeUtils.getSizeOfPrimitiveType; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; -import org.apache.fory.collection.ObjectArray; import java.lang.reflect.Field; import java.lang.reflect.Member; import java.lang.reflect.Type; @@ -260,6 +258,7 @@ public final boolean needToWriteClassDef(Serializer serializer) { * shared by both ClassResolver and XtypeResolver. * *

    Encoding: + * *

      *
    • NAMED_ENUM/NAMED_STRUCT/NAMED_EXT: namespace + typename bytes (or meta-share if enabled) *
    • NAMED_COMPATIBLE_STRUCT/COMPATIBLE_STRUCT: always meta-share @@ -267,14 +266,6 @@ public final boolean needToWriteClassDef(Serializer serializer) { *
    */ public final void writeClassInfo(MemoryBuffer buffer, ClassInfo classInfo) { - // In meta share mode, use the meta share protocol directly - // Protocol: LSB=0 means registered type by ID, LSB=1 means meta share data - if (metaContextShareEnabled) { - writeClassInfoWithMetaShare(buffer, classInfo); - return; - } - - // Non-meta share mode: write typeId followed by optional class name bytes int typeId = classInfo.getTypeId(); int internalTypeId = typeId & 0xff; buffer.writeVarUint32Small7(typeId); @@ -283,52 +274,28 @@ public final void writeClassInfo(MemoryBuffer buffer, ClassInfo classInfo) { case Types.NAMED_ENUM: case Types.NAMED_STRUCT: case Types.NAMED_EXT: - assert classInfo.namespaceBytes != null; - metaStringResolver.writeMetaStringBytes(buffer, classInfo.namespaceBytes); - assert classInfo.typeNameBytes != null; - metaStringResolver.writeMetaStringBytes(buffer, classInfo.typeNameBytes); + if (metaContextShareEnabled) { + writeSharedClassMeta(buffer, classInfo); + } else { + Preconditions.checkNotNull(classInfo.namespaceBytes); + metaStringResolver.writeMetaStringBytes(buffer, classInfo.namespaceBytes); + Preconditions.checkNotNull(classInfo.typeNameBytes); + metaStringResolver.writeMetaStringBytes(buffer, classInfo.typeNameBytes); + } + break; + case Types.NAMED_COMPATIBLE_STRUCT: + case Types.COMPATIBLE_STRUCT: + Preconditions.checkArgument( + metaContextShareEnabled, "Meta share must be enabled for compatible mode"); + if (classInfo.cls != NonexistentMetaShared.class) { + writeSharedClassMeta(buffer, classInfo); + } break; default: - // No additional data needed - type ID already written break; } } - /** - * Writes class info using meta share protocol. - * Protocol: LSB=0 means registered type by ID, LSB=1 means meta share reference/definition. - */ - private void writeClassInfoWithMetaShare(MemoryBuffer buffer, ClassInfo classInfo) { - int typeId = classInfo.getTypeId(); - // For registered types that don't need ClassDef, just write the type ID. - // The isValidRegisteredTypeId check ensures we only use the fast path for - // classes that are actually registered in this resolver's registry. - // Named types (NAMED_STRUCT, NAMED_COMPATIBLE_STRUCT, etc.) will not pass - // this check because they don't have entries in the registry. - if (isValidRegisteredTypeId(typeId) && !classInfo.needToWriteClassDef) { - buffer.writeVarUint32(typeId << 1); - return; - } - // For types that need ClassDef or are not registered, use the meta share protocol - MetaContext metaContext = fory.getSerializationContext().getMetaContext(); - assert metaContext != null : SET_META__CONTEXT_MSG; - IdentityObjectIntMap> classMap = metaContext.classMap; - int newId = classMap.size; - int id = classMap.putOrGet(classInfo.cls, newId); - if (id >= 0) { - // Reference to previously written type: (index << 1) | 1 - buffer.writeVarUint32((id << 1) | 1); - } else { - // New type: (index << 1) | 1, followed by ClassDef bytes - buffer.writeVarUint32((newId << 1) | 1); - ClassDef classDef = classInfo.classDef; - if (classDef == null) { - classDef = buildClassDef(classInfo); - } - metaContext.writingClassDefs.add(classDef); - } - } - /** * Native code for ClassResolver.writeClassInfo is too big to inline, so inline it manually. * @@ -382,117 +349,53 @@ protected final void writeSharedClassMeta(MemoryBuffer buffer, ClassInfo classIn * class info cache. */ public final ClassInfo readClassInfo(MemoryBuffer buffer) { - // In meta share mode, use the meta share protocol directly - // Protocol: LSB=0 means registered type by ID, LSB=1 means meta share data - if (metaContextShareEnabled) { - return readClassInfoWithMetaShare(buffer); - } - - // Non-meta share mode: read typeId followed by optional class name bytes int header = buffer.readVarUint32Small14(); int internalTypeId = header & 0xff; - - // Check if this is a named type (needs class name bytes) - if (isNamedType(internalTypeId)) { - // Use cache to avoid reloading dynamically created classes - // ensureSerializerForClassInfo is called within readClassInfoFromBytes - ClassInfo classInfo = readClassInfoFromBytes(buffer, classInfoCache, header); - classInfoCache = classInfo; - return classInfo; - } else { - // Lookup by type ID from registry - return getClassInfoByTypeId(header); + switch (internalTypeId) { + case Types.NAMED_ENUM: + case Types.NAMED_STRUCT: + case Types.NAMED_EXT: + if (metaContextShareEnabled) { + return readSharedClassMeta(buffer); + } + ClassInfo classInfo = readClassInfoFromBytes(buffer, classInfoCache, header); + classInfoCache = classInfo; + return classInfo; + case Types.NAMED_COMPATIBLE_STRUCT: + case Types.COMPATIBLE_STRUCT: + Preconditions.checkArgument( + metaContextShareEnabled, "Meta share must be enabled for compatible mode"); + return readSharedClassMeta(buffer); + default: + return getClassInfoByTypeId(header); } } /** - * Reads class info using meta share protocol. - * Protocol: LSB=0 means registered type by ID, LSB=1 means meta share reference/definition. + * Read class info from buffer using a target class. This is used by java serialization APIs that + * pass an expected class for meta share resolution. */ - private ClassInfo readClassInfoWithMetaShare(MemoryBuffer buffer) { - MetaContext metaContext = fory.getSerializationContext().getMetaContext(); - assert metaContext != null : SET_META__CONTEXT_MSG; + public final ClassInfo readClassInfo(MemoryBuffer buffer, Class targetClass) { int header = buffer.readVarUint32Small14(); - int id = header >>> 1; - if ((header & 0b1) == 0) { - // Registered type by ID - return getClassInfoByTypeId(id); - } - // Meta share reference or definition - ClassInfo classInfo = metaContext.readClassInfos.get(id); - if (classInfo == null) { - classInfo = readSharedClassMetaById(metaContext, id); - } - return classInfo; - } - - /** - * Read a ClassDef from meta share context and create ClassInfo. - * The ClassDef is looked up from metaContext.readClassDefs by index. - */ - protected final ClassInfo readSharedClassMetaById(MetaContext metaContext, int index) { - ClassDef classDef = metaContext.readClassDefs.get(index); - Tuple2 classDefTuple = extRegistry.classIdToDef.get(classDef.getId()); - ClassInfo classInfo; - if (classDefTuple == null || classDefTuple.f1 == null || classDefTuple.f1.serializer == null) { - classInfo = buildMetaSharedClassInfo(classDefTuple, classDef); - } else { - classInfo = classDefTuple.f1; - } - metaContext.readClassInfos.set(index, classInfo); - return classInfo; - } - - /** - * Write collected ClassDefs to buffer. ClassDefs are collected during serialization - * and written at the end of the serialization process. - */ - public final void writeClassDefs(MemoryBuffer buffer) { - MetaContext metaContext = fory.getSerializationContext().getMetaContext(); - ObjectArray writingClassDefs = metaContext.writingClassDefs; - final int size = writingClassDefs.size; - buffer.writeVarUint32Small7(size); - if (buffer.isHeapFullyWriteable()) { - for (int i = 0; i < size; i++) { - buffer.writeBytes(writingClassDefs.get(i).getEncoded()); - } - } else { - for (int i = 0; i < size; i++) { - writingClassDefs.get(i).writeClassDef(buffer); - } - } - metaContext.writingClassDefs.size = 0; - } - - /** - * Read ClassDefs from buffer and populate metaContext.readClassDefs. - * Must be called before deserializing objects in meta share mode. - */ - public final void readClassDefs(MemoryBuffer buffer) { - MetaContext metaContext = fory.getSerializationContext().getMetaContext(); - assert metaContext != null : SET_META__CONTEXT_MSG; - int numClassDefs = buffer.readVarUint32Small7(); - for (int i = 0; i < numClassDefs; i++) { - long id = buffer.readInt64(); - Tuple2 tuple2 = extRegistry.classIdToDef.get(id); - if (tuple2 != null) { - ClassDef.skipClassDef(buffer, id); - } else { - tuple2 = readClassDefFromBuffer(buffer, id); - } - metaContext.readClassDefs.add(tuple2.f0); - metaContext.readClassInfos.add(tuple2.f1); - } - } - - private Tuple2 readClassDefFromBuffer(MemoryBuffer buffer, long header) { - ClassDef readClassDef = ClassDef.readClassDef(fory, buffer, header); - Tuple2 tuple2 = extRegistry.classIdToDef.get(readClassDef.getId()); - if (tuple2 == null) { - tuple2 = Tuple2.of(readClassDef, null); - extRegistry.classIdToDef.put(readClassDef.getId(), tuple2); + int internalTypeId = header & 0xff; + switch (internalTypeId) { + case Types.NAMED_ENUM: + case Types.NAMED_STRUCT: + case Types.NAMED_EXT: + if (metaContextShareEnabled) { + return readSharedClassMeta(buffer, targetClass); + } + ClassInfo classInfo = readClassInfoFromBytes(buffer, classInfoCache, header); + classInfoCache = classInfo; + return classInfo; + case Types.NAMED_COMPATIBLE_STRUCT: + case Types.COMPATIBLE_STRUCT: + Preconditions.checkArgument( + metaContextShareEnabled, "Meta share must be enabled for compatible mode"); + return readSharedClassMeta(buffer, targetClass); + default: + return getClassInfoByTypeId(header); } - return tuple2; } /** @@ -506,21 +409,23 @@ private Tuple2 readClassDefFromBuffer(MemoryBuffer buffer, */ @CodegenInvoke public final ClassInfo readClassInfo(MemoryBuffer buffer, ClassInfo classInfoCache) { - // In meta share mode, use the meta share protocol directly - if (metaContextShareEnabled) { - return readClassInfoWithMetaShare(buffer); - } - - // Non-meta share mode: read typeId followed by optional class name bytes int header = buffer.readVarUint32Small14(); int internalTypeId = header & 0xff; - - // Check if this is a named type (needs class name bytes) - if (isNamedType(internalTypeId)) { - return readClassInfoByCache(buffer, classInfoCache, header); - } else { - // Lookup by type ID from registry - return getClassInfoByTypeId(header); + switch (internalTypeId) { + case Types.NAMED_ENUM: + case Types.NAMED_STRUCT: + case Types.NAMED_EXT: + if (metaContextShareEnabled) { + return readSharedClassMeta(buffer); + } + return readClassInfoByCache(buffer, classInfoCache, header); + case Types.NAMED_COMPATIBLE_STRUCT: + case Types.COMPATIBLE_STRUCT: + Preconditions.checkArgument( + metaContextShareEnabled, "Meta share must be enabled for compatible mode"); + return readSharedClassMeta(buffer); + default: + return getClassInfoByTypeId(header); } } @@ -534,32 +439,26 @@ public final ClassInfo readClassInfo(MemoryBuffer buffer, ClassInfo classInfoCac */ @CodegenInvoke public final ClassInfo readClassInfo(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { - // In meta share mode, use the meta share protocol directly - if (metaContextShareEnabled) { - return readClassInfoWithMetaShare(buffer); - } - - // Non-meta share mode: read typeId followed by optional class name bytes int header = buffer.readVarUint32Small14(); int internalTypeId = header & 0xff; - - // Check if this is a named type (needs class name bytes) - if (isNamedType(internalTypeId)) { - return readClassInfoFromBytes(buffer, classInfoHolder, header); - } else { - // Lookup by type ID from registry - return getClassInfoByTypeId(header); + switch (internalTypeId) { + case Types.NAMED_ENUM: + case Types.NAMED_STRUCT: + case Types.NAMED_EXT: + if (metaContextShareEnabled) { + return readSharedClassMeta(buffer); + } + return readClassInfoFromBytes(buffer, classInfoHolder, header); + case Types.NAMED_COMPATIBLE_STRUCT: + case Types.COMPATIBLE_STRUCT: + Preconditions.checkArgument( + metaContextShareEnabled, "Meta share must be enabled for compatible mode"); + return readSharedClassMeta(buffer); + default: + return getClassInfoByTypeId(header); } } - /** Helper to check if a type ID represents a named type that needs class name bytes. */ - private boolean isNamedType(int internalTypeId) { - return internalTypeId == Types.NAMED_ENUM - || internalTypeId == Types.NAMED_STRUCT - || internalTypeId == Types.NAMED_EXT - || internalTypeId == Types.NAMED_COMPATIBLE_STRUCT; - } - /** * Read class info using the provided cache. Returns cached ClassInfo if the namespace and type * name bytes match. @@ -650,13 +549,33 @@ protected final ClassInfo readSharedClassMeta(MemoryBuffer buffer) { return classInfo; } + public final ClassInfo readSharedClassMeta(MemoryBuffer buffer, Class targetClass) { + ClassInfo classInfo = readSharedClassMeta(buffer); + Class readClass = classInfo.getCls(); + // replace target class if needed + if (targetClass != readClass) { + Tuple2, Class> key = Tuple2.of(readClass, targetClass); + ClassInfo newClassInfo = extRegistry.transformedClassInfo.get(key); + if (newClassInfo == null) { + // similar to create serializer for `NonexistentMetaShared` + newClassInfo = + getMetaSharedClassInfo( + classInfo.classDef.replaceRootClassTo((ClassResolver) this, targetClass), + targetClass); + extRegistry.transformedClassInfo.put(key, newClassInfo); + } + return newClassInfo; + } + return classInfo; + } + /** * Load class info from namespace and type name bytes. Subclasses implement this to resolve the * class and create/lookup ClassInfo. * - *

    Note: This method should NOT create serializers. It's used by both readClassInfo - * (which needs serializers) and readClassInternal (which doesn't need serializers). - * Use {@link #ensureSerializerForClassInfo} after calling this if a serializer is needed. + *

    Note: This method should NOT create serializers. It's used by both readClassInfo (which + * needs serializers) and readClassInternal (which doesn't need serializers). Use {@link + * #ensureSerializerForClassInfo} after calling this if a serializer is needed. */ protected abstract ClassInfo loadBytesToClassInfo( MetaStringBytes namespaceBytes, MetaStringBytes simpleClassNameBytes); @@ -678,37 +597,17 @@ protected abstract ClassInfo loadBytesToClassInfo( protected abstract ClassInfo getClassInfoByTypeId(int typeId); /** - * Check if a typeId corresponds to a valid registered class that can be written - * using the fast path (typeId << 1) in meta share mode. + * Check if a typeId corresponds to a valid registered class that can be written using the fast + * path (typeId << 1) in meta share mode. * - *

    For ClassResolver, this checks if the typeId has an entry in registeredId2ClassInfo. - * For XtypeResolver, this checks if the typeId is a valid internal type or has a registered entry. + *

    For ClassResolver, this checks if the typeId has an entry in registeredId2ClassInfo. For + * XtypeResolver, this checks if the typeId is a valid internal type or has a registered entry. * * @param typeId the type ID to check * @return true if this typeId can use the fast path, false if meta share protocol is needed */ protected abstract boolean isValidRegisteredTypeId(int typeId); - public final ClassInfo readSharedClassMeta(MemoryBuffer buffer, Class targetClass) { - ClassInfo classInfo = readSharedClassMeta(buffer); - Class readClass = classInfo.getCls(); - // replace target class if needed - if (targetClass != readClass) { - Tuple2, Class> key = Tuple2.of(readClass, targetClass); - ClassInfo newClassInfo = extRegistry.transformedClassInfo.get(key); - if (newClassInfo == null) { - // similar to create serializer for `NonexistentMetaShared` - newClassInfo = - getMetaSharedClassInfo( - classInfo.classDef.replaceRootClassTo((ClassResolver) this, targetClass), - targetClass); - extRegistry.transformedClassInfo.put(key, newClassInfo); - } - return newClassInfo; - } - return classInfo; - } - final ClassInfo buildMetaSharedClassInfo( Tuple2 classDefTuple, ClassDef classDef) { ClassInfo classInfo; @@ -743,14 +642,27 @@ private ClassInfo getMetaSharedClassInfo(ClassDef classDef, Class clz) { } Class cls = clz; Short classId = extRegistry.registeredClassIdMap.get(cls); - ClassInfo classInfo = - new ClassInfo(this, cls, null, classId == null ? NO_CLASS_ID : classId); + int typeId = NO_CLASS_ID; + if (classId != null) { + ClassInfo registeredInfo = classInfoMap.get(cls); + typeId = registeredInfo == null ? classId : registeredInfo.typeId; + } else { + ClassInfo cachedInfo = classInfoMap.get(cls); + if (cachedInfo != null) { + typeId = cachedInfo.typeId; + } else if (cls.isEnum()) { + typeId = Types.NAMED_ENUM; + } else { + typeId = metaContextShareEnabled ? Types.NAMED_COMPATIBLE_STRUCT : Types.NAMED_STRUCT; + } + } + ClassInfo classInfo = new ClassInfo(this, cls, null, typeId); classInfo.classDef = classDef; if (NonexistentClass.class.isAssignableFrom(TypeUtils.getComponentIfArray(cls))) { if (cls == NonexistentMetaShared.class) { classInfo.setSerializer(this, new NonexistentClassSerializer(fory, classDef)); - // ensure `NonexistentMetaSharedClass` registered to write fixed-length class def, - // so we can rewrite it in `NonexistentClassSerializer`. + // Ensure NonexistentMetaShared is registered so writeClassInfo emits a placeholder typeId + // that NonexistentClassSerializer can rewrite to the original typeId. if (!fory.isCrossLanguage()) { Preconditions.checkNotNull(classId); } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java index a44fed244e..e5d54f7b25 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java @@ -28,7 +28,6 @@ import static org.apache.fory.serializer.collection.MapSerializers.HashMapSerializer; import static org.apache.fory.type.TypeUtils.qualifiedName; -import java.lang.reflect.Type; import java.math.BigDecimal; import java.math.BigInteger; import java.sql.Timestamp; @@ -55,7 +54,6 @@ import org.apache.fory.Fory; import org.apache.fory.annotation.ForyField; import org.apache.fory.annotation.Internal; -import org.apache.fory.collection.IdentityObjectIntMap; import org.apache.fory.collection.LongMap; import org.apache.fory.collection.ObjectMap; import org.apache.fory.collection.Tuple2; @@ -70,7 +68,6 @@ import org.apache.fory.meta.Encoders; import org.apache.fory.meta.MetaString; import org.apache.fory.reflect.ReflectionUtils; -import org.apache.fory.reflect.TypeRef; import org.apache.fory.serializer.ArraySerializers; import org.apache.fory.serializer.DeferedLazySerializer; import org.apache.fory.serializer.DeferedLazySerializer.DeferredLazyObjectSerializer; @@ -386,29 +383,48 @@ public void registerSerializer(Class type, Serializer serializer) { @Override public void registerInternalSerializer(Class type, Serializer serializer) { checkRegisterAllowed(); + Class unwrapped = TypeUtils.unwrap(type); + if (unwrapped == char.class + || unwrapped == void.class + || type == char[].class + || type == Character[].class) { + return; + } ClassInfo classInfo = classInfoMap.get(type); if (classInfo != null) { - classInfo.serializer = serializer; - } else { - // Determine appropriate type ID based on the type - int typeId = determineTypeIdForClass(type); - classInfo = newClassInfo(type, serializer, typeId); - classInfoMap.put(type, classInfo); + if (classInfo.serializer == null) { + classInfo.serializer = serializer; + } + return; } + // Determine appropriate type ID based on the type + int typeId = determineTypeIdForClass(type); + classInfo = newClassInfo(type, serializer, typeId); + classInfoMap.put(type, classInfo); } /** - * Determine the appropriate xlang type ID for a class. - * For collection types, use the collection-specific type IDs. - * For other types, use NAMED_STRUCT which writes namespace and typename bytes. + * Determine the appropriate xlang type ID for a class. For collection types, use the + * collection-specific type IDs. For other types, use NAMED_STRUCT which writes namespace and + * typename bytes. */ private int determineTypeIdForClass(Class type) { + if (type.isArray()) { + Class componentType = type.getComponentType(); + if (componentType.isPrimitive()) { + int elemTypeId = Types.getTypeId(fory, componentType); + return Types.getPrimitiveArrayTypeId(elemTypeId); + } + return Types.LIST; + } if (List.class.isAssignableFrom(type)) { return Types.LIST; } else if (Set.class.isAssignableFrom(type)) { return Types.SET; } else if (Map.class.isAssignableFrom(type)) { return Types.MAP; + } else if (type.isEnum()) { + return Types.ENUM; } else { // For unregistered classes, use NAMED_STRUCT so that class name is written return Types.NAMED_STRUCT; @@ -603,7 +619,8 @@ private ClassInfo buildClassInfo(Class cls) { serializer = new HashMapSerializer(fory); } else { ClassInfo classInfo = classInfoMap.get(cls); - if (classInfo != null && classInfo.serializer != null + if (classInfo != null + && classInfo.serializer != null && classInfo.serializer instanceof MapLikeSerializer && ((MapLikeSerializer) classInfo.serializer).supportCodegenHook()) { serializer = classInfo.serializer; @@ -638,7 +655,8 @@ private ClassInfo buildClassInfo(Class cls) { private Serializer getCollectionSerializer(Class cls) { ClassInfo classInfo = classInfoMap.get(cls); - if (classInfo != null && classInfo.serializer != null + if (classInfo != null + && classInfo.serializer != null && classInfo.serializer instanceof CollectionLikeSerializer && ((CollectionLikeSerializer) (classInfo.serializer)).supportCodegenHook()) { return classInfo.serializer; @@ -648,55 +666,82 @@ private Serializer getCollectionSerializer(Class cls) { private void registerDefaultTypes() { // Boolean types - registerType(Types.BOOL, Boolean.class, new PrimitiveSerializers.BooleanSerializer(fory, Boolean.class)); - registerType(Types.BOOL, boolean.class, new PrimitiveSerializers.BooleanSerializer(fory, boolean.class)); + registerType( + Types.BOOL, Boolean.class, new PrimitiveSerializers.BooleanSerializer(fory, Boolean.class)); + registerType( + Types.BOOL, boolean.class, new PrimitiveSerializers.BooleanSerializer(fory, boolean.class)); registerType(Types.BOOL, AtomicBoolean.class, new Serializers.AtomicBooleanSerializer(fory)); // Byte types - registerType(Types.UINT8, Byte.class, new PrimitiveSerializers.ByteSerializer(fory, Byte.class)); - registerType(Types.UINT8, byte.class, new PrimitiveSerializers.ByteSerializer(fory, byte.class)); + registerType( + Types.UINT8, Byte.class, new PrimitiveSerializers.ByteSerializer(fory, Byte.class)); + registerType( + Types.UINT8, byte.class, new PrimitiveSerializers.ByteSerializer(fory, byte.class)); registerType(Types.INT8, Byte.class, new PrimitiveSerializers.ByteSerializer(fory, Byte.class)); registerType(Types.INT8, byte.class, new PrimitiveSerializers.ByteSerializer(fory, byte.class)); // Short types - registerType(Types.UINT16, Short.class, new PrimitiveSerializers.ShortSerializer(fory, Short.class)); - registerType(Types.UINT16, short.class, new PrimitiveSerializers.ShortSerializer(fory, short.class)); - registerType(Types.INT16, Short.class, new PrimitiveSerializers.ShortSerializer(fory, Short.class)); - registerType(Types.INT16, short.class, new PrimitiveSerializers.ShortSerializer(fory, short.class)); + registerType( + Types.UINT16, Short.class, new PrimitiveSerializers.ShortSerializer(fory, Short.class)); + registerType( + Types.UINT16, short.class, new PrimitiveSerializers.ShortSerializer(fory, short.class)); + registerType( + Types.INT16, Short.class, new PrimitiveSerializers.ShortSerializer(fory, Short.class)); + registerType( + Types.INT16, short.class, new PrimitiveSerializers.ShortSerializer(fory, short.class)); // Integer types - registerType(Types.UINT32, Integer.class, new PrimitiveSerializers.IntSerializer(fory, Integer.class)); + registerType( + Types.UINT32, Integer.class, new PrimitiveSerializers.IntSerializer(fory, Integer.class)); registerType(Types.UINT32, int.class, new PrimitiveSerializers.IntSerializer(fory, int.class)); registerType(Types.UINT32, AtomicInteger.class, new Serializers.AtomicIntegerSerializer(fory)); - registerType(Types.INT32, Integer.class, new PrimitiveSerializers.IntSerializer(fory, Integer.class)); + registerType( + Types.INT32, Integer.class, new PrimitiveSerializers.IntSerializer(fory, Integer.class)); registerType(Types.INT32, int.class, new PrimitiveSerializers.IntSerializer(fory, int.class)); registerType(Types.INT32, AtomicInteger.class, new Serializers.AtomicIntegerSerializer(fory)); - registerType(Types.VARINT32, Integer.class, new PrimitiveSerializers.IntSerializer(fory, Integer.class)); - registerType(Types.VARINT32, int.class, new PrimitiveSerializers.IntSerializer(fory, int.class)); - registerType(Types.VARINT32, AtomicInteger.class, new Serializers.AtomicIntegerSerializer(fory)); + registerType( + Types.VARINT32, Integer.class, new PrimitiveSerializers.IntSerializer(fory, Integer.class)); + registerType( + Types.VARINT32, int.class, new PrimitiveSerializers.IntSerializer(fory, int.class)); + registerType( + Types.VARINT32, AtomicInteger.class, new Serializers.AtomicIntegerSerializer(fory)); // Long types - registerType(Types.UINT64, Long.class, new PrimitiveSerializers.LongSerializer(fory, Long.class)); - registerType(Types.UINT64, long.class, new PrimitiveSerializers.LongSerializer(fory, long.class)); + registerType( + Types.UINT64, Long.class, new PrimitiveSerializers.LongSerializer(fory, Long.class)); + registerType( + Types.UINT64, long.class, new PrimitiveSerializers.LongSerializer(fory, long.class)); registerType(Types.UINT64, AtomicLong.class, new Serializers.AtomicLongSerializer(fory)); - registerType(Types.TAGGED_UINT64, Long.class, new PrimitiveSerializers.LongSerializer(fory, Long.class)); - registerType(Types.TAGGED_UINT64, long.class, new PrimitiveSerializers.LongSerializer(fory, long.class)); + registerType( + Types.TAGGED_UINT64, Long.class, new PrimitiveSerializers.LongSerializer(fory, Long.class)); + registerType( + Types.TAGGED_UINT64, long.class, new PrimitiveSerializers.LongSerializer(fory, long.class)); registerType(Types.TAGGED_UINT64, AtomicLong.class, new Serializers.AtomicLongSerializer(fory)); - registerType(Types.INT64, Long.class, new PrimitiveSerializers.LongSerializer(fory, Long.class)); - registerType(Types.INT64, long.class, new PrimitiveSerializers.LongSerializer(fory, long.class)); + registerType( + Types.INT64, Long.class, new PrimitiveSerializers.LongSerializer(fory, Long.class)); + registerType( + Types.INT64, long.class, new PrimitiveSerializers.LongSerializer(fory, long.class)); registerType(Types.INT64, AtomicLong.class, new Serializers.AtomicLongSerializer(fory)); - registerType(Types.TAGGED_INT64, Long.class, new PrimitiveSerializers.LongSerializer(fory, Long.class)); - registerType(Types.TAGGED_INT64, long.class, new PrimitiveSerializers.LongSerializer(fory, long.class)); + registerType( + Types.TAGGED_INT64, Long.class, new PrimitiveSerializers.LongSerializer(fory, Long.class)); + registerType( + Types.TAGGED_INT64, long.class, new PrimitiveSerializers.LongSerializer(fory, long.class)); registerType(Types.TAGGED_INT64, AtomicLong.class, new Serializers.AtomicLongSerializer(fory)); - registerType(Types.VARINT64, Long.class, new PrimitiveSerializers.LongSerializer(fory, Long.class)); - registerType(Types.VARINT64, long.class, new PrimitiveSerializers.LongSerializer(fory, long.class)); + registerType( + Types.VARINT64, Long.class, new PrimitiveSerializers.LongSerializer(fory, Long.class)); + registerType( + Types.VARINT64, long.class, new PrimitiveSerializers.LongSerializer(fory, long.class)); registerType(Types.VARINT64, AtomicLong.class, new Serializers.AtomicLongSerializer(fory)); // Float types - registerType(Types.FLOAT32, Float.class, new PrimitiveSerializers.FloatSerializer(fory, Float.class)); - registerType(Types.FLOAT32, float.class, new PrimitiveSerializers.FloatSerializer(fory, float.class)); - registerType(Types.FLOAT64, Double.class, new PrimitiveSerializers.DoubleSerializer(fory, Double.class)); - registerType(Types.FLOAT64, double.class, new PrimitiveSerializers.DoubleSerializer(fory, double.class)); + registerType( + Types.FLOAT32, Float.class, new PrimitiveSerializers.FloatSerializer(fory, Float.class)); + registerType( + Types.FLOAT32, float.class, new PrimitiveSerializers.FloatSerializer(fory, float.class)); + registerType( + Types.FLOAT64, Double.class, new PrimitiveSerializers.DoubleSerializer(fory, Double.class)); + registerType( + Types.FLOAT64, double.class, new PrimitiveSerializers.DoubleSerializer(fory, double.class)); // String types registerType(Types.STRING, String.class, new StringSerializer(fory)); @@ -709,7 +754,8 @@ private void registerDefaultTypes() { registerType(Types.TIMESTAMP, Date.class, new TimeSerializers.DateSerializer(fory)); registerType(Types.TIMESTAMP, java.sql.Date.class, new TimeSerializers.SqlDateSerializer(fory)); registerType(Types.TIMESTAMP, Timestamp.class, new TimeSerializers.TimestampSerializer(fory)); - registerType(Types.TIMESTAMP, LocalDateTime.class, new TimeSerializers.LocalDateTimeSerializer(fory)); + registerType( + Types.TIMESTAMP, LocalDateTime.class, new TimeSerializers.LocalDateTimeSerializer(fory)); registerType(Types.LOCAL_DATE, LocalDate.class, new TimeSerializers.LocalDateSerializer(fory)); // Decimal types @@ -719,38 +765,60 @@ private void registerDefaultTypes() { // Binary types registerType(Types.BINARY, byte[].class, new ArraySerializers.ByteArraySerializer(fory)); @SuppressWarnings("unchecked") - Class heapByteBufferClass = (Class) Platform.HEAP_BYTE_BUFFER_CLASS; - registerType(Types.BINARY, Platform.HEAP_BYTE_BUFFER_CLASS, - new org.apache.fory.serializer.BufferSerializers.ByteBufferSerializer(fory, heapByteBufferClass)); + Class heapByteBufferClass = + (Class) Platform.HEAP_BYTE_BUFFER_CLASS; + registerType( + Types.BINARY, + Platform.HEAP_BYTE_BUFFER_CLASS, + new org.apache.fory.serializer.BufferSerializers.ByteBufferSerializer( + fory, heapByteBufferClass)); @SuppressWarnings("unchecked") - Class directByteBufferClass = (Class) Platform.DIRECT_BYTE_BUFFER_CLASS; - registerType(Types.BINARY, Platform.DIRECT_BYTE_BUFFER_CLASS, - new org.apache.fory.serializer.BufferSerializers.ByteBufferSerializer(fory, directByteBufferClass)); + Class directByteBufferClass = + (Class) Platform.DIRECT_BYTE_BUFFER_CLASS; + registerType( + Types.BINARY, + Platform.DIRECT_BYTE_BUFFER_CLASS, + new org.apache.fory.serializer.BufferSerializers.ByteBufferSerializer( + fory, directByteBufferClass)); // Primitive arrays - registerType(Types.BOOL_ARRAY, boolean[].class, new ArraySerializers.BooleanArraySerializer(fory)); + registerType( + Types.BOOL_ARRAY, boolean[].class, new ArraySerializers.BooleanArraySerializer(fory)); registerType(Types.INT16_ARRAY, short[].class, new ArraySerializers.ShortArraySerializer(fory)); registerType(Types.INT32_ARRAY, int[].class, new ArraySerializers.IntArraySerializer(fory)); registerType(Types.INT64_ARRAY, long[].class, new ArraySerializers.LongArraySerializer(fory)); - registerType(Types.FLOAT32_ARRAY, float[].class, new ArraySerializers.FloatArraySerializer(fory)); - registerType(Types.FLOAT64_ARRAY, double[].class, new ArraySerializers.DoubleArraySerializer(fory)); + registerType( + Types.FLOAT32_ARRAY, float[].class, new ArraySerializers.FloatArraySerializer(fory)); + registerType( + Types.FLOAT64_ARRAY, double[].class, new ArraySerializers.DoubleArraySerializer(fory)); // Collections registerType(Types.LIST, ArrayList.class, new ArrayListSerializer(fory)); - registerType(Types.LIST, Object[].class, new ArraySerializers.ObjectArraySerializer(fory, Object[].class)); + registerType( + Types.LIST, + Object[].class, + new ArraySerializers.ObjectArraySerializer(fory, Object[].class)); registerType(Types.LIST, List.class, new XlangListDefaultSerializer(fory, List.class)); - registerType(Types.LIST, Collection.class, new XlangListDefaultSerializer(fory, Collection.class)); + registerType( + Types.LIST, Collection.class, new XlangListDefaultSerializer(fory, Collection.class)); // Sets registerType(Types.SET, HashSet.class, new HashSetSerializer(fory)); - registerType(Types.SET, LinkedHashSet.class, - new org.apache.fory.serializer.collection.CollectionSerializers.LinkedHashSetSerializer(fory)); + registerType( + Types.SET, + LinkedHashSet.class, + new org.apache.fory.serializer.collection.CollectionSerializers.LinkedHashSetSerializer( + fory)); registerType(Types.SET, Set.class, new XlangSetDefaultSerializer(fory, Set.class)); // Maps - registerType(Types.MAP, HashMap.class, + registerType( + Types.MAP, + HashMap.class, new org.apache.fory.serializer.collection.MapSerializers.HashMapSerializer(fory)); - registerType(Types.MAP, LinkedHashMap.class, + registerType( + Types.MAP, + LinkedHashMap.class, new org.apache.fory.serializer.collection.MapSerializers.LinkedHashMapSerializer(fory)); registerType(Types.MAP, Map.class, new XlangMapSerializer(fory, Map.class)); @@ -901,6 +969,10 @@ protected ClassInfo loadBytesToClassInfo( @Override protected ClassInfo ensureSerializerForClassInfo(ClassInfo classInfo) { if (classInfo.serializer == null) { + Class cls = classInfo.cls; + if (cls != null && (ReflectionUtils.isAbstract(cls) || cls.isInterface())) { + return classInfo; + } // Get or create ClassInfo with serializer ClassInfo newClassInfo = getClassInfo(classInfo.cls); // Update the cache with the correct ClassInfo that has a serializer diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ArraySerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ArraySerializers.java index 0c16356bf3..ab3380a839 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ArraySerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ArraySerializers.java @@ -29,8 +29,8 @@ import org.apache.fory.resolver.ClassInfo; import org.apache.fory.resolver.ClassInfoHolder; import org.apache.fory.resolver.ClassResolver; -import org.apache.fory.resolver.TypeResolver; import org.apache.fory.resolver.RefResolver; +import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.collection.CollectionFlags; import org.apache.fory.serializer.collection.ForyArrayAsListSerializer; import org.apache.fory.type.GenericType; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java index 2c8256482a..4634d97198 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java @@ -32,6 +32,7 @@ import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.meta.ClassDef; +import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.MetaContext; import org.apache.fory.resolver.MetaStringResolver; import org.apache.fory.resolver.RefResolver; @@ -42,6 +43,7 @@ import org.apache.fory.type.Descriptor; import org.apache.fory.type.DescriptorGrouper; import org.apache.fory.type.Generics; +import org.apache.fory.type.Types; import org.apache.fory.util.Preconditions; import org.apache.fory.util.Utils; @@ -64,6 +66,8 @@ private ClassFieldsInfo(FieldGroups fieldGroups, int classVersionHash) { } public static final class NonexistentClassSerializer extends Serializer { + private static final int NONEXISTENT_META_SHARED_ID_SIZE = + computeVarUint32Size(ClassResolver.NONEXISTENT_META_SHARED_ID); private final ClassDef classDef; private final LongMap fieldsInfoMap; private final SerializationBinding binding; @@ -75,7 +79,8 @@ public NonexistentClassSerializer(Fory fory, ClassDef classDef) { binding = SerializationBinding.createBinding(fory); Preconditions.checkArgument(fory.getConfig().isMetaShareEnabled()); if (Utils.DEBUG_OUTPUT_ENABLED && classDef != null) { - LOG.info("========== NonexistentClassSerializer ClassDef for {} ==========", type.getName()); + LOG.info( + "========== NonexistentClassSerializer ClassDef for {} ==========", type.getName()); LOG.info("ClassDef fieldsInfo count: {}", classDef.getFieldCount()); for (int i = 0; i < classDef.getFieldsInfo().size(); i++) { LOG.info(" [{}] {}", i, classDef.getFieldsInfo().get(i)); @@ -88,8 +93,9 @@ public NonexistentClassSerializer(Fory fory, ClassDef classDef) { * classinfo by `class`, it may dispatch to same `NonexistentClassSerializer`, so we can't use * `classDef` in this serializer, but use `classDef` in `NonexistentMetaSharedClass` instead. * - *

    XtypeResolver.writeSharedClassMeta writes a stub (1-byte ref marker + stub bytes) for - * NonexistentMetaShared. This method rewinds past that stub and writes the actual classDef. + *

    NonexistentMetaShared is registered with a fixed internal typeId for dispatch. This + * serializer rewinds that placeholder typeId and writes the original class's typeId, then + * writes the shared ClassDef inline using the stream meta protocol. */ private void writeClassDef(MemoryBuffer buffer, NonexistentClass.NonexistentMetaShared value) { MetaContext metaContext = fory.getSerializationContext().getMetaContext(); @@ -107,9 +113,51 @@ private void writeClassDef(MemoryBuffer buffer, NonexistentClass.NonexistentMeta } } + private static int computeVarUint32Size(int value) { + if ((value & ~0x7f) == 0) { + return 1; + } + if ((value & ~0x3fff) == 0) { + return 2; + } + if ((value & ~0x1fffff) == 0) { + return 3; + } + if ((value & ~0xfffffff) == 0) { + return 4; + } + return 5; + } + + private int resolveTypeId(ClassDef classDef) { + int typeId = classDef.getClassSpec().typeId; + if (typeId >= 0) { + return typeId; + } + if (classDef.getClassSpec().isEnum) { + return Types.NAMED_ENUM; + } + return Types.NAMED_COMPATIBLE_STRUCT; + } + @Override public void write(MemoryBuffer buffer, Object v) { NonexistentClass.NonexistentMetaShared value = (NonexistentClass.NonexistentMetaShared) v; + int typeId = resolveTypeId(value.classDef); + int typeIdSize = computeVarUint32Size(typeId); + if (typeIdSize == NONEXISTENT_META_SHARED_ID_SIZE) { + buffer.increaseWriterIndex(-NONEXISTENT_META_SHARED_ID_SIZE); + buffer.writeVarUint32Small7(typeId); + } else { + int originalWriterIndex = buffer.writerIndex(); + int placeholderStart = originalWriterIndex - NONEXISTENT_META_SHARED_ID_SIZE; + int payloadStart = placeholderStart + NONEXISTENT_META_SHARED_ID_SIZE; + int payloadLength = originalWriterIndex - payloadStart; + byte[] payload = buffer.getBytes(payloadStart, payloadLength); + buffer.writerIndex(placeholderStart); + buffer.writeVarUint32Small7(typeId); + buffer.writeBytes(payload); + } writeClassDef(buffer, value); ClassDef classDef = value.classDef; ClassFieldsInfo fieldsInfo = getClassFieldsInfo(classDef); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java index 12006ffd9a..6091d69312 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java @@ -104,8 +104,10 @@ public ObjectSerializer(Fory fory, Class cls, boolean resolveParent) { DescriptorGrouper grouper = typeResolver.createDescriptorGrouper(descriptors, false); descriptors = grouper.getSortedDescriptors(); if (Utils.DEBUG_OUTPUT_ENABLED) { - LOG.info("========== ObjectSerializer {} sorted descriptors for {} ==========", - descriptors.size(), cls.getName()); + LOG.info( + "========== ObjectSerializer {} sorted descriptors for {} ==========", + descriptors.size(), + cls.getName()); for (Descriptor d : descriptors) { LOG.info( " {} -> {}, ref {}, nullable {}", diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java index b359b3b77f..490980e1b5 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java @@ -522,8 +522,8 @@ private static FieldTypes.FieldType buildFieldTypeFromClass(Fory fory, Class // For registered types if (fory.getClassResolver().isRegisteredById(fieldType)) { - Short classId = fory.getClassResolver().getRegisteredClassId(fieldType); - return new FieldTypes.RegisteredFieldType(true, true, classId); + int typeId = fory.getClassResolver().getTypeIdForClassDef(fieldType); + return new FieldTypes.RegisteredFieldType(true, true, typeId); } // For enums diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/OptionalSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/OptionalSerializers.java index ad51fdf50d..2c8fe9afd3 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/OptionalSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/OptionalSerializers.java @@ -25,7 +25,6 @@ import java.util.OptionalLong; import org.apache.fory.Fory; import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.TypeResolver; /** diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/PrimitiveSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/PrimitiveSerializers.java index 475656f3c5..e827c100f9 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/PrimitiveSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/PrimitiveSerializers.java @@ -27,7 +27,6 @@ import org.apache.fory.config.LongEncoding; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.Platform; -import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.util.Preconditions; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java index 80f56010a9..33c1bc73f4 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java @@ -274,7 +274,8 @@ Object readField(SerializationFieldInfo fieldInfo, RefMode refMode, MemoryBuffer } else { if (refMode != RefMode.NULL_ONLY || buffer.readByte() != Fory.NULL_FLAG) { // Preserve a dummy ref ID so ObjectSerializer.read() can pop it. - // This is needed when global ref tracking is enabled but field ref tracking is disabled. + // This is needed when global ref tracking is enabled but field ref tracking is + // disabled. refResolver.preserveRefId(-1); return fory.readNonRef(buffer, fieldInfo.classInfo); } @@ -285,7 +286,8 @@ Object readField(SerializationFieldInfo fieldInfo, RefMode refMode, MemoryBuffer } else { if (refMode != RefMode.NULL_ONLY || buffer.readByte() != Fory.NULL_FLAG) { // Preserve a dummy ref ID so ObjectSerializer.read() can pop it. - // This is needed when global ref tracking is enabled but field ref tracking is disabled. + // This is needed when global ref tracking is enabled but field ref tracking is + // disabled. refResolver.preserveRefId(-1); return fory.readNonRef(buffer, fieldInfo.classInfoHolder); } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java index 7bf15c9c30..6f9d237a66 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java @@ -49,7 +49,6 @@ import org.apache.fory.memory.Platform; import org.apache.fory.meta.ClassDef; import org.apache.fory.reflect.ReflectionUtils; -import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.util.ExceptionUtils; import org.apache.fory.util.GraalvmSupport; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/TimeSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/TimeSerializers.java index 83f6797d86..de0b6314ec 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/TimeSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/TimeSerializers.java @@ -41,7 +41,6 @@ import java.util.TimeZone; import org.apache.fory.Fory; import org.apache.fory.memory.MemoryBuffer; -import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.util.DateTimeUtils; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java index c831a5ad15..0841155757 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java @@ -273,9 +273,9 @@ private static void readAndSetFields( /** * Read and skip the layer class meta from buffer. This is used to skip over the class definition - * that was written by MetaSharedLayerSerializer.writeLayerClassMeta(). For ChildContainerSerializers, - * we use the same serializer on both write and read sides, so we just need to skip the meta - * without actually parsing it. + * that was written by MetaSharedLayerSerializer.writeLayerClassMeta(). For + * ChildContainerSerializers, we use the same serializer on both write and read sides, so we just + * need to skip the meta without actually parsing it. */ private static void readAndSkipLayerClassMeta(Fory fory, MemoryBuffer buffer) { MetaContext metaContext = fory.getSerializationContext().getMetaContext(); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java index 99aa91309c..f8477781ea 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionSerializers.java @@ -57,8 +57,8 @@ import org.apache.fory.resolver.ClassInfo; import org.apache.fory.resolver.ClassInfoHolder; import org.apache.fory.resolver.ClassResolver; -import org.apache.fory.resolver.TypeResolver; import org.apache.fory.resolver.RefResolver; +import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.ReplaceResolveSerializer; import org.apache.fory.serializer.Serializer; import org.apache.fory.serializer.Serializers; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/GuavaCollectionSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/GuavaCollectionSerializers.java index 5256d8c24d..15fc3f3bed 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/GuavaCollectionSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/GuavaCollectionSerializers.java @@ -37,7 +37,6 @@ import org.apache.fory.Fory; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.Platform; -import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.util.unsafe._JDKAccess; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ImmutableCollectionSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ImmutableCollectionSerializers.java index 29de60b9e1..72c3d4f696 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ImmutableCollectionSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ImmutableCollectionSerializers.java @@ -32,7 +32,6 @@ import org.apache.fory.Fory; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.Platform; -import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.util.unsafe._JDKAccess; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SubListSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SubListSerializers.java index 7319e22745..bcc6892794 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SubListSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SubListSerializers.java @@ -27,7 +27,6 @@ import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.reflect.ReflectionUtils; -import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.ObjectSerializer; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SynchronizedSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SynchronizedSerializers.java index f1ee86c5a2..be5d983de5 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SynchronizedSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/SynchronizedSerializers.java @@ -40,7 +40,6 @@ import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.Platform; -import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.Serializer; import org.apache.fory.util.ExceptionUtils; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/UnmodifiableSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/UnmodifiableSerializers.java index feb270f514..26f28aecb2 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/UnmodifiableSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/UnmodifiableSerializers.java @@ -39,7 +39,6 @@ import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.Platform; -import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.Serializer; import org.apache.fory.util.ExceptionUtils; diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/FinalFieldReplaceResolveSerializerTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/FinalFieldReplaceResolveSerializerTest.java index 6b5dd80cef..6030b713ef 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/FinalFieldReplaceResolveSerializerTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/FinalFieldReplaceResolveSerializerTest.java @@ -399,7 +399,7 @@ public void testNoClassNameWrittenForFinalField(boolean codegen) { byte[] bytesFinal = fory.serialize(containerFinal); byte[] bytesFinal2 = fory.serialize(containerFinal); assertEquals(bytesFinal, bytesFinal2); - assertEquals(bytesFinal.length, 108); + assertEquals(bytesFinal.length, 109); // Create a container with a non-final ImmutableList field for comparison ContainerWithNonFinalImmutableIntArray containerNonFinal = diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/NonexistentClassSerializersTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/NonexistentClassSerializersTest.java index 5ce58dca30..bed5d922ee 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/NonexistentClassSerializersTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/serializer/NonexistentClassSerializersTest.java @@ -28,7 +28,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; - import lombok.Data; import org.apache.fory.Fory; import org.apache.fory.ForyTestBase; @@ -370,7 +369,6 @@ public void testThrowExceptionIfClassNotExist() { assertThrowsCause(RuntimeException.class, () -> fory2.deserialize(bytes)); } - /** * Simple test class with primitive types for NonexistentClass serialization testing. Avoids * collection types which require complex type registration in xlang mode. diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java index 34eb2c1c1a..e03353093b 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java @@ -75,5 +75,4 @@ public void testMDArrayField() { s.arr = new int[][] {{1, 2}, {3, 4}}; serDeCheck(fory, s); } - } diff --git a/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/RecordXlangTest.java b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/RecordXlangTest.java index b707aac97e..ad985934ab 100644 --- a/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/RecordXlangTest.java +++ b/java/fory-latest-jdk-tests/src/test/java/org/apache/fory/integration_tests/RecordXlangTest.java @@ -729,8 +729,7 @@ public void testRecordRefSchemaConsistent(boolean enableCodegen) { RefOuterRecord result = (RefOuterRecord) fory.deserialize(buffer); // Verify reference identity is preserved - Assert.assertSame( - result.inner1(), result.inner2(), "inner1 and inner2 should be same object"); + Assert.assertSame(result.inner1(), result.inner2(), "inner1 and inner2 should be same object"); Assert.assertEquals(result.inner1().id(), 42); Assert.assertEquals(result.inner1().name(), "shared_inner"); } @@ -766,8 +765,7 @@ public void testRecordRefCompatible(boolean enableCodegen) { RefOuterRecord result = (RefOuterRecord) fory.deserialize(buffer); // Verify reference identity is preserved - Assert.assertSame( - result.inner1(), result.inner2(), "inner1 and inner2 should be same object"); + Assert.assertSame(result.inner1(), result.inner2(), "inner1 and inner2 should be same object"); Assert.assertEquals(result.inner1().id(), 99); Assert.assertEquals(result.inner1().name(), "compatible_shared"); } From d375a834fafd8c616ee9534edd2352eb05550ac8 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Mon, 19 Jan 2026 00:39:15 +0800 Subject: [PATCH 26/44] refactor more type system --- .../org/apache/fory/meta/ClassDefDecoder.java | 55 ++- .../org/apache/fory/meta/ClassDefEncoder.java | 2 +- .../java/org/apache/fory/meta/FieldTypes.java | 20 +- .../org/apache/fory/resolver/ClassInfo.java | 10 +- .../apache/fory/resolver/ClassResolver.java | 418 ++++++++++-------- .../apache/fory/resolver/MapRefResolver.java | 4 + .../apache/fory/resolver/TypeResolver.java | 50 ++- .../apache/fory/resolver/XtypeResolver.java | 26 +- .../fory/serializer/MetaSharedSerializer.java | 10 +- .../NonexistentClassSerializers.java | 15 +- .../serializer/ReplaceResolveSerializer.java | 3 +- .../apache/fory/serializer/Serializers.java | 8 + .../main/java/org/apache/fory/util/Utils.java | 3 +- .../fory/resolver/ClassResolverTest.java | 20 +- python/pyfory/registry.py | 34 -- python/pyfory/serialization.pyx | 54 +-- 16 files changed, 368 insertions(+), 364 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDefDecoder.java b/java/fory-core/src/main/java/org/apache/fory/meta/ClassDefDecoder.java index bda9ca420e..a756fb246e 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDefDecoder.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/ClassDefDecoder.java @@ -83,43 +83,52 @@ public static ClassDef decodeClassDef(ClassResolver resolver, MemoryBuffer buffe boolean isRegistered = (currentClassHeader & 0b1) != 0; int numFields = currentClassHeader >>> 1; if (isRegistered) { - short registeredId = (short) classDefBuf.readVarUint32Small7(); - int internalTypeId = - resolver.getFory().getConfig().isMetaShareEnabled() - ? Types.COMPATIBLE_STRUCT - : Types.STRUCT; - int typeId = (registeredId << 8) | internalTypeId; - if (resolver.getRegisteredClass(registeredId) == null) { + int typeId = classDefBuf.readVarUint32Small7(); + Class cls = resolver.getRegisteredClassByTypeId(typeId); + if (cls == null) { classSpec = new ClassSpec(NonexistentClass.NonexistentMetaShared.class, typeId); className = classSpec.entireClassName; } else { - Class cls = resolver.getRegisteredClass(registeredId); className = cls.getName(); - classSpec = new ClassSpec(cls, resolver.getTypeIdForClassDef(cls)); + classSpec = new ClassSpec(cls, typeId); } } else { String pkg = readPkgName(classDefBuf); String typeName = readTypeName(classDefBuf); - classSpec = Encoders.decodePkgAndClass(pkg, typeName); - className = classSpec.entireClassName; + ClassSpec decodedSpec = Encoders.decodePkgAndClass(pkg, typeName); + className = decodedSpec.entireClassName; if (resolver.isRegisteredByName(className)) { Class cls = resolver.getRegisteredClass(className); className = cls.getName(); classSpec = new ClassSpec(cls, resolver.getTypeIdForClassDef(cls)); } else { - int typeId = - classSpec.isEnum - ? Types.NAMED_ENUM - : (resolver.getFory().getConfig().isMetaShareEnabled() + Class cls = + resolver.loadClassForMeta( + decodedSpec.entireClassName, decodedSpec.isEnum, decodedSpec.dimension); + if (NonexistentClass.isNonexistent(cls)) { + int typeId; + if (decodedSpec.isEnum) { + typeId = Types.NAMED_ENUM; + } else { + typeId = + resolver.getFory().isCompatible() ? Types.NAMED_COMPATIBLE_STRUCT - : Types.NAMED_STRUCT); - classSpec = - new ClassSpec( - classSpec.entireClassName, - classSpec.isEnum, - classSpec.isArray, - classSpec.dimension, - typeId); + : Types.NAMED_STRUCT; + } + classSpec = + new ClassSpec( + decodedSpec.entireClassName, + decodedSpec.isEnum, + decodedSpec.isArray, + decodedSpec.dimension, + typeId); + classSpec.type = cls; + className = classSpec.entireClassName; + } else { + int typeId = resolver.getTypeIdForClassDef(cls); + classSpec = new ClassSpec(cls, typeId); + className = classSpec.entireClassName; + } } } List fieldInfos = readFieldsInfo(classDefBuf, resolver, className, numFields); diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDefEncoder.java b/java/fory-core/src/main/java/org/apache/fory/meta/ClassDefEncoder.java index e40119bc3a..f28e1f978a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDefEncoder.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/ClassDefEncoder.java @@ -178,7 +178,7 @@ public static MemoryBuffer encodeClassDef( if (classResolver.isRegisteredById(currentType)) { currentClassHeader |= 1; classDefBuf.writeVarUint32Small7(currentClassHeader); - classDefBuf.writeVarUint32Small7(classResolver.getRegisteredClassId(currentType)); + classDefBuf.writeVarUint32Small7(classResolver.getTypeIdForClassDef(currentType)); } else { classDefBuf.writeVarUint32Small7(currentClassHeader); String ns, typename; diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java index b65ffa99cf..7f5b908818 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java @@ -91,7 +91,8 @@ private static FieldType buildFieldType( typeId = Types.getTypeId(resolver.getFory(), rawType); } } else { - ClassInfo info = resolver.getClassInfo(rawType, false); + ClassInfo info = + isXlang && rawType == Object.class ? null : resolver.getClassInfo(rawType, false); if (info != null) { typeId = info.getTypeId(); } else if (isXlang) { @@ -123,13 +124,12 @@ private static FieldType buildFieldType( // For xlang: ref tracking is false by default (no shared ownership like Rust's Rc/Arc) // For native: use the type's default tracking behavior boolean trackingRef = !isXlang && genericType.trackingRef(resolver); - // For xlang: nullable is false by default (aligned with all languages) - // Exception: Optional types are nullable (like Rust's Option) - // For native: non-primitive types are nullable by default + // For xlang: nullable is false by default (aligned with all languages). + // Exception: Optional types are nullable (like Rust's Option). + // For native: non-primitive types are nullable by default. boolean nullable; if (isXlang) { - // Only Optional types and boxed types are nullable by default in xlang mode - nullable = isOptionalType(rawType) || TypeUtils.isBoxed(rawType); + nullable = isOptionalType(rawType); } else { // Primitives are never nullable, non-primitives are nullable by default // This applies to both top-level fields and nested types (in arrays, collections, maps) @@ -465,8 +465,7 @@ public TypeRef toTypeToken(TypeResolver resolver, TypeRef declared) { Preconditions.checkNotNull(xtypeInfo); cls = xtypeInfo.getCls(); } else { - int classId = typeId < ClassResolver.USER_ID_BASE ? typeId : (typeId >>> 8); - cls = ((ClassResolver) resolver).getRegisteredClass((short) classId); + cls = ((ClassResolver) resolver).getRegisteredClassByTypeId(typeId); } if (cls == null) { LOG.warn("Class {} not registered, take it as Struct type for deserialization.", typeId); @@ -484,9 +483,8 @@ public String getTypeName(TypeResolver resolver, TypeRef typeRef) { if (resolver instanceof ClassResolver) { ClassResolver classResolver = (ClassResolver) resolver; // Peer class may not register this class id, which will introduce inconsistent field order - int classId = typeId < ClassResolver.USER_ID_BASE ? typeId : (typeId >>> 8); - if (classResolver.isInternalRegistered(classId)) { - return String.valueOf(classId); + if (classResolver.isInternalRegistered(typeId)) { + return String.valueOf(typeId); } else { return "Registered"; } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassInfo.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassInfo.java index 9d3455409e..f72086fbdf 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassInfo.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassInfo.java @@ -30,7 +30,6 @@ import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.serializer.Serializer; import org.apache.fory.type.Types; -import org.apache.fory.util.Preconditions; import org.apache.fory.util.function.Functions; /** @@ -67,9 +66,6 @@ public class ClassInfo { this.isDynamicGeneratedClass = isDynamicGeneratedClass; this.typeId = typeId; this.serializer = serializer; - if (cls != null && typeId == TypeResolver.NO_CLASS_ID) { - Preconditions.checkArgument(typeNameBytes != null); - } } /** @@ -87,7 +83,7 @@ public ClassInfo(Class cls, ClassDef classDef) { this.typeNameBytes = null; this.isDynamicGeneratedClass = false; this.serializer = null; - this.typeId = TypeResolver.NO_CLASS_ID; + this.typeId = classDef == null ? Types.UNKNOWN : classDef.getClassSpec().typeId; } ClassInfo(TypeResolver classResolver, Class cls, Serializer serializer, int typeId) { @@ -107,14 +103,12 @@ public ClassInfo(Class cls, ClassDef classDef) { // - NAMED_COMPATIBLE_STRUCT: unregistered classes in compatible mode // - NAMED_ENUM, NAMED_EXT: other named types // - REPLACE_STUB_ID: for write replace class in `ClassSerializer` - // - NO_CLASS_ID: legacy support (should use NAMED_STRUCT instead) boolean isNamedType = typeId == Types.NAMED_STRUCT || typeId == Types.NAMED_COMPATIBLE_STRUCT || typeId == Types.NAMED_ENUM || typeId == Types.NAMED_EXT - || typeId == ClassResolver.REPLACE_STUB_ID - || typeId == TypeResolver.NO_CLASS_ID; + || typeId == ClassResolver.REPLACE_STUB_ID; if (cls != null && isNamedType) { Tuple2 tuple2 = Encoders.encodePkgAndClass(cls); this.namespaceBytes = diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index e104a6bda3..92e008f7a3 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -77,12 +77,8 @@ import org.apache.fory.annotation.CodegenInvoke; import org.apache.fory.annotation.ForyField; import org.apache.fory.annotation.Internal; -import org.apache.fory.builder.Generated; import org.apache.fory.builder.JITContext; import org.apache.fory.codegen.CodeGenerator; -import org.apache.fory.codegen.Expression; -import org.apache.fory.codegen.Expression.Invoke; -import org.apache.fory.codegen.Expression.Literal; import org.apache.fory.collection.ObjectMap; import org.apache.fory.collection.Tuple2; import org.apache.fory.config.Language; @@ -100,7 +96,6 @@ import org.apache.fory.serializer.ArraySerializers; import org.apache.fory.serializer.BufferSerializers; import org.apache.fory.serializer.CodegenSerializer.LazyInitBeanSerializer; -import org.apache.fory.serializer.DeferedLazySerializer; import org.apache.fory.serializer.EnumSerializer; import org.apache.fory.serializer.ExternalizableSerializer; import org.apache.fory.serializer.FinalFieldReplaceResolveSerializer; @@ -158,19 +153,9 @@ * *

    Class ID Space

    * - *

    Fory separates user class IDs from internal class IDs to provide a clean and intuitive API: - * - *

      - *
    • User ID space: IDs specified via {@link #register(Class, int)} start from 0. Valid - * range is [0, {@value Short#MAX_VALUE} - {@value #USER_ID_BASE} - 1] (i.e., [0, 32510]). - *
    • Internal ID space: Reserved for Fory's built-in types (primitives, common - * collections, etc.). These IDs are in the range [0, {@value #USER_ID_BASE} - 1] and are - * completely hidden from users. - *
    - * - *

    When users register a class with ID N, Fory internally stores it as (N + {@value - * #USER_ID_BASE}). This transformation is transparent to users - they only work with their own - * 0-based ID space. + *

    Fory separates internal IDs (built-in types) from user IDs by encoding user-registered types + * with their internal type tag (ENUM/STRUCT/EXT). User IDs start from 0 and are encoded into the + * unified type ID as {@code (userId << 8) | internalTypeId}. * *

    Registration Methods

    * @@ -187,14 +172,7 @@ public class ClassResolver extends TypeResolver { private static final Logger LOG = LoggerFactory.getLogger(ClassResolver.class); - /** Flag value indicating no class ID has been assigned. */ - public static final short NO_CLASS_ID = TypeResolver.NO_CLASS_ID; - - /** - * Base offset for user-registered class IDs. User IDs are internally stored as `userId + - * USER_ID_BASE`. 0 to `USER_ID_BASE` are reserved for Fory's internal types. - */ - public static final short USER_ID_BASE = 256; + private static final int INTERNAL_ID_LIMIT = 256; public static final int NATIVE_START_ID = Types.NAMED_EXT + 1; public static final int VOID_ID = NATIVE_START_ID; @@ -231,14 +209,13 @@ public class ClassResolver extends TypeResolver { private final Fory fory; private ClassInfo[] registeredId2ClassInfo = new ClassInfo[] {}; + private ClassInfo[] userRegisteredId2ClassInfo = new ClassInfo[] {}; private ClassInfo classInfoCache; // Every deserialization for unregistered class will query it, performance is important. private final ObjectMap compositeNameBytes2ClassInfo = new ObjectMap<>(16, foryMapLoadFactor); // classDefMap is inherited from TypeResolver private Class currentReadClass; - // class id of last default registered class. - private short innerEndClassId; private final ShimDispatcher shimDispatcher; public ClassResolver(Fory fory) { @@ -294,7 +271,6 @@ public void initialize() { registerDefaultClasses(); addDefaultSerializers(); shimDispatcher.initialize(); - innerEndClassId = extRegistry.classIdGenerator; if (GraalvmSupport.isGraalBuildtime()) { classInfoMap.forEach( (cls, classInfo) -> { @@ -410,8 +386,8 @@ private void registerDefaultClasses() { @Override public void register(Class cls) { if (!extRegistry.registeredClassIdMap.containsKey(cls)) { - while (extRegistry.userIdGenerator + USER_ID_BASE < registeredId2ClassInfo.length - && registeredId2ClassInfo[extRegistry.userIdGenerator + USER_ID_BASE] != null) { + while (extRegistry.userIdGenerator < userRegisteredId2ClassInfo.length + && userRegisteredId2ClassInfo[extRegistry.userIdGenerator] != null) { extRegistry.userIdGenerator++; } register(cls, extRegistry.userIdGenerator); @@ -444,17 +420,14 @@ public void register(String className, int classId) { /** * Registers a class with a user-specified ID. * - *

    The ID is in the user ID space, starting from 0. Fory internally transforms this to an - * internal ID by adding {@link #USER_ID_BASE}. This separation ensures user IDs never conflict - * with Fory's internal type IDs. - * - *

    Valid user ID range: [0, 32510] (i.e., [0, Short.MAX_VALUE - USER_ID_BASE - 1]) + *

    The ID is in the user ID space, starting from 0. The unified type ID is encoded as + * {@code (userId << 8) | internalTypeId}. * *

    Example: * * 

    {@code
-   * fory.register(MyClass.class, 0);      // User ID 0 -> Internal ID 256
-   * fory.register(AnotherClass.class, 1); // User ID 1 -> Internal ID 257
+   * fory.register(MyClass.class, 0);      // User ID 0 -> typeId 0x000019 (STRUCT)
+   * fory.register(AnotherClass.class, 1); // User ID 1 -> typeId 0x000119 (STRUCT)
    * }
    * * @param cls the class to register @@ -463,7 +436,7 @@ public void register(String className, int classId) { */ @Override public void register(Class cls, int id) { - registerImpl(cls, id + USER_ID_BASE); + registerUserImpl(cls, id); } /** @@ -484,17 +457,15 @@ public void register(Class cls, String namespace, String name) { if (!StringUtils.isBlank(namespace)) { fullname = namespace + "." + name; } - checkRegistration(cls, (short) -1, fullname); + checkRegistration(cls, (short) -1, fullname, false); MetaStringBytes fullNameBytes = metaStringResolver.getOrCreateMetaStringBytes( GENERIC_ENCODER.encode(fullname, MetaString.Encoding.UTF_8)); MetaStringBytes nsBytes = metaStringResolver.getOrCreateMetaStringBytes(encodePackage(namespace)); MetaStringBytes nameBytes = metaStringResolver.getOrCreateMetaStringBytes(encodeTypeName(name)); - int typeId = - cls.isEnum() - ? Types.NAMED_ENUM - : (metaContextShareEnabled ? Types.NAMED_COMPATIBLE_STRUCT : Types.NAMED_STRUCT); + ClassInfo existingInfo = classInfoMap.get(cls); + int typeId = buildUnregisteredTypeId(cls, existingInfo == null ? null : existingInfo.serializer); ClassInfo classInfo = new ClassInfo(cls, fullNameBytes, nsBytes, nameBytes, false, null, typeId); classInfoMap.put(cls, classInfo); @@ -522,16 +493,24 @@ public void registerInternal(Class... classes) { * Registers a class for internal use with an auto-assigned internal ID. * *

    Internal API: This method is for Fory's internal use only. Users should use {@link - * #register(Class)} instead. Internal IDs are in the range [0, {@link #USER_ID_BASE} - 1]. + * #register(Class)} instead. Internal IDs are in the range [0, 255]. * * @param cls the class to register */ public void registerInternal(Class cls) { if (!extRegistry.registeredClassIdMap.containsKey(cls)) { + Preconditions.checkArgument( + extRegistry.classIdGenerator < INTERNAL_ID_LIMIT, + "Internal type id overflow: %s", + extRegistry.classIdGenerator); while (extRegistry.classIdGenerator < registeredId2ClassInfo.length && registeredId2ClassInfo[extRegistry.classIdGenerator] != null) { extRegistry.classIdGenerator++; } + Preconditions.checkArgument( + extRegistry.classIdGenerator < INTERNAL_ID_LIMIT, + "Internal type id overflow: %s", + extRegistry.classIdGenerator); registerInternal(cls, extRegistry.classIdGenerator); } } @@ -542,31 +521,29 @@ public void registerInternal(Class cls) { *

    Internal API: This method is for Fory's internal use only. Users should use {@link * #register(Class, int)} instead. * - *

    Internal IDs are reserved for Fory's built-in types and must be in the range [0, {@link - * #USER_ID_BASE} - 1] (i.e., [0, 255]). User IDs start from {@link #USER_ID_BASE} and above. + *

    Internal IDs are reserved for Fory's built-in types and must be in the range [0, 255]. * * @param cls the class to register - * @param classId the internal ID, must be in range [0, {@link #USER_ID_BASE} - 1] + * @param classId the internal ID, must be in range [0, 255] * @throws IllegalArgumentException if the ID is out of range or already in use */ public void registerInternal(Class cls, int classId) { - Preconditions.checkArgument(classId >= 0 && classId < USER_ID_BASE); - registerImpl(cls, classId); + Preconditions.checkArgument(classId >= 0 && classId < INTERNAL_ID_LIMIT); + registerInternalImpl(cls, classId); } - private void registerImpl(Class cls, int classId) { + private void registerInternalImpl(Class cls, int classId) { checkRegisterAllowed(); - // class id must be less than Integer.MAX_VALUE/2 since we use bit 0 as class id flag. - Preconditions.checkArgument(classId >= 0 && classId < Short.MAX_VALUE); + Preconditions.checkArgument(classId >= 0 && classId < INTERNAL_ID_LIMIT); short id = (short) classId; - checkRegistration(cls, id, cls.getName()); + checkRegistration(cls, id, cls.getName(), true); extRegistry.registeredClassIdMap.put(cls, id); if (registeredId2ClassInfo.length <= id) { ClassInfo[] tmp = new ClassInfo[(id + 1) * 2]; System.arraycopy(registeredId2ClassInfo, 0, tmp, 0, registeredId2ClassInfo.length); registeredId2ClassInfo = tmp; } - int typeId = buildRegisteredTypeId(cls, classId, null); + int typeId = classId; ClassInfo classInfo = classInfoMap.get(cls); if (classInfo != null) { classInfo.typeId = typeId; @@ -582,10 +559,31 @@ private void registerImpl(Class cls, int classId) { GraalvmSupport.registerClass(cls, fory.getConfig().getConfigHash()); } - private int buildRegisteredTypeId(Class cls, int classId, Serializer serializer) { - if (classId < USER_ID_BASE) { - return classId; + private void registerUserImpl(Class cls, int userId) { + checkRegisterAllowed(); + Preconditions.checkArgument(userId >= 0 && userId < Short.MAX_VALUE); + short id = (short) userId; + checkRegistration(cls, id, cls.getName(), false); + extRegistry.registeredClassIdMap.put(cls, id); + if (userRegisteredId2ClassInfo.length <= id) { + ClassInfo[] tmp = new ClassInfo[(id + 1) * 2]; + System.arraycopy(userRegisteredId2ClassInfo, 0, tmp, 0, userRegisteredId2ClassInfo.length); + userRegisteredId2ClassInfo = tmp; } + int typeId = buildUserTypeId(cls, userId, null); + ClassInfo classInfo = classInfoMap.get(cls); + if (classInfo != null) { + classInfo.typeId = typeId; + } else { + classInfo = new ClassInfo(this, cls, null, typeId); + classInfoMap.put(cls, classInfo); + } + userRegisteredId2ClassInfo[id] = classInfo; + extRegistry.registeredClasses.put(cls.getName(), cls); + GraalvmSupport.registerClass(cls, fory.getConfig().getConfigHash()); + } + + private int buildUserTypeId(Class cls, int userId, Serializer serializer) { int internalTypeId; if (cls.isEnum()) { internalTypeId = Types.ENUM; @@ -594,43 +592,41 @@ private int buildRegisteredTypeId(Class cls, int classId, Serializer seria } else { internalTypeId = metaContextShareEnabled ? Types.COMPATIBLE_STRUCT : Types.STRUCT; } - return (classId << 8) | internalTypeId; + return (userId << 8) | internalTypeId; } - private int buildUnregisteredTypeId(Class cls, Serializer serializer) { - if (serializer instanceof ReplaceResolveSerializer - && !(serializer instanceof FinalFieldReplaceResolveSerializer)) { - return REPLACE_STUB_ID; - } - if (cls.isEnum()) { - return Types.NAMED_ENUM; - } - if (serializer != null && !isStructSerializer(serializer)) { + @Override + protected int buildUnregisteredTypeId(Class cls, Serializer serializer) { + if (serializer == null && !cls.isEnum() && useReplaceResolveSerializer(cls)) { return Types.NAMED_EXT; } - return metaContextShareEnabled ? Types.NAMED_COMPATIBLE_STRUCT : Types.NAMED_STRUCT; + return super.buildUnregisteredTypeId(cls, serializer); } - private boolean isStructSerializer(Serializer serializer) { - return serializer instanceof ObjectSerializer - || serializer instanceof Generated.GeneratedSerializer - || serializer instanceof DeferedLazySerializer.DeferredLazyObjectSerializer; - } - - private void checkRegistration(Class cls, short classId, String name) { + private void checkRegistration(Class cls, short classId, String name, boolean internal) { if (extRegistry.registeredClassIdMap.containsKey(cls)) { throw new IllegalArgumentException( String.format( "Class %s already registered with id %s.", cls, extRegistry.registeredClassIdMap.get(cls))); } - if (classId > 0 - && classId < registeredId2ClassInfo.length - && registeredId2ClassInfo[classId] != null) { - throw new IllegalArgumentException( - String.format( - "Class %s with id %s has been registered, registering class %s with same id are not allowed.", - registeredId2ClassInfo[classId].getCls(), classId, cls.getName())); + if (classId >= 0) { + if (internal) { + if (classId < registeredId2ClassInfo.length && registeredId2ClassInfo[classId] != null) { + throw new IllegalArgumentException( + String.format( + "Class %s with id %s has been registered, registering class %s with same id are not allowed.", + registeredId2ClassInfo[classId].getCls(), classId, cls.getName())); + } + } else { + if (classId < userRegisteredId2ClassInfo.length + && userRegisteredId2ClassInfo[classId] != null) { + throw new IllegalArgumentException( + String.format( + "Class %s with id %s has been registered, registering class %s with same id are not allowed.", + userRegisteredId2ClassInfo[classId].getCls(), classId, cls.getName())); + } + } } if (extRegistry.registeredClasses.containsKey(name) || extRegistry.registeredClasses.inverse().containsKey(cls)) { @@ -641,6 +637,14 @@ private void checkRegistration(Class cls, short classId, String name) { } } + private boolean isInternalRegisteredClassId(Class cls, short classId) { + if (classId < 0 || classId >= registeredId2ClassInfo.length) { + return false; + } + ClassInfo classInfo = registeredId2ClassInfo[classId]; + return classInfo != null && classInfo.cls == cls; + } + @Override public boolean isRegistered(Class cls) { return extRegistry.registeredClassIdMap.containsKey(cls) @@ -690,15 +694,49 @@ public Class getRegisteredClass(short id) { return null; } + public Class getRegisteredClassByTypeId(int typeId) { + ClassInfo classInfo = getRegisteredClassInfoByTypeId(typeId); + return classInfo == null ? null : classInfo.cls; + } + + public ClassInfo getRegisteredClassInfoByTypeId(int typeId) { + int internalTypeId = typeId & 0xff; + if (Types.isNamedType(internalTypeId)) { + return null; + } + if (Types.isUserDefinedType((byte) internalTypeId)) { + int userId = typeId >>> 8; + if (userId < 0 || userId >= userRegisteredId2ClassInfo.length) { + return null; + } + ClassInfo classInfo = userRegisteredId2ClassInfo[userId]; + if (classInfo != null && (classInfo.typeId & 0xff) != internalTypeId) { + return null; + } + return classInfo; + } + if (typeId < 0 || typeId >= registeredId2ClassInfo.length) { + return null; + } + return registeredId2ClassInfo[typeId]; + } + public Class getRegisteredClass(String className) { return extRegistry.registeredClasses.get(className); } public List> getRegisteredClasses() { - return Arrays.stream(registeredId2ClassInfo) - .filter(Objects::nonNull) - .map(info -> info.cls) - .collect(Collectors.toList()); + List> classes = + Arrays.stream(registeredId2ClassInfo) + .filter(Objects::nonNull) + .map(info -> info.cls) + .collect(Collectors.toList()); + for (ClassInfo info : userRegisteredId2ClassInfo) { + if (info != null) { + classes.add(info.cls); + } + } + return classes; } public String getTypeAlias(Class cls) { @@ -724,22 +762,21 @@ public int getTypeIdForClassDef(Class cls) { } Short classId = extRegistry.registeredClassIdMap.get(cls); if (classId != null) { - if (classId < USER_ID_BASE) { - return classId; + classInfo = classInfoMap.get(cls); + if (classInfo == null) { + classInfo = getClassInfo(cls); } - int internalTypeId = - cls.isEnum() - ? Types.ENUM - : (metaContextShareEnabled ? Types.COMPATIBLE_STRUCT : Types.STRUCT); - return (classId << 8) | internalTypeId; - } - if (useReplaceResolveSerializer(cls)) { - return REPLACE_STUB_ID; + return classInfo.typeId; } - if (cls.isEnum()) { - return Types.NAMED_ENUM; + int typeId = buildUnregisteredTypeId(cls, null); + classInfo = new ClassInfo(this, cls, null, typeId); + classInfoMap.put(cls, classInfo); + if (classInfo.namespaceBytes != null && classInfo.typeNameBytes != null) { + TypeNameBytes typeNameBytes = + new TypeNameBytes(classInfo.namespaceBytes.hashCode, classInfo.typeNameBytes.hashCode); + compositeNameBytes2ClassInfo.put(typeNameBytes, classInfo); } - return metaContextShareEnabled ? Types.NAMED_COMPATIBLE_STRUCT : Types.NAMED_STRUCT; + return typeId; } @Override @@ -791,13 +828,19 @@ public boolean isBuildIn(Descriptor descriptor) { } public boolean isInternalRegistered(int classId) { - return classId != NO_CLASS_ID && classId < innerEndClassId; + int internalTypeId = classId & 0xff; + if (Types.isUserDefinedType((byte) internalTypeId)) { + return false; + } + return classId > 0 + && classId < registeredId2ClassInfo.length + && registeredId2ClassInfo[classId] != null; } /** Returns true if cls is fory inner registered class. */ public boolean isInternalRegistered(Class cls) { - Short classId = extRegistry.registeredClassIdMap.get(cls); - return classId != null && classId != NO_CLASS_ID && classId < innerEndClassId; + ClassInfo classInfo = classInfoMap.get(cls); + return classInfo != null && isInternalRegistered(classInfo.typeId); } /** @@ -867,24 +910,6 @@ public void registerSerializer(Class type, Serializer serializer) { if (!serializer.getClass().getPackage().getName().startsWith("org.apache.fory")) { SerializationUtils.validate(type, serializer.getClass()); } - if (!extRegistry.registeredClassIdMap.containsKey(type) && !fory.isCrossLanguage()) { - register(type); - } - ClassInfo classInfo = classInfoMap.get(type); - if (classInfo == null) { - classInfo = getClassInfo(type); - } - if (!isStructSerializer(serializer)) { - int typeId = classInfo.typeId; - int internalTypeId = typeId & 0xff; - int userPart = typeId & 0xffffff00; - if (internalTypeId == Types.STRUCT || internalTypeId == Types.COMPATIBLE_STRUCT) { - classInfo.typeId = userPart | Types.EXT; - } else if (internalTypeId == Types.NAMED_STRUCT - || internalTypeId == Types.NAMED_COMPATIBLE_STRUCT) { - classInfo.typeId = Types.NAMED_EXT; - } - } registerSerializerImpl(type, serializer); } @@ -896,6 +921,22 @@ public void registerSerializer(Class type, Serializer serializer) { */ @Override public void registerInternalSerializer(Class type, Serializer serializer) { + Short classId = extRegistry.registeredClassIdMap.get(type); + if (classId != null && !isInternalRegisteredClassId(type, classId)) { + throw new IllegalArgumentException( + String.format( + "Class %s is not registered with an internal id (< %d).", + type, INTERNAL_ID_LIMIT)); + } + if (classId != null) { + Preconditions.checkArgument( + classId >= 0 && classId < INTERNAL_ID_LIMIT, + "Internal type id overflow: %s", + classId); + } + if (classId == null) { + registerInternal(type); + } registerSerializerImpl(type, serializer); } @@ -904,9 +945,6 @@ private void registerSerializerImpl(Class type, Serializer serializer) { if (!serializer.getClass().getPackage().getName().startsWith("org.apache.fory")) { SerializationUtils.validate(type, serializer.getClass()); } - if (!extRegistry.registeredClassIdMap.containsKey(type) && !fory.isCrossLanguage()) { - registerInternal(type); - } addSerializer(type, serializer); ClassInfo classInfo = classInfoMap.get(type); classInfoMap.put(type, classInfo); @@ -1015,15 +1053,32 @@ public void addSerializer(Class type, Serializer serializer) { Short classId = extRegistry.registeredClassIdMap.get(type); boolean registered = classId != null; if (registered) { - classInfo = registeredId2ClassInfo[classId]; - int typeId = buildRegisteredTypeId(type, classId, serializer); + int id = classId; + boolean internal = isInternalRegisteredClassId(type, (short) id); + int typeId = internal ? id : buildUserTypeId(type, id, serializer); + classInfo = classInfoMap.get(type); if (classInfo == null) { classInfo = new ClassInfo(this, type, null, typeId); classInfoMap.put(type, classInfo); - registeredId2ClassInfo[classId] = classInfo; } else { classInfo.typeId = typeId; } + if (internal) { + if (registeredId2ClassInfo.length <= id) { + ClassInfo[] tmp = new ClassInfo[(id + 1) * 2]; + System.arraycopy(registeredId2ClassInfo, 0, tmp, 0, registeredId2ClassInfo.length); + registeredId2ClassInfo = tmp; + } + registeredId2ClassInfo[id] = classInfo; + } else { + if (userRegisteredId2ClassInfo.length <= id) { + ClassInfo[] tmp = new ClassInfo[(id + 1) * 2]; + System.arraycopy( + userRegisteredId2ClassInfo, 0, tmp, 0, userRegisteredId2ClassInfo.length); + userRegisteredId2ClassInfo = tmp; + } + userRegisteredId2ClassInfo[id] = classInfo; + } } else { int typeId = buildUnregisteredTypeId(type, serializer); classInfo = classInfoMap.get(type); @@ -1380,9 +1435,12 @@ public ClassInfo getOrUpdateClassInfo(Class cls) { private ClassInfo getOrUpdateClassInfo(short classId) { ClassInfo classInfo = classInfoCache; - Short cachedId = extRegistry.registeredClassIdMap.get(classInfo.cls); - if (cachedId == null || cachedId != classId) { - classInfo = registeredId2ClassInfo[classId]; + ClassInfo internalInfo = + classId < registeredId2ClassInfo.length ? registeredId2ClassInfo[classId] : null; + Preconditions.checkArgument( + internalInfo != null, "Internal class id %s is not registered", classId); + if (classInfo != internalInfo) { + classInfo = internalInfo; if (classInfo.serializer == null) { addSerializer(classInfo.cls, createSerializer(classInfo.cls)); classInfo = classInfoMap.get(classInfo.cls); @@ -1540,18 +1598,6 @@ private boolean isSecure(Class cls) { // } } - /** - * Check if a typeId corresponds to a valid registered class in this resolver. For ClassResolver, - * this checks if the typeId is in range and has an entry in registeredId2ClassInfo. - */ - @Override - protected boolean isValidRegisteredTypeId(int typeId) { - int classId = typeId < USER_ID_BASE ? typeId : (typeId >>> 8); - return classId > 0 - && classId < registeredId2ClassInfo.length - && registeredId2ClassInfo[classId] != null; - } - /** * Write class info to buffer. TODO(chaokunyang): The method should try to write * aligned data to reduce cpu instruction overhead. `writeClassInfo` is the last step before @@ -1598,24 +1644,6 @@ protected ClassDef buildClassDef(ClassInfo classInfo) { return classDef; } - // getTypeDef is inherited from TypeResolver - - // Note: Thread safe for jit thread to call. - public Expression writeClassExpr(Expression buffer, short classId) { - Preconditions.checkArgument(classId != NO_CLASS_ID); - return writeClassExpr(buffer, Literal.ofShort(classId)); - } - - // Note: Thread safe for jit thread to call. - private Expression writeClassExpr(Expression buffer, Expression classId) { - return new Invoke(buffer, "writeVarUint32", new Expression.BitShift("<<", classId, 1)); - } - - // Note: Thread safe for jit thread to call. - public Expression skipRegisteredClassExpr(Expression buffer) { - return new Invoke(buffer, "readVarUint32Small14"); - } - /** * Write classname for java serialization. Note that the object of provided class can be * non-serializable, and class with writeReplace/readResolve defined won't be skipped. For @@ -1627,7 +1655,14 @@ public void writeClassInternal(MemoryBuffer buffer, Class cls) { Short classId = extRegistry.registeredClassIdMap.get(cls); // Don't create serializer in case the object for class is non-serializable, // Or class is abstract or interface. - classInfo = new ClassInfo(this, cls, null, classId == null ? NO_CLASS_ID : classId); + int typeId; + if (classId == null) { + typeId = buildUnregisteredTypeId(cls, null); + } else { + boolean internal = isInternalRegisteredClassId(cls, classId); + typeId = internal ? classId : buildUserTypeId(cls, classId, null); + } + classInfo = new ClassInfo(this, cls, null, typeId); classInfoMap.put(cls, classInfo); } writeClassInternal(buffer, classInfo); @@ -1636,14 +1671,10 @@ public void writeClassInternal(MemoryBuffer buffer, Class cls) { public void writeClassInternal(MemoryBuffer buffer, ClassInfo classInfo) { int typeId = classInfo.typeId; int internalTypeId = typeId & 0xff; - Short classId = extRegistry.registeredClassIdMap.get(classInfo.cls); boolean writeById = - classId != null - && typeId != NO_CLASS_ID - && typeId != REPLACE_STUB_ID - && !Types.isNamedType(internalTypeId); + typeId != REPLACE_STUB_ID && !Types.isNamedType(internalTypeId); if (writeById) { - buffer.writeVarUint32(classId << 1); + buffer.writeVarUint32(typeId << 1); } else { // let the lowermost bit of next byte be set, so the deserialization can know // whether need to read class by name in advance @@ -1667,7 +1698,7 @@ public Class readClassInternal(MemoryBuffer buffer) { MetaStringBytes simpleClassNameBytes = metaStringResolver.readMetaStringBytes(buffer); classInfo = loadBytesToClassInfo(packageBytes, simpleClassNameBytes); } else { - classInfo = registeredId2ClassInfo[header >> 1]; + classInfo = getClassInfoByTypeId(header >> 1, false); } final Class cls = classInfo.cls; currentReadClass = cls; @@ -1676,22 +1707,35 @@ public Class readClassInternal(MemoryBuffer buffer) { @Override protected ClassInfo getClassInfoByTypeId(int typeId) { - int classId = typeId < USER_ID_BASE ? typeId : (typeId >>> 8); - if (classId < 0 || classId >= registeredId2ClassInfo.length) { + return getClassInfoByTypeId(typeId, true); + } + + private ClassInfo getClassInfoByTypeId(int typeId, boolean ensureSerializer) { + int internalTypeId = typeId & 0xff; + ClassInfo classInfo = getRegisteredClassInfoByTypeId(typeId); + if (classInfo == null) { throw new IllegalStateException( String.format( "Invalid typeId %d in meta share mode. This usually indicates a protocol mismatch " - + "or buffer corruption. Expected typeId in range [0, %d). " - + "Check that the serializer and deserializer use the same protocol version.", - typeId, registeredId2ClassInfo.length)); + + "or buffer corruption. Check that the serializer and deserializer use the " + + "same protocol version.", + typeId)); + } + if (Types.isUserDefinedType((byte) internalTypeId) && !Types.isNamedType(internalTypeId)) { + int classInternalTypeId = classInfo.typeId & 0xff; + if (classInternalTypeId != internalTypeId) { + throw new IllegalStateException( + String.format( + "Type id mismatch for %s: expected internal type %d but found %d", + classInfo.cls, internalTypeId, classInternalTypeId)); + } } - ClassInfo classInfo = registeredId2ClassInfo[classId]; - // Ensure serializer is set for registered classes (they may have been registered - // without a serializer via registerInternal) - if (classInfo != null && classInfo.serializer == null) { + if (ensureSerializer && classInfo.serializer == null) { + // Ensure serializer is set for registered classes (they may have been registered + // without a serializer via registerInternal) addSerializer(classInfo.cls, createSerializer(classInfo.cls)); // Re-read from registeredId2ClassInfo since addSerializer updates it for registered classes - classInfo = registeredId2ClassInfo[classId]; + classInfo = getRegisteredClassInfoByTypeId(typeId); } return classInfo; } @@ -1742,9 +1786,10 @@ private ClassInfo populateBytesToClassInfo( metaStringResolver.getOrCreateMetaStringBytes( PACKAGE_ENCODER.encode(classSpec.entireClassName, MetaString.Encoding.UTF_8)); Class cls = loadClass(classSpec.entireClassName, classSpec.isEnum, classSpec.dimension); + int typeId = buildUnregisteredTypeId(cls, null); ClassInfo classInfo = new ClassInfo( - cls, fullClassNameBytes, packageBytes, simpleClassNameBytes, false, null, NO_CLASS_ID); + cls, fullClassNameBytes, packageBytes, simpleClassNameBytes, false, null, typeId); if (NonexistentClass.class.isAssignableFrom(TypeUtils.getComponentIfArray(cls))) { classInfo.serializer = NonexistentClassSerializers.getSerializer(fory, classSpec.entireClassName, cls); @@ -1759,6 +1804,21 @@ private ClassInfo populateBytesToClassInfo( return classInfo; } + public Class loadClassForMeta(String className, boolean isEnum, int arrayDims) { + String pkg = ReflectionUtils.getPackage(className); + String typeName = ReflectionUtils.getClassNameWithoutPackage(className); + MetaStringBytes pkgBytes = + metaStringResolver.getOrCreateMetaStringBytes(encodePackage(pkg)); + MetaStringBytes typeBytes = + metaStringResolver.getOrCreateMetaStringBytes(encodeTypeName(typeName)); + ClassInfo cachedInfo = + compositeNameBytes2ClassInfo.get(new TypeNameBytes(pkgBytes.hashCode, typeBytes.hashCode)); + if (cachedInfo != null) { + return cachedInfo.cls; + } + return loadClass(className, isEnum, arrayDims, fory.getConfig().deserializeNonexistentClass()); + } + public Class getCurrentReadClass() { return currentReadClass; } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/MapRefResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/MapRefResolver.java index e83b2a0e65..8741fe597f 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/MapRefResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/MapRefResolver.java @@ -193,6 +193,10 @@ public int lastPreservedRefId() { return readRefIds.get(readRefIds.size - 1); } + public boolean hasPreservedRefId() { + return readRefIds.size > 0; + } + @Override public void reference(Object object) { int refId = readRefIds.pop(); diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index 2be6acb12b..370d1a3be4 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -93,9 +93,8 @@ public abstract class TypeResolver { private static final Logger LOG = LoggerFactory.getLogger(ClassResolver.class); - public static final int NO_CLASS_ID = 0; static final ClassInfo NIL_CLASS_INFO = - new ClassInfo(null, null, null, null, false, null, NO_CLASS_ID); + new ClassInfo(null, null, null, null, false, null, Types.UNKNOWN); // use a lower load factor to minimize hash collision static final float foryMapLoadFactor = 0.25f; static final int estimatedNumRegistered = 150; @@ -596,18 +595,6 @@ protected abstract ClassInfo loadBytesToClassInfo( */ protected abstract ClassInfo getClassInfoByTypeId(int typeId); - /** - * Check if a typeId corresponds to a valid registered class that can be written using the fast - * path (typeId << 1) in meta share mode. - * - *

    For ClassResolver, this checks if the typeId has an entry in registeredId2ClassInfo. For - * XtypeResolver, this checks if the typeId is a valid internal type or has a registered entry. - * - * @param typeId the type ID to check - * @return true if this typeId can use the fast path, false if meta share protocol is needed - */ - protected abstract boolean isValidRegisteredTypeId(int typeId); - final ClassInfo buildMetaSharedClassInfo( Tuple2 classDefTuple, ClassDef classDef) { ClassInfo classInfo; @@ -642,18 +629,19 @@ private ClassInfo getMetaSharedClassInfo(ClassDef classDef, Class clz) { } Class cls = clz; Short classId = extRegistry.registeredClassIdMap.get(cls); - int typeId = NO_CLASS_ID; + int typeId; if (classId != null) { ClassInfo registeredInfo = classInfoMap.get(cls); - typeId = registeredInfo == null ? classId : registeredInfo.typeId; + if (registeredInfo == null) { + registeredInfo = getClassInfo(cls); + } + typeId = registeredInfo.typeId; } else { ClassInfo cachedInfo = classInfoMap.get(cls); if (cachedInfo != null) { typeId = cachedInfo.typeId; - } else if (cls.isEnum()) { - typeId = Types.NAMED_ENUM; } else { - typeId = metaContextShareEnabled ? Types.NAMED_COMPATIBLE_STRUCT : Types.NAMED_STRUCT; + typeId = buildUnregisteredTypeId(cls, null); } } ClassInfo classInfo = new ClassInfo(this, cls, null, typeId); @@ -701,6 +689,27 @@ private ClassInfo getMetaSharedClassInfo(ClassDef classDef, Class clz) { return classInfo; } + protected int buildUnregisteredTypeId(Class cls, Serializer serializer) { + if (cls.isEnum()) { + return Types.NAMED_ENUM; + } + if (serializer != null && !isStructSerializer(serializer)) { + return Types.NAMED_EXT; + } + if (fory.isCompatible()) { + return Types.NAMED_COMPATIBLE_STRUCT; + } + return Types.NAMED_STRUCT; + } + + protected static boolean isStructSerializer(Serializer serializer) { + return serializer instanceof GeneratedObjectSerializer + || serializer instanceof GeneratedMetaSharedSerializer + || serializer instanceof LazyInitBeanSerializer + || serializer instanceof ObjectSerializer + || serializer instanceof MetaSharedSerializer; + } + protected Tuple2 readClassDef(MemoryBuffer buffer, long header) { ClassDef readClassDef = ClassDef.readClassDef(fory, buffer, header); Tuple2 tuple2 = extRegistry.classIdToDef.get(readClassDef.getId()); @@ -1250,8 +1259,7 @@ public final MetaStringResolver getMetaStringResolver() { } static class ExtRegistry { - // Here we set it to 1 because `NO_CLASS_ID` is 0 to avoid calculating it again in - // `register(Class cls)`. + // Here we set it to 1 to avoid calculating it again in `register(Class cls)`. short classIdGenerator = 1; short userIdGenerator = 0; SerializerFactory serializerFactory; diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java index e5d54f7b25..1742f57c72 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java @@ -493,6 +493,9 @@ public boolean isMonomorphic(Descriptor descriptor) { return true; default: Class rawType = descriptor.getRawType(); + if (rawType == Object.class) { + return false; + } if (rawType.isEnum()) { return true; } @@ -509,6 +512,9 @@ public boolean isMonomorphic(Descriptor descriptor) { @Override public boolean isMonomorphic(Class clz) { + if (clz == Object.class) { + return false; + } if (TypeUtils.unwrap(clz).isPrimitive() || clz.isEnum() || clz == String.class) { return true; } @@ -909,23 +915,6 @@ protected ClassInfo getClassInfoByTypeId(int typeId) { } } - @Override - protected boolean isValidRegisteredTypeId(int typeId) { - // For XtypeResolver, a valid registered type ID means: - // 1. It's a built-in internal type (not UNKNOWN, and not a named type) - // 2. Or it has an entry in xtypeIdToClassMap - if (typeId == NO_CLASS_ID) { - return false; - } - int internalTypeId = typeId & 0xff; - // Named types are not "registered" - they use meta share - if (Types.isNamedType(internalTypeId)) { - return false; - } - // Check if it's in the registry - return xtypeIdToClassMap.containsKey(typeId); - } - private void throwUnexpectTypeIdException(long xtypeId) { throw new IllegalStateException(String.format("Type id %s not registered", xtypeId)); } @@ -1074,6 +1063,9 @@ public DescriptorGrouper createDescriptorGrouper( } private byte getInternalTypeId(Class cls) { + if (cls == Object.class) { + return Types.UNKNOWN; + } if (isSet(cls)) { return Types.SET; } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java index 3662411caf..ca6dbf94eb 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java @@ -33,6 +33,7 @@ import org.apache.fory.memory.Platform; import org.apache.fory.meta.ClassDef; import org.apache.fory.reflect.FieldAccessor; +import org.apache.fory.resolver.MapRefResolver; import org.apache.fory.resolver.RefResolver; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo; @@ -191,7 +192,14 @@ public T read(MemoryBuffer buffer) { Fory fory = this.fory; RefResolver refResolver = this.refResolver; SerializationBinding binding = this.binding; - refResolver.reference(targetObject); + if (refResolver instanceof MapRefResolver) { + MapRefResolver mapRefResolver = (MapRefResolver) refResolver; + if (mapRefResolver.hasPreservedRefId()) { + refResolver.reference(targetObject); + } + } else { + refResolver.reference(targetObject); + } // read order: primitive,boxed,final,other,collection,map for (SerializationFieldInfo fieldInfo : this.buildInFields) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java index 4634d97198..aabf35dfa3 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java @@ -130,14 +130,17 @@ private static int computeVarUint32Size(int value) { } private int resolveTypeId(ClassDef classDef) { - int typeId = classDef.getClassSpec().typeId; - if (typeId >= 0) { - return typeId; - } if (classDef.getClassSpec().isEnum) { - return Types.NAMED_ENUM; + if (classDef.isNamed()) { + return Types.NAMED_ENUM; + } + return (classDef.getUserTypeId() << 8) | Types.ENUM; + } + if (classDef.isNamed()) { + return classDef.isCompatible() ? Types.NAMED_COMPATIBLE_STRUCT : Types.NAMED_STRUCT; } - return Types.NAMED_COMPATIBLE_STRUCT; + int internalTypeId = classDef.isCompatible() ? Types.COMPATIBLE_STRUCT : Types.STRUCT; + return (classDef.getUserTypeId() << 8) | internalTypeId; } @Override diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java index 3d5bd29078..951024df2e 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ReplaceResolveSerializer.java @@ -240,7 +240,8 @@ public ReplaceResolveSerializer( writeClassInfo = null; } else { // FIXME new classinfo may miss serializer update in async compilation mode. - writeClassInfo = classResolver.newClassInfo(type, this, ClassResolver.NO_CLASS_ID); + int typeId = classResolver.getTypeIdForClassDef(type); + writeClassInfo = classResolver.newClassInfo(type, this, typeId); } } else { jdkMethodInfoWriteCache = null; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java index 6f9d237a66..a1716be03a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java @@ -558,6 +558,14 @@ public void write(MemoryBuffer buffer, Object value) {} public Object read(MemoryBuffer buffer) { return new Object(); } + + @Override + public void xwrite(MemoryBuffer buffer, Object value) {} + + @Override + public Object xread(MemoryBuffer buffer) { + return new Object(); + } } public static void registerDefaultSerializers(Fory fory) { diff --git a/java/fory-core/src/main/java/org/apache/fory/util/Utils.java b/java/fory-core/src/main/java/org/apache/fory/util/Utils.java index c3dd9aa8d2..17d910c15a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/util/Utils.java +++ b/java/fory-core/src/main/java/org/apache/fory/util/Utils.java @@ -21,10 +21,9 @@ /** Misc common utils. */ public class Utils { - /** Checks if ENABLE_FORY_DEBUG_OUTPUT env var is set to "1". */ public static final boolean DEBUG_OUTPUT_ENABLED; static { - DEBUG_OUTPUT_ENABLED = true; + DEBUG_OUTPUT_ENABLED = "1".equals(System.getenv("ENABLE_FORY_DEBUG_OUTPUT")); } } diff --git a/java/fory-core/src/test/java/org/apache/fory/resolver/ClassResolverTest.java b/java/fory-core/src/test/java/org/apache/fory/resolver/ClassResolverTest.java index 66620152a7..aacd2c1f0a 100644 --- a/java/fory-core/src/test/java/org/apache/fory/resolver/ClassResolverTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/resolver/ClassResolverTest.java @@ -63,6 +63,7 @@ import org.apache.fory.serializer.collection.MapSerializers; import org.apache.fory.test.bean.BeanB; import org.apache.fory.type.TypeUtils; +import org.apache.fory.type.Types; import org.testng.Assert; import org.testng.annotations.Test; @@ -121,7 +122,7 @@ public void testRegisterClass() { @Test public void testRegisterClassWithUserIds() { - // Test that user IDs 0 and 1 work correctly (mapped to internal IDs 256 and 257) + // Test that user IDs 0 and 1 work correctly with unified type IDs. Fory fory = Fory.builder().withLanguage(Language.JAVA).requireClassRegistration(true).build(); ClassResolver classResolver = fory.getClassResolver(); @@ -130,12 +131,17 @@ public void testRegisterClassWithUserIds() { // Register with user ID 1 classResolver.register(Bar.class, 1); - // Verify internal IDs are offset by USER_ID_BASE (256) - assertEquals( - classResolver.getRegisteredClassId(Foo.class).shortValue(), ClassResolver.USER_ID_BASE); - assertEquals( - classResolver.getRegisteredClassId(Bar.class).shortValue(), - (short) (ClassResolver.USER_ID_BASE + 1)); + // Verify registered IDs are user IDs and type IDs encode user ID and internal type. + assertEquals(classResolver.getRegisteredClassId(Foo.class).shortValue(), (short) 0); + assertEquals(classResolver.getRegisteredClassId(Bar.class).shortValue(), (short) 1); + int fooTypeId = classResolver.getClassInfo(Foo.class).getTypeId(); + int barTypeId = classResolver.getClassInfo(Bar.class).getTypeId(); + assertEquals(fooTypeId >>> 8, 0); + assertEquals(barTypeId >>> 8, 1); + int fooInternalType = fooTypeId & 0xff; + int barInternalType = barTypeId & 0xff; + assertTrue(fooInternalType == Types.STRUCT || fooInternalType == Types.COMPATIBLE_STRUCT); + assertTrue(barInternalType == Types.STRUCT || barInternalType == Types.COMPATIBLE_STRUCT); // Verify serialization/deserialization works Foo foo = new Foo(); diff --git a/python/pyfory/registry.py b/python/pyfory/registry.py index 15bde5b0a5..ed31d3f7f0 100644 --- a/python/pyfory/registry.py +++ b/python/pyfory/registry.py @@ -784,40 +784,6 @@ def read_shared_type_meta(self, buffer): assert meta_context is not None, "Meta context must be set when meta share is enabled" return meta_context.read_shared_typeinfo(buffer) - def write_type_defs(self, buffer): - """Write all type definitions that need to be sent.""" - meta_context = self.fory.serialization_context.meta_context - if meta_context is None: - return - writing_type_defs = meta_context.get_writing_type_defs() - buffer.write_varuint32(len(writing_type_defs)) - for type_def in writing_type_defs: - # Just copy the encoded bytes directly - buffer.write_bytes(type_def.encoded) - - def read_type_defs(self, buffer): - """Read all type definitions from the buffer.""" - meta_context = self.fory.serialization_context.meta_context - if meta_context is None: - return - - num_type_defs = buffer.read_varuint32() - for _ in range(num_type_defs): - # Read the header (first 8 bytes) to get the type ID - header = buffer.read_int64() - # Check if we already have this TypeDef cached - type_info = self._meta_shared_typeinfo.get(header) - if type_info is not None: - # Skip the rest of the TypeDef binary for faster performance - skip_typedef(buffer, header) - else: - # Read the TypeDef and create TypeInfo - type_def = decode_typedef(buffer, self, header=header) - type_info = self._build_type_info_from_typedef(type_def) - # Cache the tuple for future use - self._meta_shared_typeinfo[header] = type_info - meta_context.add_read_typeinfo(type_info) - def _build_type_info_from_typedef(self, type_def): """Build TypeInfo from TypeDef using TypeDef's create_serializer method.""" # Create serializer using TypeDef's create_serializer method diff --git a/python/pyfory/serialization.pyx b/python/pyfory/serialization.pyx index e960d6acb4..56a88d7be9 100644 --- a/python/pyfory/serialization.pyx +++ b/python/pyfory/serialization.pyx @@ -689,14 +689,6 @@ cdef class TypeResolver: typeinfo = meta_context.read_shared_typeinfo(buffer) return typeinfo - cpdef inline write_type_defs(self, Buffer buffer): - """Write all type definitions that need to be sent.""" - self._resolver.write_type_defs(buffer) - - cpdef inline read_type_defs(self, Buffer buffer): - """Read all type definitions from the buffer.""" - self._resolver.read_type_defs(buffer) - cpdef inline _read_and_build_typeinfo(self, Buffer buffer): """Read TypeDef inline from buffer and build TypeInfo.""" return self._resolver._read_and_build_typeinfo(buffer) @@ -734,7 +726,6 @@ cdef class MetaContext: flat_hash_map[uint64_t, int32_t] _c_type_map # Counter for assigning new IDs - list _writing_type_defs list _read_type_infos object fory object type_resolver @@ -742,7 +733,6 @@ cdef class MetaContext: def __cinit__(self, object fory): self.fory = fory self.type_resolver = fory.type_resolver - self._writing_type_defs = [] self._read_type_infos = [] cpdef inline void write_shared_typeinfo(self, Buffer buffer, typeinfo): @@ -772,13 +762,8 @@ cdef class MetaContext: # Write TypeDef bytes inline instead of deferring to end buffer.write_bytes(type_def.encoded) - cpdef inline list get_writing_type_defs(self): - """Get all type definitions that need to be written.""" - return self._writing_type_defs - cpdef inline reset_write(self): """Reset write state.""" - self._writing_type_defs.clear() self._c_type_map.clear() cpdef inline add_read_typeinfo(self, type_info): @@ -818,8 +803,7 @@ cdef class MetaContext: def __repr__(self): return (f"MetaContext(" - f"read_infos={self._read_type_infos}, " - f"writing_defs={self._writing_type_defs})") + f"read_infos={self._read_type_infos})") @cython.final @@ -1226,27 +1210,12 @@ cdef class Fory: set_bit(buffer, mask_index, 2) else: clear_bit(buffer, mask_index, 2) - # Reserve space for type definitions offset, similar to Java implementation - cdef int32_t type_defs_offset_pos = -1 - if self.serialization_context.scoped_meta_share_enabled: - type_defs_offset_pos = buffer.writer_index - buffer.write_int32(-1) # Reserve 4 bytes for type definitions offset - cdef int32_t start_offset if self.language == Language.PYTHON: self.write_ref(buffer, obj) else: self.xwrite_ref(buffer, obj) - # Write type definitions at the end, similar to Java implementation - if self.serialization_context.scoped_meta_share_enabled: - meta_context = self.serialization_context.meta_context - if meta_context is not None and len(meta_context.get_writing_type_defs()) > 0: - # Update the offset to point to current position - current_pos = buffer.writer_index - buffer.put_int32(type_defs_offset_pos, current_pos - type_defs_offset_pos - 4) - self.type_resolver.write_type_defs(buffer) - if buffer is not self.buffer: return buffer else: @@ -1388,32 +1357,11 @@ cdef class Fory: "produced with buffer_callback null." ) - # Read type definitions at the start, similar to Java implementation - cdef int32_t end_reader_index = -1 - if self.serialization_context.scoped_meta_share_enabled: - relative_type_defs_offset = buffer.read_int32() - if relative_type_defs_offset != -1: - # Save current reader position - current_reader_index = buffer.reader_index - # Jump to type definitions - buffer.reader_index = current_reader_index + relative_type_defs_offset - # Read type definitions - self.type_resolver.read_type_defs(buffer) - # Save the end position (after type defs) - this is the true end of serialized data - end_reader_index = buffer.reader_index - # Jump back to continue with object deserialization - buffer.reader_index = current_reader_index - if not is_target_x_lang: obj = self.read_ref(buffer) else: obj = self.xread_ref(buffer) - # After reading the object, position buffer at the end of serialized data - # (which is after the type definitions, not after the object data) - if end_reader_index != -1: - buffer.reader_index = end_reader_index - return obj cpdef inline read_ref(self, Buffer buffer): From 2efc29b1ec4e50d346a2b7b3bf1acc53572183e2 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Mon, 19 Jan 2026 01:03:16 +0800 Subject: [PATCH 27/44] fix(java): align codec builder with field types --- .../org/apache/fory/builder/BaseObjectCodecBuilder.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java index 8dd67b5c82..86d102c983 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java @@ -120,6 +120,7 @@ import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.RefResolver; import org.apache.fory.resolver.TypeResolver; +import org.apache.fory.serializer.DeferedLazySerializer.DeferredLazyObjectSerializer; import org.apache.fory.serializer.EnumSerializer; import org.apache.fory.serializer.FinalFieldReplaceResolveSerializer; import org.apache.fory.serializer.MetaSharedSerializer; @@ -793,7 +794,8 @@ private Expression getOrCreateSerializer(Class cls, boolean isField) { } if (serializerClass == LazyInitBeanSerializer.class || serializerClass == ObjectSerializer.class - || serializerClass == MetaSharedSerializer.class) { + || serializerClass == MetaSharedSerializer.class + || serializerClass == DeferredLazyObjectSerializer.class) { // field init may get jit serializer, which will cause cast exception if not use base // type. serializerClass = Serializer.class; @@ -1732,7 +1734,8 @@ protected Expression deserializeForNullableField( boolean nullable) { TypeRef typeRef = descriptor.getTypeRef(); if (typeResolver(r -> r.needToWriteRef(typeRef))) { - return readRef(buffer, callback, () -> deserializeForNotNull(buffer, typeRef, null)); + return readRef( + buffer, callback, () -> deserializeForNotNullForField(buffer, descriptor, null)); } else { if (typeRef.isPrimitive() && !nullable) { // Only skip null check if BOTH: local type is primitive AND sender didn't write null flag From d1fe4ebc7cc72fb56d3b89cfe2c24cb6c0474f67 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Mon, 19 Jan 2026 01:31:33 +0800 Subject: [PATCH 28/44] fix(java): align compatible meta share and xlang nullability --- .../java/org/apache/fory/meta/FieldTypes.java | 8 ++-- .../apache/fory/resolver/ClassResolver.java | 8 ++-- .../apache/fory/resolver/TypeResolver.java | 45 ++++++++++--------- 3 files changed, 33 insertions(+), 28 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java index 7f5b908818..d1cc6c1cae 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java @@ -124,12 +124,14 @@ private static FieldType buildFieldType( // For xlang: ref tracking is false by default (no shared ownership like Rust's Rc/Arc) // For native: use the type's default tracking behavior boolean trackingRef = !isXlang && genericType.trackingRef(resolver); - // For xlang: nullable is false by default (aligned with all languages). - // Exception: Optional types are nullable (like Rust's Option). + // For xlang: nullable is false by default for top-level fields. + // Nested element types are nullable by default to align with cross-language collection semantics. + // Optional types are nullable (like Rust's Option). // For native: non-primitive types are nullable by default. boolean nullable; if (isXlang) { - nullable = isOptionalType(rawType); + boolean nestedType = field == null; + nullable = nestedType || isOptionalType(rawType); } else { // Primitives are never nullable, non-primitives are nullable by default // This applies to both top-level fields and nested types (in arrays, collections, maps) diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 92e008f7a3..4e45ee5292 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -694,6 +694,10 @@ public Class getRegisteredClass(short id) { return null; } + public Class getRegisteredClass(String className) { + return extRegistry.registeredClasses.get(className); + } + public Class getRegisteredClassByTypeId(int typeId) { ClassInfo classInfo = getRegisteredClassInfoByTypeId(typeId); return classInfo == null ? null : classInfo.cls; @@ -721,10 +725,6 @@ public ClassInfo getRegisteredClassInfoByTypeId(int typeId) { return registeredId2ClassInfo[typeId]; } - public Class getRegisteredClass(String className) { - return extRegistry.registeredClasses.get(className); - } - public List> getRegisteredClasses() { List> classes = Arrays.stream(registeredId2ClassInfo) diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index 370d1a3be4..92400232c4 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -260,7 +260,8 @@ public final boolean needToWriteClassDef(Serializer serializer) { * *

      *
    • NAMED_ENUM/NAMED_STRUCT/NAMED_EXT: namespace + typename bytes (or meta-share if enabled) - *
    • NAMED_COMPATIBLE_STRUCT/COMPATIBLE_STRUCT: always meta-share + *
    • NAMED_COMPATIBLE_STRUCT: namespace + typename bytes (or meta-share if enabled) + *
    • COMPATIBLE_STRUCT: meta-share when enabled, otherwise only type ID *
    • Other types: just the type ID *
    */ @@ -273,6 +274,7 @@ public final void writeClassInfo(MemoryBuffer buffer, ClassInfo classInfo) { case Types.NAMED_ENUM: case Types.NAMED_STRUCT: case Types.NAMED_EXT: + case Types.NAMED_COMPATIBLE_STRUCT: if (metaContextShareEnabled) { writeSharedClassMeta(buffer, classInfo); } else { @@ -282,11 +284,8 @@ public final void writeClassInfo(MemoryBuffer buffer, ClassInfo classInfo) { metaStringResolver.writeMetaStringBytes(buffer, classInfo.typeNameBytes); } break; - case Types.NAMED_COMPATIBLE_STRUCT: case Types.COMPATIBLE_STRUCT: - Preconditions.checkArgument( - metaContextShareEnabled, "Meta share must be enabled for compatible mode"); - if (classInfo.cls != NonexistentMetaShared.class) { + if (metaContextShareEnabled && classInfo.cls != NonexistentMetaShared.class) { writeSharedClassMeta(buffer, classInfo); } break; @@ -354,17 +353,18 @@ public final ClassInfo readClassInfo(MemoryBuffer buffer) { case Types.NAMED_ENUM: case Types.NAMED_STRUCT: case Types.NAMED_EXT: + case Types.NAMED_COMPATIBLE_STRUCT: if (metaContextShareEnabled) { return readSharedClassMeta(buffer); } ClassInfo classInfo = readClassInfoFromBytes(buffer, classInfoCache, header); classInfoCache = classInfo; return classInfo; - case Types.NAMED_COMPATIBLE_STRUCT: case Types.COMPATIBLE_STRUCT: - Preconditions.checkArgument( - metaContextShareEnabled, "Meta share must be enabled for compatible mode"); - return readSharedClassMeta(buffer); + if (metaContextShareEnabled) { + return readSharedClassMeta(buffer); + } + return getClassInfoByTypeId(header); default: return getClassInfoByTypeId(header); } @@ -381,17 +381,18 @@ public final ClassInfo readClassInfo(MemoryBuffer buffer, Class targetClass) case Types.NAMED_ENUM: case Types.NAMED_STRUCT: case Types.NAMED_EXT: + case Types.NAMED_COMPATIBLE_STRUCT: if (metaContextShareEnabled) { return readSharedClassMeta(buffer, targetClass); } ClassInfo classInfo = readClassInfoFromBytes(buffer, classInfoCache, header); classInfoCache = classInfo; return classInfo; - case Types.NAMED_COMPATIBLE_STRUCT: case Types.COMPATIBLE_STRUCT: - Preconditions.checkArgument( - metaContextShareEnabled, "Meta share must be enabled for compatible mode"); - return readSharedClassMeta(buffer, targetClass); + if (metaContextShareEnabled) { + return readSharedClassMeta(buffer, targetClass); + } + return getClassInfoByTypeId(header); default: return getClassInfoByTypeId(header); } @@ -414,15 +415,16 @@ public final ClassInfo readClassInfo(MemoryBuffer buffer, ClassInfo classInfoCac case Types.NAMED_ENUM: case Types.NAMED_STRUCT: case Types.NAMED_EXT: + case Types.NAMED_COMPATIBLE_STRUCT: if (metaContextShareEnabled) { return readSharedClassMeta(buffer); } return readClassInfoByCache(buffer, classInfoCache, header); - case Types.NAMED_COMPATIBLE_STRUCT: case Types.COMPATIBLE_STRUCT: - Preconditions.checkArgument( - metaContextShareEnabled, "Meta share must be enabled for compatible mode"); - return readSharedClassMeta(buffer); + if (metaContextShareEnabled) { + return readSharedClassMeta(buffer); + } + return getClassInfoByTypeId(header); default: return getClassInfoByTypeId(header); } @@ -444,15 +446,16 @@ public final ClassInfo readClassInfo(MemoryBuffer buffer, ClassInfoHolder classI case Types.NAMED_ENUM: case Types.NAMED_STRUCT: case Types.NAMED_EXT: + case Types.NAMED_COMPATIBLE_STRUCT: if (metaContextShareEnabled) { return readSharedClassMeta(buffer); } return readClassInfoFromBytes(buffer, classInfoHolder, header); - case Types.NAMED_COMPATIBLE_STRUCT: case Types.COMPATIBLE_STRUCT: - Preconditions.checkArgument( - metaContextShareEnabled, "Meta share must be enabled for compatible mode"); - return readSharedClassMeta(buffer); + if (metaContextShareEnabled) { + return readSharedClassMeta(buffer); + } + return getClassInfoByTypeId(header); default: return getClassInfoByTypeId(header); } From 56c0ba5ab0ec3ecb3e90d100dc3b8d6fb49c47d3 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Mon, 19 Jan 2026 01:40:31 +0800 Subject: [PATCH 29/44] style(java): apply spotless --- .../java/org/apache/fory/meta/FieldTypes.java | 3 ++- .../apache/fory/resolver/ClassResolver.java | 21 +++++++------------ 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java index d1cc6c1cae..ef9af6e54c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/FieldTypes.java @@ -125,7 +125,8 @@ private static FieldType buildFieldType( // For native: use the type's default tracking behavior boolean trackingRef = !isXlang && genericType.trackingRef(resolver); // For xlang: nullable is false by default for top-level fields. - // Nested element types are nullable by default to align with cross-language collection semantics. + // Nested element types are nullable by default to align with cross-language collection + // semantics. // Optional types are nullable (like Rust's Option). // For native: non-primitive types are nullable by default. boolean nullable; diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 4e45ee5292..25ade648c6 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -98,7 +98,6 @@ import org.apache.fory.serializer.CodegenSerializer.LazyInitBeanSerializer; import org.apache.fory.serializer.EnumSerializer; import org.apache.fory.serializer.ExternalizableSerializer; -import org.apache.fory.serializer.FinalFieldReplaceResolveSerializer; import org.apache.fory.serializer.ForyCopyableSerializer; import org.apache.fory.serializer.JavaSerializer; import org.apache.fory.serializer.JdkProxySerializer; @@ -420,8 +419,8 @@ public void register(String className, int classId) { /** * Registers a class with a user-specified ID. * - *

    The ID is in the user ID space, starting from 0. The unified type ID is encoded as - * {@code (userId << 8) | internalTypeId}. + *

    The ID is in the user ID space, starting from 0. The unified type ID is encoded as {@code + * (userId << 8) | internalTypeId}. * *

    Example: * @@ -465,7 +464,8 @@ public void register(Class cls, String namespace, String name) { metaStringResolver.getOrCreateMetaStringBytes(encodePackage(namespace)); MetaStringBytes nameBytes = metaStringResolver.getOrCreateMetaStringBytes(encodeTypeName(name)); ClassInfo existingInfo = classInfoMap.get(cls); - int typeId = buildUnregisteredTypeId(cls, existingInfo == null ? null : existingInfo.serializer); + int typeId = + buildUnregisteredTypeId(cls, existingInfo == null ? null : existingInfo.serializer); ClassInfo classInfo = new ClassInfo(cls, fullNameBytes, nsBytes, nameBytes, false, null, typeId); classInfoMap.put(cls, classInfo); @@ -925,14 +925,11 @@ public void registerInternalSerializer(Class type, Serializer serializer) if (classId != null && !isInternalRegisteredClassId(type, classId)) { throw new IllegalArgumentException( String.format( - "Class %s is not registered with an internal id (< %d).", - type, INTERNAL_ID_LIMIT)); + "Class %s is not registered with an internal id (< %d).", type, INTERNAL_ID_LIMIT)); } if (classId != null) { Preconditions.checkArgument( - classId >= 0 && classId < INTERNAL_ID_LIMIT, - "Internal type id overflow: %s", - classId); + classId >= 0 && classId < INTERNAL_ID_LIMIT, "Internal type id overflow: %s", classId); } if (classId == null) { registerInternal(type); @@ -1671,8 +1668,7 @@ public void writeClassInternal(MemoryBuffer buffer, Class cls) { public void writeClassInternal(MemoryBuffer buffer, ClassInfo classInfo) { int typeId = classInfo.typeId; int internalTypeId = typeId & 0xff; - boolean writeById = - typeId != REPLACE_STUB_ID && !Types.isNamedType(internalTypeId); + boolean writeById = typeId != REPLACE_STUB_ID && !Types.isNamedType(internalTypeId); if (writeById) { buffer.writeVarUint32(typeId << 1); } else { @@ -1807,8 +1803,7 @@ private ClassInfo populateBytesToClassInfo( public Class loadClassForMeta(String className, boolean isEnum, int arrayDims) { String pkg = ReflectionUtils.getPackage(className); String typeName = ReflectionUtils.getClassNameWithoutPackage(className); - MetaStringBytes pkgBytes = - metaStringResolver.getOrCreateMetaStringBytes(encodePackage(pkg)); + MetaStringBytes pkgBytes = metaStringResolver.getOrCreateMetaStringBytes(encodePackage(pkg)); MetaStringBytes typeBytes = metaStringResolver.getOrCreateMetaStringBytes(encodeTypeName(typeName)); ClassInfo cachedInfo = From 62deeb0e69391a4b3cacce669bb5816598e7b251 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Mon, 19 Jan 2026 09:51:02 +0800 Subject: [PATCH 30/44] fix(go): align compatible read order with remote schema --- go/fory/struct.go | 48 ++++++++++++++++++++++++++++++++--------------- go/fory/types.go | 11 +++++++++++ 2 files changed, 44 insertions(+), 15 deletions(-) diff --git a/go/fory/struct.go b/go/fory/struct.go index 4a56888398..e9a82bc1fc 100644 --- a/go/fory/struct.go +++ b/go/fory/struct.go @@ -684,9 +684,9 @@ func (s *structSerializer) initFieldsFromTypeDef(typeResolver *TypeResolver) err var dispatchId DispatchId localKind := fieldType.Kind() localIsPtr := localKind == reflect.Ptr - localIsNumeric := isNumericKind(localKind) || (localIsPtr && isNumericKind(fieldType.Elem().Kind())) + localIsPrimitive := isPrimitiveDispatchKind(localKind) || (localIsPtr && isPrimitiveDispatchKind(fieldType.Elem().Kind())) - if localIsNumeric { + if localIsPrimitive { if localIsPtr { if def.nullable { // Local is *T, remote is nullable - use nullable DispatchId @@ -2350,14 +2350,10 @@ func (s *structSerializer) readFieldsInOrder(ctx *ReadContext, value reflect.Val buf := ctx.Buffer() ptr := unsafe.Pointer(value.UnsafeAddr()) err := ctx.Err() - for i := range s.fields { - field := &s.fields[i] + readField := func(field *FieldInfo) { if field.Meta.FieldIndex < 0 { s.skipField(ctx, field) - if ctx.HasError() { - return - } - continue + return } // Fast path for fixed-size primitive types (no ref flag from remote schema) @@ -2433,7 +2429,7 @@ func (s *structSerializer) readFieldsInOrder(ctx *ReadContext, value reflect.Val *v = buf.ReadFloat64(err) *(**float64)(fieldPtr) = v } - continue + return } // Fast path for varint primitive types (no ref flag from remote schema) @@ -2491,7 +2487,7 @@ func (s *structSerializer) readFieldsInOrder(ctx *ReadContext, value reflect.Val *v = uint(buf.ReadVaruint64(err)) *(**uint)(fieldPtr) = v } - continue + return } // Get field value for nullable primitives and non-primitives @@ -2503,7 +2499,7 @@ func (s *structSerializer) readFieldsInOrder(ctx *ReadContext, value reflect.Val refFlag := buf.ReadInt8(err) if refFlag == NullFlag { // Leave pointer as nil (or zero for non-pointer local types) - continue + return } // Read fixed-size value based on dispatch ID // Handle both pointer and non-pointer local field types (schema evolution) @@ -2586,7 +2582,7 @@ func (s *structSerializer) readFieldsInOrder(ctx *ReadContext, value reflect.Val fieldValue.SetFloat(v) } } - continue + return } // Handle nullable varint primitives (read ref flag + varint) @@ -2594,7 +2590,7 @@ func (s *structSerializer) readFieldsInOrder(ctx *ReadContext, value reflect.Val refFlag := buf.ReadInt8(err) if refFlag == NullFlag { // Leave pointer as nil (or zero for non-pointer local types) - continue + return } // Read varint value based on dispatch ID // Handle both pointer and non-pointer local field types (schema evolution) @@ -2656,11 +2652,11 @@ func (s *structSerializer) readFieldsInOrder(ctx *ReadContext, value reflect.Val fieldValue.SetUint(uint64(v)) } } - continue + return } if isEnumField(field) { readEnumField(ctx, field, fieldValue) - continue + return } // Slow path for non-primitives (all need ref flag per xlang spec) @@ -2671,6 +2667,28 @@ func (s *structSerializer) readFieldsInOrder(ctx *ReadContext, value reflect.Val ctx.ReadValue(fieldValue, RefModeTracking, true) } } + + for i := range s.fieldGroup.FixedFields { + field := &s.fieldGroup.FixedFields[i] + readField(field) + if ctx.HasError() { + return + } + } + for i := range s.fieldGroup.VarintFields { + field := &s.fieldGroup.VarintFields[i] + readField(field) + if ctx.HasError() { + return + } + } + for i := range s.fieldGroup.RemainingFields { + field := &s.fieldGroup.RemainingFields[i] + readField(field) + if ctx.HasError() { + return + } + } } // skipField skips a field that doesn't exist or is incompatible diff --git a/go/fory/types.go b/go/fory/types.go index 0da3b7dfc0..9205ec4760 100644 --- a/go/fory/types.go +++ b/go/fory/types.go @@ -593,6 +593,17 @@ func isNumericKind(kind reflect.Kind) bool { } } +func isPrimitiveDispatchKind(kind reflect.Kind) bool { + switch kind { + case reflect.Bool, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, + reflect.Float32, reflect.Float64: + return true + default: + return false + } +} + // getDispatchIdFromTypeId converts a TypeId to a DispatchId based on nullability. // This follows Java's DispatchId.xlangTypeIdToDispatchId pattern. func getDispatchIdFromTypeId(typeId TypeId, nullable bool) DispatchId { From 774bcfa48e60b03499a6eaca09f32ee084f17c97 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Mon, 19 Jan 2026 11:07:26 +0800 Subject: [PATCH 31/44] fix(java,kotlin): align xlang boxed nullability --- .../main/java/org/apache/fory/resolver/TypeResolver.java | 3 --- .../apache/fory/serializer/kotlin/UnsignedSerializer.kt | 8 ++++---- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index 92400232c4..6502235362 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -1108,9 +1108,6 @@ private boolean isFieldNullable(Descriptor descriptor) { // Use explicit annotation value return foryField.nullable(); } - if (TypeUtils.isBoxed(rawType)) { - return true; - } // Default for xlang: false for all non-primitives, except Optional types return TypeUtils.isOptionalType(rawType); } diff --git a/kotlin/src/main/kotlin/org/apache/fory/serializer/kotlin/UnsignedSerializer.kt b/kotlin/src/main/kotlin/org/apache/fory/serializer/kotlin/UnsignedSerializer.kt index e30f4ec591..63cf2f6e42 100644 --- a/kotlin/src/main/kotlin/org/apache/fory/serializer/kotlin/UnsignedSerializer.kt +++ b/kotlin/src/main/kotlin/org/apache/fory/serializer/kotlin/UnsignedSerializer.kt @@ -34,7 +34,7 @@ public class UByteSerializer( Serializers.CrossLanguageCompatibleSerializer( fory, UByte::class.java, - fory.isBasicTypesRefIgnored, + false, true ) { @@ -58,7 +58,7 @@ public class UShortSerializer( Serializers.CrossLanguageCompatibleSerializer( fory, UShort::class.java, - fory.isBasicTypesRefIgnored, + false, true ) { override fun write(buffer: MemoryBuffer, value: UShort) { @@ -81,7 +81,7 @@ public class UIntSerializer( Serializers.CrossLanguageCompatibleSerializer( fory, UInt::class.java, - fory.isBasicTypesRefIgnored, + false, true ) { @@ -105,7 +105,7 @@ public class ULongSerializer( Serializers.CrossLanguageCompatibleSerializer( fory, ULong::class.java, - fory.isBasicTypesRefIgnored, + false, true ) { override fun write(buffer: MemoryBuffer, value: ULong) { From c430c774782f8309fd30a6ab46e64bc3c0118432 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Mon, 19 Jan 2026 11:08:19 +0800 Subject: [PATCH 32/44] fix ci failure --- .github/workflows/ci.yml | 4 ++++ AGENTS.md | 1 + .../org.apache.fory/fory-core/native-image.properties | 1 + 3 files changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cc37b856c9..4c818f228f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -324,6 +324,7 @@ jobs: - name: Run Rust Xlang Test env: FORY_RUST_JAVA_CI: "1" + ENABLE_FORY_DEBUG_OUTPUT: "1" run: | cd java mvn -T16 --no-transfer-progress clean install -DskipTests @@ -410,6 +411,7 @@ jobs: - name: Run CPP Xlang Test env: FORY_CPP_JAVA_CI: "1" + ENABLE_FORY_DEBUG_OUTPUT: "1" run: | cd java mvn -T16 --no-transfer-progress clean install -DskipTests @@ -531,6 +533,7 @@ jobs: - name: Run Python Xlang Test env: FORY_PYTHON_JAVA_CI: "1" + ENABLE_FORY_DEBUG_OUTPUT: "1" run: | cd java mvn -T16 --no-transfer-progress clean install -DskipTests @@ -599,6 +602,7 @@ jobs: - name: Run Go Xlang Test env: FORY_GO_JAVA_CI: "1" + ENABLE_FORY_DEBUG_OUTPUT: "1" run: | cd java mvn -T16 --no-transfer-progress clean install -DskipTests diff --git a/AGENTS.md b/AGENTS.md index f93c714021..e8090ded2c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,6 +15,7 @@ While working on Fory, please remember: - **Graalvm Support using fory codegen**: For graalvm, please use `fory codegen` to generate the serializer when building graalvm native image, do not use graallvm reflect-related configuration unless for JDK `proxy`. - **Xlang Type System**: Java `native mode(xlang=false)` shares same type systems between type id from `Types.BOOL~Types.STRING` with `xlang mode(xlang=true)`, but for other types, java `native mode` has different type ids. - **Remote git repository**: `git@github.com:apache/fory.git` is remote repository, do not use other remote repository when you want to check code under `main` branch. +- **Contributor git repository**: A contributor should fork the `git@github.com:apache/fory.git` repo, and git push the code changes into their forked repo, then create a pull request from the branch in their forked repo into `git@github.com:apache/fory.git`. ## Build and Development Commands diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties index 4d4dbc324b..39cc9317f0 100644 --- a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties @@ -284,6 +284,7 @@ Args=--initialize-at-build-time=org.apache.fory.memory.MemoryBuffer,\ org.apache.fory.resolver.ClassResolver$2,\ org.apache.fory.resolver.TypeResolver$ExtRegistry,\ org.apache.fory.resolver.TypeResolver$GraalvmClassRegistry,\ + org.apache.fory.resolver.TypeResolver$1,\ org.apache.fory.resolver.ClassResolver,\ org.apache.fory.resolver.XtypeResolver,\ org.apache.fory.resolver.DisallowedList,\ From f1390ad220f41c43317934010f693794cf058eb1 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Mon, 19 Jan 2026 11:41:14 +0800 Subject: [PATCH 33/44] fix(java,scala): stabilize registered ext type ids --- AGENTS.md | 2 +- .../java/org/apache/fory/resolver/ClassResolver.java | 10 +++++++++- .../apache/fory/serializer/scala/ScalaSerializers.java | 5 +++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e8090ded2c..6e7f83f04d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,7 +15,7 @@ While working on Fory, please remember: - **Graalvm Support using fory codegen**: For graalvm, please use `fory codegen` to generate the serializer when building graalvm native image, do not use graallvm reflect-related configuration unless for JDK `proxy`. - **Xlang Type System**: Java `native mode(xlang=false)` shares same type systems between type id from `Types.BOOL~Types.STRING` with `xlang mode(xlang=true)`, but for other types, java `native mode` has different type ids. - **Remote git repository**: `git@github.com:apache/fory.git` is remote repository, do not use other remote repository when you want to check code under `main` branch. -- **Contributor git repository**: A contributor should fork the `git@github.com:apache/fory.git` repo, and git push the code changes into their forked repo, then create a pull request from the branch in their forked repo into `git@github.com:apache/fory.git`. +- **Contributor git repository**: A contributor should fork the `git@github.com:apache/fory.git` repo, and git push the code changes into their forked repo, then create a pull request from the branch in their forked repo into `git@github.com:apache/fory.git`. ## Build and Development Commands diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 25ade648c6..42b85538bb 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -714,9 +714,17 @@ public ClassInfo getRegisteredClassInfoByTypeId(int typeId) { return null; } ClassInfo classInfo = userRegisteredId2ClassInfo[userId]; - if (classInfo != null && (classInfo.typeId & 0xff) != internalTypeId) { + if (classInfo == null) { return null; } + int existingInternalTypeId = classInfo.typeId & 0xff; + if (existingInternalTypeId != internalTypeId) { + if (classInfo.serializer == null) { + classInfo.typeId = typeId; + } else { + return null; + } + } return classInfo; } if (typeId < 0 || typeId >= registeredId2ClassInfo.length) { diff --git a/scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java b/scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java index 8ac7b92811..09652446c3 100644 --- a/scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java +++ b/scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java @@ -114,6 +114,11 @@ public static void registerSerializers(Fory fory) { // Range resolver.register("scala.math.Numeric$IntIsIntegral$"); resolver.register("scala.math.Numeric$LongIsIntegral$"); + resolver.register(Range.Inclusive.class); + resolver.register(Range.Exclusive.class); + resolver.register(NumericRange.class); + resolver.register(NumericRange.Exclusive.class); + resolver.register(NumericRange.Inclusive.class); resolver.registerSerializer(Range.Inclusive.class, new RangeSerializer(fory, Range.Inclusive.class)); resolver.registerSerializer(Range.Exclusive.class, new RangeSerializer(fory, Range.Exclusive.class)); resolver.registerSerializer(NumericRange.class, new NumericRangeSerializer<>(fory, NumericRange.class)); From 133b7d6e7f9830c7915e0b443b777f6de37e3969 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Mon, 19 Jan 2026 11:58:59 +0800 Subject: [PATCH 34/44] add registerSerializerAndType API --- .../apache/fory/AbstractThreadSafeFory.java | 17 ++++++++++ .../main/java/org/apache/fory/BaseFory.java | 33 +++++++++++++++++++ .../src/main/java/org/apache/fory/Fory.java | 17 ++++++++++ .../apache/fory/resolver/TypeResolver.java | 28 ++++++++++++++++ .../serializer/scala/ScalaSerializers.java | 24 +++++++------- 5 files changed, 107 insertions(+), 12 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java b/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java index 820b98a93d..ed26ff0f3e 100644 --- a/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/AbstractThreadSafeFory.java @@ -87,6 +87,23 @@ public void registerSerializer(Class type, Function> seri registerCallback(fory -> fory.registerSerializer(type, serializerCreator.apply(fory))); } + @Override + public void registerSerializerAndType( + Class type, Class serializerClass) { + registerCallback(fory -> fory.registerSerializerAndType(type, serializerClass)); + } + + @Override + public void registerSerializerAndType(Class type, Serializer serializer) { + registerCallback(fory -> fory.registerSerializerAndType(type, serializer)); + } + + @Override + public void registerSerializerAndType( + Class type, Function> serializerCreator) { + registerCallback(fory -> fory.registerSerializerAndType(type, serializerCreator)); + } + @Override public void setSerializerFactory(SerializerFactory serializerFactory) { registerCallback(fory -> fory.setSerializerFactory(serializerFactory)); diff --git a/java/fory-core/src/main/java/org/apache/fory/BaseFory.java b/java/fory-core/src/main/java/org/apache/fory/BaseFory.java index b5150d9e9b..6a3f04e861 100644 --- a/java/fory-core/src/main/java/org/apache/fory/BaseFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/BaseFory.java @@ -138,6 +138,39 @@ public interface BaseFory { */ void registerSerializer(Class type, Function> serializerCreator); + /** + * Register a class (if not already registered) and then register its serializer class. + * + *

    NOTE: The registration order is important. If registration order is inconsistent, the + * allocated ID will be different, and the deserialization will failed !!! + * + * @param type class needed to be serialized/deserialized. + * @param serializerClass serializer class can be created with {@link Serializers#newSerializer}. + * @param type of class. + */ + void registerSerializerAndType(Class type, Class serializerClass); + + /** + * Register a class (if not already registered) and then register its serializer instance. + * + *

    NOTE: The registration order is important. If registration order is inconsistent, the + * allocated ID will be different, and the deserialization will failed !!! + */ + void registerSerializerAndType(Class type, Serializer serializer); + + /** + * Register a class (if not already registered) and then register a serializer created by + * serializerCreator when fory created. + * + *

    NOTE: The registration order is important. If registration order is inconsistent, the + * allocated ID will be different, and the deserialization will failed !!! + * + * @param type class needed to be serialized/deserialized. + * @param serializerCreator serializer creator with param {@link Fory} + */ + void registerSerializerAndType( + Class type, Function> serializerCreator); + void setSerializerFactory(SerializerFactory serializerFactory); /** diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java index 431ae9142e..ed9ac2a716 100644 --- a/java/fory-core/src/main/java/org/apache/fory/Fory.java +++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java @@ -238,6 +238,23 @@ public void registerSerializer(Class type, Function> seri _getTypeResolver().registerSerializer(type, serializerCreator.apply(this)); } + @Override + public void registerSerializerAndType( + Class type, Class serializerClass) { + _getTypeResolver().registerSerializerAndType(type, serializerClass); + } + + @Override + public void registerSerializerAndType(Class type, Serializer serializer) { + _getTypeResolver().registerSerializerAndType(type, serializer); + } + + @Override + public void registerSerializerAndType( + Class type, Function> serializerCreator) { + _getTypeResolver().registerSerializerAndType(type, serializerCreator.apply(this)); + } + @Override public void setSerializerFactory(SerializerFactory serializerFactory) { classResolver.setSerializerFactory(serializerFactory); diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index 6502235362..9170cf0458 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -194,6 +194,34 @@ public abstract void registerSerializer( */ public abstract void registerInternalSerializer(Class type, Serializer serializer); + /** + * Registers a type (if not already registered) and then registers the serializer class. + * + * @param type the class to register + * @param serializerClass the serializer class (will be instantiated by Fory) + * @param type of class + */ + public void registerSerializerAndType( + Class type, Class serializerClass) { + if (!isRegistered(type)) { + register(type); + } + registerSerializer(type, serializerClass); + } + + /** + * Registers a type (if not already registered) and then registers the serializer instance. + * + * @param type the class to register + * @param serializer the serializer instance to use + */ + public void registerSerializerAndType(Class type, Serializer serializer) { + if (!isRegistered(type)) { + register(type); + } + registerSerializer(type, serializer); + } + /** * Whether to track reference for this type. If false, reference tracing of subclasses may be * ignored too. diff --git a/scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java b/scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java index 09652446c3..f3554edd15 100644 --- a/scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java +++ b/scala/src/main/java/org/apache/fory/serializer/scala/ScalaSerializers.java @@ -114,18 +114,18 @@ public static void registerSerializers(Fory fory) { // Range resolver.register("scala.math.Numeric$IntIsIntegral$"); resolver.register("scala.math.Numeric$LongIsIntegral$"); - resolver.register(Range.Inclusive.class); - resolver.register(Range.Exclusive.class); - resolver.register(NumericRange.class); - resolver.register(NumericRange.Exclusive.class); - resolver.register(NumericRange.Inclusive.class); - resolver.registerSerializer(Range.Inclusive.class, new RangeSerializer(fory, Range.Inclusive.class)); - resolver.registerSerializer(Range.Exclusive.class, new RangeSerializer(fory, Range.Exclusive.class)); - resolver.registerSerializer(NumericRange.class, new NumericRangeSerializer<>(fory, NumericRange.class)); - resolver.registerSerializer(NumericRange.Exclusive.class, - new NumericRangeSerializer<>(fory, NumericRange.Exclusive.class)); - resolver.registerSerializer(NumericRange.Inclusive.class, - new NumericRangeSerializer<>(fory, NumericRange.Inclusive.class)); + resolver.registerSerializerAndType( + Range.Inclusive.class, new RangeSerializer(fory, Range.Inclusive.class)); + resolver.registerSerializerAndType( + Range.Exclusive.class, new RangeSerializer(fory, Range.Exclusive.class)); + resolver.registerSerializerAndType( + NumericRange.class, new NumericRangeSerializer<>(fory, NumericRange.class)); + resolver.registerSerializerAndType( + NumericRange.Exclusive.class, + new NumericRangeSerializer<>(fory, NumericRange.Exclusive.class)); + resolver.registerSerializerAndType( + NumericRange.Inclusive.class, + new NumericRangeSerializer<>(fory, NumericRange.Inclusive.class)); resolver.register(scala.collection.generic.SerializeEnd$.class); resolver.register(scala.collection.generic.DefaultSerializationProxy.class); From 57a1bf687832fa2a7675c368fac969b61a761e15 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Mon, 19 Jan 2026 12:50:41 +0800 Subject: [PATCH 35/44] refactor(java): unify resolver type id maps --- .../apache/fory/resolver/ClassResolver.java | 129 ++++--------- .../apache/fory/resolver/TypeResolver.java | 170 +++++++++++------- .../apache/fory/resolver/XtypeResolver.java | 61 +++---- 3 files changed, 156 insertions(+), 204 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 42b85538bb..160018400c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -71,7 +71,6 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import java.util.function.Supplier; -import java.util.stream.Collectors; import org.apache.fory.Fory; import org.apache.fory.ForyCopyable; import org.apache.fory.annotation.CodegenInvoke; @@ -207,8 +206,6 @@ public class ClassResolver extends TypeResolver { public static final int NONEXISTENT_META_SHARED_ID = REPLACE_STUB_ID + 1; private final Fory fory; - private ClassInfo[] registeredId2ClassInfo = new ClassInfo[] {}; - private ClassInfo[] userRegisteredId2ClassInfo = new ClassInfo[] {}; private ClassInfo classInfoCache; // Every deserialization for unregistered class will query it, performance is important. private final ObjectMap compositeNameBytes2ClassInfo = @@ -385,8 +382,7 @@ private void registerDefaultClasses() { @Override public void register(Class cls) { if (!extRegistry.registeredClassIdMap.containsKey(cls)) { - while (extRegistry.userIdGenerator < userRegisteredId2ClassInfo.length - && userRegisteredId2ClassInfo[extRegistry.userIdGenerator] != null) { + while (containsUserTypeId(extRegistry.userIdGenerator)) { extRegistry.userIdGenerator++; } register(cls, extRegistry.userIdGenerator); @@ -503,8 +499,8 @@ public void registerInternal(Class cls) { extRegistry.classIdGenerator < INTERNAL_ID_LIMIT, "Internal type id overflow: %s", extRegistry.classIdGenerator); - while (extRegistry.classIdGenerator < registeredId2ClassInfo.length - && registeredId2ClassInfo[extRegistry.classIdGenerator] != null) { + while (extRegistry.classIdGenerator < typeIdToClassInfo.length + && typeIdToClassInfo[extRegistry.classIdGenerator] != null) { extRegistry.classIdGenerator++; } Preconditions.checkArgument( @@ -538,11 +534,6 @@ private void registerInternalImpl(Class cls, int classId) { short id = (short) classId; checkRegistration(cls, id, cls.getName(), true); extRegistry.registeredClassIdMap.put(cls, id); - if (registeredId2ClassInfo.length <= id) { - ClassInfo[] tmp = new ClassInfo[(id + 1) * 2]; - System.arraycopy(registeredId2ClassInfo, 0, tmp, 0, registeredId2ClassInfo.length); - registeredId2ClassInfo = tmp; - } int typeId = classId; ClassInfo classInfo = classInfoMap.get(cls); if (classInfo != null) { @@ -554,7 +545,7 @@ private void registerInternalImpl(Class cls, int classId) { classInfoMap.put(cls, classInfo); } // serializer will be set lazily in `addSerializer` method if it's null. - registeredId2ClassInfo[id] = classInfo; + putInternalTypeInfo(id, classInfo); extRegistry.registeredClasses.put(cls.getName(), cls); GraalvmSupport.registerClass(cls, fory.getConfig().getConfigHash()); } @@ -565,11 +556,6 @@ private void registerUserImpl(Class cls, int userId) { short id = (short) userId; checkRegistration(cls, id, cls.getName(), false); extRegistry.registeredClassIdMap.put(cls, id); - if (userRegisteredId2ClassInfo.length <= id) { - ClassInfo[] tmp = new ClassInfo[(id + 1) * 2]; - System.arraycopy(userRegisteredId2ClassInfo, 0, tmp, 0, userRegisteredId2ClassInfo.length); - userRegisteredId2ClassInfo = tmp; - } int typeId = buildUserTypeId(cls, userId, null); ClassInfo classInfo = classInfoMap.get(cls); if (classInfo != null) { @@ -578,7 +564,7 @@ private void registerUserImpl(Class cls, int userId) { classInfo = new ClassInfo(this, cls, null, typeId); classInfoMap.put(cls, classInfo); } - userRegisteredId2ClassInfo[id] = classInfo; + putUserTypeInfo(id, classInfo); extRegistry.registeredClasses.put(cls.getName(), cls); GraalvmSupport.registerClass(cls, fory.getConfig().getConfigHash()); } @@ -612,19 +598,19 @@ private void checkRegistration(Class cls, short classId, String name, boolean } if (classId >= 0) { if (internal) { - if (classId < registeredId2ClassInfo.length && registeredId2ClassInfo[classId] != null) { + if (classId < typeIdToClassInfo.length && typeIdToClassInfo[classId] != null) { throw new IllegalArgumentException( String.format( "Class %s with id %s has been registered, registering class %s with same id are not allowed.", - registeredId2ClassInfo[classId].getCls(), classId, cls.getName())); + typeIdToClassInfo[classId].getCls(), classId, cls.getName())); } } else { - if (classId < userRegisteredId2ClassInfo.length - && userRegisteredId2ClassInfo[classId] != null) { + ClassInfo existingInfo = userTypeIdToClassInfo.get(classId); + if (existingInfo != null) { throw new IllegalArgumentException( String.format( "Class %s with id %s has been registered, registering class %s with same id are not allowed.", - userRegisteredId2ClassInfo[classId].getCls(), classId, cls.getName())); + existingInfo.getCls(), classId, cls.getName())); } } } @@ -638,10 +624,10 @@ private void checkRegistration(Class cls, short classId, String name, boolean } private boolean isInternalRegisteredClassId(Class cls, short classId) { - if (classId < 0 || classId >= registeredId2ClassInfo.length) { + if (classId < 0 || classId >= typeIdToClassInfo.length) { return false; } - ClassInfo classInfo = registeredId2ClassInfo[classId]; + ClassInfo classInfo = typeIdToClassInfo[classId]; return classInfo != null && classInfo.cls == cls; } @@ -685,8 +671,8 @@ public Short getRegisteredClassId(Class cls) { } public Class getRegisteredClass(short id) { - if (id < registeredId2ClassInfo.length) { - ClassInfo classInfo = registeredId2ClassInfo[id]; + if (id < typeIdToClassInfo.length) { + ClassInfo classInfo = typeIdToClassInfo[id]; if (classInfo != null) { return classInfo.cls; } @@ -710,40 +696,25 @@ public ClassInfo getRegisteredClassInfoByTypeId(int typeId) { } if (Types.isUserDefinedType((byte) internalTypeId)) { int userId = typeId >>> 8; - if (userId < 0 || userId >= userRegisteredId2ClassInfo.length) { - return null; - } - ClassInfo classInfo = userRegisteredId2ClassInfo[userId]; + ClassInfo classInfo = userTypeIdToClassInfo.get(userId); if (classInfo == null) { return null; } int existingInternalTypeId = classInfo.typeId & 0xff; if (existingInternalTypeId != internalTypeId) { - if (classInfo.serializer == null) { - classInfo.typeId = typeId; - } else { - return null; - } + return null; } return classInfo; } - if (typeId < 0 || typeId >= registeredId2ClassInfo.length) { + if (typeId < 0 || typeId >= typeIdToClassInfo.length) { return null; } - return registeredId2ClassInfo[typeId]; + return typeIdToClassInfo[typeId]; } public List> getRegisteredClasses() { - List> classes = - Arrays.stream(registeredId2ClassInfo) - .filter(Objects::nonNull) - .map(info -> info.cls) - .collect(Collectors.toList()); - for (ClassInfo info : userRegisteredId2ClassInfo) { - if (info != null) { - classes.add(info.cls); - } - } + List> classes = new ArrayList<>(extRegistry.registeredClassIdMap.size); + extRegistry.registeredClassIdMap.forEach((cls, id) -> classes.add(cls)); return classes; } @@ -841,8 +812,8 @@ public boolean isInternalRegistered(int classId) { return false; } return classId > 0 - && classId < registeredId2ClassInfo.length - && registeredId2ClassInfo[classId] != null; + && classId < typeIdToClassInfo.length + && typeIdToClassInfo[classId] != null; } /** Returns true if cls is fory inner registered class. */ @@ -1069,20 +1040,9 @@ public void addSerializer(Class type, Serializer serializer) { classInfo.typeId = typeId; } if (internal) { - if (registeredId2ClassInfo.length <= id) { - ClassInfo[] tmp = new ClassInfo[(id + 1) * 2]; - System.arraycopy(registeredId2ClassInfo, 0, tmp, 0, registeredId2ClassInfo.length); - registeredId2ClassInfo = tmp; - } - registeredId2ClassInfo[id] = classInfo; + putInternalTypeInfo(id, classInfo); } else { - if (userRegisteredId2ClassInfo.length <= id) { - ClassInfo[] tmp = new ClassInfo[(id + 1) * 2]; - System.arraycopy( - userRegisteredId2ClassInfo, 0, tmp, 0, userRegisteredId2ClassInfo.length); - userRegisteredId2ClassInfo = tmp; - } - userRegisteredId2ClassInfo[id] = classInfo; + putUserTypeInfo(id, classInfo); } } else { int typeId = buildUnregisteredTypeId(type, serializer); @@ -1380,7 +1340,7 @@ public ClassInfo getClassInfo(Class cls) { } public ClassInfo getClassInfo(short classId) { - ClassInfo classInfo = registeredId2ClassInfo[classId]; + ClassInfo classInfo = typeIdToClassInfo[classId]; assert classInfo != null : classId; if (classInfo.serializer == null) { addSerializer(classInfo.cls, createSerializer(classInfo.cls)); @@ -1441,7 +1401,7 @@ public ClassInfo getOrUpdateClassInfo(Class cls) { private ClassInfo getOrUpdateClassInfo(short classId) { ClassInfo classInfo = classInfoCache; ClassInfo internalInfo = - classId < registeredId2ClassInfo.length ? registeredId2ClassInfo[classId] : null; + classId < typeIdToClassInfo.length ? typeIdToClassInfo[classId] : null; Preconditions.checkArgument( internalInfo != null, "Internal class id %s is not registered", classId); if (classInfo != internalInfo) { @@ -1702,45 +1662,22 @@ public Class readClassInternal(MemoryBuffer buffer) { MetaStringBytes simpleClassNameBytes = metaStringResolver.readMetaStringBytes(buffer); classInfo = loadBytesToClassInfo(packageBytes, simpleClassNameBytes); } else { - classInfo = getClassInfoByTypeId(header >> 1, false); + classInfo = getClassInfoByTypeIdForReadClassInternal(header >> 1); } final Class cls = classInfo.cls; currentReadClass = cls; return cls; } - @Override - protected ClassInfo getClassInfoByTypeId(int typeId) { - return getClassInfoByTypeId(typeId, true); - } - - private ClassInfo getClassInfoByTypeId(int typeId, boolean ensureSerializer) { + private ClassInfo getClassInfoByTypeIdForReadClassInternal(int typeId) { int internalTypeId = typeId & 0xff; - ClassInfo classInfo = getRegisteredClassInfoByTypeId(typeId); - if (classInfo == null) { - throw new IllegalStateException( - String.format( - "Invalid typeId %d in meta share mode. This usually indicates a protocol mismatch " - + "or buffer corruption. Check that the serializer and deserializer use the " - + "same protocol version.", - typeId)); - } + ClassInfo classInfo; if (Types.isUserDefinedType((byte) internalTypeId) && !Types.isNamedType(internalTypeId)) { - int classInternalTypeId = classInfo.typeId & 0xff; - if (classInternalTypeId != internalTypeId) { - throw new IllegalStateException( - String.format( - "Type id mismatch for %s: expected internal type %d but found %d", - classInfo.cls, internalTypeId, classInternalTypeId)); - } - } - if (ensureSerializer && classInfo.serializer == null) { - // Ensure serializer is set for registered classes (they may have been registered - // without a serializer via registerInternal) - addSerializer(classInfo.cls, createSerializer(classInfo.cls)); - // Re-read from registeredId2ClassInfo since addSerializer updates it for registered classes - classInfo = getRegisteredClassInfoByTypeId(typeId); + classInfo = getUserTypeInfoByTypeId(typeId); + } else { + classInfo = getInternalTypeInfoByTypeId(typeId); } + Preconditions.checkArgument(classInfo != null, "Type id %s not registered", typeId); return classInfo; } diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index 9170cf0458..6d1426fde0 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -102,6 +102,7 @@ public abstract class TypeResolver { "Meta context must be set before serialization, " + "please set meta context by SerializationContext.setMetaContext"; private static final GenericType OBJECT_GENERIC_TYPE = GenericType.build(Object.class); + private static final float TYPE_ID_MAP_LOAD_FACTOR = 0.5f; final Fory fory; final boolean metaContextShareEnabled; @@ -110,6 +111,11 @@ public abstract class TypeResolver { final IdentityMap, ClassInfo> classInfoMap = new IdentityMap<>(64, foryMapLoadFactor); final ExtRegistry extRegistry; final Map, ClassDef> classDefMap = new HashMap<>(); + // Map for internal type ids (non-user-defined). + ClassInfo[] typeIdToClassInfo = new ClassInfo[] {}; + // Map for user-registered type ids, keyed by user id. + final LongMap userTypeIdToClassInfo = + new LongMap<>(estimatedNumRegistered, TYPE_ID_MAP_LOAD_FACTOR); // Cache for readClassInfo(MemoryBuffer) - persists between calls to avoid reloading // dynamically created classes that can't be found by Class.forName private ClassInfo classInfoCache; @@ -376,26 +382,9 @@ protected final void writeSharedClassMeta(MemoryBuffer buffer, ClassInfo classIn */ public final ClassInfo readClassInfo(MemoryBuffer buffer) { int header = buffer.readVarUint32Small14(); - int internalTypeId = header & 0xff; - switch (internalTypeId) { - case Types.NAMED_ENUM: - case Types.NAMED_STRUCT: - case Types.NAMED_EXT: - case Types.NAMED_COMPATIBLE_STRUCT: - if (metaContextShareEnabled) { - return readSharedClassMeta(buffer); - } - ClassInfo classInfo = readClassInfoFromBytes(buffer, classInfoCache, header); - classInfoCache = classInfo; - return classInfo; - case Types.COMPATIBLE_STRUCT: - if (metaContextShareEnabled) { - return readSharedClassMeta(buffer); - } - return getClassInfoByTypeId(header); - default: - return getClassInfoByTypeId(header); - } + ClassInfo classInfo = readClassInfoByHeader(buffer, header, classInfoCache, null, null); + classInfoCache = classInfo; + return classInfo; } /** @@ -404,26 +393,9 @@ public final ClassInfo readClassInfo(MemoryBuffer buffer) { */ public final ClassInfo readClassInfo(MemoryBuffer buffer, Class targetClass) { int header = buffer.readVarUint32Small14(); - int internalTypeId = header & 0xff; - switch (internalTypeId) { - case Types.NAMED_ENUM: - case Types.NAMED_STRUCT: - case Types.NAMED_EXT: - case Types.NAMED_COMPATIBLE_STRUCT: - if (metaContextShareEnabled) { - return readSharedClassMeta(buffer, targetClass); - } - ClassInfo classInfo = readClassInfoFromBytes(buffer, classInfoCache, header); - classInfoCache = classInfo; - return classInfo; - case Types.COMPATIBLE_STRUCT: - if (metaContextShareEnabled) { - return readSharedClassMeta(buffer, targetClass); - } - return getClassInfoByTypeId(header); - default: - return getClassInfoByTypeId(header); - } + ClassInfo classInfo = readClassInfoByHeader(buffer, header, classInfoCache, null, targetClass); + classInfoCache = classInfo; + return classInfo; } /** @@ -438,24 +410,7 @@ public final ClassInfo readClassInfo(MemoryBuffer buffer, Class targetClass) @CodegenInvoke public final ClassInfo readClassInfo(MemoryBuffer buffer, ClassInfo classInfoCache) { int header = buffer.readVarUint32Small14(); - int internalTypeId = header & 0xff; - switch (internalTypeId) { - case Types.NAMED_ENUM: - case Types.NAMED_STRUCT: - case Types.NAMED_EXT: - case Types.NAMED_COMPATIBLE_STRUCT: - if (metaContextShareEnabled) { - return readSharedClassMeta(buffer); - } - return readClassInfoByCache(buffer, classInfoCache, header); - case Types.COMPATIBLE_STRUCT: - if (metaContextShareEnabled) { - return readSharedClassMeta(buffer); - } - return getClassInfoByTypeId(header); - default: - return getClassInfoByTypeId(header); - } + return readClassInfoByHeader(buffer, header, classInfoCache, null, null); } /** @@ -469,6 +424,15 @@ public final ClassInfo readClassInfo(MemoryBuffer buffer, ClassInfo classInfoCac @CodegenInvoke public final ClassInfo readClassInfo(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { int header = buffer.readVarUint32Small14(); + return readClassInfoByHeader(buffer, header, classInfoHolder.classInfo, classInfoHolder, null); + } + + private ClassInfo readClassInfoByHeader( + MemoryBuffer buffer, + int header, + ClassInfo classInfoCache, + ClassInfoHolder classInfoHolder, + Class targetClass) { int internalTypeId = header & 0xff; switch (internalTypeId) { case Types.NAMED_ENUM: @@ -476,16 +440,31 @@ public final ClassInfo readClassInfo(MemoryBuffer buffer, ClassInfoHolder classI case Types.NAMED_EXT: case Types.NAMED_COMPATIBLE_STRUCT: if (metaContextShareEnabled) { - return readSharedClassMeta(buffer); + return targetClass == null + ? readSharedClassMeta(buffer) + : readSharedClassMeta(buffer, targetClass); } - return readClassInfoFromBytes(buffer, classInfoHolder, header); + if (classInfoHolder != null) { + return readClassInfoFromBytes(buffer, classInfoHolder, header); + } + return readClassInfoFromBytes(buffer, classInfoCache, header); case Types.COMPATIBLE_STRUCT: if (metaContextShareEnabled) { - return readSharedClassMeta(buffer); + return targetClass == null + ? readSharedClassMeta(buffer) + : readSharedClassMeta(buffer, targetClass); } - return getClassInfoByTypeId(header); + return ensureSerializerForClassInfo(requireUserTypeInfoByTypeId(header)); + case Types.ENUM: + case Types.STRUCT: + case Types.EXT: + return ensureSerializerForClassInfo(requireUserTypeInfoByTypeId(header)); + case Types.LIST: + return ensureSerializerForClassInfo(getListClassInfo()); + case Types.TIMESTAMP: + return ensureSerializerForClassInfo(getTimestampClassInfo()); default: - return getClassInfoByTypeId(header); + return ensureSerializerForClassInfo(requireInternalTypeInfoByTypeId(header)); } } @@ -619,12 +598,65 @@ protected abstract ClassInfo loadBytesToClassInfo( */ protected abstract ClassInfo ensureSerializerForClassInfo(ClassInfo classInfo); - /** - * Get ClassInfo by type ID from the registry. For internal types (0-255), returns the - * pre-registered ClassInfo. For user types, decodes the type ID and looks up in the appropriate - * registry. - */ - protected abstract ClassInfo getClassInfoByTypeId(int typeId); + protected ClassInfo getListClassInfo() { + return requireInternalTypeInfoByTypeId(Types.LIST); + } + + protected ClassInfo getTimestampClassInfo() { + return requireInternalTypeInfoByTypeId(Types.TIMESTAMP); + } + + protected final ClassInfo getInternalTypeInfoByTypeId(int typeId) { + if (typeId < 0 || typeId >= typeIdToClassInfo.length) { + return null; + } + return typeIdToClassInfo[typeId]; + } + + protected final ClassInfo getUserTypeInfoByTypeId(int typeId) { + int userId = typeId >>> 8; + ClassInfo classInfo = userTypeIdToClassInfo.get(userId); + if (classInfo == null) { + return null; + } + if ((classInfo.typeId & 0xff) != (typeId & 0xff)) { + return null; + } + return classInfo; + } + + protected final ClassInfo requireUserTypeInfoByTypeId(int typeId) { + ClassInfo classInfo = getUserTypeInfoByTypeId(typeId); + if (classInfo == null) { + throw new IllegalStateException(String.format("Type id %s not registered", typeId)); + } + return classInfo; + } + + protected final ClassInfo requireInternalTypeInfoByTypeId(int typeId) { + ClassInfo classInfo = getInternalTypeInfoByTypeId(typeId); + if (classInfo == null) { + throw new IllegalStateException(String.format("Type id %s not registered", typeId)); + } + return classInfo; + } + + protected final void putInternalTypeInfo(int typeId, ClassInfo classInfo) { + if (typeIdToClassInfo.length <= typeId) { + ClassInfo[] tmp = new ClassInfo[(typeId + 1) * 2]; + System.arraycopy(typeIdToClassInfo, 0, tmp, 0, typeIdToClassInfo.length); + typeIdToClassInfo = tmp; + } + typeIdToClassInfo[typeId] = classInfo; + } + + protected final void putUserTypeInfo(int userId, ClassInfo classInfo) { + userTypeIdToClassInfo.put(userId, classInfo); + } + + protected final boolean containsUserTypeId(int userId) { + return userTypeIdToClassInfo.containsKey(userId); + } final ClassInfo buildMetaSharedClassInfo( Tuple2 classDefTuple, ClassDef classDef) { diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java index 1742f57c72..8069a4b8f9 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java @@ -54,7 +54,6 @@ import org.apache.fory.Fory; import org.apache.fory.annotation.ForyField; import org.apache.fory.annotation.Internal; -import org.apache.fory.collection.LongMap; import org.apache.fory.collection.ObjectMap; import org.apache.fory.collection.Tuple2; import org.apache.fory.config.Config; @@ -125,9 +124,6 @@ public class XtypeResolver extends TypeResolver { private final boolean shareMeta; private int xtypeIdGenerator = 64; - // Use ClassInfo[] or LongMap? - // ClassInfo[] is faster, but we can't have bigger type id. - private final LongMap xtypeIdToClassMap = new LongMap<>(8, loadFactor); private final Set registeredTypeIds = new HashSet<>(); private final Generics generics; @@ -152,7 +148,7 @@ public void initialize() { @Override public void register(Class type) { - while (registeredTypeIds.contains(xtypeIdGenerator)) { + while (containsUserTypeId(xtypeIdGenerator)) { xtypeIdGenerator++; } register(type, xtypeIdGenerator++); @@ -164,6 +160,8 @@ public void register(Class type, int userTypeId) { // ClassInfo[] has length of max type id. If the type id is too big, Fory will waste many // memory. We can relax this limit in the future. Preconditions.checkArgument(userTypeId < MAX_TYPE_ID, "Too big type id %s", userTypeId); + Preconditions.checkArgument( + !containsUserTypeId(userTypeId), "Type id %s has been registered", userTypeId); ClassInfo classInfo = classInfoMap.get(type); if (type.isArray()) { buildClassInfo(type); @@ -288,7 +286,14 @@ private void register( } classInfoMap.put(type, classInfo); registeredTypeIds.add(xtypeId); - xtypeIdToClassMap.put(xtypeId, classInfo); + int internalTypeId = xtypeId & 0xff; + if (Types.isUserDefinedType((byte) internalTypeId) && !Types.isNamedType(internalTypeId)) { + putUserTypeInfo(xtypeId >>> 8, classInfo); + } else if (!Types.isNamedType(internalTypeId)) { + if (getInternalTypeInfoByTypeId(xtypeId) == null) { + putInternalTypeInfo(xtypeId, classInfo); + } + } } /** @@ -356,8 +361,7 @@ public void registerSerializer(Class type, Serializer serializer) { int oldTypeId = classInfo.typeId; int foryId = oldTypeId & 0xff; - if (oldTypeId != 0 && xtypeIdToClassMap.get(oldTypeId) == classInfo) { - xtypeIdToClassMap.remove(oldTypeId); + if (oldTypeId != 0) { registeredTypeIds.remove(oldTypeId); } @@ -375,7 +379,6 @@ public void registerSerializer(Class type, Serializer serializer) { int newTypeId = classInfo.typeId; if (newTypeId != 0) { - xtypeIdToClassMap.put(newTypeId, classInfo); registeredTypeIds.add(newTypeId); } } @@ -582,7 +585,7 @@ public ClassInfo getClassInfo(Class cls, ClassInfoHolder classInfoHolder) { } public ClassInfo getXtypeInfo(int typeId) { - return xtypeIdToClassMap.get(typeId); + return getInternalTypeInfoByTypeId(typeId); } public ClassInfo getUserTypeInfo(String namespace, String typeName) { @@ -591,8 +594,7 @@ public ClassInfo getUserTypeInfo(String namespace, String typeName) { } public ClassInfo getUserTypeInfo(int userTypeId) { - Preconditions.checkArgument((userTypeId & 0xff) < Types.BOUND); - return xtypeIdToClassMap.get(userTypeId); + return getUserTypeInfoByTypeId(userTypeId); } // buildGenericType methods are inherited from TypeResolver @@ -834,8 +836,8 @@ private void registerDefaultTypes() { private void registerType(int xtypeId, Class type, Serializer serializer) { ClassInfo classInfo = newClassInfo(type, serializer, (short) xtypeId); classInfoMap.put(type, classInfo); - if (!xtypeIdToClassMap.containsKey(xtypeId)) { - xtypeIdToClassMap.put(xtypeId, classInfo); + if (getInternalTypeInfoByTypeId(xtypeId) == null) { + putInternalTypeInfo(xtypeId, classInfo); } } @@ -857,7 +859,7 @@ private void registerUnionTypes() { ClassInfo classInfo = newClassInfo(cls, serializer, (short) Types.UNION); classInfoMap.put(cls, classInfo); } - xtypeIdToClassMap.put(Types.UNION, classInfoMap.get(org.apache.fory.type.union.Union.class)); + putInternalTypeInfo(Types.UNION, classInfoMap.get(org.apache.fory.type.union.Union.class)); } public ClassInfo writeClassInfo(MemoryBuffer buffer, Object obj) { @@ -899,44 +901,25 @@ public void setSerializerIfAbsent(Class cls, Serializer serializer) { // nilClassInfo and nilClassInfoHolder are inherited from TypeResolver @Override - protected ClassInfo getClassInfoByTypeId(int typeId) { - int internalTypeId = typeId & 0xff; - switch (internalTypeId) { - case Types.LIST: - return getListClassInfo(); - case Types.TIMESTAMP: - return getGenericClassInfo(); - default: - ClassInfo classInfo = xtypeIdToClassMap.get(typeId); - if (classInfo == null) { - throwUnexpectTypeIdException(typeId); - } - return classInfo; - } - } - - private void throwUnexpectTypeIdException(long xtypeId) { - throw new IllegalStateException(String.format("Type id %s not registered", xtypeId)); - } - - private ClassInfo getListClassInfo() { + protected ClassInfo getListClassInfo() { fory.incReadDepth(); GenericType genericType = generics.nextGenericType(); fory.decDepth(); if (genericType != null) { return getOrBuildClassInfo(genericType.getCls()); } - return xtypeIdToClassMap.get(Types.LIST); + return requireInternalTypeInfoByTypeId(Types.LIST); } - private ClassInfo getGenericClassInfo() { + @Override + protected ClassInfo getTimestampClassInfo() { fory.incReadDepth(); GenericType genericType = generics.nextGenericType(); fory.decDepth(); if (genericType != null) { return getOrBuildClassInfo(genericType.getCls()); } - return xtypeIdToClassMap.get(Types.TIMESTAMP); + return requireInternalTypeInfoByTypeId(Types.TIMESTAMP); } private ClassInfo getOrBuildClassInfo(Class cls) { From d5cef28c7644616c42d5a1891e2d6263a3f996ef Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Mon, 19 Jan 2026 13:05:32 +0800 Subject: [PATCH 36/44] fix(java): align user type ids with serializer --- .../src/main/java/org/apache/fory/BaseFory.java | 3 +-- .../java/org/apache/fory/resolver/ClassResolver.java | 7 ++----- .../java/org/apache/fory/resolver/TypeResolver.java | 10 ++++++++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/BaseFory.java b/java/fory-core/src/main/java/org/apache/fory/BaseFory.java index 6a3f04e861..32b0ef7a40 100644 --- a/java/fory-core/src/main/java/org/apache/fory/BaseFory.java +++ b/java/fory-core/src/main/java/org/apache/fory/BaseFory.java @@ -168,8 +168,7 @@ public interface BaseFory { * @param type class needed to be serialized/deserialized. * @param serializerCreator serializer creator with param {@link Fory} */ - void registerSerializerAndType( - Class type, Function> serializerCreator); + void registerSerializerAndType(Class type, Function> serializerCreator); void setSerializerFactory(SerializerFactory serializerFactory); diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 160018400c..7941451195 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -811,9 +811,7 @@ public boolean isInternalRegistered(int classId) { if (Types.isUserDefinedType((byte) internalTypeId)) { return false; } - return classId > 0 - && classId < typeIdToClassInfo.length - && typeIdToClassInfo[classId] != null; + return classId > 0 && classId < typeIdToClassInfo.length && typeIdToClassInfo[classId] != null; } /** Returns true if cls is fory inner registered class. */ @@ -1400,8 +1398,7 @@ public ClassInfo getOrUpdateClassInfo(Class cls) { private ClassInfo getOrUpdateClassInfo(short classId) { ClassInfo classInfo = classInfoCache; - ClassInfo internalInfo = - classId < typeIdToClassInfo.length ? typeIdToClassInfo[classId] : null; + ClassInfo internalInfo = classId < typeIdToClassInfo.length ? typeIdToClassInfo[classId] : null; Preconditions.checkArgument( internalInfo != null, "Internal class id %s is not registered", classId); if (classInfo != internalInfo) { diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index 6d1426fde0..a96341479b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -619,8 +619,14 @@ protected final ClassInfo getUserTypeInfoByTypeId(int typeId) { if (classInfo == null) { return null; } - if ((classInfo.typeId & 0xff) != (typeId & 0xff)) { - return null; + int internalTypeId = typeId & 0xff; + int registeredInternalTypeId = classInfo.typeId & 0xff; + if (registeredInternalTypeId != internalTypeId) { + if (classInfo.serializer == null) { + classInfo.typeId = typeId; + } else { + return null; + } } return classInfo; } From 5a2cc28ff6f63526e1d9d5655f4348c8613a1403 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Mon, 19 Jan 2026 13:21:11 +0800 Subject: [PATCH 37/44] perf(java): inline class info reads --- .../apache/fory/resolver/TypeResolver.java | 177 ++++++++++++++---- 1 file changed, 139 insertions(+), 38 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index a96341479b..df64c2457b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -382,7 +382,43 @@ protected final void writeSharedClassMeta(MemoryBuffer buffer, ClassInfo classIn */ public final ClassInfo readClassInfo(MemoryBuffer buffer) { int header = buffer.readVarUint32Small14(); - ClassInfo classInfo = readClassInfoByHeader(buffer, header, classInfoCache, null, null); + int internalTypeId = header & 0xff; + ClassInfo classInfo; + switch (internalTypeId) { + case Types.NAMED_ENUM: + case Types.NAMED_STRUCT: + case Types.NAMED_EXT: + case Types.NAMED_COMPATIBLE_STRUCT: + if (metaContextShareEnabled) { + classInfo = readSharedClassMeta(buffer); + } else { + classInfo = readClassInfoFromBytes(buffer, classInfoCache, header); + } + break; + case Types.COMPATIBLE_STRUCT: + if (metaContextShareEnabled) { + classInfo = readSharedClassMeta(buffer); + } else { + classInfo = requireUserTypeInfoByTypeId(header); + } + break; + case Types.ENUM: + case Types.STRUCT: + case Types.EXT: + classInfo = requireUserTypeInfoByTypeId(header); + break; + case Types.LIST: + classInfo = getListClassInfo(); + break; + case Types.TIMESTAMP: + classInfo = getTimestampClassInfo(); + break; + default: + classInfo = requireInternalTypeInfoByTypeId(header); + } + if (classInfo.serializer == null) { + classInfo = ensureSerializerForClassInfo(classInfo); + } classInfoCache = classInfo; return classInfo; } @@ -393,7 +429,43 @@ public final ClassInfo readClassInfo(MemoryBuffer buffer) { */ public final ClassInfo readClassInfo(MemoryBuffer buffer, Class targetClass) { int header = buffer.readVarUint32Small14(); - ClassInfo classInfo = readClassInfoByHeader(buffer, header, classInfoCache, null, targetClass); + int internalTypeId = header & 0xff; + ClassInfo classInfo; + switch (internalTypeId) { + case Types.NAMED_ENUM: + case Types.NAMED_STRUCT: + case Types.NAMED_EXT: + case Types.NAMED_COMPATIBLE_STRUCT: + if (metaContextShareEnabled) { + classInfo = readSharedClassMeta(buffer, targetClass); + } else { + classInfo = readClassInfoFromBytes(buffer, classInfoCache, header); + } + break; + case Types.COMPATIBLE_STRUCT: + if (metaContextShareEnabled) { + classInfo = readSharedClassMeta(buffer, targetClass); + } else { + classInfo = requireUserTypeInfoByTypeId(header); + } + break; + case Types.ENUM: + case Types.STRUCT: + case Types.EXT: + classInfo = requireUserTypeInfoByTypeId(header); + break; + case Types.LIST: + classInfo = getListClassInfo(); + break; + case Types.TIMESTAMP: + classInfo = getTimestampClassInfo(); + break; + default: + classInfo = requireInternalTypeInfoByTypeId(header); + } + if (classInfo.serializer == null) { + classInfo = ensureSerializerForClassInfo(classInfo); + } classInfoCache = classInfo; return classInfo; } @@ -410,7 +482,44 @@ public final ClassInfo readClassInfo(MemoryBuffer buffer, Class targetClass) @CodegenInvoke public final ClassInfo readClassInfo(MemoryBuffer buffer, ClassInfo classInfoCache) { int header = buffer.readVarUint32Small14(); - return readClassInfoByHeader(buffer, header, classInfoCache, null, null); + int internalTypeId = header & 0xff; + ClassInfo classInfo; + switch (internalTypeId) { + case Types.NAMED_ENUM: + case Types.NAMED_STRUCT: + case Types.NAMED_EXT: + case Types.NAMED_COMPATIBLE_STRUCT: + if (metaContextShareEnabled) { + classInfo = readSharedClassMeta(buffer); + } else { + classInfo = readClassInfoFromBytes(buffer, classInfoCache, header); + } + break; + case Types.COMPATIBLE_STRUCT: + if (metaContextShareEnabled) { + classInfo = readSharedClassMeta(buffer); + } else { + classInfo = requireUserTypeInfoByTypeId(header); + } + break; + case Types.ENUM: + case Types.STRUCT: + case Types.EXT: + classInfo = requireUserTypeInfoByTypeId(header); + break; + case Types.LIST: + classInfo = getListClassInfo(); + break; + case Types.TIMESTAMP: + classInfo = getTimestampClassInfo(); + break; + default: + classInfo = requireInternalTypeInfoByTypeId(header); + } + if (classInfo.serializer == null) { + classInfo = ensureSerializerForClassInfo(classInfo); + } + return classInfo; } /** @@ -424,48 +533,49 @@ public final ClassInfo readClassInfo(MemoryBuffer buffer, ClassInfo classInfoCac @CodegenInvoke public final ClassInfo readClassInfo(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { int header = buffer.readVarUint32Small14(); - return readClassInfoByHeader(buffer, header, classInfoHolder.classInfo, classInfoHolder, null); - } - - private ClassInfo readClassInfoByHeader( - MemoryBuffer buffer, - int header, - ClassInfo classInfoCache, - ClassInfoHolder classInfoHolder, - Class targetClass) { int internalTypeId = header & 0xff; + ClassInfo classInfo; + boolean updateCache = false; switch (internalTypeId) { case Types.NAMED_ENUM: case Types.NAMED_STRUCT: case Types.NAMED_EXT: case Types.NAMED_COMPATIBLE_STRUCT: if (metaContextShareEnabled) { - return targetClass == null - ? readSharedClassMeta(buffer) - : readSharedClassMeta(buffer, targetClass); - } - if (classInfoHolder != null) { - return readClassInfoFromBytes(buffer, classInfoHolder, header); + classInfo = readSharedClassMeta(buffer); + } else { + classInfo = readClassInfoFromBytes(buffer, classInfoHolder.classInfo, header); + updateCache = true; } - return readClassInfoFromBytes(buffer, classInfoCache, header); + break; case Types.COMPATIBLE_STRUCT: if (metaContextShareEnabled) { - return targetClass == null - ? readSharedClassMeta(buffer) - : readSharedClassMeta(buffer, targetClass); + classInfo = readSharedClassMeta(buffer); + } else { + classInfo = requireUserTypeInfoByTypeId(header); } - return ensureSerializerForClassInfo(requireUserTypeInfoByTypeId(header)); + break; case Types.ENUM: case Types.STRUCT: case Types.EXT: - return ensureSerializerForClassInfo(requireUserTypeInfoByTypeId(header)); + classInfo = requireUserTypeInfoByTypeId(header); + break; case Types.LIST: - return ensureSerializerForClassInfo(getListClassInfo()); + classInfo = getListClassInfo(); + break; case Types.TIMESTAMP: - return ensureSerializerForClassInfo(getTimestampClassInfo()); + classInfo = getTimestampClassInfo(); + break; default: - return ensureSerializerForClassInfo(requireInternalTypeInfoByTypeId(header)); + classInfo = requireInternalTypeInfoByTypeId(header); + } + if (classInfo.serializer == null) { + classInfo = ensureSerializerForClassInfo(classInfo); } + if (updateCache) { + classInfoHolder.classInfo = classInfo; + } + return classInfo; } /** @@ -477,14 +587,6 @@ protected final ClassInfo readClassInfoByCache( return readClassInfoFromBytes(buffer, classInfoCache, header); } - /** Read class info and update the ClassInfoHolder's cache. */ - protected final ClassInfo readClassInfoFromBytes( - MemoryBuffer buffer, ClassInfoHolder classInfoHolder, int header) { - ClassInfo classInfo = readClassInfoFromBytes(buffer, classInfoHolder.classInfo, header); - classInfoHolder.classInfo = classInfo; - return classInfo; - } - /** * Read class info from bytes with cache optimization. Uses the cached namespace and type name * bytes to avoid map lookups when the class is the same as the cached one (hash comparison). @@ -514,9 +616,8 @@ protected final ClassInfo readClassInfoFromBytes( simpleClassNameBytes = metaStringResolver.readMetaStringBytes(buffer); } - // Load class info from bytes (subclass-specific) and ensure serializer is set - ClassInfo classInfo = loadBytesToClassInfo(namespaceBytes, simpleClassNameBytes); - return ensureSerializerForClassInfo(classInfo); + // Load class info from bytes (subclass-specific). + return loadBytesToClassInfo(namespaceBytes, simpleClassNameBytes); } /** From 0418aa8eea8e19fb5aecce69be22d3b5a0b97096 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Mon, 19 Jan 2026 14:06:52 +0800 Subject: [PATCH 38/44] optimzie long map initial size for user type --- .../src/main/java/org/apache/fory/resolver/TypeResolver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index df64c2457b..94023d7b6c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -115,7 +115,7 @@ public abstract class TypeResolver { ClassInfo[] typeIdToClassInfo = new ClassInfo[] {}; // Map for user-registered type ids, keyed by user id. final LongMap userTypeIdToClassInfo = - new LongMap<>(estimatedNumRegistered, TYPE_ID_MAP_LOAD_FACTOR); + new LongMap<>(4, TYPE_ID_MAP_LOAD_FACTOR); // Cache for readClassInfo(MemoryBuffer) - persists between calls to avoid reloading // dynamically created classes that can't be found by Class.forName private ClassInfo classInfoCache; From 2c3056d6774612e0c116477eedaa2cf12464f16e Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Mon, 19 Jan 2026 14:09:25 +0800 Subject: [PATCH 39/44] adjust benchmakr params --- .../org/apache/fory/benchmark/UserTypeDeserializeSuite.java | 2 +- .../java/org/apache/fory/benchmark/UserTypeSerializeSuite.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/java_benchmark/src/main/java/org/apache/fory/benchmark/UserTypeDeserializeSuite.java b/benchmarks/java_benchmark/src/main/java/org/apache/fory/benchmark/UserTypeDeserializeSuite.java index 1be222ae5d..1adeed21a6 100644 --- a/benchmarks/java_benchmark/src/main/java/org/apache/fory/benchmark/UserTypeDeserializeSuite.java +++ b/benchmarks/java_benchmark/src/main/java/org/apache/fory/benchmark/UserTypeDeserializeSuite.java @@ -147,7 +147,7 @@ public Object flatbuffers_deserialize(FlatBuffersState.FlatBuffersUserTypeState public static void main(String[] args) throws IOException { if (args.length == 0) { String commandLine = - "org.apache.fory.*UserTypeDeserializeSuite.fory* -f 1 -wi 5 -i 10 -t 1 -w 2s -r 2s -rf csv " + "org.apache.fory.*UserTypeDeserializeSuite.* -f 1 -wi 3 -i 3 -t 1 -w 2s -r 2s -rf csv " + "-p objectType=MEDIA_CONTENT -p bufferType=array -p references=false"; System.out.println(commandLine); args = commandLine.split(" "); diff --git a/benchmarks/java_benchmark/src/main/java/org/apache/fory/benchmark/UserTypeSerializeSuite.java b/benchmarks/java_benchmark/src/main/java/org/apache/fory/benchmark/UserTypeSerializeSuite.java index adc0e84007..08c279eb23 100644 --- a/benchmarks/java_benchmark/src/main/java/org/apache/fory/benchmark/UserTypeSerializeSuite.java +++ b/benchmarks/java_benchmark/src/main/java/org/apache/fory/benchmark/UserTypeSerializeSuite.java @@ -157,7 +157,7 @@ public Object flatbuffers_serialize(FlatBuffersState.FlatBuffersUserTypeState st public static void main(String[] args) throws IOException { if (args.length == 0) { String commandLine = - "org.apache.fory.*UserTypeSerializeSuite.fory_serialize_compatible -f 1 -wi 5 -i 10 -t 1 -w 200s -r 2s -rf csv " + "org.apache.fory.*UserTypeSerializeSuite.* -f 1 -wi 3 -i 3 -t 1 -w 2s -r 2s -rf csv " + "-p objectType=MEDIA_CONTENT -p bufferType=array -p references=false"; System.out.println(commandLine); args = commandLine.split(" "); From 39b436b7be2d7cbbb62ecaa26e231461f7895b1a Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Mon, 19 Jan 2026 14:10:12 +0800 Subject: [PATCH 40/44] fix go skip meta --- go/fory/skip.go | 31 ++++--------------------------- go/fory/struct_test.go | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 27 deletions(-) diff --git a/go/fory/skip.go b/go/fory/skip.go index dcb8306f37..3adb00216c 100644 --- a/go/fory/skip.go +++ b/go/fory/skip.go @@ -155,33 +155,10 @@ func SkipAnyValue(ctx *ReadContext, readRefFlag bool) { nullable: true, } case COMPATIBLE_STRUCT, NAMED_COMPATIBLE_STRUCT, STRUCT, NAMED_STRUCT: - // For struct types, read meta_index to get type_info - if ctx.TypeResolver().metaShareEnabled() { - metaIndex := ctx.buffer.ReadVaruint32(err) - if ctx.HasError() { - return - } - context := ctx.TypeResolver().fory.MetaContext() - if context == nil || int(metaIndex) >= len(context.readTypeInfos) { - ctx.SetError(DeserializationErrorf("invalid meta index %d", metaIndex)) - return - } - typeInfo = context.readTypeInfos[metaIndex] - } else { - // Without share_meta, read namespace and type_name - nsBytes, nsErr := ctx.TypeResolver().metaStringResolver.ReadMetaStringBytes(ctx.buffer, err) - if nsErr != nil { - ctx.SetError(FromError(nsErr)) - return - } - typeNameBytes, tnErr := ctx.TypeResolver().metaStringResolver.ReadMetaStringBytes(ctx.buffer, err) - if tnErr != nil { - ctx.SetError(FromError(tnErr)) - return - } - // We don't have the actual type registered, so we'll have to skip fields blindly - _ = nsBytes - _ = typeNameBytes + // Read type info using the shared meta reader when enabled. + typeInfo = ctx.TypeResolver().readTypeInfoWithTypeID(ctx.buffer, typeID, err) + if ctx.HasError() { + return } fieldDef = FieldDef{ fieldType: NewSimpleFieldType(TypeId(typeID)), diff --git a/go/fory/struct_test.go b/go/fory/struct_test.go index 0bd3dff32b..1fcdff63d5 100644 --- a/go/fory/struct_test.go +++ b/go/fory/struct_test.go @@ -212,3 +212,40 @@ func TestSetFieldTypeId(t *testing.T) { } } } + +func TestSkipAnyValueReadsSharedTypeMeta(t *testing.T) { + type First struct { + ID int + } + type Second struct { + Name string + } + + f := New(WithXlang(true), WithCompatible(true)) + require.NoError(t, f.RegisterStruct(First{}, 2001)) + require.NoError(t, f.RegisterStruct(Second{}, 2002)) + + buf := NewByteBuffer(nil) + require.NoError(t, f.SerializeTo(buf, First{ID: 10})) + require.NoError(t, f.SerializeTo(buf, Second{Name: "ok"})) + + f.resetReadState() + f.readCtx.SetData(buf.Bytes()) + + isNull := readHeader(f.readCtx) + require.False(t, isNull) + SkipAnyValue(f.readCtx, true) + require.NoError(t, f.readCtx.CheckError()) + + f.resetReadState() + isNull = readHeader(f.readCtx) + require.False(t, isNull) + + var out any + f.readCtx.ReadValue(reflect.ValueOf(&out).Elem(), RefModeTracking, true) + require.NoError(t, f.readCtx.CheckError()) + + result, ok := out.(*Second) + require.True(t, ok) + require.Equal(t, "ok", result.Name) +} From d3b0d69f4b2ff2bc7611402cb8088c04fd35a8ba Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Mon, 19 Jan 2026 14:24:26 +0800 Subject: [PATCH 41/44] fix lint --- .../src/main/java/org/apache/fory/resolver/TypeResolver.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index 94023d7b6c..17a106c72c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -114,8 +114,7 @@ public abstract class TypeResolver { // Map for internal type ids (non-user-defined). ClassInfo[] typeIdToClassInfo = new ClassInfo[] {}; // Map for user-registered type ids, keyed by user id. - final LongMap userTypeIdToClassInfo = - new LongMap<>(4, TYPE_ID_MAP_LOAD_FACTOR); + final LongMap userTypeIdToClassInfo = new LongMap<>(4, TYPE_ID_MAP_LOAD_FACTOR); // Cache for readClassInfo(MemoryBuffer) - persists between calls to avoid reloading // dynamically created classes that can't be found by Class.forName private ClassInfo classInfoCache; From 72d87655de678a41ce79c1331574c04eac31c5e7 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Mon, 19 Jan 2026 14:38:31 +0800 Subject: [PATCH 42/44] update java spec --- docs/specification/java_serialization_spec.md | 197 ++++++++++++++++++ 1 file changed, 197 insertions(+) diff --git a/docs/specification/java_serialization_spec.md b/docs/specification/java_serialization_spec.md index e69de29bb2..f09a10b649 100644 --- a/docs/specification/java_serialization_spec.md +++ b/docs/specification/java_serialization_spec.md @@ -0,0 +1,197 @@ +--- +title: Java Serialization Format +sidebar_position: 1 +id: java_serialization_spec +license: | + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--- + +## Spec overview + +Apache Fory Java serialization is a dynamic binary format for Java object graphs. It supports shared references, circular +references, and polymorphism. The format is stream friendly: type metadata is written inline when needed, and there is no +end-of-stream metadata section. + +Overall layout: + +``` +| fory header | object ref meta | object type meta | object value data | +``` + +All data is encoded in little endian byte order. + +## Fory header + +Java native serialization uses a one-byte bitmap header. + +``` +| 4 bits | 1 bit | 1 bit | 1 bit | 1 bit | ++---------------+-------+-------+--------+-------+ +| reserved bits | oob | xlang | endian | null | +``` + +- null flag: 1 when object is null, 0 otherwise. If an object is null, other bits are not set. +- endian flag: 1 when data is encoded by little endian, 0 for big endian. +- xlang flag: 1 when serialization uses xlang format, 0 when serialization uses Java native format. +- oob flag: 1 when passed `BufferCallback` is not null, 0 otherwise. + +If xlang flag is set, a one-byte language ID follows the bitmap. In Java native mode (xlang flag unset), there is no +language byte and no meta start offset. + +## Reference meta + +Reference tracking uses the same flags as the xlang specification: + +| Flag | Byte Value | Description | +| ------------------- | ---------- | -------------------------------------------------------------------------------------------------------- | +| NULL FLAG | `-3` | Object is null. No further bytes are written for this object. | +| REF FLAG | `-2` | Object was already serialized. Followed by unsigned varint32 reference ID. | +| NOT_NULL VALUE FLAG | `-1` | Object is non-null but reference tracking is disabled for this type. Object data follows immediately. | +| REF VALUE FLAG | `0` | Object is referencable and this is its first occurrence. Object data follows. Assigns next reference ID. | + +When reference tracking is disabled globally or for a specific field/type, only `NULL FLAG` and `NOT_NULL VALUE FLAG` +are used. + +## Type system and type IDs + +Java native serialization uses the unified type ID layout: + +``` +full_type_id = (user_type_id << 8) | internal_type_id +``` + +- `internal_type_id` is a byte-sized ID describing the kind (enum/struct/ext, named variants, built-ins). +- `user_type_id` is the numeric registration ID (0-based) for user-defined enum/struct/ext types. + +### Shared internal type IDs + +Java shares the xlang internal IDs for the common primitive and core types. See the xlang spec for the full table: +`docs/specification/xlang_serialization_spec.md#internal-type-id-table`. + +### Java native internal type IDs + +Java native format adds Java-specific built-in type IDs starting at `Types.NAMED_EXT + 1`: + +| Type ID | Name | Description | +| ------- | -------------------------- | ------------------------------ | +| 31 | VOID_ID | java.lang.Void | +| 32 | CHAR_ID | java.lang.Character | +| 33 | PRIMITIVE_VOID_ID | void | +| 34 | PRIMITIVE_BOOL_ID | boolean | +| 35 | PRIMITIVE_INT8_ID | byte | +| 36 | PRIMITIVE_CHAR_ID | char | +| 37 | PRIMITIVE_INT16_ID | short | +| 38 | PRIMITIVE_INT32_ID | int | +| 39 | PRIMITIVE_FLOAT32_ID | float | +| 40 | PRIMITIVE_INT64_ID | long | +| 41 | PRIMITIVE_FLOAT64_ID | double | +| 42 | PRIMITIVE_BOOLEAN_ARRAY_ID | boolean[] | +| 43 | PRIMITIVE_BYTE_ARRAY_ID | byte[] | +| 44 | PRIMITIVE_CHAR_ARRAY_ID | char[] | +| 45 | PRIMITIVE_SHORT_ARRAY_ID | short[] | +| 46 | PRIMITIVE_INT_ARRAY_ID | int[] | +| 47 | PRIMITIVE_FLOAT_ARRAY_ID | float[] | +| 48 | PRIMITIVE_LONG_ARRAY_ID | long[] | +| 49 | PRIMITIVE_DOUBLE_ARRAY_ID | double[] | +| 50 | STRING_ARRAY_ID | String[] | +| 51 | OBJECT_ARRAY_ID | Object[] | +| 52 | ARRAYLIST_ID | java.util.ArrayList | +| 53 | HASHMAP_ID | java.util.HashMap | +| 54 | HASHSET_ID | java.util.HashSet | +| 55 | CLASS_ID | java.lang.Class | +| 56 | EMPTY_OBJECT_ID | empty object stub | +| 57 | LAMBDA_STUB_ID | lambda stub | +| 58 | JDK_PROXY_STUB_ID | JDK proxy stub | +| 59 | REPLACE_STUB_ID | writeReplace/readResolve stub | +| 60 | NONEXISTENT_META_SHARED_ID | meta-shared unknown class stub | + +### Named and unregistered types + +If a type is not registered by numeric ID, Java native serialization encodes it as a named type: + +- enum: `NAMED_ENUM` +- struct-like serializer: `NAMED_STRUCT` (or `NAMED_COMPATIBLE_STRUCT` when compatible mode is enabled) +- other serializers: `NAMED_EXT` + +The namespace is the package name and the type name is the simple class name. + +## Type meta encoding + +Java native serialization writes a type ID for every value, then optionally writes additional metadata depending on the +internal type ID: + +1. Write type ID using varuint32 small7 encoding. +2. For `NAMED_ENUM`, `NAMED_STRUCT`, `NAMED_EXT`, `NAMED_COMPATIBLE_STRUCT`: + - If meta share is enabled: write shared class meta (streaming format). + - Otherwise: write namespace and type name as meta strings. +3. For `COMPATIBLE_STRUCT`: + - If meta share is enabled: write shared class meta (streaming format). + - Otherwise: no extra meta (type ID is sufficient). +4. All other types: no extra meta. + +### Shared class meta (streaming) + +When meta share is enabled, Java uses a streaming shared meta protocol. It does not rely on a meta start offset in the +header. + +``` +| unsigned varint: index_marker | [class def bytes if new] | +``` + +- `index_marker` encodes both a reference flag and index: `index_marker = (index << 1) | flag`. +- If `flag == 1`, this is a reference to a previously written type, and no additional bytes follow. +- If `flag == 0`, this is a new type definition and the `class def bytes` are written inline. + +The index is assigned sequentially in the order types are first encountered. + +## ClassDef format (compatible mode) + +ClassDef encodes schema evolution metadata for compatible structs. The layout is: + +``` +| 8 bytes header | optional varuint32 extra size | class meta bytes | +``` + +### Header + +`50 bits hash + 1 bit compress flag + 1 bit has-fields-meta flag + 12 bits meta size` (lower bits on the right). + +- meta size: lower 12 bits, with an extra varuint32 written when the size exceeds the 12-bit limit. +- compress flag: set when the payload is compressed. +- has-fields-meta: set when fields metadata is included. +- hash: 50-bit hash of the payload and flags. + +### Class meta bytes + +Class meta bytes encode a linearized inheritance chain and field metadata: + +``` +| num classes | class layer 0 | class layer 1 | ... | + +class layer: +| num fields + registered flag | [type id if registered] | namespace | type name | field infos | +``` + +- num classes: number of class layers with serializable fields. +- registered flag: set when the class is registered by numeric ID. +- namespace and type name use the same meta string encoding as xlang (see below). +- field infos: for each field, the header encodes name encoding, name length/tag id, nullable flag, and ref tracking flag, + followed by the field type ID and field name (or tag id). + +### Meta string encoding + +Meta string encoding (namespace/type name/field name) follows the same encoding algorithms as the xlang specification: +`docs/specification/xlang_serialization_spec.md#meta-string`. From aee8b301a505d80e3d27c0ba260a056a675c4e50 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Mon, 19 Jan 2026 15:22:59 +0800 Subject: [PATCH 43/44] update spec doc --- docs/specification/java_serialization_spec.md | 435 ++++++++++++--- .../specification/xlang_serialization_spec.md | 504 ++++++------------ 2 files changed, 539 insertions(+), 400 deletions(-) diff --git a/docs/specification/java_serialization_spec.md b/docs/specification/java_serialization_spec.md index f09a10b649..ddfbadb3e6 100644 --- a/docs/specification/java_serialization_spec.md +++ b/docs/specification/java_serialization_spec.md @@ -7,8 +7,8 @@ license: | contributor license agreements. See the NOTICE file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at + (the "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 @@ -21,9 +21,13 @@ license: | ## Spec overview -Apache Fory Java serialization is a dynamic binary format for Java object graphs. It supports shared references, circular -references, and polymorphism. The format is stream friendly: type metadata is written inline when needed, and there is no -end-of-stream metadata section. +Apache Fory Java serialization is a dynamic binary format for Java object graphs. It supports +shared references, circular references, polymorphism, and optional schema evolution. The format is +stream friendly: shared type metadata is written inline when needed and there is no meta start +offset. + +The Java native format is an extension of the xlang wire format and reuses the same core framing +and encodings; see `docs/specification/xlang_serialization_spec.md` for the shared baseline. Overall layout: @@ -31,29 +35,30 @@ Overall layout: | fory header | object ref meta | object type meta | object value data | ``` -All data is encoded in little endian byte order. +All data is encoded in little endian byte order. When running on a big endian platform, array +serializers swap byte order on write/read so the on-wire layout remains little endian. ## Fory header -Java native serialization uses a one-byte bitmap header. +Java native serialization writes a one byte bitmap header. The header layout mirrors the xlang +bitmap and uses the same flag bits. ``` -| 4 bits | 1 bit | 1 bit | 1 bit | 1 bit | -+---------------+-------+-------+--------+-------+ -| reserved bits | oob | xlang | endian | null | +| 5 bits | 1 bit | 1 bit | 1 bit | ++--------------+-------+-------+-------+ +| reserved | oob | xlang | null | ``` -- null flag: 1 when object is null, 0 otherwise. If an object is null, other bits are not set. -- endian flag: 1 when data is encoded by little endian, 0 for big endian. +- null flag: 1 when object is null, 0 otherwise. If object is null, other bits are not set. - xlang flag: 1 when serialization uses xlang format, 0 when serialization uses Java native format. -- oob flag: 1 when passed `BufferCallback` is not null, 0 otherwise. +- oob flag: 1 when `BufferCallback` is not null, 0 otherwise. -If xlang flag is set, a one-byte language ID follows the bitmap. In Java native mode (xlang flag unset), there is no -language byte and no meta start offset. +If xlang flag is set, a one byte language ID is written after the bitmap. In Java native mode (xlang +flag unset), no language byte is written. ## Reference meta -Reference tracking uses the same flags as the xlang specification: +Reference tracking uses the same flags as the xlang specification. | Flag | Byte Value | Description | | ------------------- | ---------- | -------------------------------------------------------------------------------------------------------- | @@ -62,28 +67,66 @@ Reference tracking uses the same flags as the xlang specification: | NOT_NULL VALUE FLAG | `-1` | Object is non-null but reference tracking is disabled for this type. Object data follows immediately. | | REF VALUE FLAG | `0` | Object is referencable and this is its first occurrence. Object data follows. Assigns next reference ID. | -When reference tracking is disabled globally or for a specific field/type, only `NULL FLAG` and `NOT_NULL VALUE FLAG` -are used. +When reference tracking is disabled globally or for a specific field/type, only `NULL FLAG` and +`NOT_NULL VALUE FLAG` are used. ## Type system and type IDs -Java native serialization uses the unified type ID layout: +Java native serialization uses the unified type ID layout shared with xlang: ``` full_type_id = (user_type_id << 8) | internal_type_id ``` -- `internal_type_id` is a byte-sized ID describing the kind (enum/struct/ext, named variants, built-ins). +- `internal_type_id` is the low 8 bits describing the kind (enum/struct/ext, named variants, or a + built-in type). - `user_type_id` is the numeric registration ID (0-based) for user-defined enum/struct/ext types. - -### Shared internal type IDs - -Java shares the xlang internal IDs for the common primitive and core types. See the xlang spec for the full table: -`docs/specification/xlang_serialization_spec.md#internal-type-id-table`. - -### Java native internal type IDs - -Java native format adds Java-specific built-in type IDs starting at `Types.NAMED_EXT + 1`: +- Named types use `NAMED_*` internal IDs and carry names in metadata rather than embedding a user + ID. + +### Shared internal type IDs (0-30) + +Java native mode shares the xlang internal IDs for basic types and user-defined enum/struct/ext +tags. These IDs are stable across languages. + +| Type ID | Name | +| ------- | ----------------------- | +| 0 | UNKNOWN | +| 1 | BOOL | +| 2 | INT8 | +| 3 | INT16 | +| 4 | INT32 | +| 5 | VARINT32 | +| 6 | INT64 | +| 7 | VARINT64 | +| 8 | TAGGED_INT64 | +| 9 | UINT8 | +| 10 | UINT16 | +| 11 | UINT32 | +| 12 | VAR_UINT32 | +| 13 | UINT64 | +| 14 | VAR_UINT64 | +| 15 | TAGGED_UINT64 | +| 16 | FLOAT16 | +| 17 | FLOAT32 | +| 18 | FLOAT64 | +| 19 | STRING | +| 20 | LIST | +| 21 | SET | +| 22 | MAP | +| 23 | ENUM | +| 24 | NAMED_ENUM | +| 25 | STRUCT | +| 26 | COMPATIBLE_STRUCT | +| 27 | NAMED_STRUCT | +| 28 | NAMED_COMPATIBLE_STRUCT | +| 29 | EXT | +| 30 | NAMED_EXT | + +### Java native built-in type IDs + +Java native serialization assigns Java-specific built-ins starting at `Types.NAMED_EXT + 1`. +Type IDs greater than 30 are not shared with xlang; they are only valid in Java native mode. | Type ID | Name | Description | | ------- | -------------------------- | ------------------------------ | @@ -118,22 +161,26 @@ Java native format adds Java-specific built-in type IDs starting at `Types.NAMED | 59 | REPLACE_STUB_ID | writeReplace/readResolve stub | | 60 | NONEXISTENT_META_SHARED_ID | meta-shared unknown class stub | -### Named and unregistered types +### Registration and named types + +User-defined enum/struct/ext types can be registered by numeric ID or by name. -If a type is not registered by numeric ID, Java native serialization encodes it as a named type: +- Numeric registration: `full_type_id = (user_id << 8) | internal_type_id`. +- Name registration: type meta uses namespace and type name (see below). +- Unregistered types are encoded as named types using namespace = package name and type name = + simple class name. -- enum: `NAMED_ENUM` -- struct-like serializer: `NAMED_STRUCT` (or `NAMED_COMPATIBLE_STRUCT` when compatible mode is enabled) -- other serializers: `NAMED_EXT` +Named type selection rules for unregistered types: -The namespace is the package name and the type name is the simple class name. +- enum -> NAMED_ENUM +- struct-like serializers -> NAMED_STRUCT (or NAMED_COMPATIBLE_STRUCT in compatible mode) +- all other custom serializers -> NAMED_EXT ## Type meta encoding -Java native serialization writes a type ID for every value, then optionally writes additional metadata depending on the -internal type ID: +Every value is written with a type ID followed by optional type metadata: -1. Write type ID using varuint32 small7 encoding. +1. Write `type_id` using varuint32 small7 encoding. 2. For `NAMED_ENUM`, `NAMED_STRUCT`, `NAMED_EXT`, `NAMED_COMPATIBLE_STRUCT`: - If meta share is enabled: write shared class meta (streaming format). - Otherwise: write namespace and type name as meta strings. @@ -144,54 +191,318 @@ internal type ID: ### Shared class meta (streaming) -When meta share is enabled, Java uses a streaming shared meta protocol. It does not rely on a meta start offset in the -header. +When meta share is enabled, Java uses the streaming shared meta protocol and writes TypeDef +bytes inline on first use. ``` -| unsigned varint: index_marker | [class def bytes if new] | +| varuint32: index_marker | [class def bytes if new] | + +index_marker = (index << 1) | flag +flag = 1 -> reference +flag = 0 -> new type ``` -- `index_marker` encodes both a reference flag and index: `index_marker = (index << 1) | flag`. -- If `flag == 1`, this is a reference to a previously written type, and no additional bytes follow. -- If `flag == 0`, this is a new type definition and the `class def bytes` are written inline. +- If `flag == 1`, this is a reference to a previously written type. No class def bytes follow. +- If `flag == 0`, this is a new type definition and class def bytes are written inline. The index is assigned sequentially in the order types are first encountered. +## Schema modes + +Java native serialization supports two schema modes: + +- Schema consistent (compatible mode disabled): fields are serialized in a fixed order and no + ClassDef is required. Type meta uses `STRUCT` or `NAMED_STRUCT` for user-defined classes. +- Schema evolution (compatible mode enabled): fields are serialized with schema evolution metadata + (ClassDef). Type meta uses `COMPATIBLE_STRUCT` or `NAMED_COMPATIBLE_STRUCT`. + ## ClassDef format (compatible mode) -ClassDef encodes schema evolution metadata for compatible structs. The layout is: +ClassDef is the schema evolution metadata encoded for compatible structs. It is written inline +when shared meta is enabled, or referenced by index when already seen. + +### Binary layout ``` -| 8 bytes header | optional varuint32 extra size | class meta bytes | +| 8 bytes header | [varuint32 extra size] | class meta bytes | ``` -### Header +Header layout (lower bits on the right): -`50 bits hash + 1 bit compress flag + 1 bit has-fields-meta flag + 12 bits meta size` (lower bits on the right). +``` +| 50-bit hash | 1 bit compress | 1 bit has_fields_meta | 12-bit size | +``` -- meta size: lower 12 bits, with an extra varuint32 written when the size exceeds the 12-bit limit. -- compress flag: set when the payload is compressed. -- has-fields-meta: set when fields metadata is included. +- size: lower 12 bits. If size equals the mask (0xFFF), write extra size as varuint32 and add it. +- compress: set when payload is compressed. +- has_fields_meta: set when field metadata is present. - hash: 50-bit hash of the payload and flags. ### Class meta bytes -Class meta bytes encode a linearized inheritance chain and field metadata: +Class meta encodes a linearized class hierarchy (from parent to leaf) and field metadata: ``` -| num classes | class layer 0 | class layer 1 | ... | +| num_classes | class_layer_0 | class_layer_1 | ... | -class layer: -| num fields + registered flag | [type id if registered] | namespace | type name | field infos | +class_layer: +| num_fields << 1 | registered_flag | [type_id if registered] | +| namespace | type_name | field_infos | ``` -- num classes: number of class layers with serializable fields. -- registered flag: set when the class is registered by numeric ID. -- namespace and type name use the same meta string encoding as xlang (see below). -- field infos: for each field, the header encodes name encoding, name length/tag id, nullable flag, and ref tracking flag, - followed by the field type ID and field name (or tag id). +- `num_classes` stores `(num_layers - 1)` in a single byte. + - If it equals `0b1111`, read an extra varuint32 small7 and add it. + - The actual number of layers is `num_classes + 1`. +- `registered_flag` is 1 if the class is registered by numeric ID. +- If registered by ID, the class type ID follows (varuint32 small7). +- If registered by name or unregistered, namespace and type name are written as meta strings. + +### Field info + +Each field uses a compact header followed by its name bytes (omitted when TAG_ID is used) and its +type info: + +``` +| field_header | [field_name_bytes] | field_type | +``` + +`field_header` bits: + +- bit 0: trackingRef +- bit 1: nullable +- bits 2-3: field name encoding +- bits 4-6: name length (len-1), or tag ID when TAG_ID is used; value 7 indicates extended length +- bit 7: reserved (0) + +Field name encoding: + +- 0: UTF8 +- 1: ALL_TO_LOWER_SPECIAL +- 2: LOWER_UPPER_DIGIT_SPECIAL +- 3: TAG_ID (field name omitted, tag ID stored in size field) + +If length is extended (size==7), an extra varuint32 small7 storing `(len-1) - 7` follows. + +### Field type encoding + +Field types are encoded with a type tag and optional nested type info. For nested types, the header +includes nullable/trackingRef flags in the low bits. +Top-level field types use the tag only (no flags). + +Type tags: + +| Tag | Field type | +| --- | ----------------------------------------- | +| 0 | Object (ObjectFieldType) | +| 1 | Map (MapFieldType) | +| 2 | Collection/List/Set (CollectionFieldType) | +| 3 | Array (ArrayFieldType) | +| 4 | Enum (EnumFieldType) | +| 5+ | Registered type (RegisteredFieldType) | -### Meta string encoding +Encoding rules: -Meta string encoding (namespace/type name/field name) follows the same encoding algorithms as the xlang specification: +- ObjectFieldType: write tag 0. +- MapFieldType: write tag 1, then key type, then value type. +- CollectionFieldType: write tag 2, then element type. +- ArrayFieldType: write tag 3, then dimensions, then component type. +- EnumFieldType: write tag 4. +- RegisteredFieldType: write tag `5 + type_id`. + +For nested types, nullable/trackingRef flags are stored in the low bits of the header as +`(type_tag << 2) | (nullable << 1) | tracking_ref`. + +## Meta string encoding + +Namespace, type names, and field names use the same meta string encodings as the xlang spec. + +### Package and type names + +Header format: + +``` +| 6 bits size | 2 bits encoding | +``` + +- size is the byte length of the encoded name. +- if size == 63, write extra length `(size - 63)` as varuint32 small7. + +Encodings: + +- Package name: UTF8, ALL_TO_LOWER_SPECIAL, LOWER_UPPER_DIGIT_SPECIAL +- Type name: UTF8, LOWER_UPPER_DIGIT_SPECIAL, FIRST_TO_LOWER_SPECIAL, ALL_TO_LOWER_SPECIAL + +### Field names + +Field name encoding is described in the ClassDef field header section. When using TAG_ID, the +field name bytes are omitted and the tag ID is stored in the size field. + +### Encoding algorithms + +See the xlang specification for encoding algorithms and tables: `docs/specification/xlang_serialization_spec.md#meta-string`. + +## Value encodings + +This section describes the byte layouts for common built-in serializers used in Java native +serialization. Custom serializers (EXT) may define additional formats but must still follow the +reference and type meta rules described above. + +### Primitives + +- boolean: 1 byte (0x00 or 0x01). +- byte: 1 byte. +- short: 2 bytes little endian. +- char: 2 bytes little endian (UTF-16 code unit). +- int: + - fixed: 4 bytes little endian. + - varint: signed varint32 (ZigZag) when `compressInt` is enabled. +- long: + - fixed: 8 bytes little endian. + - varint: signed varint64 (ZigZag) when `longEncoding=VARINT`. + - tagged: tagged int64 when `longEncoding=TAGGED`. +- float: IEEE 754 float32, little endian. +- double: IEEE 754 float64, little endian. + +Varint encodings follow the xlang spec: +`docs/specification/xlang_serialization_spec.md#unsigned-varint32`. + +### String + +Strings are encoded as: + +``` +| varuint36_small: (num_bytes << 2) | coder | string bytes | +``` + +- coder: 2-bit value + - 0: LATIN1 + - 1: UTF16 + - 2: UTF8 +- num_bytes: byte length of the encoded string payload. + +UTF16 is encoded as little endian 2-byte code units. + +### Enum + +- If `serializeEnumByName` is enabled: write enum name as a meta string. +- Otherwise: write enum ordinal as varuint32 small7. + +### Binary (byte[]) + +Primitive byte arrays are encoded as: + +``` +| varuint32: num_bytes | raw bytes | +``` + +### Primitive arrays + +Primitive arrays use `writePrimitiveArrayWithSize` unless compression is enabled: + +``` +| varuint32: byte_length | raw bytes | +``` + +- `compressIntArray`: int[] encoded as `| varuint32: length | varint32... |`. +- `compressLongArray`: long[] encoded as `| varuint32: length | varint64/tagged... |`. + +### Object arrays + +Object arrays encode length and a monomorphic flag: + +``` +| varuint32_small7: (length << 1) | mono_flag | +``` + +- If `mono_flag == 1`, all elements share a known component serializer. Each element uses ref + flags and the component serializer writes the value. +- If `mono_flag == 0`, each element uses ref flags and writes its own class info and data. + +### Collections (List/Set) + +Collections encode length and a one-byte elements header: + +``` +| varuint32_small7: length | elements_header | [elem_class_info] | elements... | +``` + +`elements_header` bits (see `CollectionFlags`): + +- bit 0: TRACKING_REF +- bit 1: HAS_NULL +- bit 2: IS_DECL_ELEMENT_TYPE +- bit 3: IS_SAME_TYPE + +If `IS_SAME_TYPE` is set and `IS_DECL_ELEMENT_TYPE` is not set, the element class info is written +once before the elements. Element values then follow with either ref flags (if TRACKING_REF) or +per-element null flags (if HAS_NULL). + +If `IS_SAME_TYPE` is not set, each element is written with its own class info and data (and +optionally ref flags). + +### Maps + +Maps encode entry count and then a sequence of chunks. Each chunk groups entries that share key +and value types. + +``` +| varuint32_small7: size | chunk_1 | chunk_2 | ... | + +chunk (non-null entries): +| header | chunk_size | [key_class_info] | [value_class_info] | entries... | +``` + +`header` bits (see `MapFlags`): + +- bit 0: TRACKING_KEY_REF +- bit 1: KEY_HAS_NULL +- bit 2: KEY_DECL_TYPE +- bit 3: TRACKING_VALUE_REF +- bit 4: VALUE_HAS_NULL +- bit 5: VALUE_DECL_TYPE + +If `KEY_DECL_TYPE` or `VALUE_DECL_TYPE` is unset, the corresponding class info is written once at +the start of the chunk. `chunk_size` is a single byte (1..255) and `MAX_CHUNK_SIZE` is 255. + +#### Null key/value entries + +Entries with null key or null value are encoded as special single-entry chunks without a +`chunk_size` byte: + +- null key, non-null value: `NULL_KEY_VALUE_DECL_TYPE*` flags, then value payload +- null value, non-null key: `NULL_VALUE_KEY_DECL_TYPE*` flags, then key payload +- null key and null value: `KV_NULL` header only + +These chunks always represent exactly one entry. + +### Objects and structs + +Object values are encoded as: + +``` +| ref meta | type meta | field data | +``` + +Field data is written by the serializer selected by the class info. For standard object +serialization: + +- Fields are sorted deterministically using `DescriptorGrouper` order: + primitives, boxed primitives, built-ins, collections, maps, then other fields, with names sorted + within each category. +- For compatible mode, `MetaSharedSerializer` uses ClassDef field metadata to read and skip + unknown fields. +- For each field, the serializer uses field metadata (nullable, trackingRef, polymorphic) to decide + whether to write ref flags and/or type meta before the field value. + +### Extensions (EXT) + +Extension types are encoded by their registered serializer. Type meta is still written before the +value as described above. The serializer is responsible for the value layout. + +## Out-of-band buffers + +When a `BufferCallback` is provided, the oob flag is set in the header and serializers may emit +buffer references instead of inline bytes (for example, large primitive arrays). The out-of-band +buffer protocol is specific to the callback implementation; the main stream only contains +references to those buffers. diff --git a/docs/specification/xlang_serialization_spec.md b/docs/specification/xlang_serialization_spec.md index d5c26a98f7..92aac982fc 100644 --- a/docs/specification/xlang_serialization_spec.md +++ b/docs/specification/xlang_serialization_spec.md @@ -148,10 +148,13 @@ Such information can be provided in other languages too: ### Type ID -All internal data types are expressed using an ID in range `0~64`. Users can use IDs in range `0~8192` for registering their -custom types (struct/ext/enum). User type IDs are in a separate namespace and combined with internal type IDs via bit shifting: +All internal data types use an 8-bit internal ID (`0~255`, with `0~50` defined here). Users can +register types by numeric ID (`0~4095` in current implementations). User IDs are encoded together +with the internal type ID: `(user_type_id << 8) | internal_type_id`. +Named types (`NAMED_*`) do not embed a user ID; their names are carried in metadata instead. + #### Internal Type ID Table | Type ID | Name | Description | @@ -250,9 +253,9 @@ The data are serialized using little endian byte order for all types. Fory header format for xlang serialization: ``` -| 1 byte bitmap | 1 byte | optional 4 bytes | -+--------------------------------+------------+------------------------------------+ -| 4 bits reserved | 4 bits meta | language | unsigned int for meta start offset | +| 1 byte bitmap | 1 byte | ++--------------------------------+------------+ +| flags | language | ``` Detailed byte layout: @@ -264,7 +267,6 @@ Byte 0: Bitmap flags - Bit 2: oob flag (0x04) - Bits 3-7: reserved Byte 1: Language ID (only present when xlang flag is set) -Byte 2-5: Meta start offset (only present when meta share mode is enabled) ``` - **null flag** (bit 0): 1 when object is null, 0 otherwise. If an object is null, only this flag is set. @@ -288,12 +290,6 @@ All data is encoded in little-endian format. | RUST | 6 | | DART | 7 | -### Meta Start Offset - -In non-streaming compatible mode, an uncompressed unsigned int32 (4 bytes, little endian) is appended to indicate the start offset of metadata. During serialization, this is initially written as a placeholder (e.g., `-1` or `0`), then updated after all objects are serialized and metadata is collected. - -**Note:** In streaming mode, the meta start offset is omitted because type metadata is written inline during serialization rather than being deferred to the end of the buffer. - ## Reference Meta Reference tracking handles whether the object is null, and whether to track reference for the object by writing @@ -410,260 +406,151 @@ explicit smart pointers (`Rc`, `Arc`). ## Type Meta -For every type to be serialized, it have a type id to indicate its type. - -- basic types: the type id -- enum: - - `Type.ENUM` + registered id - - `Type.NAMED_ENUM` + registered namespace+typename -- list: `Type.List` -- set: `Type.SET` -- map: `Type.MAP` -- ext: - - `Type.EXT` + registered id - - `Type.NAMED_EXT` + registered namespace+typename -- struct: - - `Type.STRUCT` + struct meta - - `Type.NAMED_STRUCT` + struct meta - -Every type must be registered with an ID or name first. The registration can be used for security check and type -identification. - -Struct is a special type, depending whether schema compatibility is enabled, Fory will write struct meta -differently. +Every non-primitive value begins with a type ID that identifies its concrete type. The type ID is +followed by optional type-specific metadata. -Only ext/enum/struct can be registered using namespaced type. +### Type ID encoding -### Struct Schema consistent +- The type ID is written as an unsigned varint32 (small7). +- Internal types use their internal type ID directly (low 8 bits). +- User-registered types use a full type ID: `(user_type_id << 8) | internal_type_id`. + - `user_type_id` is a numeric ID (0-4095 in current implementations). + - `internal_type_id` is one of `ENUM`, `STRUCT`, `COMPATIBLE_STRUCT`, or `EXT`. +- Named types do not embed a user ID. They use `NAMED_*` internal type IDs and carry a namespace + and type name (or shared TypeDef) instead. -- If schema consistent mode is enabled globally when creating fory, type meta will be written as a fory unsigned varint - of `type_id`. Schema evolution related meta will be ignored. -- If schema evolution mode is enabled globally when creating fory, and current class is configured to use schema - consistent mode like `struct` vs `table` in flatbuffers: - - Type meta will be add to `captured_type_defs`: `captured_type_defs[type def stub] = map size` ahead when - registering type. - - Get index of the meta in `captured_type_defs`, write that index as `| unsigned varint: index |`. +### Type meta payload -### Struct Schema evolution +After the type ID: -If schema evolution mode is enabled globally when creating fory, and enabled for current type, type meta will be written -using one of the following mode. Which mode to use is configured when creating fory. +- **ENUM / STRUCT / EXT**: no extra bytes (registration by ID required on both sides). +- **COMPATIBLE_STRUCT**: + - If meta share is enabled, write a shared TypeDef entry (see below). + - If meta share is disabled, no extra bytes. +- **NAMED_ENUM / NAMED_STRUCT / NAMED_COMPATIBLE_STRUCT / NAMED_EXT**: + - If meta share is disabled, write `namespace` and `type_name` as meta strings. + - If meta share is enabled, write a shared TypeDef entry (see below). +- **LIST / SET / MAP / ARRAY / primitives**: no extra bytes at this layer. -- Normal mode(meta share not enabled): - - If type meta hasn't been written before, add `type def` - to `captured_type_defs`: `captured_type_defs[type def] = map size`. - - Get index of the meta in `captured_type_defs`, write that index as `| unsigned varint: index |`. - - After finished the serialization of the object graph, fory will start to write `captured_type_defs`: - - Firstly, set current to `meta start offset` of fory header - - Then write `captured_type_defs` one by one: +Unregistered types are serialized as named types: - ```python - buffer.write_var_uint32(len(writting_type_defs) - len(schema_consistent_type_def_stubs)) - for type_meta in writting_type_defs: - if not type_meta.is_stub(): - type_meta.write_type_def(buffer) - writing_type_defs = copy(schema_consistent_type_def_stubs) - ``` +- Enums -> `NAMED_ENUM` +- Struct-like classes -> `NAMED_STRUCT` (or `NAMED_COMPATIBLE_STRUCT` when meta share is enabled) +- Custom extension types -> `NAMED_EXT` -- Meta share mode: the writing steps are same as the normal mode, but `captured_type_defs` will be shared across - multiple serializations of different objects. For example, suppose we have a batch to serialize: +The namespace is the package/module name and the type name is the simple class name. - ```python - captured_type_defs = {} - stream = ... - # add `Type1` to `captured_type_defs` and write `Type1` - fory.serialize(stream, [Type1()]) - # add `Type2` to `captured_type_defs` and write `Type2`, `Type1` is written before. - fory.serialize(stream, [Type1(), Type2()]) - # `Type1` and `Type2` are written before, no need to write meta. - fory.serialize(stream, [Type1(), Type2()]) - ``` +### Shared Type Meta (streaming) -- Streaming mode(streaming mode doesn't support meta share): - - If type meta hasn't been written before, the data will be written as: +When meta share is enabled, TypeDef metadata is written inline the first time a type is +encountered, and subsequent occurrences only reference it. - ``` - | unsigned varint: index << 1 | type def bytes | - ``` +Encoding: - The LSB=0 indicates this is a new type definition. The `index` is the sequential index - assigned to this type (starting from 0), and `type def bytes` contains the complete - TypeDef including the 8-byte global header. +- `marker = (index << 1) | flag` +- `flag = 0`: new type definition follows +- `flag = 1`: reference to a previously written type definition +- `index` is the sequential index assigned to this type (starting from 0). - - If type meta has been written before, the data will be written as: +Write algorithm: - ``` - | unsigned varint: (index << 1) | 1 | - ``` +1. Look up the class in the per-stream meta context map. +2. If found, write `(index << 1) | 1`. +3. If not found: + - assign `index = next_id` + - write `(index << 1)` + - write the encoded TypeDef bytes immediately after - The LSB=1 indicates this is a reference to a previously written type. The `index` is - the same sequential index that was assigned when the type was first written. +Read algorithm: - - With this mode, `meta start offset` can be omitted since all type metadata is written - inline during serialization rather than being deferred to the end. +1. Read `marker` as varuint32. +2. `flag = marker & 1`, `index = marker >>> 1`. +3. If `flag == 1`, use the cached TypeDef at `index`. +4. If `flag == 0`, read a TypeDef, cache it at `index`, and use it. -> The normal mode and meta share mode will forbid streaming writing since it needs to look back for update the start -> offset after the whole object graph writing and meta collecting is finished. Only in this way we can ensure -> deserialization failure in meta share mode doesn't lost shared meta. +TypeDef bytes include the 8-byte global header and optional size extension. -#### Type Def +### TypeDef (schema evolution metadata) -Here we mainly describe the meta layout for schema evolution mode: +TypeDef describes a struct-like type (or a named enum/ext) for schema evolution and name +resolution. It is encoded as: ``` -| 8 bytes header | variable bytes | variable bytes | -+----------------------+--------------------+-------------------+ -| global binary header | meta header | fields meta | +| 8-byte global header | [optional size varuint] | TypeDef body | ``` -For languages which support inheritance, if parent class and subclass has fields with same name, using field in -subclass. +#### Global header -##### Global binary header +The 8-byte header is a little-endian uint64: -`50 bits hash + 1bit compress flag + write fields meta + 12 bits meta size`. Right is the lower bits. +- Low 12 bits: meta size (number of bytes in the TypeDef body). + - If meta size >= 0xFFF, the low 12 bits are set to 0xFFF and an extra + `varuint32(meta_size - 0xFFF)` follows immediately after the header. +- Bit 12: `HAS_FIELDS_META` (1 = fields metadata present). +- Bit 13: `COMPRESS_META` (1 = body is compressed; decompress before parsing). +- High 50 bits: hash of the TypeDef body. -- lower 12 bits are used to encode meta size. If meta size `>= 0b1111_1111_1111`, then write - `meta_ size - 0b1111_1111_1111` next. -- 13rd bit is used to indicate whether to write fields meta. When this class is schema-consistent or use registered - serializer, fields meta will be skipped. Class Meta will be used for share namespace + type name only. -- 14rd bit is used to indicate whether meta is compressed. -- Other 50 bits is used to store the unique hash of `flags + all layers class meta`. +#### TypeDef body -##### Meta header - -Meta header is a 8 bits number value. - -- Lowest 5 digits `0b00000~0b11110` are used to record num fields. `0b11111` is preserved to indicate that Fory need to - read more bytes for length using Fory unsigned int encoding. Note that num_fields is the number of compatible fields. - Users can use tag id to mark some fields as compatible fields in schema consistent context. In such cases, schema - consistent fields will be serialized first, then compatible fields will be serialized next. At deserialization, - Fory will use fields info of those fields which aren't annotated by tag id for deserializing schema consistent - fields, then use fields info in meta for deserializing compatible fields. -- The 6th bit: 0 for registered by id, 1 for registered by name. -- Remaining 2 bits are reserved for future extension. - -##### Fields meta - -Format: +TypeDef body has a single layer (fields are flattened in class hierarchy order): ``` -| field info: variable bytes | variable bytes | ... | -+---------------------------------+-----------------+-----+ -| header + type info + field name | next field info | ... | +| meta header (1 byte) | type spec | field info ... | ``` -###### Field Header - -Field Header is 8 bits, annotation can be used to provide more specific info. If annotation not exists, fory will infer -those info automatically. - -The format for field header is: - -``` -2 bits field name encoding + 4 bits size + nullability flag + ref tracking flag -``` - -Detailed spec: - -- 2 bits field name encoding: - - encoding: `UTF8/ALL_TO_LOWER_SPECIAL/LOWER_UPPER_DIGIT_SPECIAL/TAG_ID` - - If tag id is used, field name will be written by an unsigned varint tag id, and 2 bits encoding will be `11`. -- size of field name: - - The `4 bits size: 0~14` will be used to indicate length `1~15`, the value `15` indicates to read more bytes, - the encoding will encode `size - 15` as a varint next. - - If encoding is `TAG_ID`, then num_bytes of field name will be used to store tag id. -- ref tracking: when set to 1, ref tracking will be enabled for this field. -- nullability: when set to 1, this field can be null. - -###### Field Type Info - -Field type info is written as unsigned int8. Detailed id spec is: - -- For struct registered by id, it will be `Type.STRUCT`. -- For struct registered by name, it will be `Type.NAMED_STRUCT`. -- For enum registered by id, it will be `Type.ENUM`. -- For enum registered by name, it will be `Type.NAMED_ENUM`. -- For ext type registered by id, it will be `Type.EXT`. -- For ext type registered by name, it will be `Type.NAMED_EXT`. -- For list/set type, it will be written as `Type.LIST/SET`, then write element type recursively. -- For 1D primitive array type, it will be written as `Type.XXX_ARRAY`. -- For multi-dimensional primitive array type with same size on each dim, it will be written as `Type.TENSOR`. -- For other array type, it will be written as `Type.LIST`, then write element type recursively. -- For map type, it will be written as `Type.MAP`, then write key and value type recursively. -- For other types supported by fory directly, it will be fory type id for that type. -- For other types not determined at compile time, write `Type.UNKNOWN` instead. For such types, actual type - will be written when serializing such field values. - -Polymorphism spec: - -- `struct/named_struct/ext/named_ext` are taken as polymorphic, the meta for those types are written separately - instead of inlining here to reduce meta space cost if object of this type is serialized in current object graph - multiple times, and the field value may be null too. -- `enum` is taken as dynamic, if deserialization doesn't have this field, or the type is not enum, enum value - will be skipped. -- `list/map/set` are taken as dynamic, when serializing values of those type, the concrete types won't be written - again. -- Other types that fory supported are taken as dynamic too. - -List/Set/Map nested type spec: - -- `list`: `| list type id | nested type id << 2 + nullability flag + ref tracking flag | ... multi-layer type info |` -- `set`: `| set type id | nested type id << 2 + nullability flag + ref tracking flag | ... multi-layer type info |` -- `map`: `| set type id | key type info | value type info |` - - Key type format: `| nested type id << 2 + nullability flag + ref tracking flag | ... multi-layer type info |` - - Value type format: `| nested type id << 2 + nullability flag + ref tracking flag | ... multi-layer type info |` - -###### Field Name +Meta header byte: -If tag id is set, tag id will be used instead. Otherwise meta string of field name will be written instead. +- Bits 0-4: `num_fields` (0-30). + - If `num_fields == 31`, read an extra `varuint32` and add it. +- Bit 5: `REGISTER_BY_NAME` (1 = namespace + type name, 0 = numeric type ID). +- Bits 6-7: reserved. -###### Field order +Type spec: -Field order are left as implementation details, which is not exposed to specification, the deserialization need to -resort fields based on Fory fields sort algorithms. In this way, fory can compute statistics for field names or types and -using a more compact encoding. +- If `REGISTER_BY_NAME` is set: + - `namespace` meta string + - `type_name` meta string +- Otherwise: + - `type_id` as `varuint32` (small7) -## Extended Type Meta with Inheritance support +Field info list: -If one want to support inheritance for struct, one can implement following spec. - -### Schema consistent - -Fields are serialized from parent type to leaf type. Fields are sorted using fory struct fields sort algorithms. - -### Schema Evolution - -Meta layout for schema evolution mode: +Each field is encoded as: ``` -| 8 bytes header | variable bytes | variable bytes | variable bytes | variable bytes | -+----------------------+----------------+----------------+--------------------+--------------------+ -| global binary header | meta header | fields meta | parent meta header | parent fields meta | +| field header (1 byte) | field type info | [field name bytes] | ``` -#### Meta header +Field header layout: -Meta header is a 64 bits number value encoded in little endian order. +- Bits 6-7: field name encoding (`UTF8`, `ALL_TO_LOWER_SPECIAL`, + `LOWER_UPPER_DIGIT_SPECIAL`, or `TAG_ID`) +- Bits 2-5: size + - For name encoding: `size = (name_bytes_length - 1)` + - For tag ID: `size = tag_id` + - If `size == 0b1111`, read `varuint32(size - 15)` and add it +- Bit 1: nullable flag +- Bit 0: reference tracking flag -- Lowest 4 digits `0b0000~0b1110` are used to record num classes. `0b1111` is preserved to indicate that Fory need to - read more bytes for length using Fory unsigned int encoding. If current type doesn't has parent type, or parent - type doesn't have fields to serialize, or we're in a context which serialize fields of current type - only, num classes will be 1. -- The 5th bit is used to indicate whether this type needs schema evolution. -- Other 56 bits are used to store the unique hash of `flags + all layers type meta`. +Field type info: -#### Single layer type meta +- The top-level field type is written as `varuint32(type_id)` (small7) without flags. +- For `LIST` / `SET`, an element type follows, encoded as + `(nested_type_id << 2) | (nullable << 1) | tracking_ref`. +- For `MAP`, key type and value type follow, both encoded the same way. +- One-dimensional primitive arrays use `*_ARRAY` type IDs; other arrays are encoded as `LIST`. -``` -| unsigned varint | var uint | field info: variable bytes | variable bytes | ... | -+-----------------+----------+-------------------------------+-----------------+-----+ -| num_fields | type id | header + type id + field name | next field info | ... | -``` +Field names: + +- If `TAG_ID` encoding is used, no name bytes are written. +- Otherwise, write the encoded field name bytes as a meta string. +- For xlang, field names are converted to `snake_case` before encoding for + cross-language compatibility. -#### Other layers type meta +Field order: -Same encoding algorithm as the previous layer. +Field order is implementation-defined. Decoders must match fields by name or tag ID rather than +position. Fory uses a stable grouping and sorting order to produce deterministic TypeDefs. ## Meta String @@ -1234,16 +1121,10 @@ then copy the whole buffer into the stream. Such serialization won't compress the array. If users want to compress primitive array, users need to register custom serializers for such types or mark it as list type. -#### Tensor +#### Multi-dimensional arrays -Tensor is a special primitive multi-dimensional array which all dimensions have same size and type. The serialization -format is: - -``` -| num_dims(unsigned varint) | shape[0](unsigned varint) | shape[...] | shape[N] | element type | data | -``` - -The data is continuous to reduce copy and may zero-copy in some cases. +Xlang does not define a dedicated tensor encoding. Multi-dimensional arrays are serialized as +nested lists, while one-dimensional primitive arrays use the `*_ARRAY` type IDs. #### object array @@ -1339,98 +1220,46 @@ Not supported for now. ### struct -Struct means object of `class/pojo/struct/bean/record` type. -Struct will be serialized by writing its fields data in fory order. - -Depending on schema compatibility, structs will have different formats. - -#### field order - -Field will be ordered as following, every group of fields will have its own order: - -- primitive fields: - - larger size type first, smaller later, variable size type last. - - when same size, sort by type id - - when same size and type id, sort by snake case field name - - types: bool/int8/int16/int32/var32/int64/var64/h64/float16/float32/float64 -- nullable primitive fields: same order as primitive fields -- other internal type fields: sort by type id then snake case field name -- list fields: sort by snake case field name -- set fields: sort by snake case field name -- map fields: sort by snake case field name -- other fields: sort by snake case field name - -If two fields have same type, then sort by snake_case styled field name. - -#### schema consistent - -Object will be written as: - -``` -| 4 byte | variable bytes | -+---------------+------------------+ -| type hash | field values | -``` - -Type hash is used to check the type schema consistency across languages. Type hash will be the first 32 bits of 56 bits -value of the type meta. - -Object fields will be serialized one by one using following format: - -``` -not null primitive field value: -| var bytes | -+----------------+ -| value data | -+----------------+ -nullable primitive field value: -| one byte | var bytes | -+-----------+---------------+ -| null flag | field value | -+-----------+---------------+ -other interal types supported by fory -| var bytes | var objects | -+-----------+-------------+ -| null flag | value data | -+-----------+-------------+ -list field type: -| one byte | var objects | -+-----------+-------------+ -| ref meta | value data | -set field type: -| one byte | var objects | -+-----------+-------------+ -| ref meta | value data | -map field type: -| one byte | var objects | -+-----------+-------------+ -| ref meta | value data | -+-----------+-------------+-------------+ -other types such as enum/struct/ext -| one byte | var bytes | var objects | -+-----------+------------+------------+ -| ref flag | type meta | value data | -+-----------+------------+------------+ -``` - -Type hash algorithm: - -- Sort fields by fields sort algorithm -- Start with string `""` -- Iterate every field, append string by: - - `snow_case(field_name),`. For camelcase name, convert it to snow_case first. - - `$type_id,`, for other fields, use type id `TypeId::UNKNOWN` instead. - - `$nullable;`, `1` if nullable, `0` otherwise. -- Then convert string to utf8 bytes -- Compute murmurhash3_x64_128, and use first 32 bits - -#### Schema evolution - -Schema evolution have similar format as schema consistent mode for object except: - -- For the object type, `schema consistent` mode will write type by id only, but `schema evolution` mode will - write type consisting of field names, types and other meta too, see [Type meta](#type-meta). -- Type meta of `final custom type` needs to be written too, because peers may not have this type defined. +Struct means object of `class/pojo/struct/bean/record` type. Struct values are serialized by writing +fields in Fory order. The type meta before the value is written according to the rules in +[Type Meta](#type-meta). + +#### Field order + +Fory uses `DescriptorGrouper` to build a deterministic order: + +- primitive/boxed/built-in fields first +- collections and maps next +- other user-defined fields last + +Within each group, descriptors are sorted by a stable comparator (type ID and name). The exact +ordering is implementation-defined but stable within a release. + +#### Schema consistent (meta share disabled) + +Object value layout: + +``` +| [optional 4-byte schema hash] | field values | +``` + +The schema hash is written only when class-version checking is enabled. It is the low 32 bits of a +MurmurHash3 x64_128 of the struct fingerprint string: + +- For each field, build `,,,;`. +- Field identifier is the tag ID if present, otherwise the snake_case field name. +- Sort by field identifier lexicographically before concatenation. + +Field values are serialized in Fory order. Primitive fields are written as raw values (nullable +primitives include a null flag). Non-primitive fields write ref/null flags as needed and then the +value; polymorphic fields include type meta. + +#### Compatible mode (meta share enabled) + +The field value layout is the same as schema-consistent mode, but the type meta for +`COMPATIBLE_STRUCT` and `NAMED_COMPATIBLE_STRUCT` uses shared TypeDef entries. Deserializers use +TypeDef to map fields by name or tag ID and to honor nullable/ref flags from metadata; unknown fields +are skipped. ### Type @@ -1534,9 +1363,8 @@ This section provides a step-by-step guide for implementing Fory xlang serializa - [ ] Optionally implement Hybrid encoding (TAGGED_INT64/TAGGED_UINT64) for int64 3. **Header Handling** - - [ ] Write/read bitmap flags (null, endian, xlang, oob) - - [ ] Write/read language ID - - [ ] Handle meta start offset placeholder (for schema evolution) + - [ ] Write/read bitmap flags (null, xlang, oob) + - [ ] Write/read language ID (when xlang flag is set) ### Phase 2: Basic Type Serializers @@ -1605,26 +1433,26 @@ Meta strings are required for enum and struct serialization (encoding field name - [ ] Generate type IDs: `(user_id << 8) | internal_type_id` 14. **Field Ordering** - - [ ] Implement Fory field ordering algorithm - - [ ] Sort primitives by size (larger first), then type ID, then name - - [ ] Handle nullable vs non-nullable fields - - [ ] Convert field names to snake_case for sorting + - [ ] Implement DescriptorGrouper ordering (primitive/boxed/built-in, collections/maps, other) + - [ ] Use a stable comparator within each group (type ID and name) + - [ ] Use tag ID or snake_case field name as field identifier for fingerprints 15. **Schema Consistent Mode** - - [ ] Compute type hash (MurmurHash3 of field info string) - - [ ] Write 4-byte type hash before fields + - [ ] If class-version check is enabled, compute schema hash from field identifiers + - [ ] Write 4-byte schema hash before fields - [ ] Serialize fields in Fory order -16. **Schema Evolution Mode** (Optional) - - [ ] Implement type meta writing - - [ ] Support field addition/removal - - [ ] Handle unknown fields (skip during read) +16. **Compatible/Meta Share Mode** + - [ ] Implement shared TypeDef stream (inline new TypeDefs, index references) + - [ ] Map fields by name or tag ID, skip unknown fields + - [ ] Apply nullable/ref flags from TypeDef metadata ### Phase 7: Other types 17. **Binary/Array Types** - - [ ] Primitive arrays (direct buffer copy) - - [ ] Tensor (multi-dimensional arrays) + +- [ ] Primitive arrays (direct buffer copy) +- [ ] Multi-dimensional arrays as nested lists (no tensor encoding) ### Testing Strategy @@ -1680,8 +1508,8 @@ Meta strings are required for enum and struct serialization (encoding field name 1. **Byte Order**: Always use little-endian for multi-byte values 2. **Varint Sign Extension**: Ensure proper handling of signed vs unsigned varints 3. **Reference ID Ordering**: IDs must be assigned in serialization order -4. **Field Order Consistency**: Must match exactly across languages (schema consistent mode only; in evolution mode, deserialization follows serialization field order from type meta) +4. **Field Order Consistency**: Must match exactly across languages in schema-consistent mode; in compatible mode, match by TypeDef field names or tag IDs 5. **String Encoding**: Use best encoding for current language 6. **Null Handling**: Different languages represent null differently 7. **Empty Collections**: Still write length (0) and header byte -8. **Type Hash Calculation**: Must use exact same algorithm across languages +8. **Schema Hash Calculation**: Must use the same fingerprint and MurmurHash3 algorithm across languages when enabled From 123582536fca34274cc53b49f36f802026c54736 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Mon, 19 Jan 2026 15:44:37 +0800 Subject: [PATCH 44/44] update field order --- .../specification/xlang_serialization_spec.md | 65 +++++++++++++++++-- 1 file changed, 58 insertions(+), 7 deletions(-) diff --git a/docs/specification/xlang_serialization_spec.md b/docs/specification/xlang_serialization_spec.md index 92aac982fc..5882cffc66 100644 --- a/docs/specification/xlang_serialization_spec.md +++ b/docs/specification/xlang_serialization_spec.md @@ -1226,14 +1226,65 @@ fields in Fory order. The type meta before the value is written according to the #### Field order -Fory uses `DescriptorGrouper` to build a deterministic order: +Field order must be deterministic and identical across languages. This section defines the +language-neutral ordering algorithm; implementations must follow the rules here rather than any +language-specific helper classes. -- primitive/boxed/built-in fields first -- collections and maps next -- other user-defined fields last +##### Step 1: Field identifier -Within each group, descriptors are sorted by a stable comparator (type ID and name). The exact -ordering is implementation-defined but stable within a release. +For every field, compute a stable identifier used for ordering: + +- If a tag ID is configured (e.g., `@ForyField(id=...)`), use the tag ID as a decimal string. +- Otherwise, use the field name converted to `snake_case`. + +Tag IDs must be unique within a type; duplicate tag IDs are invalid. + +##### Step 2: Group assignment + +Assign each field to exactly one group in the following order: + +1. **Primitive (non-nullable)**: primitive or boxed numeric/boolean types with `nullable=false`. +2. **Primitive (nullable)**: primitive or boxed numeric/boolean types with `nullable=true`. +3. **Built-in (non-container)**: internal type IDs that are not user-defined and not UNKNOWN, + excluding collections and maps (for example: STRING, TIME types, UNION, primitive arrays). +4. **Collection**: list/set/object-array fields. Non-primitive arrays are treated as LIST for + ordering purposes. +5. **Map**: map fields. +6. **Other**: user-defined enum/struct/ext and UNKNOWN types. + +##### Step 3: Intra-group ordering + +Within each group, apply the following sort keys in order until a difference is found: + +**Primitive groups (1 and 2):** + +1. **Compression category**: fixed-size numeric and boolean types first, then compressed numeric + types (`VARINT32`, `VAR_UINT32`, `VARINT64`, `VAR_UINT64`, `TAGGED_INT64`, `TAGGED_UINT64`). +2. **Primitive size** (descending): 8-byte > 4-byte > 2-byte > 1-byte. +3. **Internal type ID** (descending) as a tie-breaker for equal sizes. +4. **Field identifier** (lexicographic ascending). + +**Built-in / Collection / Map groups (3-5):** + +1. **Internal type ID** (ascending). +2. **Field identifier** (lexicographic ascending). + +**Other group (6):** + +1. **Field identifier** (lexicographic ascending). + +If two fields still compare equal after the rules above, preserve a deterministic order by +comparing declaring class name and then the original field name. This tie-breaker should be +reachable only in invalid schemas (e.g., duplicate tag IDs). + +##### Notes + +- The ordering above is used for serialization order and TypeDef field lists. Schema hashes use + the field identifier ordering described in the schema hash section. +- Collection/map normalization is required so peers with different concrete types (e.g., + `List` vs `Collection`) still agree on ordering. +- The compressed numeric rule is critical for cross-language consistency: compressed integer + fields are always placed after all fixed-width integer fields. #### Schema consistent (meta share disabled) @@ -1433,7 +1484,7 @@ Meta strings are required for enum and struct serialization (encoding field name - [ ] Generate type IDs: `(user_id << 8) | internal_type_id` 14. **Field Ordering** - - [ ] Implement DescriptorGrouper ordering (primitive/boxed/built-in, collections/maps, other) + - [ ] Implement the spec-defined grouping and ordering (primitive/boxed/built-in, collections/maps, other) - [ ] Use a stable comparator within each group (type ID and name) - [ ] Use tag ID or snake_case field name as field identifier for fingerprints