From d6f8de0fe1987f8163d547d19b27fa7ecf4511c0 Mon Sep 17 00:00:00 2001 From: Anurag Raut Date: Sat, 14 Nov 2026 16:15:19 +0530 Subject: [PATCH] GH-55503: [C++] Fix Float16 memo table keys built from raw bits --- cpp/src/arrow/array/array_dict.cc | 15 ++++++++++-- cpp/src/arrow/array/array_dict_test.cc | 23 +++++++++++++++++++ .../arrow/compute/kernels/aggregate_basic.cc | 7 +++++- .../arrow/compute/kernels/aggregate_test.cc | 9 ++++++++ .../arrow/compute/kernels/codegen_internal.h | 2 +- 5 files changed, 52 insertions(+), 4 deletions(-) diff --git a/cpp/src/arrow/array/array_dict.cc b/cpp/src/arrow/array/array_dict.cc index 6b3511721d28..c93691e701d7 100644 --- a/cpp/src/arrow/array/array_dict.cc +++ b/cpp/src/arrow/array/array_dict.cc @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -39,6 +40,7 @@ #include "arrow/util/bit_util.h" #include "arrow/util/bitmap_ops.h" #include "arrow/util/checked_cast.h" +#include "arrow/util/float16.h" #include "arrow/util/int_util.h" #include "arrow/util/logging_internal.h" #include "arrow/visit_type_inline.h" @@ -358,18 +360,27 @@ class DictionaryUnifierImpl : public DictionaryUnifier { dictionary.type()->ToString()); } const ArrayType& values = checked_cast(dictionary); + // The half float memo table is keyed on `util::Float16`, whose converting + // constructor would read the raw uint16_t storage value as a number. + auto get_key = [&](int64_t i) { + if constexpr (std::is_same_v) { + return ::arrow::util::Float16::FromBits(values.GetView(i)); + } else { + return values.GetView(i); + } + }; if (out != nullptr) { ARROW_ASSIGN_OR_RAISE(auto result, AllocateBuffer(dictionary.length() * sizeof(int32_t), pool_)); auto result_raw = reinterpret_cast(result->mutable_data()); for (int64_t i = 0; i < values.length(); ++i) { - RETURN_NOT_OK(memo_table_.GetOrInsert(values.GetView(i), &result_raw[i])); + RETURN_NOT_OK(memo_table_.GetOrInsert(get_key(i), &result_raw[i])); } *out = std::move(result); } else { for (int64_t i = 0; i < values.length(); ++i) { int32_t unused_memo_index; - RETURN_NOT_OK(memo_table_.GetOrInsert(values.GetView(i), &unused_memo_index)); + RETURN_NOT_OK(memo_table_.GetOrInsert(get_key(i), &unused_memo_index)); } } return Status::OK(); diff --git a/cpp/src/arrow/array/array_dict_test.cc b/cpp/src/arrow/array/array_dict_test.cc index 23335ebb008a..372ea7a5fff9 100644 --- a/cpp/src/arrow/array/array_dict_test.cc +++ b/cpp/src/arrow/array/array_dict_test.cc @@ -1655,6 +1655,29 @@ TEST(TestDictionaryUnifier, Numeric) { CheckTransposeMap(*b3, {3, 5}); } +TEST(TestDictionaryUnifier, HalfFloat) { + // The unified dictionary must keep the values, not their bit patterns read as numbers + auto dict_ty = float16(); + auto d1 = ArrayFromJSON(dict_ty, "[1.0, 2.5, 65504.0]"); + auto d2 = ArrayFromJSON(dict_ty, "[0.5, 65504.0, 1.0]"); + auto expected = dictionary(int8(), dict_ty); + auto expected_dict = ArrayFromJSON(dict_ty, "[1.0, 2.5, 65504.0, 0.5]"); + + ASSERT_OK_AND_ASSIGN(auto unifier, DictionaryUnifier::Make(dict_ty)); + std::shared_ptr b1, b2; + ASSERT_OK(unifier->Unify(*d1, &b1)); + ASSERT_OK(unifier->Unify(*d2, &b2)); + + std::shared_ptr out_type; + std::shared_ptr out_dict; + ASSERT_OK(unifier->GetResult(&out_type, &out_dict)); + ASSERT_TRUE(out_type->Equals(*expected)); + AssertArraysEqual(*expected_dict, *out_dict, /*verbose=*/true); + + CheckTransposeMap(*b1, {0, 1, 2}); + CheckTransposeMap(*b2, {3, 2, 0}); +} + TEST(TestDictionaryUnifier, String) { auto dict_ty = utf8(); diff --git a/cpp/src/arrow/compute/kernels/aggregate_basic.cc b/cpp/src/arrow/compute/kernels/aggregate_basic.cc index f7ff72bc114a..3211f1604f33 100644 --- a/cpp/src/arrow/compute/kernels/aggregate_basic.cc +++ b/cpp/src/arrow/compute/kernels/aggregate_basic.cc @@ -172,7 +172,12 @@ struct CountDistinctImpl : public ScalarAggregator { auto visit_null = []() { return Status::OK(); }; auto visit_value = [&](VisitorArgType arg) { int32_t y; - return memo_table_->GetOrInsert(arg, &y); + if constexpr (std::is_same_v) { + // `arg` holds the raw bits, don't let `Float16` read them as a number. + return memo_table_->GetOrInsert(::arrow::util::Float16::FromBits(arg), &y); + } else { + return memo_table_->GetOrInsert(arg, &y); + } }; RETURN_NOT_OK(VisitArraySpanInline(arr, visit_value, visit_null)); } else { diff --git a/cpp/src/arrow/compute/kernels/aggregate_test.cc b/cpp/src/arrow/compute/kernels/aggregate_test.cc index 4ef2b2f88a51..fc5ad0eeec79 100644 --- a/cpp/src/arrow/compute/kernels/aggregate_test.cc +++ b/cpp/src/arrow/compute/kernels/aggregate_test.cc @@ -1194,6 +1194,15 @@ TEST_F(TestCountDistinctKernel, AllScalarTypesWithNulls) { Check(decimal256(13, 3), sample); } +TEST_F(TestCountDistinctKernel, HalfFloat) { + // 1.0, 1.0009765625 and 1.001953125 have the consecutive bit patterns + // 0x3C00, 0x3C01 and 0x3C02 and must be counted as distinct values. + Check(float16(), "[1.0, 1.0009765625, 1.001953125, 1.0, null]", 4); + Check(float16(), "[1.0, 1.0009765625, 1.001953125, 1.0]", 3, /*has_nulls=*/false); + CheckChunkedArr(float16(), {"[1.0, 1.0009765625]", "[1.001953125, null]"}, 4); + Check(float16(), "1.5"); +} + TEST_F(TestCountDistinctKernel, Random) { UInt32Builder builder; std::unordered_set memo; diff --git a/cpp/src/arrow/compute/kernels/codegen_internal.h b/cpp/src/arrow/compute/kernels/codegen_internal.h index d51fd7c9a0e0..6abab0fcc270 100644 --- a/cpp/src/arrow/compute/kernels/codegen_internal.h +++ b/cpp/src/arrow/compute/kernels/codegen_internal.h @@ -432,7 +432,7 @@ template <> struct UnboxScalar { using T = Float16; static T Unbox(const Scalar& val) { - return T(checked_cast(val).value); + return T::FromBits(checked_cast(val).value); } };