Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 36 additions & 1 deletion cpp/src/gandiva/engine.cc
Original file line number Diff line number Diff line change
Expand Up @@ -117,9 +117,44 @@

namespace gandiva {

// Oldest macOS release for each architecture. Used as a floor and as the fallback when
// the Darwin kernel version is missing or can't be converted.
constexpr unsigned kMinMacOSMajorArm64 = 11; // first Apple Silicon release
constexpr unsigned kMinMacOSMinorArm64 = 0;
constexpr unsigned kMinMacOSMajorX86 = 10;
constexpr unsigned kMinMacOSMinorX86 = 15;

extern const unsigned char kPrecompiledBitcode[];
extern const size_t kPrecompiledBitcodeSize;

llvm::Triple NormalizeDarwinTriple(llvm::Triple triple) {

@dmitry-chirkov-dremio dmitry-chirkov-dremio Nov 13, 2026 •

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Part of me wants to do something like this which is conditioned on llvm/llvm-project#210674 making it into LLVM 24 (and then us supporting it). Thoughts?

This is a bit wider change to clean up assumptions of the past since we are here anyway. This would replace the manual CPU/feature setup with LLVM’s detectHost(), but preserves the commit’s opt-out by clearing CPU/features when target_host_cpu(false), and adds the macOS triple correction.

Result<llvm::orc::JITTargetMachineBuilder> MakeTargetMachineBuilder(
    const Configuration& conf) {
  auto detected = llvm::orc::JITTargetMachineBuilder::detectHost();
  ARROW_ASSIGN_OR_RAISE(
      auto jtmb, AsArrowResult(detected, "Could not detect host: "));

  if (!conf.target_host_cpu()) {
    jtmb.setCPU("");
    jtmb.setFeatures("");
  }

#if defined(__APPLE__) && LLVM_VERSION_MAJOR < 24
  auto& triple = jtmb.getTargetTriple();
  if (triple.getOS() == llvm::Triple::Darwin) {
    char version[64]{};
    size_t size = sizeof(version);
    if (sysctlbyname("kern.osproductversion", version, &size,
                    nullptr, 0) != 0) {
      return arrow::internal::IOErrorFromErrno(
          errno, "sysctlbyname failed for 'kern.osproductversion'");
    }
    triple.setOSName(std::string("macosx") + version);
  }
#endif

  jtmb.setRelocationModel(llvm::Reloc::PIC_);

#if LLVM_VERSION_MAJOR >= 18
  using CodeGenOptLevel = llvm::CodeGenOptLevel;
#else
  using CodeGenOptLevel = llvm::CodeGenOpt::Level;
#endif
  jtmb.setCodeGenOptLevel(
      conf.optimize() ? CodeGenOptLevel::Aggressive : CodeGenOptLevel::None);

  return jtmb;
}

// LLVM's default host triple names the OS "darwin<kernel version>", but LLVM only
// enables macOS libcalls (e.g. __exp10 for llvm.exp10) for the "macosx" OS.
// clang's driver does this rewrite; the JIT does not, so do it here.
if (triple.getOS() != llvm::Triple::Darwin) {
return triple;
}

const llvm::VersionTuple min_version =
triple.isAArch64() ? llvm::VersionTuple(kMinMacOSMajorArm64, kMinMacOSMinorArm64)
: llvm::VersionTuple(kMinMacOSMajorX86, kMinMacOSMinorX86);

// Only "at least macOS 10.9" matters for libcall selection, so an approximate
// kernel->macOS mapping is acceptable here.
llvm::VersionTuple version = min_version;
llvm::VersionTuple converted;
if (triple.getMacOSXVersion(converted) && converted >= min_version) {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This conversion can advertise a newer macOS than the machine actually runs. With LLVM 21.1.8, darwin27.2.0 converts to macosx28.0 on my macOS 27.2.1 machine. Although that enables exp10, the target version affects other libcall availability decisions too. Could we instead take the macOS OS/version component from Gandiva’s embedded bitcode target triple, after validating its platform and architecture? Clang already records that target. I tested that approach with LLVM 21.1.8, and it fixes the reported cases without the approximate kernel-version conversion.

version = converted;
}

// getMacOSXVersion() leaves the minor unset for darwin20+; print it as "11.0", not
// "11".
const unsigned minor = version.getMinor().value_or(0);

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

getMinor() returns llvm::Optional on LLVM 14, which has no value_or(). CMake still accepts those versions, and this function compiles on every platform. Let's use the portable form.

const auto minor = version.getMinor();
const unsigned minor_value = minor ? *minor : 0;

triple.setOSName("macosx" +
llvm::VersionTuple(version.getMajor(), minor).getAsString());
return triple;
}

namespace {

std::once_flag llvm_init_once_flag;
Expand All @@ -142,7 +177,7 @@ arrow::Result<T> AsArrowResult(llvm::Expected<T>& expected,
Result<llvm::orc::JITTargetMachineBuilder> MakeTargetMachineBuilder(
const Configuration& conf) {
llvm::orc::JITTargetMachineBuilder jtmb(
(llvm::Triple(llvm::sys::getDefaultTargetTriple())));
(NormalizeDarwinTriple(llvm::Triple(llvm::sys::getDefaultTargetTriple()))));
jtmb.setRelocationModel(llvm::Reloc::PIC_);
if (conf.target_host_cpu()) {
jtmb.setCPU(cpu_name.str());
Expand Down
5 changes: 5 additions & 0 deletions cpp/src/gandiva/engine.h
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,11 @@ class LLJIT;

namespace gandiva {

/// \brief Rewrite a "darwin<kernel version>" host triple to "macosx<version>".
/// LLVM only enables macOS libcalls (e.g. __exp10) for the "macosx" OS name.
/// Non-Darwin triples are returned unchanged.
GANDIVA_EXPORT llvm::Triple NormalizeDarwinTriple(llvm::Triple triple);

/// \brief LLVM Execution engine wrapper.
class GANDIVA_EXPORT Engine {
public:
Expand Down
54 changes: 54 additions & 0 deletions cpp/src/gandiva/engine_llvm_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,10 @@ using add_vector_func_t = int64_t (*)(int64_t*, int);

class TestEngine : public ::testing::Test {
protected:
static std::string Normalize(const std::string& triple) {
return NormalizeDarwinTriple(llvm::Triple(triple)).str();
}

std::string BuildVecAdd(Engine* gdv_engine) {
auto types = gdv_engine->types();
llvm::IRBuilder<>* builder = gdv_engine->ir_builder();
Expand Down Expand Up @@ -133,4 +137,54 @@ TEST_F(TestEngine, TestAddOptimised) {
EXPECT_EQ(add_func(my_array, 5), 17);
}

TEST_F(TestEngine, NormalizeDarwinTripleDarwinBecomesMacOSX) {
llvm::Triple arm = NormalizeDarwinTriple(llvm::Triple("arm64-apple-darwin27.2.0"));
EXPECT_EQ(arm.getOS(), llvm::Triple::MacOSX);
EXPECT_TRUE(arm.isAArch64());
llvm::VersionTuple v;
ASSERT_TRUE(arm.getMacOSXVersion(v));
EXPECT_GE(v, llvm::VersionTuple(11, 0));

EXPECT_EQ(Normalize("x86_64-apple-darwin19.0.0"), "x86_64-apple-macosx10.15");
}

TEST_F(TestEngine, NormalizeDarwinTripleUnversionedOrOldUsesFloor) {
EXPECT_EQ(Normalize("arm64-apple-darwin"), "arm64-apple-macosx11.0");
EXPECT_EQ(Normalize("arm64-apple-darwin8.0.0"), "arm64-apple-macosx11.0");
EXPECT_EQ(Normalize("x86_64-apple-darwin3.0.0"), "x86_64-apple-macosx10.15");
}

TEST_F(TestEngine, NormalizeDarwinTripleOtherTriplesUnchanged) {
// Already-macosx triples are left as-is
EXPECT_EQ(Normalize("arm64-apple-macosx26.0.0"), "arm64-apple-macosx26.0.0");
// Non-Darwin Apple OSes
EXPECT_EQ(Normalize("arm64-apple-ios17.0.0"), "arm64-apple-ios17.0.0");
// Linux
EXPECT_EQ(Normalize("x86_64-unknown-linux-gnu"), "x86_64-unknown-linux-gnu");
EXPECT_EQ(Normalize("aarch64-unknown-linux-musl"), "aarch64-unknown-linux-musl");
// Windows
EXPECT_EQ(Normalize("x86_64-pc-windows-msvc"), "x86_64-pc-windows-msvc");
}

TEST_F(TestEngine, NormalizeDarwinTripleFloorBoundary) {
// darwin19 is macOS 10.15: x86 stays at its floor, arm64 is raised to 11.0
EXPECT_EQ(Normalize("x86_64-apple-darwin19.0.0"), "x86_64-apple-macosx10.15");
EXPECT_EQ(Normalize("arm64-apple-darwin19.0.0"), "arm64-apple-macosx11.0");
EXPECT_EQ(Normalize("arm64-apple-darwin20.0.0"), "arm64-apple-macosx11.0");
}

TEST_F(TestEngine, NormalizeDarwinTripleArchAliases) {
EXPECT_EQ(Normalize("aarch64-apple-darwin"), "aarch64-apple-macosx11.0");
EXPECT_EQ(Normalize("arm64e-apple-darwin"), "arm64e-apple-macosx11.0");
EXPECT_EQ(Normalize("i386-apple-darwin"), "i386-apple-macosx10.15");
}

TEST_F(TestEngine, NormalizeDarwinTripleIdempotent) {
for (const char* triple : {"arm64-apple-darwin23.1.0", "arm64-apple-darwin",
"x86_64-apple-darwin8", "x86_64-unknown-linux-gnu"}) {
std::string once = Normalize(triple);
EXPECT_EQ(Normalize(once), once) << triple;
}
}

} // namespace gandiva
95 changes: 95 additions & 0 deletions cpp/src/gandiva/tests/decimal_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,23 @@ class TestDecimal : public ::testing::Test {
int32_t scale);

protected:
// Compiles round(float64, int32) through the Gandiva JIT and evaluates it on the

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: One of my favorite things to do is test-only first commit that fails the CI however here I still don't know what runs on pull requests before the green light from the overseer

// inputs. Unlike the native tests in extended_math_ops_test.cc, this compiles the whole
// round_float64_int32, including the powl(10, scale) fallback in get_scale_multiplier,
// which LLVM lowers to exp10. That is what the Darwin triple fix in engine.cc covers.
// The decimal128 round tests in decimal_test.cc use integer arithmetic instead and
// never reach exp10.
void EvaluateRoundFloat64(const std::vector<double>& values,
const std::vector<int32_t>& scales, bool optimize,
std::shared_ptr<arrow::Array>* result);

// Runs the case with IR optimization off and on, and checks the result each time.
// Without the Darwin triple fix, compiling round(float64, int32) aborts the process
// on macOS ("no libcall available for fexp10"), whatever the scale values are.
void CheckRoundFloat64(const std::vector<double>& values,
const std::vector<int32_t>& scales,
const std::vector<double>& expected_values);

arrow::MemoryPool* pool_;
};

Expand All @@ -67,6 +84,50 @@ std::vector<Decimal128> TestDecimal::MakeDecimalVector(std::vector<std::string>
return ret;
}

void TestDecimal::EvaluateRoundFloat64(const std::vector<double>& values,
const std::vector<int32_t>& scales, bool optimize,
std::shared_ptr<arrow::Array>* result) {
auto field_value = arrow::field("value", arrow::float64());
auto field_scale = arrow::field("scale", arrow::int32());
auto schema = arrow::schema({field_value, field_scale});
auto field_result = arrow::field("result", arrow::float64());

auto round_fn = TreeExprBuilder::MakeFunction(
"round",
{TreeExprBuilder::MakeField(field_value), TreeExprBuilder::MakeField(field_scale)},
arrow::float64());
auto expr = TreeExprBuilder::MakeExpression(round_fn, field_result);
auto configuration = ConfigurationBuilder().build(optimize);

std::shared_ptr<Projector> projector;
auto status = Projector::Make(schema, {expr}, configuration, &projector);
ASSERT_TRUE(status.ok()) << status.message();

std::vector<bool> valid(values.size(), true);
auto in_batch = arrow::RecordBatch::Make(
schema, static_cast<int>(values.size()),
{MakeArrowArrayFloat64(values, valid), MakeArrowArrayInt32(scales, valid)});

arrow::ArrayVector outputs;
status = projector->Evaluate(*in_batch, pool_, &outputs);
ASSERT_TRUE(status.ok()) << status.message();
ASSERT_EQ(outputs.size(), 1u);
*result = outputs.at(0);
}

void TestDecimal::CheckRoundFloat64(const std::vector<double>& values,
const std::vector<int32_t>& scales,
const std::vector<double>& expected_values) {
auto expected = MakeArrowArrayFloat64(expected_values,
std::vector<bool>(expected_values.size(), true));
for (bool optimize : {false, true}) {
SCOPED_TRACE(optimize ? "optimize=true" : "optimize=false");
std::shared_ptr<arrow::Array> result;
ASSERT_NO_FATAL_FAILURE(EvaluateRoundFloat64(values, scales, optimize, &result));
EXPECT_ARROW_ARRAY_EQUALS(expected, result);
}
}

TEST_F(TestDecimal, CastLargeBatch) {
// Allocating decimal return slots in the record loop can exhaust the stack
// when optimization is disabled. A single int32-to-decimal cast is sufficient.
Expand Down Expand Up @@ -1349,4 +1410,38 @@ TEST_F(TestDecimal, TestCastDecimalVarCharInvalidInputInvalidOutput) {
ASSERT_NOT_OK(status);
ASSERT_THAT(status.message(), ::testing::HasSubstr("not a valid decimal128 number"));
}

TEST_F(TestDecimal, TestRoundFloat64ScaleInLookupTable) {
// Scales served by the lookup table
CheckRoundFloat64({1.26, 2.34}, {1, 1}, {1.3, 2.3});
}

TEST_F(TestDecimal, TestRoundFloat64ScaleAtTableBoundary) {
// The first scale outside the lookup table (20); values come back unchanged
CheckRoundFloat64({1.26, -1.26}, {20, 20}, {1.26, -1.26});
}

TEST_F(TestDecimal, TestRoundFloat64LargeScale) {
// Scales well above the table need no rounding
CheckRoundFloat64({1.26, 2.34, -1.26}, {25, 30, 25}, {1.26, 2.34, -1.26});
}

TEST_F(TestDecimal, TestRoundFloat64NegativeScale) {
// Negative scales are also outside the table; these inputs round to 0
CheckRoundFloat64({1.26, 2.34}, {-2, -1}, {0.0, 0.0});
}

TEST_F(TestDecimal, TestRoundFloat64MixedScalesInOneBatch) {
// Table path (1), exp10 path (25, -1) and the table's last entry (19) in one batch
CheckRoundFloat64({1.26, 1.26, 2.5, 1.26}, {1, 25, -1, 19}, {1.3, 1.26, 0.0, 1.26});
}

TEST_F(TestDecimal, TestRoundFloat64LargeValuesSkipRounding) {
// |x| >= 2^52 / 10^scale is returned unchanged (the DX-90291 overflow guard)
CheckRoundFloat64({1e300, -1e300, 4.5e15}, {1, 1, 1}, {1e300, -1e300, 4.5e15});
}

TEST_F(TestDecimal, TestRoundFloat64ZeroValue) {
CheckRoundFloat64({0.0, 0.0, 0.0}, {0, 25, -3}, {0.0, 0.0, 0.0});
}
} // namespace gandiva
Loading