GH-55478: [R] Vendored dependencies to enable parquet (smallest change, biggest tarball)

ci/scripts/r_test.sh

@@ -158,6 +158,26 @@ if [ "${ARROW_R_TEST_OFFLINE:-false}" = "true" ]; then
   cp /etc/resolv.conf{.bak,}
 fi
 
+if [ -n "${ARROW_R_EXPECTED_FEATURES:-}" ]; then
+  # The tests skip what the build doesn't support, so make sure the features
+  # we expect (e.g. from the vendored thirdparty sources) were actually built.
+  # ARROW_R_EXPECTED_FEATURES is a comma-separated list of
+  # names(arrow::arrow_info()$capabilities) and memory pool backends
+  # (e.g. mimalloc).
+  ${R_BIN} -s -e "
+    library(arrow, lib.loc = 'check/arrow.Rcheck')
+    info <- arrow_info()
+    print(info\$capabilities)
+    print(info\$memory_pool\$available_backends)
+    available <- c(names(which(info\$capabilities)), info\$memory_pool\$available_backends)
+    expected <- strsplit(Sys.getenv('ARROW_R_EXPECTED_FEATURES'), ',')[[1]]
+    missing <- setdiff(expected, available)
+    if (length(missing) > 0) {
+      stop('Expected features are not available: ', paste(missing, collapse = ', '))
+    }
+  "
+fi
+
 AFTER=$(ls -alh ~/)
 if [ "$NOT_CRAN" != "true" ] && [ "$BEFORE" != "$AFTER" ]; then
   # Ignore ~/.TinyTex/ and ~/R/ because it has many files.

cpp/cmake_modules/ThirdpartyToolchain.cmake

@@ -469,6 +469,12 @@ foreach(_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT})
     continue()
   endif()
 
+  # Allow overriding checksums by environment variables, e.g. for a source
+  # archive given by ARROW_$LIBRARY_URL that differs from the upstream one
+  if(_VARIABLE_NAME MATCHES "_CHECKSUM$" AND DEFINED ENV{${_VARIABLE_NAME}})
+    set(_VARIABLE_VALUE "$ENV{${_VARIABLE_NAME}}")
+  endif()
+
   # For debugging
   message(STATUS "${_VARIABLE_NAME}: ${_VARIABLE_VALUE}")
 

dev/release/rat_exclude_files.txt

@@ -97,6 +97,7 @@ r/tests/testthat/_snaps/*.md
 r/inst/include/cpp11.hpp
 r/inst/include/cpp11/*.hpp
 r/tools/nixlibs-allowlist.txt
+r/inst/COPYRIGHTS
 .gitattributes
 ruby/red-arrow/.yardopts
 ruby/red-arrow-format/lib/arrow-format/org/*

dev/tasks/tasks.yml

@@ -633,14 +633,28 @@ tasks:
     template: r/github.linux.offline.build.yml
 
   # Simulate a CRAN builder without network access: the build should detect
-  # that it is offline and fall back to a minimal build without downloading
-  # anything.
-  test-r-offline-minimal:
-    ci: github
-    template: docker-tests/github.linux.yml
-    params:
-      flags: '-e ARROW_R_TEST_OFFLINE=true -e NOT_CRAN=false -e SKIP_VIGNETTES=true'
+  # that it is offline and build libarrow without downloading anything, using
+  # the thirdparty sources vendored in the package (tools/vendored) to still
+  # include Parquet, Dataset, JSON, the compression codecs, etc.
+  # ARROW_SOURCE_HOME='' makes it use the C++ source copied into tools/cpp
+  # (like the CRAN package does), whose versions.txt has the checksums of the
+  # trimmed tarballs. bz2 is off by default on Linux, so it is turned on to make
+  # sure every vendored tarball is built. Brotli and Zstandard aren't vendored.
+  test-r-offline-vendored:
+    ci: github
+    template: docker-tests/github.linux.yml
+    params:
+      flags: >-
+        -e ARROW_R_TEST_OFFLINE=true
+        -e NOT_CRAN=false
+        -e SKIP_VIGNETTES=true
+        -e ARROW_SOURCE_HOME=''
+        -e ARROW_WITH_BZ2=ON
+        -e ARROW_R_EXPECTED_FEATURES=dataset,parquet,json,mimalloc,re2,utf8proc,snappy,gzip,lz4,bz2
       image: r
+      # A full libarrow build with 2 cores (NOT_CRAN=false) and no sccache
+      # (it's offline)
+      timeout: 180
 
   # This one has -flto=auto
   test-r-ubuntu-22.04:

docs/source/developers/cpp/building.rst

@@ -581,6 +581,10 @@ internet.
 To enable offline builds, you can download the source artifacts yourself and
 use environment variables of the form ``ARROW_$LIBRARY_URL`` to direct the
 build system to read from a local file rather than accessing the internet.
+Each archive is checked against the SHA256 checksum in
+``thirdparty/versions.txt``; if you use an archive that differs from the
+upstream one, you can set the matching ``ARROW_$LIBRARY_BUILD_SHA256_CHECKSUM``
+environment variable to its checksum.
 
 To make this easier for you, we have prepared a script
 ``thirdparty/download_dependencies.sh`` which will download the correct version

r/.Rbuildignore

@@ -32,3 +32,5 @@ STYLE.md
 ^inst/__pycache__$
 ^bootstrap.R$
 air.toml
+^tools/vendored-downloads$
+^tools/trim$

r/.gitignore

@@ -35,3 +35,5 @@ extra-tests/files
 # Source tarballs of required dependencies that we ship for offline builds.
 # They're downloaded by `make sync-cpp` (see tools/vendor_dependencies.sh).
 /tools/vendored/
+# Cache of the upstream (untrimmed) tarballs used by tools/vendor_dependencies.sh
+/tools/vendored-downloads/

r/DESCRIPTION

@@ -24,6 +24,7 @@ Description: 'Apache' 'Arrow' <https://arrow.apache.org/> is a cross-language
     package provides an interface to the 'Arrow C++' library.
 Depends: R (>= 4.2)
 License: Apache License (>= 2.0)
+Copyright: See file inst/COPYRIGHTS
 URL: https://github.com/apache/arrow/, https://arrow.apache.org/docs/r/
 BugReports: https://github.com/apache/arrow/issues
 Encoding: UTF-8

r/inst/COPYRIGHTS

@@ -0,0 +1,71 @@
+The arrow R package is Copyright The Apache Software Foundation and is licensed
+under the Apache License, Version 2.0. The Arrow C++ library source copied into
+tools/cpp includes code from other projects; see tools/LICENSE.txt and
+tools/NOTICE.txt in the source package.
+
+To build the Arrow C++ library without network access, the source package also
+includes (trimmed) source tarballs of these thirdparty dependencies in
+tools/vendored. They are only used when the dependency isn't found on the
+system, and are statically linked into the package when used. Each tarball
+contains the dependency's full license text.
+
+Boost (the subset of libraries used by Apache Thrift; Boost.Locale and the
+Boost libraries it needs are compiled and statically linked, the rest are
+header-only)
+  Copyright the Boost authors
+  Boost Software License 1.0
+
+bzip2
+  Copyright (C) 1996-2019 Julian R Seward
+  BSD-style license (see LICENSE in the tarball)
+
+LZ4 (lib/ only)
+  Copyright (c) 2011-2020, Yann Collet
+  BSD 2-Clause License
+
+mimalloc
+  Copyright (c) 2018-2025 Microsoft Corporation, Daan Leijen
+  MIT License
+
+RapidJSON
+  Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip
+  MIT License (with code under the BSD and JSON licenses, see license.txt in
+  the tarball)
+
+RE2
+  Copyright (c) 2009 The RE2 Authors
+  BSD 3-Clause License
+
+simdjson
+  Copyright 2018-2025 The simdjson authors
+  Apache License 2.0 or MIT License
+
+Snappy
+  Copyright 2011, Google Inc.
+  BSD 3-Clause License
+
+Apache Thrift
+  Copyright (C) 2006-2019, The Apache Software Foundation
+  Apache License 2.0
+
+uriparser (library sources only)
+  Copyright (C) 2007, Weijia Song and Sebastian Pipping
+  BSD 3-Clause License
+
+utf8proc
+  Copyright (c) 2014-2021 Steven G. Johnson, Jiahao Chen, Tony Kelman,
+  Jonas Fonseca, and other contributors
+  Copyright (c) 2009, 2013 Public Software Group e. V., Berlin, Germany
+  MIT License, with Unicode data under the Unicode Data Files and Software
+  License
+
+xsimd
+  Copyright (c) 2016, Johan Mabille, Sylvain Corlay, Wolf Vollprecht and
+  Martin Renou
+  Copyright (c) 2016, QuantStack
+  Copyright (c) 2018, Serge Guelton
+  BSD 3-Clause License
+
+zlib
+  Copyright (C) 1995-2022 Jean-loup Gailly and Mark Adler
+  zlib License