ARROW-10856: [R] CC and CXX environment variables passing to cmake

Ensure that the environment variables are properly passed to cmake. Closes apache#8932 from jonkeane/centos-ARROW-10856 Authored-by: Jonathan Keane <[email protected]> Signed-off-by: Neal Richardson <[email protected]>
arw2019 · Dec 19, 2020 · 5819943 · 5819943
1 parent 5c15af9
commit 5819943
Show file tree

Hide file tree

Showing 9 changed files with 39 additions and 4 deletions.
diff --git a/.env b/.env
@@ -47,3 +47,5 @@ ARROW_R_DEV=TRUE
 R_ORG=rhub
 R_IMAGE=ubuntu-gcc-release
 R_TAG=latest
+# -1 does not attempt to install a devtoolset version, any positive integer will install devtoolset-n
+DEVTOOLSET_VERSION=-1
diff --git a/ci/docker/linux-r.dockerfile b/ci/docker/linux-r.dockerfile
@@ -27,6 +27,9 @@ ENV R_BIN=${r_bin}
 ARG r_dev=FALSE
 ENV ARROW_R_DEV=${r_dev}
 
+ARG devtoolset_version=-1
+ENV DEVTOOLSET_VERSION=${devtoolset_version}
+
 # Make sure R is on the path for the R-hub devel versions (where RPREFIX is set in its dockerfile)
 ENV PATH "${RPREFIX}/bin:${PATH}"
 

diff --git a/ci/scripts/r_docker_configure.sh b/ci/scripts/r_docker_configure.sh
@@ -39,6 +39,18 @@ if [ "$RHUB_PLATFORM" = "linux-x86_64-fedora-clang" ]; then
   rm -rf $(${R_BIN} RHOME)/etc/Makeconf.bak
 fi
 
+# Special hacking to try to reproduce quirks on centos using non-default build
+# tooling.
+if [[ "$DEVTOOLSET_VERSION" -gt 0 ]]; then
+  if [ "`which dnf`" ]; then
+    dnf install -y centos-release-scl
+    dnf install -y "devtoolset-$DEVTOOLSET_VERSION"
+  else
+    yum install -y centos-release-scl
+    yum install -y "devtoolset-$DEVTOOLSET_VERSION"
+  fi
+fi
+
 # Install openssl for S3 support
 if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_R_DEV" == "TRUE" ]; then
   if [ "`which dnf`" ]; then

diff --git a/ci/scripts/r_test.sh b/ci/scripts/r_test.sh
@@ -52,6 +52,11 @@ export TEXMFVAR=/tmp/texmf-var
 # Not all Suggested packages are needed for checking, so in case they aren't installed don't fail
 export _R_CHECK_FORCE_SUGGESTS_=FALSE
 
+if [[ "$DEVTOOLSET_VERSION" -gt 0 ]]; then
+  # enable the devtoolset version to use it
+  source /opt/rh/devtoolset-$DEVTOOLSET_VERSION/enable
+fi
+
 # Make sure we aren't writing to the home dir (CRAN _hates_ this but there is no official check)
 BEFORE=$(ls -alh ~/)
 

diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt
@@ -24,7 +24,7 @@
 # format).
 
 ARROW_ABSL_BUILD_VERSION=2eba343b51e0923cd3fb919a6abd6120590fc059
-ARROW_AWSSDK_BUILD_VERSION=1.8.57
+ARROW_AWSSDK_BUILD_VERSION=1.8.90
 ARROW_AWS_CHECKSUMS_BUILD_VERSION=v0.1.5
 ARROW_AWS_C_COMMON_BUILD_VERSION=v0.4.59
 ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION=v0.1.5

diff --git a/dev/tasks/r/azure.linux.yml b/dev/tasks/r/azure.linux.yml
@@ -36,6 +36,7 @@ jobs:
         export R_ORG={{ r_org }}
         export R_IMAGE={{ r_image }}
         export R_TAG={{ r_tag }}
+        export DEVTOOLSET_VERSION={{ devtoolset_version|default("-1") }}
         docker-compose pull --ignore-pull-failures r
         docker-compose build r
       displayName: Docker build

diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
@@ -1766,6 +1766,16 @@ tasks:
       r_tag: 3.6-centos8
       not_cran: "TRUE"
 
+  test-r-rstudio-r-base-3.6-centos7-devtoolset-8:
+    ci: azure
+    template: r/azure.linux.yml
+    params:
+      r_org: rstudio
+      r_image: r-base
+      r_tag: 3.6-centos7
+      not_cran: "TRUE"
+      devtoolset_version: 8
+
   test-r-rstudio-r-base-3.6-opensuse15:
     ci: azure
     template: r/azure.linux.yml

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -867,13 +867,15 @@ services:
       args:
         base: ${R_ORG}/${R_IMAGE}:${R_TAG}
         r_dev: ${ARROW_R_DEV}
+        devtoolset_version: ${DEVTOOLSET_VERSION}
     shm_size: *shm-size
     environment:
       LIBARROW_DOWNLOAD: "false"
       ARROW_HOME: "/arrow"
       ARROW_R_DEV: ${ARROW_R_DEV}
       # To test for CRAN release, delete ^^ these two env vars so we download the Apache release
       ARROW_USE_PKG_CONFIG: "false"
+      DEVTOOLSET_VERSION: ${DEVTOOLSET_VERSION}
     volumes:
       - .:/arrow:delegated
     command: >

diff --git a/r/tools/linuxlibs.R b/r/tools/linuxlibs.R
@@ -384,7 +384,7 @@ with_s3_support <- function(env_vars) {
   if (arrow_s3) {
     # User wants S3 support. Let's make sure they're not on gcc < 4.9
     # and make sure that we have curl and openssl system libs
-    info <- system(paste(env_vars, "&& $CMAKE --system-information"), intern = TRUE)
+    info <- system(paste("export", env_vars, "&& $CMAKE --system-information"), intern = TRUE)
     info <- grep("^[A-Z_]* .*$", info, value = TRUE)
     vals <- as.list(sub('^.*? "?(.*?)"?$', "\\1", info))
     names(vals) <- sub("^(.*?) .*$", "\\1", info)
@@ -410,9 +410,9 @@ cmake_find_package <- function(pkg, version = NULL, env_vars) {
   find_package <- paste0("find_package(", pkg, " ", version, " REQUIRED)")
   writeLines(find_package, file.path(td, "CMakeLists.txt"))
   cmake_cmd <- paste0(
-    env_vars,
+    "export ", env_vars,
     " && cd ", td,
-    " && $CMAKE",
+    " && $CMAKE ",
     " -DCMAKE_EXPORT_NO_PACKAGE_REGISTRY=ON",
     " -DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON",
     " ."