EXP: [C++] Move bundled google-cloud-cpp libraries

ci/scripts/cpp_test.sh

@@ -147,18 +147,28 @@ if [ "${ARROW_USE_MESON:-OFF}" = "OFF" ] && \
     -S "${source_dir}/examples/minimal_build" \
     -B "${build_dir}/examples/minimal_build" \
     -DARROW_EXAMPLE_S3_STATIC_CHECK=ON \
+    -DARROW_EXAMPLE_GCS_STATIC_CHECK=OFF \
     -DCMAKE_PREFIX_PATH="${CMAKE_PREFIX_PATH}"
+  # Check S3 before discovering GCS dependencies
+  cmake --build "${build_dir}/examples/minimal_build"
+  cmake -S "${source_dir}/examples/minimal_build" \
+    -B "${build_dir}/examples/minimal_build" \
+    -DARROW_EXAMPLE_GCS_STATIC_CHECK=ON
   cmake --build "${build_dir}/examples/minimal_build"
   pushd "${source_dir}/examples/minimal_build"
   # PATH= is for Windows.
   PATH="${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}}/bin:${PATH}" \
     "${build_dir}/examples/minimal_build/arrow-example"
-  # Test static linking with S3
-  s3_example="${build_dir}/examples/minimal_build/arrow-example-s3-static"
-  if [ -x "${s3_example}" ]; then
-    PATH="${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}}/bin:${PATH}" \
-      "${s3_example}"
-  fi
+  # Test static filesystem linking
+  for static_example in arrow-example-s3-static \
+                        arrow-example-gcs-static \
+                        arrow-example-s3-gcs-static; do
+    example="${build_dir}/examples/minimal_build/${static_example}"
+    if [ -x "${example}" ]; then
+      PATH="${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}}/bin:${PATH}" \
+        "${example}"
+    fi
+  done
   popd
 fi
 

cpp/cmake_modules/ThirdpartyToolchain.cmake

@@ -18,11 +18,12 @@
 include(ProcessorCount)
 processorcount(NPROC)
 
-# Accumulate all bundled targets and we will splice them together later as
+# Accumulate core bundled targets and we will splice them together later as
 # libarrow_bundled_dependencies.a so that third party libraries have something
 # usable to create statically-linked builds with some BUNDLED dependencies,
 # including allocators like jemalloc and mimalloc
 set(ARROW_BUNDLED_STATIC_LIBS)
+set(ARROW_GCS_BUNDLED_STATIC_LIBS)
 
 # Accumulate all system dependencies to provide suitable static link
 # parameters to the third party libraries.
@@ -1942,6 +1943,9 @@ function(build_absl)
     # When ABSL_ENABLE_INSTALL is ON, the real target is "time" not "absl_time"
     # Cannot use set_property on alias targets (absl::time is an alias)
     target_link_libraries(time INTERFACE ${CoreFoundation})
+    set(ARROW_PC_LIBS_PRIVATE
+        "${ARROW_PC_LIBS_PRIVATE} -framework CoreFoundation"
+        PARENT_SCOPE)
   endif()
 
   list(PREPEND
@@ -3877,13 +3881,8 @@ function(build_google_cloud_cpp_storage)
   # 141MB -> 79MB
   file(REMOVE_RECURSE "${google_cloud_cpp_SOURCE_DIR}/ci")
 
-  list(PREPEND
-       ARROW_BUNDLED_STATIC_LIBS
-       google-cloud-cpp::storage
-       google-cloud-cpp::rest_internal
-       google-cloud-cpp::common)
-  set(ARROW_BUNDLED_STATIC_LIBS
-      ${ARROW_BUNDLED_STATIC_LIBS}
+  set(ARROW_GCS_BUNDLED_STATIC_LIBS
+      google-cloud-cpp::storage google-cloud-cpp::rest_internal google-cloud-cpp::common
       PARENT_SCOPE)
 
   list(POP_BACK CMAKE_MESSAGE_INDENT)
@@ -4526,3 +4525,4 @@ if(ARROW_FLIGHT_SQL_ODBC)
 endif()
 
 message(STATUS "All bundled static libraries: ${ARROW_BUNDLED_STATIC_LIBS}")
+message(STATUS "GCS bundled static libraries: ${ARROW_GCS_BUNDLED_STATIC_LIBS}")

cpp/examples/minimal_build/CMakeLists.txt

@@ -67,3 +67,30 @@ elseif(ARROW_EXAMPLE_S3_STATIC_CHECK)
   message(STATUS "Skipping arrow-example-s3-static: "
                  "requires ARROW_S3 and Arrow::arrow_static")
 endif()
+
+option(ARROW_EXAMPLE_GCS_STATIC_CHECK "Verify GCS and static linking" OFF)
+if(ARROW_EXAMPLE_GCS_STATIC_CHECK
+   AND ARROW_GCS
+   AND TARGET Arrow::arrow_static)
+  find_package(ArrowGcs REQUIRED)
+  if(NOT TARGET ArrowGcs::arrow_gcs_static)
+    message(FATAL_ERROR "ArrowGcs::arrow_gcs_static not found")
+  endif()
+  add_executable(arrow-example-gcs-static example.cc)
+  target_compile_definitions(arrow-example-gcs-static
+                             PRIVATE ARROW_EXAMPLE_GCS_STATIC_CHECK)
+  target_link_libraries(arrow-example-gcs-static PRIVATE ArrowGcs::arrow_gcs_static)
+
+  if(TARGET ArrowS3::arrow_s3_static)
+    add_executable(arrow-example-s3-gcs-static example.cc)
+    target_compile_definitions(arrow-example-s3-gcs-static
+                               PRIVATE ARROW_EXAMPLE_S3_STATIC_CHECK
+                                       ARROW_EXAMPLE_GCS_STATIC_CHECK)
+    target_link_libraries(arrow-example-s3-gcs-static PRIVATE ArrowS3::arrow_s3_static
+                                                              ArrowGcs::arrow_gcs_static)
+  endif()
+
+elseif(ARROW_EXAMPLE_GCS_STATIC_CHECK)
+  message(STATUS "Skipping GCS static examples: requires ARROW_GCS and Arrow::arrow_static"
+  )
+endif()

cpp/examples/minimal_build/README.md

@@ -85,4 +85,10 @@ system. To run this configuration, set
 docker compose run --rm static-system-dependency
 ```
 
+### Static filesystem examples
+
+Enable `ARROW_EXAMPLE_S3_STATIC_CHECK` or `ARROW_EXAMPLE_GCS_STATIC_CHECK`
+to test static linking. Enable both to also test S3 and GCS together.
+Run the examples from this directory.
+
 [docker-compose]: https://docs.docker.com/compose/

cpp/examples/minimal_build/example.cc

@@ -16,6 +16,9 @@
 // under the License.
 
 #include <arrow/csv/api.h>
+#ifdef ARROW_EXAMPLE_GCS_STATIC_CHECK
+#  include <arrow/filesystem/gcsfs.h>
+#endif
 #ifdef ARROW_EXAMPLE_S3_STATIC_CHECK
 #  include <arrow/filesystem/s3fs.h>
 #endif
@@ -63,6 +66,12 @@ Status RunMain(int argc, char** argv) {
   ARROW_RETURN_NOT_OK(arrow::fs::EnsureS3Finalized());
 #endif
 
+#ifdef ARROW_EXAMPLE_GCS_STATIC_CHECK
+  std::cerr << "* Verify GCS filesystem construction" << std::endl;
+  ARROW_RETURN_NOT_OK(
+      arrow::fs::GcsFileSystem::Make(arrow::fs::GcsOptions::Anonymous()).status());
+#endif
+
   return Status::OK();
 }
 

cpp/src/arrow/ArrowConfig.cmake.in

@@ -128,15 +128,26 @@ if(TARGET Arrow::arrow_static AND NOT TARGET Arrow::arrow_bundled_dependencies)
     )
   endforeach()
 
+  if(ARROW_BUNDLED_STATIC_LIBS)
+    get_target_property(arrow_bundled_dependencies_link_libs Arrow::arrow_static
+                        INTERFACE_LINK_LIBRARIES)
+    list(REMOVE_ITEM arrow_bundled_dependencies_link_libs
+         Arrow::arrow_bundled_dependencies)
+    target_link_libraries(Arrow::arrow_bundled_dependencies
+                          INTERFACE ${arrow_bundled_dependencies_link_libs})
+  endif()
+
   # CMP0057: Support new if() IN_LIST operator.
   # https://cmake.org/cmake/help/latest/policy/CMP0057.html
   cmake_policy(PUSH)
   cmake_policy(SET CMP0057 NEW)
+  if(APPLE AND ("aws-c-common" IN_LIST ARROW_BUNDLED_STATIC_LIBS
+                OR "absl::time" IN_LIST ARROW_BUNDLED_STATIC_LIBS))
+    find_library(CORE_FOUNDATION CoreFoundation)
+    target_link_libraries(Arrow::arrow_bundled_dependencies INTERFACE ${CORE_FOUNDATION})
+  endif()
   if("aws-c-common" IN_LIST ARROW_BUNDLED_STATIC_LIBS)
     if(APPLE)
-      find_library(CORE_FOUNDATION CoreFoundation)
-      target_link_libraries(Arrow::arrow_bundled_dependencies
-                            INTERFACE ${CORE_FOUNDATION})
       find_library(NETWORK Network)
       target_link_libraries(Arrow::arrow_bundled_dependencies INTERFACE ${NETWORK})
       find_library(SECURITY Security)

cpp/src/arrow/ArrowGcsConfig.cmake.in

@@ -27,6 +27,7 @@
 @PACKAGE_INIT@
 
 set(ARROW_GCS_SYSTEM_DEPENDENCIES "@ARROW_GCS_SYSTEM_DEPENDENCIES@")
+set(ARROW_GCS_BUNDLED_STATIC_LIBS "@ARROW_GCS_BUNDLED_STATIC_LIBS@")
 
 include(CMakeFindDependencyMacro)
 find_dependency(Arrow CONFIG)
@@ -37,6 +38,40 @@ endif()
 
 include("${CMAKE_CURRENT_LIST_DIR}/ArrowGcsTargets.cmake")
 
+if(TARGET ArrowGcs::arrow_gcs_static
+   AND ARROW_GCS_BUNDLED_STATIC_LIBS
+   AND NOT TARGET ArrowGcs::arrow_gcs_bundled_dependencies)
+  add_library(ArrowGcs::arrow_gcs_bundled_dependencies STATIC IMPORTED)
+  get_target_property(arrow_gcs_bundled_dependencies_link_libs ArrowGcs::arrow_gcs_static
+                      INTERFACE_LINK_LIBRARIES)
+  list(REMOVE_ITEM arrow_gcs_bundled_dependencies_link_libs
+       ArrowGcs::arrow_gcs_bundled_dependencies Arrow::arrow_static)
+  if("@absl_SOURCE@" STREQUAL "BUNDLED")
+    list(APPEND arrow_gcs_bundled_dependencies_link_libs
+         Arrow::arrow_bundled_dependencies)
+  endif()
+  if(NOT WIN32)
+    list(APPEND arrow_gcs_bundled_dependencies_link_libs OpenSSL::Crypto)
+  endif()
+  target_link_libraries(ArrowGcs::arrow_gcs_bundled_dependencies
+                        INTERFACE ${arrow_gcs_bundled_dependencies_link_libs})
+  get_target_property(arrow_gcs_static_configurations ArrowGcs::arrow_gcs_static
+                      IMPORTED_CONFIGURATIONS)
+  foreach(CONFIGURATION ${arrow_gcs_static_configurations})
+    string(TOUPPER "${CONFIGURATION}" CONFIGURATION)
+    get_target_property(arrow_gcs_static_location ArrowGcs::arrow_gcs_static
+                        IMPORTED_LOCATION_${CONFIGURATION})
+    get_filename_component(arrow_gcs_lib_dir "${arrow_gcs_static_location}" DIRECTORY)
+    set_property(TARGET ArrowGcs::arrow_gcs_bundled_dependencies
+                 APPEND
+                 PROPERTY IMPORTED_CONFIGURATIONS ${CONFIGURATION})
+    set_target_properties(ArrowGcs::arrow_gcs_bundled_dependencies
+                          PROPERTIES IMPORTED_LOCATION_${CONFIGURATION}
+                                     "${arrow_gcs_lib_dir}/${CMAKE_STATIC_LIBRARY_PREFIX}arrow_gcs_bundled_dependencies${CMAKE_STATIC_LIBRARY_SUFFIX}"
+    )
+  endforeach()
+endif()
+
 arrow_keep_backward_compatibility(ArrowGcs arrow_gcs)
 
 check_required_components(ArrowGcs)

cpp/src/arrow/CMakeLists.txt

@@ -970,10 +970,37 @@ if(ARROW_FILESYSTEM)
   endif()
   if(ARROW_GCS)
     set(ARROW_GCS_STATIC_INSTALL_INTERFACE_LIBS Arrow::arrow_static)
+    if(ARROW_BUILD_STATIC AND ARROW_GCS_BUNDLED_STATIC_LIBS)
+      arrow_car(_FIRST_GCS_LIB ${ARROW_GCS_BUNDLED_STATIC_LIBS})
+      arrow_cdr(_OTHER_GCS_LIBS ${ARROW_GCS_BUNDLED_STATIC_LIBS})
+      arrow_create_merged_static_lib(arrow_gcs_bundled_dependencies
+                                     NAME
+                                     arrow_gcs_bundled_dependencies
+                                     ROOT
+                                     ${_FIRST_GCS_LIB}
+                                     TO_MERGE
+                                     ${_OTHER_GCS_LIBS})
+      get_target_property(arrow_gcs_bundled_dependencies_path
+                          arrow_gcs_bundled_dependencies IMPORTED_LOCATION)
+      install(FILES ${arrow_gcs_bundled_dependencies_path} ${INSTALL_IS_OPTIONAL}
+              DESTINATION ${CMAKE_INSTALL_LIBDIR})
+      string(PREPEND ARROW_GCS_PC_LIBS_PRIVATE " -larrow_gcs_bundled_dependencies")
+      # Resolve bundled Abseil after SDK
+      list(PREPEND ARROW_GCS_STATIC_INSTALL_INTERFACE_LIBS
+           ArrowGcs::arrow_gcs_bundled_dependencies)
+    endif()
     if(google_cloud_cpp_storage_SOURCE STREQUAL "SYSTEM")
       list(APPEND ARROW_GCS_STATIC_INSTALL_INTERFACE_LIBS google-cloud-cpp::storage)
     elseif(google_cloud_cpp_storage_SOURCE STREQUAL "BUNDLED")
       list(APPEND ARROW_GCS_STATIC_INSTALL_INTERFACE_LIBS CURL::libcurl)
+      if(WIN32)
+        list(APPEND
+             ARROW_GCS_STATIC_INSTALL_INTERFACE_LIBS
+             ws2_32
+             bcrypt
+             crypt32)
+        string(APPEND ARROW_GCS_PC_LIBS_PRIVATE " -lws2_32 -lbcrypt -lcrypt32")
+      endif()
       if(absl_SOURCE STREQUAL "SYSTEM")
         # Bundled google-cloud-cpp with system Abseil needs these for static consumers.
         list(APPEND

dev/tasks/linux-packages/apache-arrow/debian/libarrow-gcs-dev.install

@@ -1,4 +1,4 @@
 usr/lib/*/cmake/ArrowGcs/
-usr/lib/*/libarrow_gcs.a
+usr/lib/*/libarrow_gcs*.a
 usr/lib/*/libarrow_gcs.so
 usr/lib/*/pkgconfig/arrow-gcs.pc

dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in

@@ -387,7 +387,7 @@ Libraries and header files for Apache Arrow GCS filesystem.
 %doc README.md
 %license LICENSE.txt NOTICE.txt
 %{_libdir}/cmake/ArrowGcs/
-%{_libdir}/libarrow_gcs.a
+%{_libdir}/libarrow_gcs*.a
 %{_libdir}/libarrow_gcs.so
 %{_libdir}/pkgconfig/arrow-gcs.pc
 %endif