diff --git a/ci/scripts/cpp_test.sh b/ci/scripts/cpp_test.sh index 0863afc13a2..d1be11f204e 100755 --- a/ci/scripts/cpp_test.sh +++ b/ci/scripts/cpp_test.sh @@ -147,18 +147,28 @@ if [ "${ARROW_USE_MESON:-OFF}" = "OFF" ] && \ -S "${source_dir}/examples/minimal_build" \ -B "${build_dir}/examples/minimal_build" \ -DARROW_EXAMPLE_S3_STATIC_CHECK=ON \ + -DARROW_EXAMPLE_GCS_STATIC_CHECK=OFF \ -DCMAKE_PREFIX_PATH="${CMAKE_PREFIX_PATH}" + # Check S3 before discovering GCS dependencies + cmake --build "${build_dir}/examples/minimal_build" + cmake -S "${source_dir}/examples/minimal_build" \ + -B "${build_dir}/examples/minimal_build" \ + -DARROW_EXAMPLE_GCS_STATIC_CHECK=ON cmake --build "${build_dir}/examples/minimal_build" pushd "${source_dir}/examples/minimal_build" # PATH= is for Windows. PATH="${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}}/bin:${PATH}" \ "${build_dir}/examples/minimal_build/arrow-example" - # Test static linking with S3 - s3_example="${build_dir}/examples/minimal_build/arrow-example-s3-static" - if [ -x "${s3_example}" ]; then - PATH="${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}}/bin:${PATH}" \ - "${s3_example}" - fi + # Test static filesystem linking + for static_example in arrow-example-s3-static \ + arrow-example-gcs-static \ + arrow-example-s3-gcs-static; do + example="${build_dir}/examples/minimal_build/${static_example}" + if [ -x "${example}" ]; then + PATH="${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}}/bin:${PATH}" \ + "${example}" + fi + done popd fi diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 40343415b36..2e8a7b80533 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -18,11 +18,12 @@ include(ProcessorCount) processorcount(NPROC) -# Accumulate all bundled targets and we will splice them together later as +# Accumulate core bundled targets and we will splice them together later as # libarrow_bundled_dependencies.a so that third party libraries have something # usable to create statically-linked builds with some BUNDLED dependencies, # including allocators like jemalloc and mimalloc set(ARROW_BUNDLED_STATIC_LIBS) +set(ARROW_GCS_BUNDLED_STATIC_LIBS) # Accumulate all system dependencies to provide suitable static link # parameters to the third party libraries. @@ -1942,6 +1943,9 @@ function(build_absl) # When ABSL_ENABLE_INSTALL is ON, the real target is "time" not "absl_time" # Cannot use set_property on alias targets (absl::time is an alias) target_link_libraries(time INTERFACE ${CoreFoundation}) + set(ARROW_PC_LIBS_PRIVATE + "${ARROW_PC_LIBS_PRIVATE} -framework CoreFoundation" + PARENT_SCOPE) endif() list(PREPEND @@ -3877,13 +3881,8 @@ function(build_google_cloud_cpp_storage) # 141MB -> 79MB file(REMOVE_RECURSE "${google_cloud_cpp_SOURCE_DIR}/ci") - list(PREPEND - ARROW_BUNDLED_STATIC_LIBS - google-cloud-cpp::storage - google-cloud-cpp::rest_internal - google-cloud-cpp::common) - set(ARROW_BUNDLED_STATIC_LIBS - ${ARROW_BUNDLED_STATIC_LIBS} + set(ARROW_GCS_BUNDLED_STATIC_LIBS + google-cloud-cpp::storage google-cloud-cpp::rest_internal google-cloud-cpp::common PARENT_SCOPE) list(POP_BACK CMAKE_MESSAGE_INDENT) @@ -4526,3 +4525,4 @@ if(ARROW_FLIGHT_SQL_ODBC) endif() message(STATUS "All bundled static libraries: ${ARROW_BUNDLED_STATIC_LIBS}") +message(STATUS "GCS bundled static libraries: ${ARROW_GCS_BUNDLED_STATIC_LIBS}") diff --git a/cpp/examples/minimal_build/CMakeLists.txt b/cpp/examples/minimal_build/CMakeLists.txt index e1a9217401f..0b7cf9080b4 100644 --- a/cpp/examples/minimal_build/CMakeLists.txt +++ b/cpp/examples/minimal_build/CMakeLists.txt @@ -67,3 +67,30 @@ elseif(ARROW_EXAMPLE_S3_STATIC_CHECK) message(STATUS "Skipping arrow-example-s3-static: " "requires ARROW_S3 and Arrow::arrow_static") endif() + +option(ARROW_EXAMPLE_GCS_STATIC_CHECK "Verify GCS and static linking" OFF) +if(ARROW_EXAMPLE_GCS_STATIC_CHECK + AND ARROW_GCS + AND TARGET Arrow::arrow_static) + find_package(ArrowGcs REQUIRED) + if(NOT TARGET ArrowGcs::arrow_gcs_static) + message(FATAL_ERROR "ArrowGcs::arrow_gcs_static not found") + endif() + add_executable(arrow-example-gcs-static example.cc) + target_compile_definitions(arrow-example-gcs-static + PRIVATE ARROW_EXAMPLE_GCS_STATIC_CHECK) + target_link_libraries(arrow-example-gcs-static PRIVATE ArrowGcs::arrow_gcs_static) + + if(TARGET ArrowS3::arrow_s3_static) + add_executable(arrow-example-s3-gcs-static example.cc) + target_compile_definitions(arrow-example-s3-gcs-static + PRIVATE ARROW_EXAMPLE_S3_STATIC_CHECK + ARROW_EXAMPLE_GCS_STATIC_CHECK) + target_link_libraries(arrow-example-s3-gcs-static PRIVATE ArrowS3::arrow_s3_static + ArrowGcs::arrow_gcs_static) + endif() + +elseif(ARROW_EXAMPLE_GCS_STATIC_CHECK) + message(STATUS "Skipping GCS static examples: requires ARROW_GCS and Arrow::arrow_static" + ) +endif() diff --git a/cpp/examples/minimal_build/README.md b/cpp/examples/minimal_build/README.md index 3403ce95e5b..ca76c33cd70 100644 --- a/cpp/examples/minimal_build/README.md +++ b/cpp/examples/minimal_build/README.md @@ -85,4 +85,10 @@ system. To run this configuration, set docker compose run --rm static-system-dependency ``` +### Static filesystem examples + +Enable `ARROW_EXAMPLE_S3_STATIC_CHECK` or `ARROW_EXAMPLE_GCS_STATIC_CHECK` +to test static linking. Enable both to also test S3 and GCS together. +Run the examples from this directory. + [docker-compose]: https://docs.docker.com/compose/ diff --git a/cpp/examples/minimal_build/example.cc b/cpp/examples/minimal_build/example.cc index f600f9429d6..3652bc2e434 100644 --- a/cpp/examples/minimal_build/example.cc +++ b/cpp/examples/minimal_build/example.cc @@ -16,6 +16,9 @@ // under the License. #include +#ifdef ARROW_EXAMPLE_GCS_STATIC_CHECK +# include +#endif #ifdef ARROW_EXAMPLE_S3_STATIC_CHECK # include #endif @@ -63,6 +66,12 @@ Status RunMain(int argc, char** argv) { ARROW_RETURN_NOT_OK(arrow::fs::EnsureS3Finalized()); #endif +#ifdef ARROW_EXAMPLE_GCS_STATIC_CHECK + std::cerr << "* Verify GCS filesystem construction" << std::endl; + ARROW_RETURN_NOT_OK( + arrow::fs::GcsFileSystem::Make(arrow::fs::GcsOptions::Anonymous()).status()); +#endif + return Status::OK(); } diff --git a/cpp/src/arrow/ArrowConfig.cmake.in b/cpp/src/arrow/ArrowConfig.cmake.in index c12b3ce7103..6bbb53a2f76 100644 --- a/cpp/src/arrow/ArrowConfig.cmake.in +++ b/cpp/src/arrow/ArrowConfig.cmake.in @@ -128,15 +128,26 @@ if(TARGET Arrow::arrow_static AND NOT TARGET Arrow::arrow_bundled_dependencies) ) endforeach() + if(ARROW_BUNDLED_STATIC_LIBS) + get_target_property(arrow_bundled_dependencies_link_libs Arrow::arrow_static + INTERFACE_LINK_LIBRARIES) + list(REMOVE_ITEM arrow_bundled_dependencies_link_libs + Arrow::arrow_bundled_dependencies) + target_link_libraries(Arrow::arrow_bundled_dependencies + INTERFACE ${arrow_bundled_dependencies_link_libs}) + endif() + # CMP0057: Support new if() IN_LIST operator. # https://cmake.org/cmake/help/latest/policy/CMP0057.html cmake_policy(PUSH) cmake_policy(SET CMP0057 NEW) + if(APPLE AND ("aws-c-common" IN_LIST ARROW_BUNDLED_STATIC_LIBS + OR "absl::time" IN_LIST ARROW_BUNDLED_STATIC_LIBS)) + find_library(CORE_FOUNDATION CoreFoundation) + target_link_libraries(Arrow::arrow_bundled_dependencies INTERFACE ${CORE_FOUNDATION}) + endif() if("aws-c-common" IN_LIST ARROW_BUNDLED_STATIC_LIBS) if(APPLE) - find_library(CORE_FOUNDATION CoreFoundation) - target_link_libraries(Arrow::arrow_bundled_dependencies - INTERFACE ${CORE_FOUNDATION}) find_library(NETWORK Network) target_link_libraries(Arrow::arrow_bundled_dependencies INTERFACE ${NETWORK}) find_library(SECURITY Security) diff --git a/cpp/src/arrow/ArrowGcsConfig.cmake.in b/cpp/src/arrow/ArrowGcsConfig.cmake.in index 9e82b03a1e0..2fc68387a4d 100644 --- a/cpp/src/arrow/ArrowGcsConfig.cmake.in +++ b/cpp/src/arrow/ArrowGcsConfig.cmake.in @@ -27,6 +27,7 @@ @PACKAGE_INIT@ set(ARROW_GCS_SYSTEM_DEPENDENCIES "@ARROW_GCS_SYSTEM_DEPENDENCIES@") +set(ARROW_GCS_BUNDLED_STATIC_LIBS "@ARROW_GCS_BUNDLED_STATIC_LIBS@") include(CMakeFindDependencyMacro) find_dependency(Arrow CONFIG) @@ -37,6 +38,40 @@ endif() include("${CMAKE_CURRENT_LIST_DIR}/ArrowGcsTargets.cmake") +if(TARGET ArrowGcs::arrow_gcs_static + AND ARROW_GCS_BUNDLED_STATIC_LIBS + AND NOT TARGET ArrowGcs::arrow_gcs_bundled_dependencies) + add_library(ArrowGcs::arrow_gcs_bundled_dependencies STATIC IMPORTED) + get_target_property(arrow_gcs_bundled_dependencies_link_libs ArrowGcs::arrow_gcs_static + INTERFACE_LINK_LIBRARIES) + list(REMOVE_ITEM arrow_gcs_bundled_dependencies_link_libs + ArrowGcs::arrow_gcs_bundled_dependencies Arrow::arrow_static) + if("@absl_SOURCE@" STREQUAL "BUNDLED") + list(APPEND arrow_gcs_bundled_dependencies_link_libs + Arrow::arrow_bundled_dependencies) + endif() + if(NOT WIN32) + list(APPEND arrow_gcs_bundled_dependencies_link_libs OpenSSL::Crypto) + endif() + target_link_libraries(ArrowGcs::arrow_gcs_bundled_dependencies + INTERFACE ${arrow_gcs_bundled_dependencies_link_libs}) + get_target_property(arrow_gcs_static_configurations ArrowGcs::arrow_gcs_static + IMPORTED_CONFIGURATIONS) + foreach(CONFIGURATION ${arrow_gcs_static_configurations}) + string(TOUPPER "${CONFIGURATION}" CONFIGURATION) + get_target_property(arrow_gcs_static_location ArrowGcs::arrow_gcs_static + IMPORTED_LOCATION_${CONFIGURATION}) + get_filename_component(arrow_gcs_lib_dir "${arrow_gcs_static_location}" DIRECTORY) + set_property(TARGET ArrowGcs::arrow_gcs_bundled_dependencies + APPEND + PROPERTY IMPORTED_CONFIGURATIONS ${CONFIGURATION}) + set_target_properties(ArrowGcs::arrow_gcs_bundled_dependencies + PROPERTIES IMPORTED_LOCATION_${CONFIGURATION} + "${arrow_gcs_lib_dir}/${CMAKE_STATIC_LIBRARY_PREFIX}arrow_gcs_bundled_dependencies${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + endforeach() +endif() + arrow_keep_backward_compatibility(ArrowGcs arrow_gcs) check_required_components(ArrowGcs) diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index e71d92fdb88..6ceca297be3 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -970,10 +970,37 @@ if(ARROW_FILESYSTEM) endif() if(ARROW_GCS) set(ARROW_GCS_STATIC_INSTALL_INTERFACE_LIBS Arrow::arrow_static) + if(ARROW_BUILD_STATIC AND ARROW_GCS_BUNDLED_STATIC_LIBS) + arrow_car(_FIRST_GCS_LIB ${ARROW_GCS_BUNDLED_STATIC_LIBS}) + arrow_cdr(_OTHER_GCS_LIBS ${ARROW_GCS_BUNDLED_STATIC_LIBS}) + arrow_create_merged_static_lib(arrow_gcs_bundled_dependencies + NAME + arrow_gcs_bundled_dependencies + ROOT + ${_FIRST_GCS_LIB} + TO_MERGE + ${_OTHER_GCS_LIBS}) + get_target_property(arrow_gcs_bundled_dependencies_path + arrow_gcs_bundled_dependencies IMPORTED_LOCATION) + install(FILES ${arrow_gcs_bundled_dependencies_path} ${INSTALL_IS_OPTIONAL} + DESTINATION ${CMAKE_INSTALL_LIBDIR}) + string(PREPEND ARROW_GCS_PC_LIBS_PRIVATE " -larrow_gcs_bundled_dependencies") + # Resolve bundled Abseil after SDK + list(PREPEND ARROW_GCS_STATIC_INSTALL_INTERFACE_LIBS + ArrowGcs::arrow_gcs_bundled_dependencies) + endif() if(google_cloud_cpp_storage_SOURCE STREQUAL "SYSTEM") list(APPEND ARROW_GCS_STATIC_INSTALL_INTERFACE_LIBS google-cloud-cpp::storage) elseif(google_cloud_cpp_storage_SOURCE STREQUAL "BUNDLED") list(APPEND ARROW_GCS_STATIC_INSTALL_INTERFACE_LIBS CURL::libcurl) + if(WIN32) + list(APPEND + ARROW_GCS_STATIC_INSTALL_INTERFACE_LIBS + ws2_32 + bcrypt + crypt32) + string(APPEND ARROW_GCS_PC_LIBS_PRIVATE " -lws2_32 -lbcrypt -lcrypt32") + endif() if(absl_SOURCE STREQUAL "SYSTEM") # Bundled google-cloud-cpp with system Abseil needs these for static consumers. list(APPEND diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-gcs-dev.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-gcs-dev.install index ccce70981d2..432005e7b8b 100644 --- a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-gcs-dev.install +++ b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-gcs-dev.install @@ -1,4 +1,4 @@ usr/lib/*/cmake/ArrowGcs/ -usr/lib/*/libarrow_gcs.a +usr/lib/*/libarrow_gcs*.a usr/lib/*/libarrow_gcs.so usr/lib/*/pkgconfig/arrow-gcs.pc diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in index 3932effdcb3..6b79eda6afb 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in +++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in @@ -387,7 +387,7 @@ Libraries and header files for Apache Arrow GCS filesystem. %doc README.md %license LICENSE.txt NOTICE.txt %{_libdir}/cmake/ArrowGcs/ -%{_libdir}/libarrow_gcs.a +%{_libdir}/libarrow_gcs*.a %{_libdir}/libarrow_gcs.so %{_libdir}/pkgconfig/arrow-gcs.pc %endif