Added download-dependencies build step for pre-seeding dependency source files in build-artifacts-cache

build-scripts/download-dependencies

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+#
+# This script downloads dependency source files to be bundled with CFEngine.
+# It calls install-dependencies with the argument --download-only.
+# This is a separate file to make it possible for bootstrap-pr in jenkins to selectively call this function only when --download-only is supported in install-dependencies script.
+#
+# See install-dependencies for expectations and usage
+
+# Here we provide JOB_BASE_NAME=label=NO_LABEL_DOWNLOAD_ONLY so that sftp cache is used, the label won't be used because we aren't dealing with built packages for deps, only the source files download.
+# We also explicitly call install-dependencies with EXPLICIT_ROLE set to agent AND hub to ensure all dependencies are cached.
+EXPLICIT_ROLE=hub JOB_BASE_NAME=label=NO_LABEL_DOWNLOAD_ONLY "$(dirname "$0")"/install-dependencies --download-only
+EXPLICIT_ROLE=agent JOB_BASE_NAME=label=NO_LABEL_DOWNLOAD_ONLY "$(dirname "$0")"/install-dependencies --download-only

build-scripts/install-dependencies

@@ -4,6 +4,10 @@
 # This script installs the dependencies to be bundled with CFEngine.
 # The dependencies are built into you working directory.
 #
+# This script can take one or more arguments for specific dependencies to install.
+# By default without arguments it will install all dependencies as appropriate.
+# The --download-only first argument will only download the source files and seed the build-artifacts-cache sftp cache.
+#
 # The script expects the following repositories to exist side by side:
 # .
 # ├── buildscripts
@@ -17,9 +21,16 @@
 # The script can be run as described in the usage message below:
 
 usage() {
-    echo "Usage: PROJECT=[nova|community] EXPLICIT_ROLE=[hub|agent] BUILD_TYPE=[DEBUG|RELEASE] $0"
+    echo "Usage: PROJECT=[nova|community] EXPLICIT_ROLE=[hub|agent] BUILD_TYPE=[DEBUG|RELEASE] $0 [--download-only|dependency-name]"
 }
 
+# the --download-only option is for bootstrap-pr job and similar where we want to seed the sftp build-artifacts-cache with dependency files ONCE to be re-used by many platforms
+# this avoids 429 too many requests troubles we have had with lmdb as well as increases efficiency and resilience to network flakiness of third party sources.
+if [ "$1" = "--download-only" ]; then
+    DOWNLOAD_ONLY="yes"
+    shift # get rid of this first argument so that any other arguments are used as a list of dependencies to work on
+fi
+
 . "$(dirname "$0")"/functions
 . detect-environment
 . compile-options
@@ -120,7 +131,9 @@ check_and_install_perl() {
     export PERL
 }
 
-check_and_install_perl
+if [ "$DOWNLOAD_ONLY" != "yes" ]; then
+    check_and_install_perl
+fi
 
 # non-gnu exotics don't support sed -i,
 # and only hub or deb packages use files mentioned below
@@ -188,6 +201,11 @@ for dep in $DEPS; do
     fi
     version=0+$revision$versuffix$crossver$safe_prefix
 
+    if [ "$DOWNLOAD_ONLY" = "yes" ]; then
+        pkg-get-src "$BASEDIR/buildscripts/deps-packaging/$dep"
+        continue
+    fi
+
     # FETCH dependency packages if needed
     package_files=$(pkg-cache listpkgfiles "$dep-$version")