diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index 1349358ad4..d751247b60 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -50,6 +50,8 @@ RAPIDS_DATASET_ROOT_DIR="$(realpath datasets)" export RAPIDS_DATASET_ROOT_DIR if [[ "${CUOPT_CPP_TEST_GROUP}" != "mathopt" ]]; then + rapids-logger "Test routing dataset downloader" + python -m unittest discover -s ci/tests -p test_get_test_data.py rapids-logger "Download routing datasets" pushd "${RAPIDS_DATASET_ROOT_DIR}" ./get_test_data.sh diff --git a/ci/tests/test_get_test_data.py b/ci/tests/test_get_test_data.py new file mode 100644 index 0000000000..e2433f5888 --- /dev/null +++ b/ci/tests/test_get_test_data.py @@ -0,0 +1,217 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import io +import os +import subprocess +import tarfile +import tempfile +import unittest +import zipfile +from pathlib import Path + +SCRIPT = Path(__file__).resolve().parents[2] / "datasets" / "get_test_data.sh" + + +def fake_tool(directory, name, body): + path = directory / name + path.write_text("#!/bin/bash\nset -eu\n" + body) + path.chmod(0o755) + + +def make_tar(path, member, contents=b"new dataset"): + with tarfile.open(path, "w:gz") as archive: + item = tarfile.TarInfo(member) + item.size = len(contents) + archive.addfile(item, io.BytesIO(contents)) + + +class DatasetDownloadTests(unittest.TestCase): + def setUp(self): + temp = tempfile.TemporaryDirectory() + self.addCleanup(temp.cleanup) + self.root = Path(temp.name) + self.bindir = self.root / "bin" + self.bindir.mkdir() + self.archives = self.root / "archives" + self.archives.mkdir() + self.workdir = self.root / "work" + self.workdir.mkdir() + fake_tool( + self.bindir, + "wget", + """url=${!#} +name=${url##*/} +case "$FAKE_FETCH_MODE" in + fail) exit 8 ;; + corrupt) printf 'not an archive' > "$name" ;; + fixtures) cp "$FAKE_ARCHIVES/$name" "$name" ;; +esac +""", + ) + self.env = { + key: value + for key, value in os.environ.items() + if not key.startswith(("AWS_", "CUOPT_")) + } + self.env.update( + PATH=f"{self.bindir}:{self.env['PATH']}", + FAKE_ARCHIVES=str(self.archives), + ) + + def run_script(self, option, mode): + env = self.env | {"FAKE_FETCH_MODE": mode} + return subprocess.run( + ["bash", str(SCRIPT), option], + cwd=self.workdir, + env=env, + text=True, + capture_output=True, + timeout=30, + ) + + def add_cvrptw_archives(self): + for count in range(200, 1001, 200): + with zipfile.ZipFile( + self.archives / f"homberger_{count}_customer_instances.zip", + "w", + ) as archive: + archive.writestr(f"{count}.txt", str(count)) + with zipfile.ZipFile( + self.archives / "solomon-100.zip", "w" + ) as archive: + archive.writestr("solomon.txt", "solomon") + + def enable_fake_s3(self, body): + fake_tool(self.bindir, "aws", body) + self.env.update( + CUOPT_S3_URI="s3://example/", + CUOPT_AWS_ACCESS_KEY_ID="test", + CUOPT_AWS_SECRET_ACCESS_KEY="test", + ) + + def test_http_failures_preserve_existing_data_and_tmp(self): + (self.workdir / "cvrp").mkdir() + (self.workdir / "cvrp" / "existing.txt").write_text("keep me") + (self.workdir / "tmp").mkdir() + (self.workdir / "tmp" / "unrelated.txt").write_text("also keep me") + for mode in ("fail", "corrupt"): + with self.subTest(mode=mode): + result = self.run_script("--cvrp", mode) + self.assertNotEqual( + result.returncode, 0, result.stdout + result.stderr + ) + self.assertEqual( + (self.workdir / "cvrp" / "existing.txt").read_text(), + "keep me", + ) + self.assertEqual( + (self.workdir / "tmp" / "unrelated.txt").read_text(), + "also keep me", + ) + + def test_success_replaces_selected_dataset_after_extraction(self): + make_tar(self.archives / "Vrp-Set-X.tgz", "new.txt") + (self.workdir / "cvrp").mkdir() + (self.workdir / "cvrp" / "old.txt").write_text("old") + (self.workdir / "tmp").mkdir() + (self.workdir / "tmp" / "unrelated.txt").write_text("keep me") + result = self.run_script("--cvrp", "fixtures") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual( + (self.workdir / "cvrp" / "new.txt").read_text(), "new dataset" + ) + self.assertFalse((self.workdir / "cvrp" / "old.txt").exists()) + self.assertEqual( + (self.workdir / "tmp" / "unrelated.txt").read_text(), "keep me" + ) + + def test_multiple_archives_share_a_destination(self): + self.add_cvrptw_archives() + result = self.run_script("--cvrptw", "fixtures") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual( + sorted(path.name for path in (self.workdir / "cvrptw").iterdir()), + ["1000.txt", "200.txt", "400.txt", "600.txt", "800.txt"], + ) + self.assertEqual( + (self.workdir / "solomon" / "solomon.txt").read_text(), "solomon" + ) + + def test_failed_s3_and_http_do_not_modify_existing_data(self): + self.enable_fake_s3("""if [[ "$1 $2" == "sts get-caller-identity" ]]; then exit 0; fi +if [[ "$1 $2" == "s3 cp" ]]; then printf 'corrupt'; exit 1; fi +if [[ "$1 $2" == "s3 sync" ]]; then mkdir -p "$4"; printf 'partial' > "$4/partial.txt"; exit 1; fi +exit 1 +""") + (self.workdir / "cvrp").mkdir() + (self.workdir / "cvrp" / "existing.txt").write_text("keep me") + result = self.run_script("--cvrp", "fail") + self.assertNotEqual( + result.returncode, 0, result.stdout + result.stderr + ) + self.assertEqual( + (self.workdir / "cvrp" / "existing.txt").read_text(), "keep me" + ) + self.assertFalse((self.workdir / "cvrp" / "partial.txt").exists()) + + def test_successful_s3_archive_merges_without_following_destination_symlink( + self, + ): + make_tar(self.archives / "s3.tar.gz", "cvrp/from_s3.txt", b"from s3") + self.enable_fake_s3("""if [[ "$1 $2" == "sts get-caller-identity" ]]; then exit 0; fi +if [[ "$1 $2" == "s3 cp" ]]; then cat "$FAKE_ARCHIVES/s3.tar.gz"; exit 0; fi +exit 1 +""") + (self.workdir / "cvrp").mkdir() + (self.workdir / "cvrp" / "existing.txt").write_text("keep me") + outside = self.root / "outside.txt" + outside.write_text("outside") + (self.workdir / "cvrp" / "from_s3.txt").symlink_to(outside) + result = self.run_script("--cvrp", "fail") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual( + (self.workdir / "cvrp" / "from_s3.txt").read_text(), "from s3" + ) + self.assertEqual( + (self.workdir / "cvrp" / "existing.txt").read_text(), "keep me" + ) + self.assertEqual(outside.read_text(), "outside") + + def test_publish_failure_restores_all_original_destinations(self): + self.add_cvrptw_archives() + fake_tool( + self.bindir, + "mv", + """if [[ "$2" == */data/solomon && "$3" == solomon ]]; then exit 1; fi +exec /usr/bin/mv "$@" +""", + ) + for name in ("cvrptw", "solomon"): + (self.workdir / name).mkdir() + (self.workdir / name / "existing.txt").write_text(name) + result = self.run_script("--cvrptw", "fixtures") + self.assertNotEqual( + result.returncode, 0, result.stdout + result.stderr + ) + for name in ("cvrptw", "solomon"): + self.assertEqual( + (self.workdir / name / "existing.txt").read_text(), name + ) + self.assertFalse(list(self.workdir.glob(".cuopt-datasets.*"))) + + def test_s3_sync_succeeds_after_archive_failure(self): + self.enable_fake_s3("""if [[ "$1 $2" == "sts get-caller-identity" ]]; then exit 0; fi +if [[ "$1 $2" == "s3 cp" ]]; then printf 'corrupt'; exit 1; fi +if [[ "$1 $2" == "s3 sync" ]]; then mkdir -p "$4"; printf 'synced' > "$4/from_sync.txt"; exit 0; fi +exit 1 +""") + result = self.run_script("--cvrp", "fail") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual( + (self.workdir / "cvrp" / "from_sync.txt").read_text(), "synced" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/datasets/get_test_data.sh b/datasets/get_test_data.sh index 07bfd39baa..7fbd41cc2a 100755 --- a/datasets/get_test_data.sh +++ b/datasets/get_test_data.sh @@ -189,63 +189,150 @@ URLS=($(echo "$DATASET_DATA"|awk '{if (NR%4 == 3) print $0}')) # extract 3rd fi # shellcheck disable=SC2207 DESTDIRS=($(echo "$DATASET_DATA"|awk '{if (NR%4 == 0) print $0}')) # extract 4th fields to a bash array -# Try S3 download first with selected directories -if try_download_from_s3 "${DESTDIRS[@]}"; then - echo "Datasets successfully retrieved from S3, skipping HTTP download." - exit 0 -fi - -echo "Downloading from HTTP sources..." - -# Download all tarfiles to a tmp dir -rm -rf tmp -mkdir tmp -cd tmp -# Loop through URLs with error handling -for url in "${URLS[@]}"; do - # Try up to 3 times with continue option - wget -4 --tries=3 --continue --progress=dot:mega --retry-connrefused "${url}" || { - echo "Failed to download: ${url}" - continue - } +# Stage all downloads and extraction in a unique sibling directory. The existing +# datasets must remain untouched until every selected dataset is ready. +work_dir=$(mktemp -d "${PWD}/.cuopt-datasets.XXXXXXXX") +backup_dir="${work_dir}/backup" +mkdir -p "${backup_dir}" +unique_destinations=() +backed_up=() +installed=() +committed=false + +for dir in "${DESTDIRS[@]}"; do + seen=false + for previous in "${unique_destinations[@]}"; do + if [[ "${dir}" == "${previous}" ]]; then + seen=true + break + fi + done + if ! $seen; then unique_destinations+=("${dir}"); fi done -cd .. -# Setup the destination dirs, removing any existing ones first! -for index in ${!DESTDIRS[*]}; do - rm -rf "${DESTDIRS[$index]}" -done -for index in ${!DESTDIRS[*]}; do - mkdir -p "${DESTDIRS[$index]}" -done +cleanup() { + local status=$? + local preserve=false + trap - EXIT + if ! $committed; then + if [[ ${#installed[@]} -gt 0 ]]; then + mkdir -p "${work_dir}/uncommitted" || preserve=true + fi + for dir in "${installed[@]}"; do + if [[ -e "${dir}" || -L "${dir}" ]]; then + mv -- "${dir}" "${work_dir}/uncommitted/${dir}" || preserve=true + fi + done + for dir in "${backed_up[@]}"; do + if [[ -e "${backup_dir}/${dir}" || -L "${backup_dir}/${dir}" ]]; then + if [[ -e "${dir}" || -L "${dir}" ]]; then + preserve=true + else + mv -- "${backup_dir}/${dir}" "${dir}" || preserve=true + fi + fi + done + fi + if $preserve; then + echo "Could not restore all dataset directories; recovery data retained at ${work_dir}" >&2 + else + rm -rf -- "${work_dir}" + fi + exit "${status}" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +s3_dir="${work_dir}/s3" +mkdir -p "${s3_dir}" +data_dir="" +# The S3 attempt may partially extract or sync data before failing; isolate it +# from both the existing destinations and the separate HTTP fallback. +if (cd "${s3_dir}" && try_download_from_s3 "${DESTDIRS[@]}"); then + s3_complete=true + for dir in "${unique_destinations[@]}"; do + if [[ ! -d "${s3_dir}/${dir}" ]] || [[ -z "$(find "${s3_dir}/${dir}" -type f -print -quit)" ]]; then + echo "S3 dataset is empty or missing: ${dir}; falling back to HTTP" >&2 + s3_complete=false + break + fi + done + if $s3_complete; then + # S3 sync previously merged into existing datasets. Preserve that behavior + # on a staged copy, without following destination symlinks on overwrite. + merged_dir="${work_dir}/merged" + mkdir -p "${merged_dir}" + for dir in "${unique_destinations[@]}"; do + if [[ -L "${dir}" || ( -e "${dir}" && ! -d "${dir}" ) ]]; then + echo "Cannot safely merge S3 data into non-directory destination: ${dir}" >&2 + exit 1 + fi + if [[ -d "${dir}" ]]; then + cp -a --reflink=auto "${dir}" "${merged_dir}/${dir}" + else + mkdir -p "${merged_dir}/${dir}" + fi + cp -a --remove-destination "${s3_dir}/${dir}/." "${merged_dir}/${dir}/" + done + data_dir="${merged_dir}" + fi +fi -# Iterate over the arrays and untar the nth tarfile to the nth dest directory. -# The tarfile name is derived from the download url. -echo Decompressing ... -set +e # Disable exit on error for the entire script +if [[ -z "${data_dir}" ]]; then + echo "Downloading from HTTP sources..." + downloads_dir="${work_dir}/downloads" + data_dir="${work_dir}/data" + mkdir -p "${downloads_dir}" "${data_dir}" -for index in ${!DESTDIRS[*]}; do - tfname=$(basename "${URLS[$index]}") + for url in "${URLS[@]}"; do + if ! (cd "${downloads_dir}" && wget -4 --tries=3 --continue --progress=dot:mega --retry-connrefused "${url}"); then + echo "Failed to download: ${url}" >&2 + exit 1 + fi + done + + echo "Decompressing ..." + for index in "${!DESTDIRS[@]}"; do + dir="${DESTDIRS[$index]}" + archive="${downloads_dir}/$(basename "${URLS[$index]}")" + mkdir -p "${data_dir}/${dir}" + if [[ ! -s "${archive}" ]]; then + echo "Downloaded archive is empty or missing: ${archive}" >&2 + exit 1 + fi + mime_type=$(file --mime-type -b "${archive}") + case "${mime_type}" in + application/gzip|application/x-gzip) tar xzf "${archive}" -C "${data_dir}/${dir}" ;; + application/zip|application/x-zip-compressed) unzip -oq "${archive}" -d "${data_dir}/${dir}" ;; + *) tar xf "${archive}" -C "${data_dir}/${dir}" ;; + esac + for compressed in "${data_dir}/${dir}"/*.gz; do + if [[ -f "${compressed}" ]]; then gzip -d -- "${compressed}"; fi + done + done - if file --mime-type "tmp/${tfname}" | grep -q gzip$; then - tar xvzf tmp/"${tfname}" -C "${DESTDIRS[$index]}" || true - elif file --mime-type "tmp/${tfname}" | grep -q zip$; then - unzip tmp/"${tfname}" -d "${DESTDIRS[$index]}" || true - else - tar xvf tmp/"${tfname}" -C "${DESTDIRS[$index]}" || true + if [[ -d "${data_dir}/pdptw" ]]; then + for dataset in "${data_dir}/pdptw"/*.txt; do + if [[ -f "${dataset}" ]]; then mv -- "${dataset}" "${dataset%.txt}.pdptw"; fi + done fi +fi - if ls "${DESTDIRS[$index]}"/*.gz >/dev/null 2>&1; then - gzip -d "${DESTDIRS[$index]}"/* || true +# Validate every selected destination before moving any existing dataset. +for dir in "${unique_destinations[@]}"; do + if [[ ! -d "${data_dir}/${dir}" ]] || [[ -z "$(find "${data_dir}/${dir}" -type f -print -quit)" ]]; then + echo "Dataset is empty or missing after download: ${dir}" >&2 + exit 1 fi done -rm -rf tmp - -if [ -d "./pdptw" ]; then - cd pdptw - # change file extensions - for f in $(shopt -s nullglob; echo *.txt); do - mv -- "$f" "${f%.txt}.pdptw" - done -fi +for dir in "${unique_destinations[@]}"; do + if [[ -e "${dir}" || -L "${dir}" ]]; then + mv -- "${dir}" "${backup_dir}/${dir}" + backed_up+=("${dir}") + fi + mv -- "${data_dir}/${dir}" "${dir}" + installed+=("${dir}") +done +committed=true