diff --git a/README.md b/README.md index f773ccf..710fe67 100644 --- a/README.md +++ b/README.md @@ -166,7 +166,7 @@ This module is based on the following research and datasets: - [GEDTM30](https://github.com/openlandmap/GEDTM30) for slope - License: Creative Commons Attribution 4.0 International -- [GlobCover land cover data](https://due.esrin.esa.int/page_globcover.php) +- [GlobCover land cover data](https://doi.org/10.1594/PANGAEA.787668) - License: "You may use the GlobCover land cover map for educational and/or scientific purposes, without any fee on the condition that you credit ESA and the Université Catholique de Louvain as the source of the GlobCover products." - [GEBCO (General Bathymetric Chart of the Oceans)](https://www.gebco.net/data-products/gridded-bathymetry-data) 15 arc-second data - License: "The GEBCO Grid is placed in the public domain and may be used free of charge. [...] Users must: Acknowledge the source of The GEBCO Grid. A suitable form of attribution is given in the documentation that accompanies The GEBCO Grid." diff --git a/tests/integration/Snakefile b/tests/integration/Snakefile index 819f364..db2cfeb 100644 --- a/tests/integration/Snakefile +++ b/tests/integration/Snakefile @@ -13,7 +13,7 @@ rule download_netherlands_shapes: "Download and unzip the Netherlands shapes." shell: """ - curl -sSLo {output:q} "https://zenodo.org/records/16684683/files/NLD.parquet" + curl -fsSLo {output:q} "https://zenodo.org/records/16684683/files/NLD.parquet" """ @@ -28,7 +28,7 @@ rule download_netherlands_protected_areas: "Download a dummy drop-in dataset for Netherlands protected areas (not based on WDPA)." shell: """ - curl -sSLo {output:q} "https://zenodo.org/records/16684513/files/wdpa-nl-dropin.gdb.zip" + curl -fsSLo {output:q} "https://zenodo.org/records/16684513/files/wdpa-nl-dropin.gdb.zip" """ diff --git a/workflow/internal/settings.yaml b/workflow/internal/settings.yaml index 2edfab2..051370e 100644 --- a/workflow/internal/settings.yaml +++ b/workflow/internal/settings.yaml @@ -6,10 +6,8 @@ resources: ## slope: "https://zenodo.org/records/14920379/files/slope.in.degree_edtm_m_240m_s_20000101_20221231_go_epsg.4326_v20241230.tif" bathymetry: "https://zenodo.org/records/15741950/files/gebco_2024_sub_ice_cog.tif" - globcover: "https://due.esrin.esa.int/files/Globcover2009_V2.3_Global_.zip" - globcover_landcover_tif: "GLOBCOVER_L4_200901_200912_V2.3.tif" - ghsl: "https://jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/GHSL/GHS_BUILT_S_GLOBE_R2023A/GHS_BUILT_S_E2025_GLOBE_R2023A_4326_30ss/V1-0/GHS_BUILT_S_E2025_GLOBE_R2023A_4326_30ss_V1_0.zip" - ghsl_tif: "GHS_BUILT_S_E2025_GLOBE_R2023A_4326_30ss_V1_0.tif" + globcover: "https://hs.pangaea.de/Projects/GLOBCOVER/GLOBCOVER_L4_200901_200912_V2.3.tif" + ghsl: "https://jrc-ghsl.s3.eu-central-1.amazonaws.com/ghs-built-s/r2023a/4326/30asec/2025/GHS_BUILT_S_E2025_GLOBE_R2023A_4326_30ss_V1_0.tif" ship_travel: "https://datacatalogfiles.worldbank.org/ddh-published/0037580/5/DR0045406/shipdensity_global.zip" ship_travel_tif: "shipdensity_global.tif" diff --git a/workflow/rules/automatic.smk b/workflow/rules/automatic.smk index 479d6a1..1a379c8 100644 --- a/workflow/rules/automatic.smk +++ b/workflow/rules/automatic.smk @@ -3,7 +3,7 @@ if config.get("tiny_files", False): ## - # Directly download clipped slope and bathymetry data + # Directly download clipped datasets when possible ## rule clip_slope: @@ -36,10 +36,40 @@ if config.get("tiny_files", False): wrapper: "v9.14.0/geo/rasterio/clip" + rule clip_landcover: + input: + like_vector=rules.normalise_shapes.output.shapes, + output: + path="/automatic/cutout/{shape}/landcover.tif", + log: + "/{shape}/clip_landcover.log", + localrule: True + params: + cog_url=internal["resources"]["automatic"]["globcover"], + message: + "Download landcover data covering the bounds of the input shapefile." + wrapper: + "v9.14.0/geo/rasterio/clip" + + rule clip_settlement: + input: + like_vector=rules.normalise_shapes.output.shapes, + output: + path="/automatic/cutout/{shape}/settlement.tif", + log: + "/{shape}/clip_settlement.log", + localrule: True + params: + cog_url=internal["resources"]["automatic"]["ghsl"], + message: + "Cut settlement data to the bounds of the input shapefile." + wrapper: + "v9.14.0/geo/rasterio/clip" + else: ## - # Download global slope and bathymetry data, then clip the files locally + # Download the full datasets, then clip the files locally ## rule download_slope: @@ -56,7 +86,7 @@ else: "Download global slope data." shell: """ - curl -sSLo {output:q} {params.url:q} >{log:q} 2>&1 + curl -fsSLo {output:q} {params.url:q} >{log:q} 2>&1 """ rule download_bathymetry: @@ -73,7 +103,41 @@ else: "Download global bathymetry data." shell: """ - curl -sSLo {output:q} {params.url:q} >{log:q} 2>&1 + curl -fsSLo {output:q} {params.url:q} >{log:q} 2>&1 + """ + + rule download_landcover: + output: + "/automatic/global/globcover-landcover.tif", + log: + "/download_landcover.log", + localrule: True + conda: + "../envs/module.yaml" + params: + url=internal["resources"]["automatic"]["globcover"], + message: + "Download the GlobCover land cover data (~380 MB)." + shell: + """ + curl -fsSLo {output:q} {params.url:q} >{log:q} 2>&1 + """ + + rule download_settlement: + output: + "/automatic/global/ghsl_built_s.tif", + log: + "/download_settlement.log", + localrule: True + conda: + "../envs/module.yaml" + params: + url=internal["resources"]["automatic"]["ghsl"], + message: + "Download the GHSL (Global Human Settlement Layer) built-up surface data." + shell: + """ + curl -fsSLo {output:q} {params.url:q} >{log:q} 2>&1 """ rule clip_slope: @@ -102,62 +166,31 @@ else: wrapper: "v9.14.0/geo/rasterio/clip" + rule clip_landcover: + input: + like_vector=rules.normalise_shapes.output.shapes, + raster=rules.download_landcover.output[0], + output: + path="/automatic/cutout/{shape}/landcover.tif", + log: + "/{shape}/clip_landcover.log", + message: + "Cut land cover data to the bounds of the input shapefile." + wrapper: + "v9.14.0/geo/rasterio/clip" -## -# Globcover -## - - -rule download_globcover: - output: - "/automatic/global/globcover.zip", - log: - "/download_globcover.log", - localrule: True - conda: - "../envs/module.yaml" - params: - url=internal["resources"]["automatic"]["globcover"], - message: - "Download the GlobCover land cover data (~380 MB)." - shell: - """ - curl -sSLo {output:q} {params.url:q} >{log:q} 2>&1 - """ - - -rule unzip_globcover: - input: - script=workflow.source_path("../scripts/unzip_like.py"), - zipfile=rules.download_globcover.output, - output: - "/automatic/global/globcover-landcover.tif", - log: - "/unzip_globcover.log", - conda: - "../envs/module.yaml" - params: - target_file=internal["resources"]["automatic"]["globcover_landcover_tif"], - message: - "Unzip the relevant TIF files from the GlobCover zip file." - shell: - """ - python {input.script:q} {input.zipfile:q} -f {params.target_file:q} -o {output:q} >{log:q} 2>&1 - """ - - -rule clip_landcover: - input: - like_vector=rules.normalise_shapes.output.shapes, - raster=rules.unzip_globcover.output[0], - output: - path="/automatic/cutout/{shape}/landcover.tif", - log: - "/{shape}/clip_landcover.log", - message: - "Cut land cover data to the bounds of the input shapefile." - wrapper: - "v9.14.0/geo/rasterio/clip" + rule clip_settlement: + input: + like_vector=rules.normalise_shapes.output.shapes, + raster=rules.download_settlement.output[0], + output: + path="/automatic/cutout/{shape}/settlement.tif", + log: + "/{shape}/clip_settlement.log", + message: + "Cut settlement data to the bounds of the input shapefile." + wrapper: + "v9.14.0/geo/rasterio/clip" ## @@ -213,63 +246,6 @@ rule clip_ship_travel: "v9.14.0/geo/rasterio/clip" -## -# Global Human Settlement Layer (GHSL) -## - - -rule download_ghsl: - output: - "/automatic/global/ghsl_built_s.zip", - log: - "/download_ghsl.log", - localrule: True - conda: - "../envs/module.yaml" - params: - url=internal["resources"]["automatic"]["ghsl"], - message: - "Download the GHSL (Global Human Settlement Layer) built-up surface data." - shell: - """ - curl -sSLo {output:q} {params.url:q} >{log:q} 2>&1 - """ - - -rule unzip_ghsl: - input: - script=workflow.source_path("../scripts/unzip_like.py"), - zipfile=rules.download_ghsl.output, - output: - "/automatic/global/ghsl_built_s.tif", - log: - "/unzip_ghsl.log", - conda: - "../envs/module.yaml" - params: - target_file=internal["resources"]["automatic"]["ghsl_tif"], - message: - "Unzip the relevant TIF file from the GHSL data." - shell: - """ - python {input.script:q} {input.zipfile:q} -f {params.target_file:q} -o {output:q} >{log:q} 2>&1 - """ - - -rule clip_settlement: - input: - like_vector=rules.normalise_shapes.output.shapes, - raster=rules.unzip_ghsl.output[0], - output: - path="/automatic/cutout/{shape}/settlement.tif", - log: - "/{shape}/clip_settlement.log", - message: - "Cut settlement data to the bounds of the input shapefile." - wrapper: - "v9.14.0/geo/rasterio/clip" - - ## # Protected Areas (WDPA) ##