From c4599bf9b65ba24f9ea73406b8ab9cbfba6181f2 Mon Sep 17 00:00:00 2001 From: Ivan Ruiz Manuel <72193617+irm-codebase@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:07:00 +0100 Subject: [PATCH 1/4] Use PANGAEA version of globcover --- README.md | 2 +- workflow/internal/settings.yaml | 3 +- workflow/rules/automatic.smk | 100 ++++++++++++++------------------ 3 files changed, 46 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index f773ccf..710fe67 100644 --- a/README.md +++ b/README.md @@ -166,7 +166,7 @@ This module is based on the following research and datasets: - [GEDTM30](https://github.com/openlandmap/GEDTM30) for slope - License: Creative Commons Attribution 4.0 International -- [GlobCover land cover data](https://due.esrin.esa.int/page_globcover.php) +- [GlobCover land cover data](https://doi.org/10.1594/PANGAEA.787668) - License: "You may use the GlobCover land cover map for educational and/or scientific purposes, without any fee on the condition that you credit ESA and the Université Catholique de Louvain as the source of the GlobCover products." - [GEBCO (General Bathymetric Chart of the Oceans)](https://www.gebco.net/data-products/gridded-bathymetry-data) 15 arc-second data - License: "The GEBCO Grid is placed in the public domain and may be used free of charge. [...] Users must: Acknowledge the source of The GEBCO Grid. A suitable form of attribution is given in the documentation that accompanies The GEBCO Grid." diff --git a/workflow/internal/settings.yaml b/workflow/internal/settings.yaml index 2edfab2..3aa7c87 100644 --- a/workflow/internal/settings.yaml +++ b/workflow/internal/settings.yaml @@ -6,8 +6,7 @@ resources: ## slope: "https://zenodo.org/records/14920379/files/slope.in.degree_edtm_m_240m_s_20000101_20221231_go_epsg.4326_v20241230.tif" bathymetry: "https://zenodo.org/records/15741950/files/gebco_2024_sub_ice_cog.tif" - globcover: "https://due.esrin.esa.int/files/Globcover2009_V2.3_Global_.zip" - globcover_landcover_tif: "GLOBCOVER_L4_200901_200912_V2.3.tif" + globcover: "https://hs.pangaea.de/Projects/GLOBCOVER/GLOBCOVER_L4_200901_200912_V2.3.tif" ghsl: "https://jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/GHSL/GHS_BUILT_S_GLOBE_R2023A/GHS_BUILT_S_E2025_GLOBE_R2023A_4326_30ss/V1-0/GHS_BUILT_S_E2025_GLOBE_R2023A_4326_30ss_V1_0.zip" ghsl_tif: "GHS_BUILT_S_E2025_GLOBE_R2023A_4326_30ss_V1_0.tif" ship_travel: "https://datacatalogfiles.worldbank.org/ddh-published/0037580/5/DR0045406/shipdensity_global.zip" diff --git a/workflow/rules/automatic.smk b/workflow/rules/automatic.smk index 479d6a1..4b4952c 100644 --- a/workflow/rules/automatic.smk +++ b/workflow/rules/automatic.smk @@ -36,10 +36,25 @@ if config.get("tiny_files", False): wrapper: "v9.14.0/geo/rasterio/clip" + rule clip_landcover: + input: + like_vector=rules.normalise_shapes.output.shapes, + output: + path="/automatic/cutout/{shape}/landcover.tif", + log: + "/{shape}/clip_landcover.log", + localrule: True + params: + cog_url=internal["resources"]["automatic"]["globcover"], + message: + "Download bathymetry data covering the bounds of the input shapefile." + wrapper: + "v9.14.0/geo/rasterio/clip" + else: ## - # Download global slope and bathymetry data, then clip the files locally + # Download global slope, bathymetry and landcover data, then clip the files locally ## rule download_slope: @@ -102,62 +117,35 @@ else: wrapper: "v9.14.0/geo/rasterio/clip" + rule download_landcover: + output: + "/automatic/global/globcover-landcover.tif", + log: + "/download_landcover.log", + localrule: True + conda: + "../envs/module.yaml" + params: + url=internal["resources"]["automatic"]["globcover"], + message: + "Download the GlobCover land cover data (~380 MB)." + shell: + """ + curl -sSLo {output:q} {params.url:q} >{log:q} 2>&1 + """ -## -# Globcover -## - - -rule download_globcover: - output: - "/automatic/global/globcover.zip", - log: - "/download_globcover.log", - localrule: True - conda: - "../envs/module.yaml" - params: - url=internal["resources"]["automatic"]["globcover"], - message: - "Download the GlobCover land cover data (~380 MB)." - shell: - """ - curl -sSLo {output:q} {params.url:q} >{log:q} 2>&1 - """ - - -rule unzip_globcover: - input: - script=workflow.source_path("../scripts/unzip_like.py"), - zipfile=rules.download_globcover.output, - output: - "/automatic/global/globcover-landcover.tif", - log: - "/unzip_globcover.log", - conda: - "../envs/module.yaml" - params: - target_file=internal["resources"]["automatic"]["globcover_landcover_tif"], - message: - "Unzip the relevant TIF files from the GlobCover zip file." - shell: - """ - python {input.script:q} {input.zipfile:q} -f {params.target_file:q} -o {output:q} >{log:q} 2>&1 - """ - - -rule clip_landcover: - input: - like_vector=rules.normalise_shapes.output.shapes, - raster=rules.unzip_globcover.output[0], - output: - path="/automatic/cutout/{shape}/landcover.tif", - log: - "/{shape}/clip_landcover.log", - message: - "Cut land cover data to the bounds of the input shapefile." - wrapper: - "v9.14.0/geo/rasterio/clip" + rule clip_landcover: + input: + like_vector=rules.normalise_shapes.output.shapes, + raster=rules.download_landcover.output[0], + output: + path="/automatic/cutout/{shape}/landcover.tif", + log: + "/{shape}/clip_landcover.log", + message: + "Cut land cover data to the bounds of the input shapefile." + wrapper: + "v9.14.0/geo/rasterio/clip" ## From e1d2fb0f9f24f2d09d7a7cb31b53f175f02dfcbe Mon Sep 17 00:00:00 2001 From: Ivan Ruiz Manuel <72193617+irm-codebase@users.noreply.github.com> Date: Tue, 27 Oct 2026 20:01:25 +0100 Subject: [PATCH 2/4] Use unzip wrapper for GHSL --- workflow/rules/automatic.smk | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/workflow/rules/automatic.smk b/workflow/rules/automatic.smk index 4b4952c..358c849 100644 --- a/workflow/rules/automatic.smk +++ b/workflow/rules/automatic.smk @@ -226,22 +226,17 @@ rule download_ghsl: rule unzip_ghsl: input: - script=workflow.source_path("../scripts/unzip_like.py"), - zipfile=rules.download_ghsl.output, + rules.download_ghsl.output[0], output: "/automatic/global/ghsl_built_s.tif", log: "/unzip_ghsl.log", - conda: - "../envs/module.yaml" params: - target_file=internal["resources"]["automatic"]["ghsl_tif"], + internal_paths=internal["resources"]["automatic"]["ghsl_tif"], message: "Unzip the relevant TIF file from the GHSL data." - shell: - """ - python {input.script:q} {input.zipfile:q} -f {params.target_file:q} -o {output:q} >{log:q} 2>&1 - """ + wrapper: + "v9.8.0/utils/libarchive/extract" rule clip_settlement: From 3dbbad9c5e500d042661174e1eebf943c7ad46dc Mon Sep 17 00:00:00 2001 From: Ivan Ruiz Manuel <72193617+irm-codebase@users.noreply.github.com> Date: Tue, 27 Oct 2026 22:29:37 +0100 Subject: [PATCH 3/4] Add tiny processing for GHSL --- tests/integration/Snakefile | 4 +- workflow/internal/settings.yaml | 3 +- workflow/rules/automatic.smk | 142 +++++++++++++++----------------- 3 files changed, 71 insertions(+), 78 deletions(-) diff --git a/tests/integration/Snakefile b/tests/integration/Snakefile index 819f364..db2cfeb 100644 --- a/tests/integration/Snakefile +++ b/tests/integration/Snakefile @@ -13,7 +13,7 @@ rule download_netherlands_shapes: "Download and unzip the Netherlands shapes." shell: """ - curl -sSLo {output:q} "https://zenodo.org/records/16684683/files/NLD.parquet" + curl -fsSLo {output:q} "https://zenodo.org/records/16684683/files/NLD.parquet" """ @@ -28,7 +28,7 @@ rule download_netherlands_protected_areas: "Download a dummy drop-in dataset for Netherlands protected areas (not based on WDPA)." shell: """ - curl -sSLo {output:q} "https://zenodo.org/records/16684513/files/wdpa-nl-dropin.gdb.zip" + curl -fsSLo {output:q} "https://zenodo.org/records/16684513/files/wdpa-nl-dropin.gdb.zip" """ diff --git a/workflow/internal/settings.yaml b/workflow/internal/settings.yaml index 3aa7c87..051370e 100644 --- a/workflow/internal/settings.yaml +++ b/workflow/internal/settings.yaml @@ -7,8 +7,7 @@ resources: slope: "https://zenodo.org/records/14920379/files/slope.in.degree_edtm_m_240m_s_20000101_20221231_go_epsg.4326_v20241230.tif" bathymetry: "https://zenodo.org/records/15741950/files/gebco_2024_sub_ice_cog.tif" globcover: "https://hs.pangaea.de/Projects/GLOBCOVER/GLOBCOVER_L4_200901_200912_V2.3.tif" - ghsl: "https://jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/GHSL/GHS_BUILT_S_GLOBE_R2023A/GHS_BUILT_S_E2025_GLOBE_R2023A_4326_30ss/V1-0/GHS_BUILT_S_E2025_GLOBE_R2023A_4326_30ss_V1_0.zip" - ghsl_tif: "GHS_BUILT_S_E2025_GLOBE_R2023A_4326_30ss_V1_0.tif" + ghsl: "https://jrc-ghsl.s3.eu-central-1.amazonaws.com/ghs-built-s/r2023a/4326/30asec/2025/GHS_BUILT_S_E2025_GLOBE_R2023A_4326_30ss_V1_0.tif" ship_travel: "https://datacatalogfiles.worldbank.org/ddh-published/0037580/5/DR0045406/shipdensity_global.zip" ship_travel_tif: "shipdensity_global.tif" diff --git a/workflow/rules/automatic.smk b/workflow/rules/automatic.smk index 358c849..c5ffe46 100644 --- a/workflow/rules/automatic.smk +++ b/workflow/rules/automatic.smk @@ -3,7 +3,7 @@ if config.get("tiny_files", False): ## - # Directly download clipped slope and bathymetry data + # Directly download clipped datasets when possible ## rule clip_slope: @@ -47,14 +47,29 @@ if config.get("tiny_files", False): params: cog_url=internal["resources"]["automatic"]["globcover"], message: - "Download bathymetry data covering the bounds of the input shapefile." + "Download landcover data covering the bounds of the input shapefile." + wrapper: + "v9.14.0/geo/rasterio/clip" + + rule clip_settlement: + input: + like_vector=rules.normalise_shapes.output.shapes, + output: + path="/automatic/cutout/{shape}/settlement.tif", + log: + "/{shape}/clip_settlement.log", + localrule: True + params: + cog_url=internal["resources"]["automatic"]["ghsl"], + message: + "Cut settlement data to the bounds of the input shapefile." wrapper: "v9.14.0/geo/rasterio/clip" else: ## - # Download global slope, bathymetry and landcover data, then clip the files locally + # Download the full datasets, then clip the files locally ## rule download_slope: @@ -71,7 +86,7 @@ else: "Download global slope data." shell: """ - curl -sSLo {output:q} {params.url:q} >{log:q} 2>&1 + curl -fsSLo {output:q} {params.url:q} >{log:q} 2>&1 """ rule download_bathymetry: @@ -88,9 +103,44 @@ else: "Download global bathymetry data." shell: """ - curl -sSLo {output:q} {params.url:q} >{log:q} 2>&1 + curl -fsSLo {output:q} {params.url:q} >{log:q} 2>&1 """ + rule download_landcover: + output: + "/automatic/global/globcover-landcover.tif", + log: + "/download_landcover.log", + localrule: True + conda: + "../envs/module.yaml" + params: + url=internal["resources"]["automatic"]["globcover"], + message: + "Download the GlobCover land cover data (~380 MB)." + shell: + """ + curl -fsSLo {output:q} {params.url:q} >{log:q} 2>&1 + """ + + rule download_settlement: + output: + "/automatic/global/ghsl_built_s.tif", + log: + "/download_settlement.log", + localrule: True + conda: + "../envs/module.yaml" + params: + url=internal["resources"]["automatic"]["ghsl"], + message: + "Download the GHSL (Global Human Settlement Layer) built-up surface data." + shell: + """ + curl -fsSLo {output:q} {params.url:q} >{log:q} 2>&1 + """ + + rule clip_slope: input: like_vector=rules.normalise_shapes.output.shapes, @@ -117,23 +167,6 @@ else: wrapper: "v9.14.0/geo/rasterio/clip" - rule download_landcover: - output: - "/automatic/global/globcover-landcover.tif", - log: - "/download_landcover.log", - localrule: True - conda: - "../envs/module.yaml" - params: - url=internal["resources"]["automatic"]["globcover"], - message: - "Download the GlobCover land cover data (~380 MB)." - shell: - """ - curl -sSLo {output:q} {params.url:q} >{log:q} 2>&1 - """ - rule clip_landcover: input: like_vector=rules.normalise_shapes.output.shapes, @@ -147,6 +180,19 @@ else: wrapper: "v9.14.0/geo/rasterio/clip" + rule clip_settlement: + input: + like_vector=rules.normalise_shapes.output.shapes, + raster=rules.download_settlement.output[0], + output: + path="/automatic/cutout/{shape}/settlement.tif", + log: + "/{shape}/clip_settlement.log", + message: + "Cut settlement data to the bounds of the input shapefile." + wrapper: + "v9.14.0/geo/rasterio/clip" + ## # Global Ship Traffic Density @@ -201,58 +247,6 @@ rule clip_ship_travel: "v9.14.0/geo/rasterio/clip" -## -# Global Human Settlement Layer (GHSL) -## - - -rule download_ghsl: - output: - "/automatic/global/ghsl_built_s.zip", - log: - "/download_ghsl.log", - localrule: True - conda: - "../envs/module.yaml" - params: - url=internal["resources"]["automatic"]["ghsl"], - message: - "Download the GHSL (Global Human Settlement Layer) built-up surface data." - shell: - """ - curl -sSLo {output:q} {params.url:q} >{log:q} 2>&1 - """ - - -rule unzip_ghsl: - input: - rules.download_ghsl.output[0], - output: - "/automatic/global/ghsl_built_s.tif", - log: - "/unzip_ghsl.log", - params: - internal_paths=internal["resources"]["automatic"]["ghsl_tif"], - message: - "Unzip the relevant TIF file from the GHSL data." - wrapper: - "v9.8.0/utils/libarchive/extract" - - -rule clip_settlement: - input: - like_vector=rules.normalise_shapes.output.shapes, - raster=rules.unzip_ghsl.output[0], - output: - path="/automatic/cutout/{shape}/settlement.tif", - log: - "/{shape}/clip_settlement.log", - message: - "Cut settlement data to the bounds of the input shapefile." - wrapper: - "v9.14.0/geo/rasterio/clip" - - ## # Protected Areas (WDPA) ## From 5b1e0870d292160b62d06c6965a0985a8357c628 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 21:30:10 +0000 Subject: [PATCH 4/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- workflow/rules/automatic.smk | 1 - 1 file changed, 1 deletion(-) diff --git a/workflow/rules/automatic.smk b/workflow/rules/automatic.smk index c5ffe46..1a379c8 100644 --- a/workflow/rules/automatic.smk +++ b/workflow/rules/automatic.smk @@ -140,7 +140,6 @@ else: curl -fsSLo {output:q} {params.url:q} >{log:q} 2>&1 """ - rule clip_slope: input: like_vector=rules.normalise_shapes.output.shapes,