Use PANGAEA version of globcover and AWS for GHSL settlement

README.md

@@ -166,7 +166,7 @@ This module is based on the following research and datasets:
 
 - [GEDTM30](https://github.com/openlandmap/GEDTM30) for slope
     - License: Creative Commons Attribution 4.0 International
-- [GlobCover land cover data](https://due.esrin.esa.int/page_globcover.php)
+- [GlobCover land cover data](https://doi.org/10.1594/PANGAEA.787668)
     - License: "You may use the GlobCover land cover map for educational and/or scientific purposes, without any fee on the condition that you credit ESA and the Université Catholique de Louvain as the source of the GlobCover products."
 - [GEBCO (General Bathymetric Chart of the Oceans)](https://www.gebco.net/data-products/gridded-bathymetry-data) 15 arc-second data
     - License: "The GEBCO Grid is placed in the public domain and may be used free of charge. [...] Users must: Acknowledge the source of The GEBCO Grid. A suitable form of attribution is given in the documentation that accompanies The GEBCO Grid."

tests/integration/Snakefile

@@ -13,7 +13,7 @@ rule download_netherlands_shapes:
         "Download and unzip the Netherlands shapes."
     shell:
         """
-        curl -sSLo {output:q} "https://zenodo.org/records/16684683/files/NLD.parquet"
+        curl -fsSLo {output:q} "https://zenodo.org/records/16684683/files/NLD.parquet"
         """
 
 
@@ -28,7 +28,7 @@ rule download_netherlands_protected_areas:
         "Download a dummy drop-in dataset for Netherlands protected areas (not based on WDPA)."
     shell:
         """
-        curl -sSLo {output:q} "https://zenodo.org/records/16684513/files/wdpa-nl-dropin.gdb.zip"
+        curl -fsSLo {output:q} "https://zenodo.org/records/16684513/files/wdpa-nl-dropin.gdb.zip"
         """
 
 

workflow/internal/settings.yaml

@@ -6,10 +6,8 @@ resources:
     ##
     slope: "https://zenodo.org/records/14920379/files/slope.in.degree_edtm_m_240m_s_20000101_20221231_go_epsg.4326_v20241230.tif"
     bathymetry: "https://zenodo.org/records/15741950/files/gebco_2024_sub_ice_cog.tif"
-    globcover: "https://due.esrin.esa.int/files/Globcover2009_V2.3_Global_.zip"
-    globcover_landcover_tif: "GLOBCOVER_L4_200901_200912_V2.3.tif"
-    ghsl: "https://jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/GHSL/GHS_BUILT_S_GLOBE_R2023A/GHS_BUILT_S_E2025_GLOBE_R2023A_4326_30ss/V1-0/GHS_BUILT_S_E2025_GLOBE_R2023A_4326_30ss_V1_0.zip"
-    ghsl_tif: "GHS_BUILT_S_E2025_GLOBE_R2023A_4326_30ss_V1_0.tif"
+    globcover: "https://hs.pangaea.de/Projects/GLOBCOVER/GLOBCOVER_L4_200901_200912_V2.3.tif"
+    ghsl: "https://jrc-ghsl.s3.eu-central-1.amazonaws.com/ghs-built-s/r2023a/4326/30asec/2025/GHS_BUILT_S_E2025_GLOBE_R2023A_4326_30ss_V1_0.tif"
     ship_travel: "https://datacatalogfiles.worldbank.org/ddh-published/0037580/5/DR0045406/shipdensity_global.zip"
     ship_travel_tif: "shipdensity_global.tif"
 

workflow/rules/automatic.smk

@@ -3,7 +3,7 @@
 if config.get("tiny_files", False):
 
     ##
-    # Directly download clipped slope and bathymetry data
+    # Directly download clipped datasets when possible
     ##
 
     rule clip_slope:
@@ -36,10 +36,40 @@ if config.get("tiny_files", False):
         wrapper:
             "v9.14.0/geo/rasterio/clip"
 
+    rule clip_landcover:
+        input:
+            like_vector=rules.normalise_shapes.output.shapes,
+        output:
+            path="<resources>/automatic/cutout/{shape}/landcover.tif",
+        log:
+            "<logs>/{shape}/clip_landcover.log",
+        localrule: True
+        params:
+            cog_url=internal["resources"]["automatic"]["globcover"],
+        message:
+            "Download landcover data covering the bounds of the input shapefile."
+        wrapper:
+            "v9.14.0/geo/rasterio/clip"
+
+    rule clip_settlement:
+        input:
+            like_vector=rules.normalise_shapes.output.shapes,
+        output:
+            path="<resources>/automatic/cutout/{shape}/settlement.tif",
+        log:
+            "<logs>/{shape}/clip_settlement.log",
+        localrule: True
+        params:
+            cog_url=internal["resources"]["automatic"]["ghsl"],
+        message:
+            "Cut settlement data to the bounds of the input shapefile."
+        wrapper:
+            "v9.14.0/geo/rasterio/clip"
+
 else:
 
     ##
-    # Download global slope and bathymetry data, then clip the files locally
+    # Download the full datasets, then clip the files locally
     ##
 
     rule download_slope:
@@ -56,7 +86,7 @@ else:
             "Download global slope data."
         shell:
             """
-            curl -sSLo {output:q} {params.url:q} >{log:q} 2>&1
+            curl -fsSLo {output:q} {params.url:q} >{log:q} 2>&1
             """
 
     rule download_bathymetry:
@@ -73,7 +103,41 @@ else:
             "Download global bathymetry data."
         shell:
             """
-            curl -sSLo {output:q} {params.url:q} >{log:q} 2>&1
+            curl -fsSLo {output:q} {params.url:q} >{log:q} 2>&1
+            """
+
+    rule download_landcover:
+        output:
+            "<resources>/automatic/global/globcover-landcover.tif",
+        log:
+            "<logs>/download_landcover.log",
+        localrule: True
+        conda:
+            "../envs/module.yaml"
+        params:
+            url=internal["resources"]["automatic"]["globcover"],
+        message:
+            "Download the GlobCover land cover data (~380 MB)."
+        shell:
+            """
+            curl -fsSLo {output:q} {params.url:q} >{log:q} 2>&1
+            """
+
+    rule download_settlement:
+        output:
+            "<resources>/automatic/global/ghsl_built_s.tif",
+        log:
+            "<logs>/download_settlement.log",
+        localrule: True
+        conda:
+            "../envs/module.yaml"
+        params:
+            url=internal["resources"]["automatic"]["ghsl"],
+        message:
+            "Download the GHSL (Global Human Settlement Layer) built-up surface data."
+        shell:
+            """
+            curl -fsSLo {output:q} {params.url:q} >{log:q} 2>&1
             """
 
     rule clip_slope:
@@ -102,62 +166,31 @@ else:
         wrapper:
             "v9.14.0/geo/rasterio/clip"
 
+    rule clip_landcover:
+        input:
+            like_vector=rules.normalise_shapes.output.shapes,
+            raster=rules.download_landcover.output[0],
+        output:
+            path="<resources>/automatic/cutout/{shape}/landcover.tif",
+        log:
+            "<logs>/{shape}/clip_landcover.log",
+        message:
+            "Cut land cover data to the bounds of the input shapefile."
+        wrapper:
+            "v9.14.0/geo/rasterio/clip"
 
-##
-# Globcover
-##
-
-
-rule download_globcover:
-    output:
-        "<resources>/automatic/global/globcover.zip",
-    log:
-        "<logs>/download_globcover.log",
-    localrule: True
-    conda:
-        "../envs/module.yaml"
-    params:
-        url=internal["resources"]["automatic"]["globcover"],
-    message:
-        "Download the GlobCover land cover data (~380 MB)."
-    shell:
-        """
-        curl -sSLo {output:q} {params.url:q} >{log:q} 2>&1
-        """
-
-
-rule unzip_globcover:
-    input:
-        script=workflow.source_path("../scripts/unzip_like.py"),
-        zipfile=rules.download_globcover.output,
-    output:
-        "<resources>/automatic/global/globcover-landcover.tif",
-    log:
-        "<logs>/unzip_globcover.log",
-    conda:
-        "../envs/module.yaml"
-    params:
-        target_file=internal["resources"]["automatic"]["globcover_landcover_tif"],
-    message:
-        "Unzip the relevant TIF files from the GlobCover zip file."
-    shell:
-        """
-        python {input.script:q} {input.zipfile:q} -f {params.target_file:q} -o {output:q} >{log:q} 2>&1
-        """
-
-
-rule clip_landcover:
-    input:
-        like_vector=rules.normalise_shapes.output.shapes,
-        raster=rules.unzip_globcover.output[0],
-    output:
-        path="<resources>/automatic/cutout/{shape}/landcover.tif",
-    log:
-        "<logs>/{shape}/clip_landcover.log",
-    message:
-        "Cut land cover data to the bounds of the input shapefile."
-    wrapper:
-        "v9.14.0/geo/rasterio/clip"
+    rule clip_settlement:
+        input:
+            like_vector=rules.normalise_shapes.output.shapes,
+            raster=rules.download_settlement.output[0],
+        output:
+            path="<resources>/automatic/cutout/{shape}/settlement.tif",
+        log:
+            "<logs>/{shape}/clip_settlement.log",
+        message:
+            "Cut settlement data to the bounds of the input shapefile."
+        wrapper:
+            "v9.14.0/geo/rasterio/clip"
 
 
 ##
@@ -213,63 +246,6 @@ rule clip_ship_travel:
         "v9.14.0/geo/rasterio/clip"
 
 
-##
-# Global Human Settlement Layer (GHSL)
-##
-
-
-rule download_ghsl:
-    output:
-        "<resources>/automatic/global/ghsl_built_s.zip",
-    log:
-        "<logs>/download_ghsl.log",
-    localrule: True
-    conda:
-        "../envs/module.yaml"
-    params:
-        url=internal["resources"]["automatic"]["ghsl"],
-    message:
-        "Download the GHSL (Global Human Settlement Layer) built-up surface data."
-    shell:
-        """
-        curl -sSLo {output:q} {params.url:q} >{log:q} 2>&1
-        """
-
-
-rule unzip_ghsl:
-    input:
-        script=workflow.source_path("../scripts/unzip_like.py"),
-        zipfile=rules.download_ghsl.output,
-    output:
-        "<resources>/automatic/global/ghsl_built_s.tif",
-    log:
-        "<logs>/unzip_ghsl.log",
-    conda:
-        "../envs/module.yaml"
-    params:
-        target_file=internal["resources"]["automatic"]["ghsl_tif"],
-    message:
-        "Unzip the relevant TIF file from the GHSL data."
-    shell:
-        """
-        python {input.script:q} {input.zipfile:q} -f {params.target_file:q} -o {output:q} >{log:q} 2>&1
-        """
-
-
-rule clip_settlement:
-    input:
-        like_vector=rules.normalise_shapes.output.shapes,
-        raster=rules.unzip_ghsl.output[0],
-    output:
-        path="<resources>/automatic/cutout/{shape}/settlement.tif",
-    log:
-        "<logs>/{shape}/clip_settlement.log",
-    message:
-        "Cut settlement data to the bounds of the input shapefile."
-    wrapper:
-        "v9.14.0/geo/rasterio/clip"
-
-
 ##
 # Protected Areas (WDPA)
 ##