From c58cc0ebff32312fa519880d028c7cfa8966047b Mon Sep 17 00:00:00 2001 From: Joaquim Date: Tue, 17 Nov 2026 23:04:39 +0000 Subject: [PATCH] Add NDMI and Landsat L2 thermal; fix band order --- docs/src/gallery/HLS/cloud-native-hls-data.md | 26 ++++--- src/RemoteS.jl | 2 +- src/spectral_indices.jl | 38 ++++++++-- src/utils.jl | 75 +++++++++++++++++-- test/runtests.jl | 2 +- 5 files changed, 117 insertions(+), 26 deletions(-) diff --git a/docs/src/gallery/HLS/cloud-native-hls-data.md b/docs/src/gallery/HLS/cloud-native-hls-data.md index 1733cd2..d550c6d 100644 --- a/docs/src/gallery/HLS/cloud-native-hls-data.md +++ b/docs/src/gallery/HLS/cloud-native-hls-data.md @@ -2,7 +2,9 @@ Extracting an EVI Time Series from Harmonized Landsat-8 Sentinel-2 (HLS) data in the Cloud using CMR's SpatioTemporal Asset Catalog (CMR-STAC). This tutorial demonstrates how to work with the -HLS Landsat 8 (HLSL30.015) and Sentinel-2 (HLSS30.015) data products. +HLS Landsat 8/9 (HLSL30) and Sentinel-2 (HLSS30) v2.0 data products. + +#### NOTE: Since September 2024 the CMR-STAC collection IDs are `HLSL30_2.0` and `HLSS30_2.0` (the old `HLSL30.v1.5`/`HLSS30.v1.5` collections are retired and the `.v2.0` form is gone). The code below uses the current IDs; the printed outputs are from the tutorial's original 2021 run and still show the old names. #### NOTE: This tutorial is adapted from [Getting Started with Cloud-Native HLS Data in Python](https://lpdaac.usgs.gov/resources/e-learning/getting-started-cloud-native-hls-data-python/) @@ -106,7 +108,7 @@ collections[2] "description" => "The Harmonized Landsat and Sentinel-2 (HLS) project provid… "extent" => Dict{String, Any}("spatial"=>Dict{String, Any}("bbox"=>Any[… -### In CMR, ```id``` is used to query by a specific product, so be sure to save the ID for the HLS S30 and L30 V1.5 products below: +### In CMR, ```id``` is used to query by a specific product, so be sure to save the ID for the HLS S30 and L30 v2.0 products below: ```julia # Search available collections for HLS and print them out @@ -126,13 +128,13 @@ hls_collections = [c for c in collections if contains(c["title"], "HLS")]; nothing nothing -> Note that the ```id``` shortname is in the format: productshortname.vVVV (where VVV = product version) +> Note that the ```id``` shortname is in the format: productshortname_V.V (where V.V = product version, e.g. HLSS30_2.0) ### Explore the attributes contained in the HLSS30 Collection. ```julia # Grab HLSS30 collection -s30 = [h for h in hls_collections if h["id"] == "HLSS30.v1.5"][1]; +s30 = [h for h in hls_collections if h["id"] == "HLSS30_2.0"][1]; # Check out the extent of this collection for k in keys(s30["extent"]) println(k, " : ", s30["extent"][k]) end @@ -415,11 +417,11 @@ params ### As of October 4, 2021, the HLS Operational Land Imager Surface Reflectance and TOA Brightness (HLSL30) product has been provisionally released, and this tutorial has been updated to show how to combine observations from both products into a time series. -Next, add the ```shortname``` for the HLSS30 v1.5 product (HLSS30.v1.5) to the params dictionary, and query +Next, add the ```shortname``` for the HLSS30 v2.0 product (HLSS30_2.0) to the params dictionary, and query the CMR-STAC LPCLOUD search endpoint for just HLSS30 items. ```julia -s30_id = "HLSS30.v1.5" +s30_id = "HLSS30_2.0" params["collections"] = [s30_id]; ``` @@ -433,10 +435,10 @@ length(s30_items) 61 -Append the HLSL30 V1.5 Product shortname (ID) to the list under the ```collections``` parameter. +Append the HLSL30 v2.0 Product shortname (ID) to the list under the ```collections``` parameter. ```julia -l30_id = "HLSL30.v1.5"; +l30_id = "HLSL30_2.0"; append!(params["collections"], [l30_id]) params ``` @@ -500,8 +502,8 @@ h = hls_items[1] ```julia evi_band_links = [] -# Define which HLS product is being accessed -if (split(h["assets"]["browse"]["href"], '/')[5] == "HLSS30.015") +# Define which HLS product is being accessed (the item ID starts with HLS.S30 or HLS.L30) +if startswith(h["id"], "HLS.S30") evi_bands = ["B8A", "B04", "B02", "Fmask"] # NIR RED BLUE Quality for S30 else evi_bands = ["B05", "B04", "B02", "Fmask"] # NIR RED BLUE Quality for L30 @@ -661,7 +663,7 @@ for (j, h) in enumerate(hls_items) try evi_band_links = [] # Define which HLS product is being accessed - if (split(h["assets"]["browse"]["href"], '/')[5] == "HLSS30.015") + if startswith(h["id"], "HLS.S30") evi_bands = ["B8A", "B04", "B02", "Fmask"] # NIR RED BLUE Quality for S30 else evi_bands = ["B05", "B04", "B02", "Fmask"] # NIR RED BLUE Quality for L30 @@ -728,7 +730,7 @@ eviFiles = [o for o in readdir() if endswith(o, "EVI.grd")]; df = dateformat"yyyymmddTHHMMSS"; time = Vector{TimeType}(undef, length(eviFiles)); for (i, e) in enumerate(eviFiles) - t = split(split(e, ".v1.5")[1], ".")[end]; + t = split(split(e, ".v2.0")[1], ".")[end]; # Julia doesn't parse a date in yyyyjjjT... so must resort to brewed functions mon, day = monthday(doy2date(t[5:7], t[1:4])); time[i] = DateTime(@sprintf("%s%.02d%.02d%s", t[1:4], mon, day, t[8:end]), df); diff --git a/src/RemoteS.jl b/src/RemoteS.jl index 9cd3434..de15bec 100644 --- a/src/RemoteS.jl +++ b/src/RemoteS.jl @@ -12,7 +12,7 @@ end export cutcube, subcube, dn2temperature, dn2radiance, dn2reflectance, reflectance_surf, grid_at_sensor, truecolor, - clg, clre, evi, evi2, gli, gndvi, mndwi, mtci, mcari, msavi, nbri, ndvi, ndwi, ndwi2, ndrei1, + clg, clre, evi, evi2, gli, gndvi, mndwi, mtci, mcari, msavi, nbri, ndmi, ndvi, ndwi, ndwi2, ndrei1, ndrei2, satvi, savi, slavi, tgi, vari, classify, train_raster, clip_orbits, findscenes, sat_scenes, sat_tracks, reportbands diff --git a/src/spectral_indices.jl b/src/spectral_indices.jl index 354c58e..c14e056 100644 --- a/src/spectral_indices.jl +++ b/src/spectral_indices.jl @@ -187,7 +187,7 @@ MNDWI = (green-swir2) / (green+swir2) $(generic_docs) """ -mndwi(green, swir2; kw...) = sp_indices(swir2, green; index="MNDWI", kw...) +mndwi(green, swir2; kw...) = sp_indices(green, swir2; index="MNDWI", kw...) mndwi(cube::String; bands::Vector{Int}=Int[], layers::Vector{Int}=Int[], bandnames::Vector{String}=String[], kw...) = helper_si_method(cube, "MNDWI"; bands=bands, layers=layers, bandnames=bandnames, defbandnames=["green", "swir2"], kw...) mndwi(cube::Union{GMT.GMTimage{UInt16, 3}, AbstractArray{<:AbstractFloat, 3}}; @@ -259,7 +259,7 @@ Normalised Burn Ratio Index. Garcia 1991 NBRI = (nir - swir2) / (nir + swir2) """ -nbri(nir, swir2; kw...) = sp_indices(swir2, nir; index="NBRI", kw...) +nbri(nir, swir2; kw...) = sp_indices(nir, swir2; index="NBRI", kw...) nbri(cube::String; bands::Vector{Int}=Int[], layers::Vector{Int}=Int[], bandnames::Vector{String}=String[], kw...) = helper_si_method(cube, "NBRI"; bands=bands, layers=layers, bandnames=bandnames, defbandnames=["nir", "swir2"], kw...) @@ -301,7 +301,7 @@ NDWI = (green - nir)/(green + nir) $(generic_docs) """ -ndwi(green, nir; kw...) = sp_indices(nir, green; index="NDWI", kw...) +ndwi(green, nir; kw...) = sp_indices(green, nir; index="NDWI", kw...) ndwi(cube::String; bands::Vector{Int}=Int[], layers::Vector{Int}=Int[], bandnames::Vector{String}=String[], kw...) = helper_si_method(cube, "NDWI"; bands=bands, layers=layers, bandnames=bandnames, defbandnames=["green", "nir"], kw...) ndwi(cube::Union{GMT.GMTimage{UInt16, 3}, AbstractArray{<:AbstractFloat, 3}}; @@ -322,13 +322,34 @@ NDWI2 = (nir - swir2)/(nir + swir2) $(generic_docs) """ -ndwi2(nir, swir2; kw...) = sp_indices(swir2, nir; index="NDWI2", kw...) +ndwi2(nir, swir2; kw...) = sp_indices(nir, swir2; index="NDWI2", kw...) ndwi2(cube::String; bands::Vector{Int}=Int[], layers::Vector{Int}=Int[], bandnames::Vector{String}=String[], kw...) = helper_si_method(cube, "NDWI2"; bands=bands, layers=layers, bandnames=bandnames, defbandnames=["nir", "swir2"], kw...) ndwi2(cube::Union{GMT.GMTimage{UInt16, 3}, AbstractArray{<:AbstractFloat, 3}}; bands::Vector{Int}=Int[], layers::Vector{Int}=Int[], bandnames::Vector{String}=String[], kw...) = helper_si_method(cube, "NDWI2"; bands=bands, layers=layers, bandnames=bandnames, defbandnames=["nir", "swir2"], kw...) +# ---------------------------------------------------------------------------------------------------------- +""" + NDMI = ndmi(nir, swir1; kw...) +or + + NDMI = ndmi(cube::Union{String, GMTgrid}; [bands=Int[], bandnames=String[], layers=Int[]], kwargs...) + +Normalized difference moisture index. Gao 1996 (NDWI with the 1.6 μm SWIR band). Leaf water content, +crop water stress. Landsat 8/9 bands 5 and 6; Sentinel-2 bands 8A and 11. + +NDMI = (nir - swir1)/(nir + swir1) + +$(generic_docs) +""" +ndmi(nir, swir1; kw...) = sp_indices(nir, swir1; index="NDMI", kw...) +ndmi(cube::String; bands::Vector{Int}=Int[], layers::Vector{Int}=Int[], bandnames::Vector{String}=String[], kw...) = + helper_si_method(cube, "NDMI"; bands=bands, layers=layers, bandnames=bandnames, defbandnames=["nir", "swir1"], kw...) +ndmi(cube::Union{GMT.GMTimage{UInt16, 3}, AbstractArray{<:AbstractFloat, 3}}; + bands::Vector{Int}=Int[], layers::Vector{Int}=Int[], bandnames::Vector{String}=String[], kw...) = + helper_si_method(cube, "NDMI"; bands=bands, layers=layers, bandnames=bandnames, defbandnames=["nir", "swir1"], kw...) + # ---------------------------------------------------------------------------------------------------------- """ NDREI1 = ndrei1(redEdge1, redEdge2; kw...) @@ -683,7 +704,7 @@ function sp_indices(bnd1, bnd2, bnd3=nothing; index::String="", kwargs...) end helper_si!(img, threshold, classes) # Threshold or Classes if one of them is != nothing end - elseif (index == "GNDVI" || index == "MNDWI" || index == "NBRI" || index == "NDVI" || index == "NDWI" || index == "NDWI2" || index == "NDREI1" || index == "NDREI2") + elseif (index == "GNDVI" || index == "MNDWI" || index == "NBRI" || index == "NDVI" || index == "NDWI" || index == "NDWI2" || index == "NDMI" || index == "NDREI1" || index == "NDREI2") # green Normalized diff vegetation index: more sensitive to cholorphyll than ndvi. Gitelson, A., and M. Merzlyak. # GNDVI => (nir - green) / (nir + green) # Modified Normalised Difference Water Index. Xu2006; MNDWI => (green-swir2) / (green+swir2) @@ -691,11 +712,14 @@ function sp_indices(bnd1, bnd2, bnd3=nothing; index::String="", kwargs...) # Normalized difference water index. McFeeters 1996. NDWI => (green - nir)/(green + nir) # NDVI Normalised Difference Vegetation Index. Rouse 1974. (nir - red) / (nir + red) # NDBI, LSWI. Normalized difference water index. Gao 1996, Chen 2005; NDWI2 => (nir - swir2)/(nir + swir2) + # Normalized difference moisture index. Gao 1996; NDMI => (nir - swir1)/(nir + swir1) # Normalized difference red edge index. Gitelson and Merzlyak 1994; (redEdge2 - redEdge1)/(redEdge2 + redEdge1) # Normalized difference red edge index 2. Barnes et al 2000; (redEdge3 - redEdge1)/(redEdge3 + redEdge1) - # Need to swap bnd1 and bnd2 for indices that are defined as (shorter_cdo - longer_cdo) - _b1, _b2 = (index == "MNDWI" || index == "NBRI" || index == "NDWI" || index == "NDWI2" || index == "NDWI2") ? (bnd2, bnd1) : (bnd1, bnd2) + # Need to swap bnd1 and bnd2 for indices that are defined as (shorter_cdo - longer_cdo). The bands + # always arrive in the order of the index's own signature (= its defbandnames), for the band-matrix + # methods and the cube methods alike, so this swap is the ONLY place the order is changed. + _b1, _b2 = (index == "MNDWI" || index == "NBRI" || index == "NDWI" || index == "NDWI2" || index == "NDMI") ? (bnd2, bnd1) : (bnd1, bnd2) if (ismask) @inbounds Threads.@threads for k = 1:mn t1 = _b1[k]*i_tmax; t2 = _b2[k]*i_tmax diff --git a/src/utils.jl b/src/utils.jl index 3fddb51..aecddfe 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -11,6 +11,8 @@ struct MTL_short sun_dist::Float64 K1::Float64 K2::Float64 + temp_mul::Float64 # Level-2 ST_B10: T(K) = DN * temp_mul + temp_add. 0 = not a Level-2 thermal band + temp_add::Float64 end const KW = Dict{Symbol,Any} @@ -483,6 +485,9 @@ function read_mtl(fname::String, mtl::String=""; get_full::Bool=false) (ind === nothing) && return nothing # Only happens when get_full = true and name is no Landsat if (mtl == "") mtl = joinpath(pato, _fname[1:ind[1]] * "MTL.txt") + # A Level-2 band (…_T1_SR_B5.TIF, …_T1_ST_B10.TIF): its MTL is …_T1_MTL.txt, without the SR/ST. + (!isfile(mtl) && occursin(r"_(SR|ST)_B\d+"i, _fname)) && + (mtl = joinpath(pato, replace(_fname, r"_(SR|ST)_B\d+.*$"i => "_MTL.txt"))) if (!isfile(mtl)) lst = filter(x -> endswith(x, "MTL.txt"), readdir((pato == "") ? "." : pato)) if (length(lst) == 1 && startswith(lst[1], _fname[1:16])) @@ -520,12 +525,23 @@ function parse_mtl(mtl::Vector{<:AbstractString}, band::Int) sun_dist = get_par("EARTH_SUN") K1 = (band >= 10) ? get_par("K1_CONSTANT_BAND_$(band)") : 0.0 K2 = (band >= 10) ? get_par("K2_CONSTANT_BAND_$(band)") : 0.0 - MTL_short(band, rad_mul, rad_add, rad_max, reflect_mul, reflect_add, reflect_max, sun_azim, sun_elev, sun_dist, K1, K2) + # A Level-2 thermal band (…_ST_B10.TIF) is NOT Level-1 DNs: it is surface temperature already, stored + # as T(K) = DN * TEMPERATURE_MULT_BAND_ST_B10 + TEMPERATURE_ADD_BAND_ST_B10 (LEVEL2_SURFACE_TEMPERATURE_ + # PARAMETERS). Taking those DNs through the Level-1 radiance + K1/K2 inversion gave ~35-45 C too hot. + temp_mul, temp_add = 0.0, 0.0 + if (band >= 10 && any(l -> occursin("TEMPERATURE_MULT_BAND_ST_B$(band)", l), mtl)) + temp_mul = get_par("TEMPERATURE_MULT_BAND_ST_B$(band)") + temp_add = get_par("TEMPERATURE_ADD_BAND_ST_B$(band)") + end + MTL_short(band, rad_mul, rad_add, rad_max, reflect_mul, reflect_add, reflect_max, sun_azim, sun_elev, sun_dist, K1, K2, temp_mul, temp_add) end # ---------------------------------------------------------------------------------------------------------- function helper1_sats(fname::String, band_layer::Int) - I::GMT.GMTimage{UInt16, 2} = (band_layer == 0) ? gmtread(fname) : gmtread(fname, layer=band_layer, layout="TRB") + # "TRB" for a single band file too, as for a cube layer: the plain reader's image label does not + # describe its memory, and the grid built on it (`mat2grid(o, I)`) came out with another geometry + # description than a grid read of the same file (pixel-registered edges vs gridline nodes). + I::GMT.GMTimage{UInt16, 2} = (band_layer == 0) ? gmtread(fname, layout="TRB") : gmtread(fname, layer=band_layer, layout="TRB") indNaN = isnodata(I) o = Matrix{Float32}(undef, size(I)) return I, indNaN, o @@ -618,17 +634,58 @@ T = dn2temperature(cube, band=10) ``` """ function dn2temperature(fname::String; band::Int=0, mtl::String="", save::String="") + _is_trad(fname) && return _trad_out(_trad2(fname, mtl, true), save) band_layer, pars, = parse_lsat8_file(fname, band=band, mtl=mtl) (pars.band < 10) && throw(ArgumentError("Brightness temperature is only for bands 10 or 11. Not this one: $(pars.band)")) I, indNaN, o = helper1_sats(fname, band_layer) - @inbounds Threads.@threads for k = 1:size(I,1)*size(I,2) - o[k] = indNaN[k] ? NaN32 : pars.K2 / (log(pars.K1 / (I.image[k] * pars.rad_mul + pars.rad_add) + 1.0)) - 273.15 + if (pars.temp_mul != 0.0) # Level-2 ST band: the DN IS the temperature, linearly scaled + tm, ta = pars.temp_mul, pars.temp_add - 273.15 + @inbounds Threads.@threads for k = 1:size(I,1)*size(I,2) + o[k] = indNaN[k] ? NaN32 : I.image[k] * tm + ta + end + else + @inbounds Threads.@threads for k = 1:size(I,1)*size(I,2) + o[k] = indNaN[k] ? NaN32 : pars.K2 / (log(pars.K1 / (I.image[k] * pars.rad_mul + pars.rad_add) + 1.0)) - 273.15 + end end G = mat2grid(o, I) (save != "") && gdaltranslate(G, dest=save) return (save != "") ? nothing : G end +# ---------------------------------------------------------------------------------------------------------- +# Landsat Collection 2 Level-2 `…_ST_TRAD.TIF`: band 10's thermal radiance at the sensor (the product's +# thermal band set; Landsat 9 L2 ships no B11). INT16, W/(m² sr µm) = DN * 0.001, fill -9999. The scale is +# not in the MTL nor in the file: it is the USGS Landsat 8-9 Collection 2 Level-2 Science Product Guide's. +# Temperature is the brightness temperature of band 10, T = K2 / ln(K1/L + 1), K1/K2 of BAND_10 from the MTL. +const TRAD_SCALE = 0.001 +_is_trad(fname::String)::Bool = occursin(r"_ST_TRAD\.TIF$"i, basename(fname)) + +function _trad2(fname::String, mtl::String, temperature::Bool)::GMTgrid + if (mtl == "") + pato, _fname = splitdir(fname) + mtl = joinpath(pato, replace(_fname, r"_ST_TRAD.*$"i => "_MTL.txt")) + end + isfile(mtl) || error("The MTL file of $(basename(fname)) was not found. Pass it with `mtl=`.") + pars = parse_mtl(readlines(mtl), 10) + G = gmtread(fname, layout="TRB")::GMTgrid + z = G.z + K1, K2 = pars.K1, pars.K2 + @inbounds Threads.@threads for k = 1:length(z) + v = z[k] + z[k] = (isnan(v) || v == -9999) ? NaN32 : + temperature ? K2 / log(K1 / (v * TRAD_SCALE) + 1.0) - 273.15 : v * TRAD_SCALE + end + G.nodata = NaN + GMT.setgrdminmax!(G) + return G +end + +function _trad_out(G::GMTgrid, save::String) + (save != "") && (gdaltranslate(G, dest=save); return nothing) + return G +end + # ---------------------------------------------------------------------------------------------------------- function helper_dns_op(I::GMTimage, mul::Float64, add::Float64, indNaN::Matrix{Bool}, o::Matrix{Float32}) # Helper function for the dn2radiance, dn2reflectance functions. Avoid IF branch if no NaNs @@ -657,6 +714,7 @@ R = dn2radiance("LC08_L1TP_204033_20210525_20210529_02_T1_B2.TIF") ``` """ function dn2radiance(fname::String; band::Int=0, bandname::String="", bandnames::String="", mtl::String="", save::String="") + _is_trad(fname) && return _trad_out(_trad2(fname, mtl, false), save) dn2aux(fname, "radiance"; band=band, bandname=bandname, bandnames=bandnames, mtl=mtl, save=save) end @@ -683,7 +741,14 @@ function dn2aux(fname::String, fun::String; band::Int=0, bandname::String="", ba function inn_helper_rad(fname, band, bandname, bandnames, mtl) band_layer, pars = helper_dns(fname, band, bandname, bandnames, mtl) I, indNaN, o = helper1_sats(fname, band_layer) - helper_dns_op(I, pars.rad_mul, pars.rad_add, indNaN, o) + (pars.temp_mul == 0.0) && return helper_dns_op(I, pars.rad_mul, pars.rad_add, indNaN, o) + # Level-2 ST band: its DNs are temperature, not L1 counts. T(K) = DN*temp_mul + temp_add, then the + # inverse of the K1/K2 relation dn2temperature uses on Level-1, L = K1 / (exp(K2/T) - 1). + K1, K2, tm, ta = pars.K1, pars.K2, pars.temp_mul, pars.temp_add + @inbounds Threads.@threads for k = 1:size(I,1)*size(I,2) + o[k] = indNaN[k] ? NaN32 : K1 / (exp(K2 / (I.image[k] * tm + ta)) - 1.0) + end + mat2grid(o, I) end function inn_helper_ref(fname, band, bandname, bandnames, mtl) band_layer, pars = helper_dns(fname, band, bandname, bandnames, mtl) diff --git a/test/runtests.jl b/test/runtests.jl index 2f53fb6..8df3e1d 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -21,7 +21,7 @@ truecolor(mat2img(rand(UInt16, 16, 16), noconv=1), mat2img(rand(UInt16, 16, 16), #sat_tracks(position=true, tle=["1 27424U 02022A 21245.83760660 .00000135 00000-0 39999-4 0 9997"; "2 27424 98.2123 186.0654 0002229 67.6025 313.3829 14.57107527 28342"]); @test_throws ErrorException("Bad input type Symbol. Must be a DateTime, a String or a Tuple(Int)") getitDTime(:val) -@test get_sat_name(Dict(:sat => "TERRA")) == "TERRA" +@test get_sat_name(Dict{Symbol,Any}(:sat => "TERRA")) == "TERRA" @test get_MODIS_scene_name(datetime2julian(DateTime("2020-09-20")), "A") == "AQUA_MODIS.20200920T000001.L2.SST.NRT.nc" @test get_MODIS_scene_name(datetime2julian(DateTime("2020-09-20")), "A", false) == "A2020264000000.L2_LAC_OC.nc"