Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions mapchete_eo/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,7 @@ def read(
sort: Optional[SortMethodConfig] = None,
nodatavals: NodataVals = None,
raise_empty: bool = True,
clear_caches: bool = True,
**kwargs,
) -> xr.Dataset:
"""
Expand All @@ -180,6 +181,7 @@ def read(
raise_empty=raise_empty,
product_read_kwargs=kwargs,
sort=sort,
clear_caches=clear_caches,
**self.default_read_values(
merge_products_by=merge_products_by,
merge_method=merge_method,
Expand All @@ -202,6 +204,7 @@ def read_np_array(
sort: Optional[SortMethodConfig] = None,
nodatavals: NodataVals = None,
raise_empty: bool = True,
clear_caches: bool = True,
**kwargs,
) -> ma.MaskedArray:
"""
Expand All @@ -220,6 +223,7 @@ def read_np_array(
product_read_kwargs=kwargs,
raise_empty=raise_empty,
sort=sort,
clear_caches=clear_caches,
**self.default_read_values(
merge_products_by=merge_products_by,
merge_method=merge_method,
Expand Down Expand Up @@ -247,6 +251,7 @@ def read_levelled(
band_axis_name: str = "bands",
x_axis_name: str = "x",
y_axis_name: str = "y",
clear_caches: bool = True,
**kwargs,
) -> xr.Dataset:
return read_levelled_cube_to_xarray(
Expand All @@ -267,6 +272,7 @@ def read_levelled(
x_axis_name=x_axis_name,
y_axis_name=y_axis_name,
sort=sort,
clear_caches=clear_caches,
**self.default_read_values(
merge_products_by=merge_products_by,
merge_method=merge_method,
Expand All @@ -290,6 +296,7 @@ def read_levelled_np_array(
merge_method: Optional[MergeMethod] = None,
sort: SortMethodConfig = TargetDateSort(),
raise_empty: bool = True,
clear_caches: bool = True,
**kwargs,
) -> ma.MaskedArray:
"""
Expand Down Expand Up @@ -327,6 +334,7 @@ def read_levelled_np_array(
raise_empty=raise_empty,
product_read_kwargs=kwargs,
sort=sort,
clear_caches=clear_caches,
**self.default_read_values(
merge_products_by=merge_products_by,
merge_method=merge_method,
Expand Down
15 changes: 9 additions & 6 deletions mapchete_eo/io/levelled_cubes.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ def read_levelled_cube_to_np_array(
out_dtype: DTypeLike = np.uint16,
out_fill_value: NodataVal = 0,
read_mask: Optional[np.ndarray] = None,
clear_caches: bool = True,
) -> ma.MaskedArray:
"""
Read products as slices into a cube by filling up nodata gaps with next slice.
Expand Down Expand Up @@ -99,7 +100,7 @@ def _cube_read_mask() -> np.ndarray:
slices_read_count, slices_skip_count = 0, 0

# pick slices one by one
for slice_count, slice_ in enumerate(slices, 1):
for slice_count, slice in enumerate(slices, 1):
# all filled up? let's get outta here!
if not out.mask.any():
logger.debug("cube has no pixels to be filled, quitting!")
Expand All @@ -113,11 +114,11 @@ def _cube_read_mask() -> np.ndarray:
logger.debug(
"see if slice %s %s has some of the %s unmasked pixels for cube",
slice_count,
slice_,
slice,
cube_nodata_mask.sum(),
)
with slice_.cached():
slice_array = slice_.read(
with slice.cached(active=clear_caches):
slice_array = slice.read(
merge_method=merge_method,
product_read_kwargs=dict(
product_read_kwargs,
Expand All @@ -133,12 +134,12 @@ def _cube_read_mask() -> np.ndarray:
)
slices_read_count += 1
except (EmptySliceException, CorruptedSlice) as exc:
logger.debug("skipped slice %s: %s", slice_, str(exc))
logger.warning("skipped slice %s: %s", slice, str(exc))
slices_skip_count += 1
continue

# if slice was not empty, fill pixels into cube
logger.debug("add slice %s array to cube", slice_)
logger.debug("add slice %s array to cube", slice)

# iterate through layers of cube
for layer_index in range(target_height):
Expand Down Expand Up @@ -223,6 +224,7 @@ def read_levelled_cube_to_xarray(
x_axis_name: str = "x",
y_axis_name: str = "y",
read_mask: Optional[np.ndarray] = None,
clear_caches: bool = True,
) -> xr.Dataset:
"""
Read products as slices into a cube by filling up nodata gaps with next slice.
Expand All @@ -245,6 +247,7 @@ def read_levelled_cube_to_xarray(
product_read_kwargs=product_read_kwargs,
raise_empty=raise_empty,
read_mask=read_mask,
clear_caches=clear_caches,
),
slice_names=[f"layer-{ii}" for ii in range(target_height)],
band_names=variables,
Expand Down
28 changes: 20 additions & 8 deletions mapchete_eo/io/products.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ def products_to_np_array(
raise_empty: bool = True,
out_dtype: Optional[DTypeLike] = None,
read_mask: Optional[np.ndarray] = None,
clear_caches: bool = True,
) -> ma.MaskedArray:
"""Read grid window of EOProducts and merge into a 4D xarray."""
return ma.stack(
Expand All @@ -69,6 +70,7 @@ def products_to_np_array(
product_read_kwargs=product_read_kwargs,
raise_empty=raise_empty,
read_mask=read_mask,
clear_caches=clear_caches,
)
]
)
Expand All @@ -91,6 +93,7 @@ def products_to_xarray(
raise_empty: bool = True,
product_read_kwargs: dict = {},
read_mask: Optional[np.ndarray] = None,
clear_caches: bool = True,
) -> xr.Dataset:
"""Read grid window of EOProducts and merge into a 4D xarray."""
data_vars = [
Expand All @@ -108,6 +111,7 @@ def products_to_xarray(
product_read_kwargs=product_read_kwargs,
raise_empty=raise_empty,
read_mask=read_mask,
clear_caches=clear_caches,
)
]
if merge_products_by and merge_products_by not in ["date", "datetime"]:
Expand Down Expand Up @@ -181,13 +185,20 @@ def properties(self) -> Dict[str, Any]:
return properties

@contextmanager
def cached(self) -> Generator[Slice, None, None]:
def cached(self, active: bool = True) -> Generator[Slice, None, None]:
"""Clear caches and run garbage collector when context manager is closed."""
yield self
with Timer() as tt:
self.clear_cached_data()
gc.collect()
logger.debug("Slice cache cleared and garbage collected in %s", tt)
if active:
with Timer() as duration:
self.clear_cached_data()
# triggering the garbage collector helps fixing a memory leak on longer running
# process workers
cleared_objects = gc.collect()
logger.debug(
"Slice cache cleared and garbage collector cleared %s objects in %s",
cleared_objects,
duration,
)

def clear_cached_data(self):
logger.debug("clear caches of all products in slice")
Expand Down Expand Up @@ -360,6 +371,7 @@ def generate_slice_dataarrays(
product_read_kwargs: dict = {},
raise_empty: bool = True,
read_mask: Optional[np.ndarray] = None,
clear_caches: bool = True,
) -> Iterator[xr.DataArray]:
"""
Yield products or merged products into slices as DataArrays.
Expand Down Expand Up @@ -392,7 +404,7 @@ def generate_slice_dataarrays(
try:
# if merge_products_by is none, each slice contains just one product
# so nothing will have to be merged anyways
with slice.cached():
with slice.cached(active=clear_caches):
yield to_dataarray(
merge_products(
products=slice.products,
Expand All @@ -416,8 +428,8 @@ def generate_slice_dataarrays(
)
# if at least one slice can be yielded, the stack is not empty
stack_empty = False
except (EmptySliceException, CorruptedSlice) as exception:
logger.warning(exception)
except (EmptySliceException, CorruptedSlice) as exc:
logger.warning("skipped slice %s: %s", slice, str(exc))

if stack_empty:
raise EmptyStackException("all slices are empty")
9 changes: 6 additions & 3 deletions mapchete_eo/platforms/sentinel2/masks.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ def masks_to_xarray(
sort: Optional[SortMethodConfig] = None,
raise_empty: bool = True,
product_read_kwargs: dict = {},
clear_caches: bool = True,
) -> xr.Dataset:
"""
Read masks of products and merge into an xarray.Dataset.
Expand All @@ -74,6 +75,7 @@ def masks_to_xarray(
sort=sort,
product_read_kwargs=product_read_kwargs,
raise_empty=raise_empty,
clear_caches=clear_caches,
)
]
if merge_products_by and merge_products_by not in ["date", "datetime"]:
Expand Down Expand Up @@ -256,6 +258,7 @@ def generate_slice_masks_dataarrays(
mask_name: List = ["EOxCloudless_masks"],
product_read_kwargs: dict = {},
raise_empty: bool = True,
clear_caches: bool = True,
) -> Iterator[xr.DataArray]:
"""
Yield products or merged products into slices as DataArrays.
Expand All @@ -279,7 +282,7 @@ def generate_slice_masks_dataarrays(
try:
# if merge_products_by is none, each slice contains just one product
# so nothing will have to be merged anyways
with slice.cached():
with slice.cached(active=clear_caches):
yield to_dataarray(
merge_products_masks(
products=slice.products,
Expand All @@ -297,8 +300,8 @@ def generate_slice_masks_dataarrays(
)
# if at least one slice can be yielded, the stack is not empty
stack_empty = False
except (EmptySliceException, CorruptedSlice):
pass
except (EmptySliceException, CorruptedSlice) as exc:
logger.warning("skipped slice %s: %s", slice, str(exc))

if stack_empty:
raise EmptyStackException("all slices are empty")
Expand Down
6 changes: 4 additions & 2 deletions mapchete_eo/platforms/sentinel2/product.py
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,8 @@ def from_stac_item(

@property
def item(self) -> Item:
if not self._item:
if self._item is None:
logger.debug("%s no cached STAC Item found, recreating ...", self.id)
if self._item_dict:
self._item = Item.from_dict(self._item_dict)
else:
Expand All @@ -213,7 +214,8 @@ def item(self) -> Item:

@property
def metadata(self) -> S2Metadata:
if not self._metadata:
if self._metadata is None:
logger.debug("%s no cached metadata found, recreating ...", self.id)
if self._metadata_mapper:
self._metadata = self._metadata_mapper(self.item)
else:
Expand Down
Loading