From 6c85770d5314979541f4315115382c570c9a1ae1 Mon Sep 17 00:00:00 2001 From: Joachim Ungar Date: Mon, 16 Nov 2026 10:02:13 +0100 Subject: [PATCH 1/2] make clearing product caches optional --- mapchete_eo/base.py | 8 ++++++++ mapchete_eo/io/levelled_cubes.py | 15 +++++++++------ mapchete_eo/io/products.py | 22 ++++++++++++++-------- mapchete_eo/platforms/sentinel2/masks.py | 9 ++++++--- mapchete_eo/platforms/sentinel2/product.py | 6 ++++-- 5 files changed, 41 insertions(+), 19 deletions(-) diff --git a/mapchete_eo/base.py b/mapchete_eo/base.py index 225b5936..73466140 100644 --- a/mapchete_eo/base.py +++ b/mapchete_eo/base.py @@ -162,6 +162,7 @@ def read( sort: Optional[SortMethodConfig] = None, nodatavals: NodataVals = None, raise_empty: bool = True, + clear_caches: bool = True, **kwargs, ) -> xr.Dataset: """ @@ -180,6 +181,7 @@ def read( raise_empty=raise_empty, product_read_kwargs=kwargs, sort=sort, + clear_caches=clear_caches, **self.default_read_values( merge_products_by=merge_products_by, merge_method=merge_method, @@ -202,6 +204,7 @@ def read_np_array( sort: Optional[SortMethodConfig] = None, nodatavals: NodataVals = None, raise_empty: bool = True, + clear_caches: bool = True, **kwargs, ) -> ma.MaskedArray: """ @@ -220,6 +223,7 @@ def read_np_array( product_read_kwargs=kwargs, raise_empty=raise_empty, sort=sort, + clear_caches=clear_caches, **self.default_read_values( merge_products_by=merge_products_by, merge_method=merge_method, @@ -247,6 +251,7 @@ def read_levelled( band_axis_name: str = "bands", x_axis_name: str = "x", y_axis_name: str = "y", + clear_caches: bool = True, **kwargs, ) -> xr.Dataset: return read_levelled_cube_to_xarray( @@ -267,6 +272,7 @@ def read_levelled( x_axis_name=x_axis_name, y_axis_name=y_axis_name, sort=sort, + clear_caches=clear_caches, **self.default_read_values( merge_products_by=merge_products_by, merge_method=merge_method, @@ -290,6 +296,7 @@ def read_levelled_np_array( merge_method: Optional[MergeMethod] = None, sort: SortMethodConfig = TargetDateSort(), raise_empty: bool = True, + clear_caches: bool = True, **kwargs, ) -> ma.MaskedArray: """ @@ -327,6 +334,7 @@ def read_levelled_np_array( raise_empty=raise_empty, product_read_kwargs=kwargs, sort=sort, + clear_caches=clear_caches, **self.default_read_values( merge_products_by=merge_products_by, merge_method=merge_method, diff --git a/mapchete_eo/io/levelled_cubes.py b/mapchete_eo/io/levelled_cubes.py index 082b99b8..72ba13e5 100644 --- a/mapchete_eo/io/levelled_cubes.py +++ b/mapchete_eo/io/levelled_cubes.py @@ -41,6 +41,7 @@ def read_levelled_cube_to_np_array( out_dtype: DTypeLike = np.uint16, out_fill_value: NodataVal = 0, read_mask: Optional[np.ndarray] = None, + clear_caches: bool = True, ) -> ma.MaskedArray: """ Read products as slices into a cube by filling up nodata gaps with next slice. @@ -99,7 +100,7 @@ def _cube_read_mask() -> np.ndarray: slices_read_count, slices_skip_count = 0, 0 # pick slices one by one - for slice_count, slice_ in enumerate(slices, 1): + for slice_count, slice in enumerate(slices, 1): # all filled up? let's get outta here! if not out.mask.any(): logger.debug("cube has no pixels to be filled, quitting!") @@ -113,11 +114,11 @@ def _cube_read_mask() -> np.ndarray: logger.debug( "see if slice %s %s has some of the %s unmasked pixels for cube", slice_count, - slice_, + slice, cube_nodata_mask.sum(), ) - with slice_.cached(): - slice_array = slice_.read( + with slice.cached(active=clear_caches): + slice_array = slice.read( merge_method=merge_method, product_read_kwargs=dict( product_read_kwargs, @@ -133,12 +134,12 @@ def _cube_read_mask() -> np.ndarray: ) slices_read_count += 1 except (EmptySliceException, CorruptedSlice) as exc: - logger.debug("skipped slice %s: %s", slice_, str(exc)) + logger.warning("skipped slice %s: %s", slice, str(exc)) slices_skip_count += 1 continue # if slice was not empty, fill pixels into cube - logger.debug("add slice %s array to cube", slice_) + logger.debug("add slice %s array to cube", slice) # iterate through layers of cube for layer_index in range(target_height): @@ -223,6 +224,7 @@ def read_levelled_cube_to_xarray( x_axis_name: str = "x", y_axis_name: str = "y", read_mask: Optional[np.ndarray] = None, + clear_caches: bool = True, ) -> xr.Dataset: """ Read products as slices into a cube by filling up nodata gaps with next slice. @@ -245,6 +247,7 @@ def read_levelled_cube_to_xarray( product_read_kwargs=product_read_kwargs, raise_empty=raise_empty, read_mask=read_mask, + clear_caches=clear_caches, ), slice_names=[f"layer-{ii}" for ii in range(target_height)], band_names=variables, diff --git a/mapchete_eo/io/products.py b/mapchete_eo/io/products.py index 424a7333..8fe528ec 100644 --- a/mapchete_eo/io/products.py +++ b/mapchete_eo/io/products.py @@ -51,6 +51,7 @@ def products_to_np_array( raise_empty: bool = True, out_dtype: Optional[DTypeLike] = None, read_mask: Optional[np.ndarray] = None, + clear_caches: bool = True, ) -> ma.MaskedArray: """Read grid window of EOProducts and merge into a 4D xarray.""" return ma.stack( @@ -69,6 +70,7 @@ def products_to_np_array( product_read_kwargs=product_read_kwargs, raise_empty=raise_empty, read_mask=read_mask, + clear_caches=clear_caches, ) ] ) @@ -91,6 +93,7 @@ def products_to_xarray( raise_empty: bool = True, product_read_kwargs: dict = {}, read_mask: Optional[np.ndarray] = None, + clear_caches: bool = True, ) -> xr.Dataset: """Read grid window of EOProducts and merge into a 4D xarray.""" data_vars = [ @@ -108,6 +111,7 @@ def products_to_xarray( product_read_kwargs=product_read_kwargs, raise_empty=raise_empty, read_mask=read_mask, + clear_caches=clear_caches, ) ] if merge_products_by and merge_products_by not in ["date", "datetime"]: @@ -181,13 +185,14 @@ def properties(self) -> Dict[str, Any]: return properties @contextmanager - def cached(self) -> Generator[Slice, None, None]: + def cached(self, active: bool = True) -> Generator[Slice, None, None]: """Clear caches and run garbage collector when context manager is closed.""" yield self - with Timer() as tt: - self.clear_cached_data() - gc.collect() - logger.debug("Slice cache cleared and garbage collected in %s", tt) + if active: + with Timer() as tt: + self.clear_cached_data() + gc.collect() + logger.debug("Slice cache cleared and garbage collected in %s", tt) def clear_cached_data(self): logger.debug("clear caches of all products in slice") @@ -360,6 +365,7 @@ def generate_slice_dataarrays( product_read_kwargs: dict = {}, raise_empty: bool = True, read_mask: Optional[np.ndarray] = None, + clear_caches: bool = True, ) -> Iterator[xr.DataArray]: """ Yield products or merged products into slices as DataArrays. @@ -392,7 +398,7 @@ def generate_slice_dataarrays( try: # if merge_products_by is none, each slice contains just one product # so nothing will have to be merged anyways - with slice.cached(): + with slice.cached(active=clear_caches): yield to_dataarray( merge_products( products=slice.products, @@ -416,8 +422,8 @@ def generate_slice_dataarrays( ) # if at least one slice can be yielded, the stack is not empty stack_empty = False - except (EmptySliceException, CorruptedSlice) as exception: - logger.warning(exception) + except (EmptySliceException, CorruptedSlice) as exc: + logger.warning("skipped slice %s: %s", slice, str(exc)) if stack_empty: raise EmptyStackException("all slices are empty") diff --git a/mapchete_eo/platforms/sentinel2/masks.py b/mapchete_eo/platforms/sentinel2/masks.py index 0357fb0f..1aab5c8d 100644 --- a/mapchete_eo/platforms/sentinel2/masks.py +++ b/mapchete_eo/platforms/sentinel2/masks.py @@ -60,6 +60,7 @@ def masks_to_xarray( sort: Optional[SortMethodConfig] = None, raise_empty: bool = True, product_read_kwargs: dict = {}, + clear_caches: bool = True, ) -> xr.Dataset: """ Read masks of products and merge into an xarray.Dataset. @@ -74,6 +75,7 @@ def masks_to_xarray( sort=sort, product_read_kwargs=product_read_kwargs, raise_empty=raise_empty, + clear_caches=clear_caches, ) ] if merge_products_by and merge_products_by not in ["date", "datetime"]: @@ -256,6 +258,7 @@ def generate_slice_masks_dataarrays( mask_name: List = ["EOxCloudless_masks"], product_read_kwargs: dict = {}, raise_empty: bool = True, + clear_caches: bool = True, ) -> Iterator[xr.DataArray]: """ Yield products or merged products into slices as DataArrays. @@ -279,7 +282,7 @@ def generate_slice_masks_dataarrays( try: # if merge_products_by is none, each slice contains just one product # so nothing will have to be merged anyways - with slice.cached(): + with slice.cached(active=clear_caches): yield to_dataarray( merge_products_masks( products=slice.products, @@ -297,8 +300,8 @@ def generate_slice_masks_dataarrays( ) # if at least one slice can be yielded, the stack is not empty stack_empty = False - except (EmptySliceException, CorruptedSlice): - pass + except (EmptySliceException, CorruptedSlice) as exc: + logger.warning("skipped slice %s: %s", slice, str(exc)) if stack_empty: raise EmptyStackException("all slices are empty") diff --git a/mapchete_eo/platforms/sentinel2/product.py b/mapchete_eo/platforms/sentinel2/product.py index 88a69317..a3470ee3 100644 --- a/mapchete_eo/platforms/sentinel2/product.py +++ b/mapchete_eo/platforms/sentinel2/product.py @@ -201,7 +201,8 @@ def from_stac_item( @property def item(self) -> Item: - if not self._item: + if self._item is None: + logger.debug("%s no cached STAC Item found, recreating ...", self.id) if self._item_dict: self._item = Item.from_dict(self._item_dict) else: @@ -213,7 +214,8 @@ def item(self) -> Item: @property def metadata(self) -> S2Metadata: - if not self._metadata: + if self._metadata is None: + logger.debug("%s no cached metadata found, recreating ...", self.id) if self._metadata_mapper: self._metadata = self._metadata_mapper(self.item) else: From ba351cc177740803937513087356fb15b3aed778 Mon Sep 17 00:00:00 2001 From: Joachim Ungar Date: Mon, 16 Nov 2026 10:14:38 +0100 Subject: [PATCH 2/2] add explanatory comment and report cleared objects in log message --- mapchete_eo/io/products.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/mapchete_eo/io/products.py b/mapchete_eo/io/products.py index 8fe528ec..f16598f5 100644 --- a/mapchete_eo/io/products.py +++ b/mapchete_eo/io/products.py @@ -189,10 +189,16 @@ def cached(self, active: bool = True) -> Generator[Slice, None, None]: """Clear caches and run garbage collector when context manager is closed.""" yield self if active: - with Timer() as tt: + with Timer() as duration: self.clear_cached_data() - gc.collect() - logger.debug("Slice cache cleared and garbage collected in %s", tt) + # triggering the garbage collector helps fixing a memory leak on longer running + # process workers + cleared_objects = gc.collect() + logger.debug( + "Slice cache cleared and garbage collector cleared %s objects in %s", + cleared_objects, + duration, + ) def clear_cached_data(self): logger.debug("clear caches of all products in slice")