Source code for satpy.readers.olci_nc

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (c) 2016-2023 Satpy developers
#
# This file is part of satpy.
#
# satpy is free software: you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation, either version 3 of the License, or (at your option) any later
# version.
#
# satpy is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with
# satpy.  If not, see <http://www.gnu.org/licenses/>.
"""Sentinel-3 OLCI reader.

This reader supports an optional argument to choose the 'engine' for reading
OLCI netCDF4 files. By default, this reader uses the default xarray choice of
engine, as defined in the :func:`xarray.open_dataset` documentation`.

As an alternative, the user may wish to use the 'h5netcdf' engine, but that is
not default as it typically prints many non-fatal but confusing error messages
to the terminal.
To choose between engines the user can  do as follows for the default::

    scn = Scene(filenames=my_files, reader='olci_l1b')

or as follows for the h5netcdf engine::

    scn = Scene(filenames=my_files,
                reader='olci_l1b', reader_kwargs={'engine': 'h5netcdf'})

References:
    - :func:`xarray.open_dataset`

"""


import logging
from functools import reduce

import dask.array as da
import numpy as np
import xarray as xr

from satpy._compat import cached_property
from satpy.readers import open_file_or_filename
from satpy.readers.file_handlers import BaseFileHandler
from satpy.utils import angle2xyz, get_legacy_chunk_size, xyz2angle

# the order of the L1B quality flags are from highest 32nd bit to the lowest 1 bit
# https://sentinel.esa.int/documents/247904/1872756/Sentinel-3-OLCI-Product-Data-Format-Specification-OLCI-Level-1
L1B_QUALITY_FLAGS = ["saturated@Oa21", "saturated@Oa20", "saturated@Oa19", "saturated@Oa18",
                     "saturated@Oa17", "saturated@Oa16", "saturated@Oa15", "saturated@Oa14",
                     "saturated@Oa13", "saturated@Oa12", "saturated@Oa11", "saturated@Oa10",
                     "saturated@Oa09", "saturated@Oa08", "saturated@Oa07", "saturated@Oa06",
                     "saturated@Oa05", "saturated@Oa04", "saturated@Oa03", "saturated@Oa02",
                     "saturated@Oa01", "dubious", "sun-glint_risk", "duplicated",
                     "cosmetic", "invalid", "straylight_risk", "bright",
                     "tidal_region", "fresh_inland_water", "coastline", "land"]

DEFAULT_L1B_MASK_ITEMS = ["dubious", "sun-glint_risk", "duplicated", "cosmetic", "invalid",
    "straylight_risk", "bright", "tidal_region", "coastline", "land"]

WQSF_FLAG_LIST = ["INVALID", "WATER", "LAND", "CLOUD", "SNOW_ICE", "INLAND_WATER", "TIDAL",
    "COSMETIC", "SUSPECT", "HISOLZEN", "SATURATED", "MEGLINT", "HIGHGLINT", "WHITECAPS", "ADJAC",
    "WV_FAIL", "PAR_FAIL", "AC_FAIL", "OC4ME_FAIL", "OCNN_FAIL", "Extra_1", "KDM_FAIL", "Extra_2",
    "CLOUD_AMBIGUOUS", "CLOUD_MARGIN", "BPAC_ON", "WHITE_SCATT", "LOWRW", "HIGHRW"]

DEFAULT_WQSF_MASK_ITEMS = ["INVALID", "SNOW_ICE", "INLAND_WATER", "SUSPECT", "AC_FAIL", "CLOUD",
    "HISOLZEN", "OCNN_FAIL", "CLOUD_MARGIN", "CLOUD_AMBIGUOUS", "LOWRW", "LAND"]

logger = logging.getLogger(__name__)

CHUNK_SIZE = get_legacy_chunk_size()

PLATFORM_NAMES = {"S3A": "Sentinel-3A",
                  "S3B": "Sentinel-3B",
                  "ENV": "Environmental Satellite"}


[docs] class BitFlags: """Manipulate flags stored bitwise.""" def __init__(self, value, flag_list=None): """Init the flags.""" self._value = value if flag_list is None: try: meanings = value.attrs["flag_meanings"].split() masks = value.attrs["flag_masks"] except (AttributeError, KeyError): meanings = WQSF_FLAG_LIST self.meaning = {meaning: mask for mask, meaning in enumerate(meanings)} else: self.meaning = {meaning: int(np.log2(mask)) for meaning, mask in zip(meanings, masks)} else: self.meaning = {meaning: mask for mask, meaning in enumerate(flag_list)} def __getitem__(self, item): """Get the item.""" pos = self.meaning[item] data = self._value if isinstance(data, xr.DataArray): data = data.data res = ((data >> pos) % 2).astype(bool) res = xr.DataArray(res, coords=self._value.coords, attrs=self._value.attrs, dims=self._value.dims) else: res = ((data >> pos) % 2).astype(bool) return res
[docs] class NCOLCIBase(BaseFileHandler): """The OLCI reader base.""" rows_name = "rows" cols_name = "columns" def __init__(self, filename, filename_info, filetype_info, engine=None, **kwargs): """Init the olci reader base.""" super().__init__(filename, filename_info, filetype_info) self._engine = engine self._start_time = filename_info["start_time"] self._end_time = filename_info["end_time"] # TODO: get metadata from the manifest file (xfdumanifest.xml) self.platform_name = PLATFORM_NAMES[filename_info["mission_id"]] self.sensor = "olci" @cached_property def nc(self): """Get the nc xr dataset.""" f_obj = open_file_or_filename(self.filename) dataset = xr.open_dataset(f_obj, decode_cf=True, mask_and_scale=True, engine=self._engine, chunks={self.cols_name: CHUNK_SIZE, self.rows_name: CHUNK_SIZE}) return dataset.rename({self.cols_name: "x", self.rows_name: "y"}) @property def start_time(self): """Start time property.""" return self._start_time @property def end_time(self): """End time property.""" return self._end_time
[docs] def get_dataset(self, key, info): """Load a dataset.""" logger.debug("Reading %s.", key["name"]) variable = self.nc[key["name"]] return variable
[docs] class NCOLCICal(NCOLCIBase): """Dummy class for calibration."""
[docs] class NCOLCIGeo(NCOLCIBase): """Dummy class for navigation."""
[docs] class NCOLCIChannelBase(NCOLCIBase): """Base class for channel reading.""" def __init__(self, filename, filename_info, filetype_info, engine=None): """Init the file handler.""" super().__init__(filename, filename_info, filetype_info, engine) self.channel = filename_info.get("dataset_name") self.reflectance_prefix = "Oa" self.reflectance_suffix = "_reflectance"
[docs] class NCOLCI1B(NCOLCIChannelBase): """File handler for OLCI l1b.""" def __init__(self, filename, filename_info, filetype_info, cal=None, engine=None, mask_items=None): """Init the file handler.""" super().__init__(filename, filename_info, filetype_info, engine) if cal is not None: self.cal = cal.nc self.mask_items = mask_items
[docs] @staticmethod def _get_items(idx, solar_flux): """Get items.""" return solar_flux[idx]
[docs] def _get_solar_flux(self, band): """Get the solar flux for the band.""" solar_flux = self.cal["solar_flux"].isel(bands=band).values d_index = self.cal["detector_index"].fillna(0).astype(int) return da.map_blocks(self._get_items, d_index.data, solar_flux=solar_flux, dtype=solar_flux.dtype)
[docs] def getbitmask(self, quality_flags, items=None): """Get the quality flags bitmask.""" if items is None: items = DEFAULT_L1B_MASK_ITEMS bflags = BitFlags(quality_flags, flag_list=L1B_QUALITY_FLAGS) return reduce(np.logical_or, [bflags[item] for item in items])
[docs] def get_dataset(self, key, info): """Load a dataset.""" if self.channel is not None and self.channel != key["name"]: return logger.debug("Reading %s.", key["name"]) if key["name"] == "quality_flags": dataset = self.nc["quality_flags"] elif key["name"] == "mask": dataset = self.getbitmask(self.nc["quality_flags"], self.mask_items) else: dataset = self.nc[self.channel + "_radiance"] if key["calibration"] == "reflectance": idx = int(key["name"][2:]) - 1 sflux = self._get_solar_flux(idx) dataset = dataset / sflux * np.pi * 100 dataset.attrs["units"] = "%" dataset.attrs["platform_name"] = self.platform_name dataset.attrs["sensor"] = self.sensor dataset.attrs.update(key.to_dict()) return dataset
[docs] class NCOLCI2(NCOLCIChannelBase): """File handler for OLCI l2.""" def __init__(self, filename, filename_info, filetype_info, engine=None, unlog=False, mask_items=None): """Init the file handler.""" super().__init__(filename, filename_info, filetype_info, engine) self.unlog = unlog self.mask_items = mask_items
[docs] def get_dataset(self, key, info): """Load a dataset.""" if self.channel is not None and self.channel != key["name"]: return logger.debug("Reading %s.", key["name"]) if self.channel is not None and self.channel.startswith(self.reflectance_prefix): dataset = self.nc[self.channel + self.reflectance_suffix] else: dataset = self.nc[info["nc_key"]] if key["name"] == "wqsf": dataset.attrs["_FillValue"] = 1 elif key["name"] == "mask": dataset = self.getbitmask(dataset, self.mask_items) dataset.attrs["platform_name"] = self.platform_name dataset.attrs["sensor"] = self.sensor dataset.attrs.update(key.to_dict()) if self.unlog: dataset = self.delog(dataset) return dataset
[docs] def delog(self, data_array): """Remove log10 from the units and values.""" units = data_array.attrs["units"] if units.startswith("lg("): data_array = 10 ** data_array data_array.attrs["units"] = units.split("lg(re ")[1].strip(")") return data_array
[docs] def getbitmask(self, wqsf, items=None): """Get the bitmask.""" if items is None: items = DEFAULT_WQSF_MASK_ITEMS bflags = BitFlags(wqsf, WQSF_FLAG_LIST) return reduce(np.logical_or, [bflags[item] for item in items])
[docs] class NCOLCILowResData(NCOLCIBase): """Handler for low resolution data.""" rows_name = "tie_rows" cols_name = "tie_columns" def __init__(self, filename, filename_info, filetype_info, engine=None, **kwargs): """Init the file handler.""" super().__init__(filename, filename_info, filetype_info, engine) @property def l_step(self): """Get the line step.""" return self.nc.attrs["al_subsampling_factor"] @property def c_step(self): """Get the column step.""" return self.nc.attrs["ac_subsampling_factor"]
[docs] def _do_interpolate(self, data): if not isinstance(data, tuple): data = (data,) shape = data[0].shape from geotiepoints.interpolator import Interpolator tie_lines = np.arange(0, (shape[0] - 1) * self.l_step + 1, self.l_step) tie_cols = np.arange(0, (shape[1] - 1) * self.c_step + 1, self.c_step) lines = np.arange((shape[0] - 1) * self.l_step + 1) cols = np.arange((shape[1] - 1) * self.c_step + 1) along_track_order = 1 cross_track_order = 3 satint = Interpolator([x.values for x in data], (tie_lines, tie_cols), (lines, cols), along_track_order, cross_track_order) int_data = satint.interpolate() return [xr.DataArray(da.from_array(x, chunks=(CHUNK_SIZE, CHUNK_SIZE)), dims=["y", "x"]) for x in int_data]
@property def _need_interpolation(self): return (self.c_step != 1 or self.l_step != 1)
[docs] class NCOLCIAngles(NCOLCILowResData): """File handler for the OLCI angles.""" datasets = {"satellite_azimuth_angle": "OAA", "satellite_zenith_angle": "OZA", "solar_azimuth_angle": "SAA", "solar_zenith_angle": "SZA"}
[docs] def get_dataset(self, key, info): """Load a dataset.""" if key["name"] not in self.datasets: return logger.debug("Reading %s.", key["name"]) if self._need_interpolation: if key["name"].startswith("satellite"): azi, zen = self.satellite_angles elif key["name"].startswith("solar"): azi, zen = self.sun_angles else: raise NotImplementedError("Don't know how to read " + key["name"]) if "zenith" in key["name"]: values = zen elif "azimuth" in key["name"]: values = azi else: raise NotImplementedError("Don't know how to read " + key["name"]) else: values = self.nc[self.datasets[key["name"]]] values.attrs["platform_name"] = self.platform_name values.attrs["sensor"] = self.sensor values.attrs.update(key.to_dict()) return values
@cached_property def sun_angles(self): """Return the sun angles.""" zen = self.nc[self.datasets["solar_zenith_angle"]] azi = self.nc[self.datasets["solar_azimuth_angle"]] azi, zen = self._interpolate_angles(azi, zen) return azi, zen @cached_property def satellite_angles(self): """Return the satellite angles.""" zen = self.nc[self.datasets["satellite_zenith_angle"]] azi = self.nc[self.datasets["satellite_azimuth_angle"]] azi, zen = self._interpolate_angles(azi, zen) return azi, zen
[docs] def _interpolate_angles(self, azi, zen): aattrs = azi.attrs zattrs = zen.attrs x, y, z = angle2xyz(azi, zen) x, y, z = self._do_interpolate((x, y, z)) azi, zen = xyz2angle(x, y, z) azi.attrs = aattrs zen.attrs = zattrs return azi, zen
[docs] class NCOLCIMeteo(NCOLCILowResData): """File handler for the OLCI meteo data.""" datasets = ["humidity", "sea_level_pressure", "total_columnar_water_vapour", "total_ozone"] def __init__(self, filename, filename_info, filetype_info, engine=None): """Init the file handler.""" super().__init__(filename, filename_info, filetype_info, engine) self.cache = {} # TODO: the following depends on more than columns, rows # float atmospheric_temperature_profile(tie_rows, tie_columns, tie_pressure_levels) ; # float horizontal_wind(tie_rows, tie_columns, wind_vectors) ; # float reference_pressure_level(tie_pressure_levels) ;
[docs] def get_dataset(self, key, info): """Load a dataset.""" if key["name"] not in self.datasets: return logger.debug("Reading %s.", key["name"]) if self._need_interpolation and self.cache.get(key["name"]) is None: data = self.nc[key["name"]] values, = self._do_interpolate(data) values.attrs = data.attrs self.cache[key["name"]] = values elif key["name"] in self.cache: values = self.cache[key["name"]] else: values = self.nc[key["name"]] values.attrs["platform_name"] = self.platform_name values.attrs["sensor"] = self.sensor values.attrs.update(key.to_dict()) return values