#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (c) 2016-2023 Satpy developers
#
# This file is part of satpy.
#
# satpy is free software: you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation, either version 3 of the License, or (at your option) any later
# version.
#
# satpy is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with
# satpy. If not, see <http://www.gnu.org/licenses/>.
"""Sentinel-3 OLCI reader.
This reader supports an optional argument to choose the 'engine' for reading
OLCI netCDF4 files. By default, this reader uses the default xarray choice of
engine, as defined in the :func:`xarray.open_dataset` documentation`.
As an alternative, the user may wish to use the 'h5netcdf' engine, but that is
not default as it typically prints many non-fatal but confusing error messages
to the terminal.
To choose between engines the user can do as follows for the default::
scn = Scene(filenames=my_files, reader='olci_l1b')
or as follows for the h5netcdf engine::
scn = Scene(filenames=my_files,
reader='olci_l1b', reader_kwargs={'engine': 'h5netcdf'})
References:
- :func:`xarray.open_dataset`
"""
import logging
from functools import reduce
import dask.array as da
import numpy as np
import xarray as xr
from satpy._compat import cached_property
from satpy.readers import open_file_or_filename
from satpy.readers.file_handlers import BaseFileHandler
from satpy.utils import angle2xyz, get_legacy_chunk_size, xyz2angle
# the order of the L1B quality flags are from highest 32nd bit to the lowest 1 bit
# https://sentinel.esa.int/documents/247904/1872756/Sentinel-3-OLCI-Product-Data-Format-Specification-OLCI-Level-1
L1B_QUALITY_FLAGS = ["saturated@Oa21", "saturated@Oa20", "saturated@Oa19", "saturated@Oa18",
"saturated@Oa17", "saturated@Oa16", "saturated@Oa15", "saturated@Oa14",
"saturated@Oa13", "saturated@Oa12", "saturated@Oa11", "saturated@Oa10",
"saturated@Oa09", "saturated@Oa08", "saturated@Oa07", "saturated@Oa06",
"saturated@Oa05", "saturated@Oa04", "saturated@Oa03", "saturated@Oa02",
"saturated@Oa01", "dubious", "sun-glint_risk", "duplicated",
"cosmetic", "invalid", "straylight_risk", "bright",
"tidal_region", "fresh_inland_water", "coastline", "land"]
DEFAULT_L1B_MASK_ITEMS = ["dubious", "sun-glint_risk", "duplicated", "cosmetic", "invalid",
"straylight_risk", "bright", "tidal_region", "coastline", "land"]
WQSF_FLAG_LIST = ["INVALID", "WATER", "LAND", "CLOUD", "SNOW_ICE", "INLAND_WATER", "TIDAL",
"COSMETIC", "SUSPECT", "HISOLZEN", "SATURATED", "MEGLINT", "HIGHGLINT", "WHITECAPS", "ADJAC",
"WV_FAIL", "PAR_FAIL", "AC_FAIL", "OC4ME_FAIL", "OCNN_FAIL", "Extra_1", "KDM_FAIL", "Extra_2",
"CLOUD_AMBIGUOUS", "CLOUD_MARGIN", "BPAC_ON", "WHITE_SCATT", "LOWRW", "HIGHRW"]
DEFAULT_WQSF_MASK_ITEMS = ["INVALID", "SNOW_ICE", "INLAND_WATER", "SUSPECT", "AC_FAIL", "CLOUD",
"HISOLZEN", "OCNN_FAIL", "CLOUD_MARGIN", "CLOUD_AMBIGUOUS", "LOWRW", "LAND"]
logger = logging.getLogger(__name__)
CHUNK_SIZE = get_legacy_chunk_size()
PLATFORM_NAMES = {"S3A": "Sentinel-3A",
"S3B": "Sentinel-3B",
"ENV": "Environmental Satellite"}
[docs]
class BitFlags:
"""Manipulate flags stored bitwise."""
def __init__(self, value, flag_list=None):
"""Init the flags."""
self._value = value
if flag_list is None:
try:
meanings = value.attrs["flag_meanings"].split()
masks = value.attrs["flag_masks"]
except (AttributeError, KeyError):
meanings = WQSF_FLAG_LIST
self.meaning = {meaning: mask for mask, meaning in enumerate(meanings)}
else:
self.meaning = {meaning: int(np.log2(mask)) for meaning, mask in zip(meanings, masks)}
else:
self.meaning = {meaning: mask for mask, meaning in enumerate(flag_list)}
def __getitem__(self, item):
"""Get the item."""
pos = self.meaning[item]
data = self._value
if isinstance(data, xr.DataArray):
data = data.data
res = ((data >> pos) % 2).astype(bool)
res = xr.DataArray(res, coords=self._value.coords, attrs=self._value.attrs,
dims=self._value.dims)
else:
res = ((data >> pos) % 2).astype(bool)
return res
[docs]
class NCOLCIBase(BaseFileHandler):
"""The OLCI reader base."""
rows_name = "rows"
cols_name = "columns"
def __init__(self, filename, filename_info, filetype_info, engine=None, **kwargs):
"""Init the olci reader base."""
super().__init__(filename, filename_info, filetype_info)
self._engine = engine
self._start_time = filename_info["start_time"]
self._end_time = filename_info["end_time"]
# TODO: get metadata from the manifest file (xfdumanifest.xml)
self.platform_name = PLATFORM_NAMES[filename_info["mission_id"]]
self.sensor = "olci"
@cached_property
def nc(self):
"""Get the nc xr dataset."""
f_obj = open_file_or_filename(self.filename)
dataset = xr.open_dataset(f_obj,
decode_cf=True,
mask_and_scale=True,
engine=self._engine,
chunks={self.cols_name: CHUNK_SIZE,
self.rows_name: CHUNK_SIZE})
return dataset.rename({self.cols_name: "x", self.rows_name: "y"})
@property
def start_time(self):
"""Start time property."""
return self._start_time
@property
def end_time(self):
"""End time property."""
return self._end_time
[docs]
def get_dataset(self, key, info):
"""Load a dataset."""
logger.debug("Reading %s.", key["name"])
variable = self.nc[key["name"]]
return variable
[docs]
class NCOLCICal(NCOLCIBase):
"""Dummy class for calibration."""
[docs]
class NCOLCIGeo(NCOLCIBase):
"""Dummy class for navigation."""
[docs]
class NCOLCIChannelBase(NCOLCIBase):
"""Base class for channel reading."""
def __init__(self, filename, filename_info, filetype_info, engine=None):
"""Init the file handler."""
super().__init__(filename, filename_info, filetype_info, engine)
self.channel = filename_info.get("dataset_name")
self.reflectance_prefix = "Oa"
self.reflectance_suffix = "_reflectance"
[docs]
class NCOLCI1B(NCOLCIChannelBase):
"""File handler for OLCI l1b."""
def __init__(self, filename, filename_info, filetype_info, cal=None, engine=None, mask_items=None):
"""Init the file handler."""
super().__init__(filename, filename_info, filetype_info, engine)
if cal is not None:
self.cal = cal.nc
self.mask_items = mask_items
[docs]
@staticmethod
def _get_items(idx, solar_flux):
"""Get items."""
return solar_flux[idx]
[docs]
def _get_solar_flux(self, band):
"""Get the solar flux for the band."""
solar_flux = self.cal["solar_flux"].isel(bands=band).values
d_index = self.cal["detector_index"].fillna(0).astype(int)
return da.map_blocks(self._get_items, d_index.data,
solar_flux=solar_flux, dtype=solar_flux.dtype)
[docs]
def getbitmask(self, quality_flags, items=None):
"""Get the quality flags bitmask."""
if items is None:
items = DEFAULT_L1B_MASK_ITEMS
bflags = BitFlags(quality_flags, flag_list=L1B_QUALITY_FLAGS)
return reduce(np.logical_or, [bflags[item] for item in items])
[docs]
def get_dataset(self, key, info):
"""Load a dataset."""
if self.channel is not None and self.channel != key["name"]:
return
logger.debug("Reading %s.", key["name"])
if key["name"] == "quality_flags":
dataset = self.nc["quality_flags"]
elif key["name"] == "mask":
dataset = self.getbitmask(self.nc["quality_flags"], self.mask_items)
else:
dataset = self.nc[self.channel + "_radiance"]
if key["calibration"] == "reflectance":
idx = int(key["name"][2:]) - 1
sflux = self._get_solar_flux(idx)
dataset = dataset / sflux * np.pi * 100
dataset.attrs["units"] = "%"
dataset.attrs["platform_name"] = self.platform_name
dataset.attrs["sensor"] = self.sensor
dataset.attrs.update(key.to_dict())
return dataset
[docs]
class NCOLCI2(NCOLCIChannelBase):
"""File handler for OLCI l2."""
def __init__(self, filename, filename_info, filetype_info, engine=None, unlog=False, mask_items=None):
"""Init the file handler."""
super().__init__(filename, filename_info, filetype_info, engine)
self.unlog = unlog
self.mask_items = mask_items
[docs]
def get_dataset(self, key, info):
"""Load a dataset."""
if self.channel is not None and self.channel != key["name"]:
return
logger.debug("Reading %s.", key["name"])
if self.channel is not None and self.channel.startswith(self.reflectance_prefix):
dataset = self.nc[self.channel + self.reflectance_suffix]
else:
dataset = self.nc[info["nc_key"]]
if key["name"] == "wqsf":
dataset.attrs["_FillValue"] = 1
elif key["name"] == "mask":
dataset = self.getbitmask(dataset, self.mask_items)
dataset.attrs["platform_name"] = self.platform_name
dataset.attrs["sensor"] = self.sensor
dataset.attrs.update(key.to_dict())
if self.unlog:
dataset = self.delog(dataset)
return dataset
[docs]
def delog(self, data_array):
"""Remove log10 from the units and values."""
units = data_array.attrs["units"]
if units.startswith("lg("):
data_array = 10 ** data_array
data_array.attrs["units"] = units.split("lg(re ")[1].strip(")")
return data_array
[docs]
def getbitmask(self, wqsf, items=None):
"""Get the bitmask."""
if items is None:
items = DEFAULT_WQSF_MASK_ITEMS
bflags = BitFlags(wqsf, WQSF_FLAG_LIST)
return reduce(np.logical_or, [bflags[item] for item in items])
[docs]
class NCOLCILowResData(NCOLCIBase):
"""Handler for low resolution data."""
rows_name = "tie_rows"
cols_name = "tie_columns"
def __init__(self, filename, filename_info, filetype_info,
engine=None, **kwargs):
"""Init the file handler."""
super().__init__(filename, filename_info, filetype_info, engine)
@property
def l_step(self):
"""Get the line step."""
return self.nc.attrs["al_subsampling_factor"]
@property
def c_step(self):
"""Get the column step."""
return self.nc.attrs["ac_subsampling_factor"]
[docs]
def _do_interpolate(self, data):
if not isinstance(data, tuple):
data = (data,)
shape = data[0].shape
from geotiepoints.interpolator import Interpolator
tie_lines = np.arange(0, (shape[0] - 1) * self.l_step + 1, self.l_step)
tie_cols = np.arange(0, (shape[1] - 1) * self.c_step + 1, self.c_step)
lines = np.arange((shape[0] - 1) * self.l_step + 1)
cols = np.arange((shape[1] - 1) * self.c_step + 1)
along_track_order = 1
cross_track_order = 3
satint = Interpolator([x.values for x in data],
(tie_lines, tie_cols),
(lines, cols),
along_track_order,
cross_track_order)
int_data = satint.interpolate()
return [xr.DataArray(da.from_array(x, chunks=(CHUNK_SIZE, CHUNK_SIZE)),
dims=["y", "x"]) for x in int_data]
@property
def _need_interpolation(self):
return (self.c_step != 1 or self.l_step != 1)
[docs]
class NCOLCIAngles(NCOLCILowResData):
"""File handler for the OLCI angles."""
datasets = {"satellite_azimuth_angle": "OAA",
"satellite_zenith_angle": "OZA",
"solar_azimuth_angle": "SAA",
"solar_zenith_angle": "SZA"}
[docs]
def get_dataset(self, key, info):
"""Load a dataset."""
if key["name"] not in self.datasets:
return
logger.debug("Reading %s.", key["name"])
if self._need_interpolation:
if key["name"].startswith("satellite"):
azi, zen = self.satellite_angles
elif key["name"].startswith("solar"):
azi, zen = self.sun_angles
else:
raise NotImplementedError("Don't know how to read " + key["name"])
if "zenith" in key["name"]:
values = zen
elif "azimuth" in key["name"]:
values = azi
else:
raise NotImplementedError("Don't know how to read " + key["name"])
else:
values = self.nc[self.datasets[key["name"]]]
values.attrs["platform_name"] = self.platform_name
values.attrs["sensor"] = self.sensor
values.attrs.update(key.to_dict())
return values
@cached_property
def sun_angles(self):
"""Return the sun angles."""
zen = self.nc[self.datasets["solar_zenith_angle"]]
azi = self.nc[self.datasets["solar_azimuth_angle"]]
azi, zen = self._interpolate_angles(azi, zen)
return azi, zen
@cached_property
def satellite_angles(self):
"""Return the satellite angles."""
zen = self.nc[self.datasets["satellite_zenith_angle"]]
azi = self.nc[self.datasets["satellite_azimuth_angle"]]
azi, zen = self._interpolate_angles(azi, zen)
return azi, zen
[docs]
def _interpolate_angles(self, azi, zen):
aattrs = azi.attrs
zattrs = zen.attrs
x, y, z = angle2xyz(azi, zen)
x, y, z = self._do_interpolate((x, y, z))
azi, zen = xyz2angle(x, y, z)
azi.attrs = aattrs
zen.attrs = zattrs
return azi, zen
[docs]
class NCOLCIMeteo(NCOLCILowResData):
"""File handler for the OLCI meteo data."""
datasets = ["humidity", "sea_level_pressure", "total_columnar_water_vapour", "total_ozone"]
def __init__(self, filename, filename_info, filetype_info,
engine=None):
"""Init the file handler."""
super().__init__(filename, filename_info, filetype_info, engine)
self.cache = {}
# TODO: the following depends on more than columns, rows
# float atmospheric_temperature_profile(tie_rows, tie_columns, tie_pressure_levels) ;
# float horizontal_wind(tie_rows, tie_columns, wind_vectors) ;
# float reference_pressure_level(tie_pressure_levels) ;
[docs]
def get_dataset(self, key, info):
"""Load a dataset."""
if key["name"] not in self.datasets:
return
logger.debug("Reading %s.", key["name"])
if self._need_interpolation and self.cache.get(key["name"]) is None:
data = self.nc[key["name"]]
values, = self._do_interpolate(data)
values.attrs = data.attrs
self.cache[key["name"]] = values
elif key["name"] in self.cache:
values = self.cache[key["name"]]
else:
values = self.nc[key["name"]]
values.attrs["platform_name"] = self.platform_name
values.attrs["sensor"] = self.sensor
values.attrs.update(key.to_dict())
return values