Source code for satpy.composites.config_loader

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (c) 2020-2021 Satpy developers
#
# This file is part of satpy.
#
# satpy is free software: you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation, either version 3 of the License, or (at your option) any later
# version.
#
# satpy is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with
# satpy.  If not, see <http://www.gnu.org/licenses/>.
"""Classes for loading compositor and modifier configuration files."""
from __future__ import annotations

import logging
import os
import warnings
from functools import lru_cache, update_wrapper
from typing import Callable, Iterable

import yaml
from yaml import UnsafeLoader

import satpy
from satpy import DataID, DataQuery
from satpy._config import config_search_paths, get_entry_points_config_dirs, glob_config
from satpy.dataset.dataid import minimal_default_keys_config
from satpy.utils import recursive_dict_update

logger = logging.getLogger(__name__)


def _convert_dep_info_to_data_query(dep_info):
    key_item = dep_info.copy()
    key_item.pop('prerequisites', None)
    key_item.pop('optional_prerequisites', None)
    if 'modifiers' in key_item:
        key_item['modifiers'] = tuple(key_item['modifiers'])
    key = DataQuery.from_dict(key_item)
    return key


class _CompositeConfigHelper:
    """Helper class for parsing composite configurations.

    The provided `loaded_compositors` dictionary is updated inplace.

    """

    def __init__(self, loaded_compositors, sensor_id_keys):
        self.loaded_compositors = loaded_compositors
        self.sensor_id_keys = sensor_id_keys

    def _create_comp_from_info(self, composite_info, loader):
        key = DataID(self.sensor_id_keys, **composite_info)
        comp = loader(_satpy_id=key, **composite_info)
        return key, comp

    def _handle_inline_comp_dep(self, dep_info, dep_num, parent_name):
        # Create an unique temporary name for the composite
        sub_comp_name = '_' + parent_name + '_dep_{}'.format(dep_num)
        dep_info['name'] = sub_comp_name
        self._load_config_composite(dep_info)

    @staticmethod
    def _get_compositor_loader_from_config(composite_name, composite_info):
        try:
            loader = composite_info.pop('compositor')
        except KeyError:
            raise ValueError("'compositor' key missing or empty for '{}'. Option keys = {}".format(
                composite_name, str(composite_info.keys())))
        return loader

    def _process_composite_deps(self, composite_info):
        dep_num = -1
        for prereq_type in ['prerequisites', 'optional_prerequisites']:
            prereqs = []
            for dep_info in composite_info.get(prereq_type, []):
                dep_num += 1
                if not isinstance(dep_info, dict):
                    prereqs.append(dep_info)
                    continue
                elif 'compositor' in dep_info:
                    self._handle_inline_comp_dep(
                        dep_info, dep_num, composite_info['name'])
                prereq_key = _convert_dep_info_to_data_query(dep_info)
                prereqs.append(prereq_key)
            composite_info[prereq_type] = prereqs

    def _load_config_composite(self, composite_info):
        composite_name = composite_info['name']
        loader = self._get_compositor_loader_from_config(composite_name, composite_info)
        self._process_composite_deps(composite_info)
        key, comp = self._create_comp_from_info(composite_info, loader)
        self.loaded_compositors[key] = comp

    def _load_config_composites(self, configured_composites):
        for composite_name, composite_info in configured_composites.items():
            composite_info['name'] = composite_name
            self._load_config_composite(composite_info)

    def parse_config(self, configured_composites, composite_configs):
        """Parse composite configuration dictionary."""
        try:
            self._load_config_composites(configured_composites)
        except (ValueError, KeyError):
            raise RuntimeError("Failed to load composites from configs "
                               "'{}'".format(composite_configs))


class _ModifierConfigHelper:
    """Helper class for parsing modifier configurations.

    The provided `loaded_modifiers` dictionary is updated inplace.

    """

    def __init__(self, loaded_modifiers, sensor_id_keys):
        self.loaded_modifiers = loaded_modifiers
        self.sensor_id_keys = sensor_id_keys

    @staticmethod
    def _get_modifier_loader_from_config(modifier_name, modifier_info):
        try:
            loader = modifier_info.pop('modifier', None)
            if loader is None:
                loader = modifier_info.pop('compositor')
                warnings.warn("Modifier '{}' uses deprecated 'compositor' "
                              "key to point to Python class, replace "
                              "with 'modifier'.".format(modifier_name))
        except KeyError:
            raise ValueError("'modifier' key missing or empty for '{}'. Option keys = {}".format(
                modifier_name, str(modifier_info.keys())))
        return loader

    def _process_modifier_deps(self, modifier_info):
        for prereq_type in ['prerequisites', 'optional_prerequisites']:
            prereqs = []
            for dep_info in modifier_info.get(prereq_type, []):
                if not isinstance(dep_info, dict):
                    prereqs.append(dep_info)
                    continue
                prereq_key = _convert_dep_info_to_data_query(dep_info)
                prereqs.append(prereq_key)
            modifier_info[prereq_type] = prereqs

    def _load_config_modifier(self, modifier_info):
        modifier_name = modifier_info['name']
        loader = self._get_modifier_loader_from_config(modifier_name, modifier_info)
        self._process_modifier_deps(modifier_info)
        self.loaded_modifiers[modifier_name] = (loader, modifier_info)

    def _load_config_modifiers(self, configured_modifiers):
        for modifier_name, modifier_info in configured_modifiers.items():
            modifier_info['name'] = modifier_name
            self._load_config_modifier(modifier_info)

    def parse_config(self, configured_modifiers, composite_configs):
        """Parse modifier configuration dictionary."""
        try:
            self._load_config_modifiers(configured_modifiers)
        except (ValueError, KeyError):
            raise RuntimeError("Failed to load modifiers from configs "
                               "'{}'".format(composite_configs))


def _load_config(composite_configs):
    if not isinstance(composite_configs, (list, tuple)):
        composite_configs = [composite_configs]

    conf = {}
    for composite_config in composite_configs:
        with open(composite_config, 'r', encoding='utf-8') as conf_file:
            conf = recursive_dict_update(conf, yaml.load(conf_file, Loader=UnsafeLoader))
    try:
        sensor_name = conf['sensor_name']
    except KeyError:
        logger.debug('No "sensor_name" tag found in %s, skipping.',
                     composite_configs)
        return {}, {}, {}

    sensor_compositors = {}
    sensor_modifiers = {}

    dep_id_keys = None
    sensor_deps = sensor_name.split('/')[:-1]
    if sensor_deps:
        # get dependent
        for sensor_dep in sensor_deps:
            dep_comps, dep_mods, dep_id_keys = load_compositor_configs_for_sensor(sensor_dep)
        # the last parent should include all of its parents so only add the last one
        sensor_compositors.update(dep_comps)
        sensor_modifiers.update(dep_mods)

    id_keys = _get_sensor_id_keys(conf, dep_id_keys)
    mod_config_helper = _ModifierConfigHelper(sensor_modifiers, id_keys)
    configured_modifiers = conf.get('modifiers', {})
    mod_config_helper.parse_config(configured_modifiers, composite_configs)

    comp_config_helper = _CompositeConfigHelper(sensor_compositors, id_keys)
    configured_composites = conf.get('composites', {})
    comp_config_helper.parse_config(configured_composites, composite_configs)
    return sensor_compositors, sensor_modifiers, id_keys


def _get_sensor_id_keys(conf, parent_id_keys):
    try:
        id_keys = conf['composite_identification_keys']
    except KeyError:
        id_keys = parent_id_keys
        if not id_keys:
            id_keys = minimal_default_keys_config
    return id_keys


def _lru_cache_with_config_path(func: Callable):
    """Use lru_cache but include satpy's current config_path."""
    @lru_cache()
    def _call_without_config_path_wrapper(sensor_name, _):
        return func(sensor_name)

    def _add_config_path_wrapper(sensor_name: str):
        config_path = satpy.config.get("config_path")
        # make sure config_path is hashable, but keep original order since it matters
        config_path = tuple(config_path)
        return _call_without_config_path_wrapper(sensor_name, config_path)

    wrapper = update_wrapper(_add_config_path_wrapper, func)
    wrapper = _update_cached_wrapper(wrapper, _call_without_config_path_wrapper)
    return wrapper


def _update_cached_wrapper(wrapper, cached_func):
    for meth_name in ("cache_clear", "cache_parameters", "cache_info"):
        if hasattr(cached_func, meth_name):
            setattr(wrapper, meth_name, getattr(cached_func, meth_name))
    return wrapper


[docs]@_lru_cache_with_config_path def load_compositor_configs_for_sensor(sensor_name: str) -> tuple[dict[str, dict], dict[str, dict], dict]: """Load compositor, modifier, and DataID key information from configuration files for the specified sensor. Args: sensor_name: Sensor name that has matching ``sensor_name.yaml`` config files. Returns: (comps, mods, data_id_keys): Where `comps` is a dictionary: composite ID -> compositor object And `mods` is a dictionary: modifier name -> (modifier class, modifiers options) Add `data_id_keys` is a dictionary: DataID key -> key properties """ config_filename = sensor_name + ".yaml" logger.debug("Looking for composites config file %s", config_filename) paths = get_entry_points_config_dirs('satpy.composites') composite_configs = config_search_paths( os.path.join("composites", config_filename), search_dirs=paths, check_exists=True) if not composite_configs: logger.debug("No composite config found called %s", config_filename) return {}, {}, minimal_default_keys_config return _load_config(composite_configs)
[docs]def load_compositor_configs_for_sensors(sensor_names: Iterable[str]) -> tuple[dict[str, dict], dict[str, dict]]: """Load compositor and modifier configuration files for the specified sensors. Args: sensor_names (list of strings): Sensor names that have matching ``sensor_name.yaml`` config files. Returns: (comps, mods): Where `comps` is a dictionary: sensor_name -> composite ID -> compositor object And `mods` is a dictionary: sensor_name -> modifier name -> (modifier class, modifiers options) """ comps = {} mods = {} for sensor_name in sensor_names: sensor_comps, sensor_mods = load_compositor_configs_for_sensor(sensor_name)[:2] comps[sensor_name] = sensor_comps mods[sensor_name] = sensor_mods return comps, mods
[docs]def all_composite_sensors(): """Get all sensor names from available composite configs.""" paths = get_entry_points_config_dirs('satpy.composites') composite_configs = glob_config( os.path.join("composites", "*.yaml"), search_dirs=paths) yaml_names = set([os.path.splitext(os.path.basename(fn))[0] for fn in composite_configs]) non_sensor_yamls = ('visir',) sensor_names = [x for x in yaml_names if x not in non_sensor_yamls] return sensor_names