Source code for openquake.hazardlib.source_reader

# -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2019, GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

import time
import copy
import hashlib
import json
import os.path
import pickle
import operator
import logging
import numpy

from openquake.baselib import parallel, performance, general, hdf5
from openquake.hazardlib import (
    geo, nrml, source, sourceconverter, valid, InvalidFile, calc)
from openquake.hazardlib.source_group import (
    CompositeSourceModel, SourceGroup, get_unique)
from openquake.hazardlib.source.multi_fault import save_and_split
from openquake.hazardlib.source.base import ts_samples_dt
from openquake.hazardlib.lt import (
    apply_uncertainties, check_correlated, source_ids, get_bset_value,
    restrict_ts_samples)
from openquake.hazardlib.rmap_maker import get_unique_inverse
from openquake.hazardlib.valid import basename

TWO24 = 2**24
TWO32 = 2**32

CLASSICAL_MODES = ('classical', 'classical_risk', 'classical_damage',
                   'classical_bcr', 'disaggregation', 'preclassical')
U16 = numpy.uint16
U32 = numpy.uint32
F32 = numpy.float32
bybranch = operator.attrgetter('branch')
checksum = operator.attrgetter('checksum')

# in classical calculations the source model files bigger than BIG_FILE
# are read by STRIDE tasks, each one keeping one source over STRIDE: the
# order of the sources changes, which is irrelevant for classical
BIG_FILE = 2**20
STRIDE = 5

source_info_dt = numpy.dtype([
    ('source_id', hdf5.vstr),          # 0
    ('grp_id', U16),                   # 1
    ('code', (numpy.bytes_, 1)),       # 2
    ('calc_time', F32),                # 3
    ('num_ctxs', numpy.uint64),        # 4
    ('est_ctxs', numpy.uint64),        # 5
    ('num_ruptures', U32),             # 6
    ('weight', F32),                   # 7
    ('mul', U16),                      # 8
])


[docs]def ts_samples(samples, trt_smr): """ :returns: a structured array (trt_smr, samples) of length 1 """ return numpy.array([(trt_smr, samples, 0)], ts_samples_dt)
def _copy_uncertainty_attrs(src, segment): # the segments of a split source must keep the ts_samples and the # flag bysrc_unc, otherwise the uncertainties are applied to the # base realization only, see get_ts_sets segment.ts_samples = src.ts_samples segment.bysrc_unc = src.bysrc_unc # NB: blocksize is chosen so that event_based/case_35 works
[docs]def splitMF(sources, disagg_by_src, blocksize=1000): """ Split the MultiPointSources to avoid oceanic sources with 160M ruptures hanging during rupture sampling """ splits = [] for src in sources: if src.code == b'M' and len(src) > blocksize: for i, slc in enumerate(general.gen_slices(0, len(src), blocksize)): segment = source.MultiPointSource( source_id=f'{src.source_id}:{i}', name=src.name, tectonic_region_type=src.tectonic_region_type, mfd=src.mfd[slc], magnitude_scaling_relationship=( src.magnitude_scaling_relationship), rupture_aspect_ratio=src.rupture_aspect_ratio, upper_seismogenic_depth=src.upper_seismogenic_depth, lower_seismogenic_depth=src.lower_seismogenic_depth, nodal_plane_distribution=src.nodal_plane_distribution, hypocenter_distribution=src.hypocenter_distribution, mesh=geo.Mesh(src.mesh.lons[slc], src.mesh.lats[slc]), temporal_occurrence_model=src.temporal_occurrence_model) _copy_uncertainty_attrs(src, segment) splits.append(segment) elif src.code == b'F' and not disagg_by_src: # use the colon convention only in absence of kendra-splitting for segment in src: _copy_uncertainty_attrs(src, segment) splits.append(segment) else: splits.append(src) sources[:] = splits
[docs]def check_unique(ids, msg='', strict=True): """ Raise a DuplicatedID exception if there are duplicated IDs """ if isinstance(ids, dict): # ids by key all_ids = sum(ids.values(), []) unique, counts = numpy.unique(all_ids, return_counts=True) for dupl in unique[counts > 1]: keys = [k for k in ids if dupl in ids[k]] if keys: errmsg = '%r appears in %s %s' % (dupl, keys, msg) if strict: raise nrml.DuplicatedID(errmsg) else: logging.info('*' * 60 + ' DuplicatedID:\n' + errmsg) return unique, counts = numpy.unique(ids, return_counts=True) for u, c in zip(unique, counts): if c > 1: errmsg = '%r appears %d times %s' % (u, c, msg) if strict: raise nrml.DuplicatedID(errmsg) else: logging.info('*' * 60 + ' DuplicatedID:\n' + errmsg)
[docs]def create_source_info(csm, h5): """ Creates source_info, trt_smrs, toms """ csm.source_info = csm.get_source_info() num_srcs = len(csm.source_info) # avoid hdf5 damned bug by creating source_info in advance h5.create_dataset('source_info', (num_srcs,), source_info_dt)
[docs]def trt_smrs(src): return tuple(src.trt_smrs)
def _sample_atomic_groups(src_groups, sample, applied): # Atomic groups are sampled as units; applied sources are always retained. atomic = [sg for sg in src_groups if sg.atomic] if not atomic: return src_groups selected = general.random_filter(atomic, float(sample)) selected.extend(sg for sg in atomic if any( src.source_id in applied for src in sg)) if not selected: selected = [atomic[0]] selected_ids = {id(sg) for sg in selected} return [sg for sg in src_groups if not sg.atomic or id(sg) in selected_ids]
[docs]def read_source_model(fname, branch, converter, applied, sample, stride, monitor): """ :param fname: path to a source model XML file :param branch: source model logic tree branch ID :param converter: SourceConverter :param applied: list of source IDs within applyToSources :param sample: a string with the sampling factor (if any) :param stride: None or a pair (k, n) to keep only the k-th source over n :param monitor: a Monitor instance :returns: a dictionary (fname, k) -> SourceModel """ t0 = time.time() [sm] = nrml.read_source_models([fname], converter, stride) sm.branch = branch if sample: sm.src_groups = _sample_atomic_groups(sm.src_groups, sample, applied) for sg in sm.src_groups: if sample and not sg.atomic: srcs = [] for src in sg: if src.source_id in applied: srcs.append(src) else: srcs.extend(calc.filters.split_source(src)) if srcs: kept = [src for src in srcs if src.source_id in applied] rand = general.random_filter(srcs, float(sample)) sg.sources = (kept + rand) or [srcs[0]] else: sg.sources = [] # compute the content hashes here, in parallel, since they are needed # by add_bangs and check_same_sources (40 seconds in AUS) for sg in sm.src_groups: for src in sg: src.checksum = content_hash(src) sm.rtime = time.time() - t0 # save the read time return {(fname, stride[0] if stride else 0): sm}
# merge the source models of the same file read with different strides, # i.e. convert a dictionary (fname, k) -> sm into a dictionary fname -> sm; # the groups are simply concatenated, since they are rebuilt anyway by # build_groups; sorting by k makes the order independent from the order # in which the tasks finish def _merge_strides(dic): out = {} for (fname, _k), sm in sorted(dic.items()): if fname not in out: out[fname] = sm else: out[fname].src_groups.extend(sm.src_groups) out[fname].rtime += sm.rtime return out # NB:this is called after reduce_sources, so ";" is not added # if the same source appears multiple times, i.e. len(srcs) == 1
[docs]def add_semicolons(src_groups): """ Add semicolons to differentiate different sources with the same source_id and then sort the sources in each group by extended source_id """ sources = general.AccumDict(accum=[]) for sg in src_groups: for src in sg: sources[src.source_id].append(src) for src_id, srcs in sources.items(): srcs = get_unique(srcs) if len(srcs) > 1: # happens in logictree/case_01/rup.ini for i, src in enumerate(srcs): src.source_id = '%s;%d' % (src.source_id, i) for sg in src_groups: sg.sources.sort(key=operator.attrgetter('source_id'))
# tested in logictree/case_05 with 9 variations of # AreaSource 1 and SimpleFaultSource 2
[docs]def check_branchID(branchID): """ Forbids invalid characters .:; used in fragmentno """ if '.' in branchID: raise InvalidFile('branchID %r contains an invalid "."' % branchID) elif ':' in branchID: raise InvalidFile('branchID %r contains an invalid ":"' % branchID) elif ';' in branchID: raise InvalidFile('branchID %r contains an invalid ";"' % branchID)
[docs]def check_duplicates(smdict, strict): # check_duplicates in the same file for sm in smdict.values(): srcids = [] for sg in sm.src_groups: srcids.extend(src.source_id for src in sg) if sg.src_interdep == 'mutex': # mutex sources in the same group must have all the same # basename, i.e. the colon convention must be used basenames = set(map(basename, sg)) assert len(basenames) == 1, basenames check_unique(srcids, 'in ' + sm.fname, strict) # check duplicates in different files but in the same branch # the problem was discovered in the DOM model for branch, sms in general.groupby(smdict.values(), bybranch).items(): srcids = general.AccumDict(accum=[]) fnames = [] for sm in sms: if isinstance(sm, nrml.GeometryModel): # the section IDs are not checked since they not count # as real sources continue for sg in sm.src_groups: srcids[sm.fname].extend(src.source_id for src in sg) fnames.append(sm.fname) check_unique(srcids, 'in branch %s' % branch, strict=strict)
[docs]def save_read_times(dstore, source_models): """ Store how many seconds it took to read each source model file in a table (fname, rtime) """ dt = [('fname', hdf5.vstr), ('rtime', float)] arr = numpy.array([(sm.fname, sm.rtime) for sm in source_models], dt) dstore.create_dset('source_model_read_times', arr)
[docs]def get_csm(oq, full_lt, dstore=None): """ Build a CompositeSourceModel without applying the uncertainties, that are applied in the workers, see modified_groups. """ converter = sourceconverter.SourceConverter( oq.investigation_time, oq.rupture_mesh_spacing, oq.complex_fault_mesh_spacing, oq.width_of_mfd_bin, oq.area_source_discretization, oq.minimum_magnitude, oq.source_id, discard_trts=[s.strip() for s in oq.discard_trts.split(',')], floating_x_step=oq.floating_x_step, floating_y_step=oq.floating_y_step, source_nodes=oq.source_nodes, infer_occur_rates=oq.infer_occur_rates, filter_sourcecodes=oq.filter_sourcecodes) full_lt.ses_seed = oq.ses_seed logging.info('Reading the source model(s) in parallel') # NB: the source models file must be in the shared directory # NB: dstore is None in logictree_test.py allargs = [] sdata = full_lt.source_model_lt.source_data allpaths = set(full_lt.source_model_lt.info.smpaths) dic = general.group_array(sdata, 'fname') smpaths = [] ss = os.environ.get('OQ_SAMPLE_SOURCES') applied = set() for srcs in full_lt.source_model_lt.info.applytosources.values(): applied.update(srcs) # striding is disabled for event based, where the order of the sources # matters, and for correlated branchsets, since check_correlated needs # the full groups correlated = any(bset.correlated for bset in full_lt.source_model_lt.branchsets) strided = (oq.calculation_mode in CLASSICAL_MODES and not ss and not correlated) for fname, rows in dic.items(): path = os.path.abspath( os.path.join(full_lt.source_model_lt.basepath, fname)) smpaths.append(path) big = strided and os.path.getsize(path) > BIG_FILE for stride in ([(k, STRIDE) for k in range(STRIDE)] if big else [None]): allargs.append((path, rows[0]['branch'], converter, applied, ss, stride)) for path in allpaths - set(smpaths): # geometry models allargs.append((path, '', converter, applied, ss, None)) # read the biggest files first, so that they do not end up last allargs.sort(key=lambda args: os.path.getsize(args[0]), reverse=True) smdict = _merge_strides(parallel.Starmap( read_source_model, allargs, h5=dstore if dstore else None).reduce()) parallel.Starmap.shutdown() # save memory smdict = {k: smdict[k] for k in sorted(smdict)} if dstore: save_read_times(dstore, smdict.values()) check_duplicates(smdict, strict=oq.disagg_by_src) found = add_bangs(smdict) if found: logging.info('Found different sources with same ID %s', general.shortlist(found)) smlt = oq.inputs.get('source_model_logic_tree', oq.inputs['job_ini']) if oq.strict: check_same_sources(smdict, smlt) if found: check_mfd_variants(smdict, smlt) # checking ps_grid_spacing pointlike_sources = 0 for sm in smdict.values(): for sg in sm.src_groups: for src in sg: if src.code in b'PAM': pointlike_sources += 1 break if (oq.strict and oq.mosaic_model and pointlike_sources and 'classical' in oq.calculation_mode and oq.ps_grid_spacing == 0 and not oq.auto_grid and not oq.sites and not oq.disagg_by_src): raise InvalidFile(f'{oq.inputs["job_ini"]}: ' 'missing ps_grid_spacing') return build_csm(oq, full_lt, smdict, dstore)
[docs]def get_bset_values(full_lt, sources): """ :param full_lt: a FullLogicTree instance :param sources: a SourceGroup or a list of sources of the same group :returns: the dictionary of uncertainties to apply expected by modified_groups, i.e. one entry for each set of realizations with the same uncertainties; the uncertainties of a set are the ones of its first realization NB: only the uncertainties relevant for the given sources are returned, i.e. the ones of their realizations, without the branchsets applied to other sources, since the logic tree can be huge and the dictionary is sent to the workers """ ordinals = {trt_smrs[0] % TWO24 for src in sources for trt_smrs in src.get_ts_sets()} ids = source_ids(sources) return {ordinal: [(bset, value) for bset, value in full_lt.get_bset_values(ordinal) if _may_apply(bset, value, ids)] for ordinal in sorted(ordinals)}
def _may_apply(bset, value, ids): # False if the branchset certainly does not apply to the sources with # the given IDs and basenames (see source_ids and get_bset_value) if bset.correlated: return not ids.isdisjoint(value) ats = bset.filters.get('applyToSources') return not ats or not ids.isdisjoint(ats)
[docs]def modified_groups(sources, bset_values): """ Apply the uncertainties to a group of sources built *without* them, as needed by the workers computing the rates or the ruptures: this is done one set of realizations at a time, i.e. one set with the same uncertainties at a time (see build_groups). :param sources: a SourceGroup or a list of sources of the same group :param bset_values: the uncertainties to apply, as returned by get_bset_values; it can be empty, if there are no uncertainties at all :returns: a generator of (trt_smrs, group) pairs, one for each set of realizations with the same uncertainties, with the uncertainties applied and the ts_samples restricted to the set of realizations """ for trt_smrs, srcs in _subsets_by_unc(sources).items(): grp = _restricted_group(sources, srcs, trt_smrs) # NB: the trt_smrs are trti * TWO24 + ordinal, see gen_groups bvals = bset_values[trt_smrs[0] % TWO24] if bset_values else [] # NB: check=False since the group is a fragment of the original # one (split by weight in the preclassical), so the correlated # branchsets were already checked at build time, see build_groups grp = apply_uncertainties(bvals, grp, check=False) yield trt_smrs, grp
def _subsets_by_unc(sources): """ :returns: a dictionary trt_smrs -> sources, i.e. the sources grouped by set of realizations with the same uncertainties """ if getattr(sources, 'atomic', False): # the sources of an atomic group are mutually exclusive (or belong # to a cluster), so they must be kept together, see sample_cluster # and cmakers_groups_gkeys; the sets of realizations are the same for # all of them, since they belong to the same source model return {trt_smrs: list(sources) for trt_smrs in sources[0].get_ts_sets()} subsets = {} for src in sources: for trt_smrs in src.get_ts_sets(): subsets.setdefault(trt_smrs, []).append(src) return subsets def _restricted_group(sources, srcs, trt_smrs): """ :returns: a group with the given sources, i.e. copies of the sources of `sources` with the ts_samples restricted to trt_smrs """ restricted = [restrict_ts_samples(src, trt_smrs) for src in srcs] if hasattr(sources, 'sources'): # keep the attributes of the group grp = copy.copy(sources) grp.sources = restricted else: # a plain list of sources, e.g. a block of sources grp = SourceGroup(sources[0].tectonic_region_type, restricted) return grp
[docs]def unc_signature(bset_values, src): """ :returns: a tuple identifying the uncertainties applied to src """ sig = [] for bset, value in bset_values: ok, val = get_bset_value(bset, value, src) if ok: sig.append((bset.id, str(val))) return tuple(sig)
[docs]def collect_sources(full_lt, rlz_groups): """ :param full_lt: a FullLogicTree instance :param rlz_groups: an iterator of (sm_rlz, source group) pairs :returns: a dictionary id(grp) -> [grp, {id(src): (src, pairs)}], where pairs is a list of (trt_smr, samples, signature) tuples, one per realization of the source """ dic = {} correlated = {} # ordinal -> (bset_values, source IDs of the rlz) for rlz, grp in rlz_groups: trti = full_lt.trti.get(grp.trt, 0) trt_smr = trti * TWO24 + rlz.ordinal bset_values = full_lt.get_bset_values(rlz.ordinal) # NB: the uncertainties are applied later, in the workers, on # groups split by weight, so the correlated branchsets are checked # here, where the groups are still whole; the sources of a # correlated branchset can be in different groups of the # realization, so they are checked at the end if any(bset.correlated for bset, _ in bset_values): srcids = correlated.setdefault( rlz.ordinal, (bset_values, set()))[1] srcids.update(source_ids(grp)) # NB: the sources are collected by id(grp), since the group objects # are shared by all the realizations selecting the same source model # file (see gen_groups); the groups built from them are then merged # by trt_smrs below. The dicts preserve the order of first # appearance, making the groups and the sources inside them # reproducible. srcs = dic.setdefault(id(grp), [grp, {}])[1] for src in grp: sig = unc_signature(bset_values, src) pairs = srcs.setdefault(id(src), (src, []))[1] pairs.append((trt_smr, rlz.samples, sig)) for bset_values, srcids in correlated.values(): check_correlated(bset_values, srcids) return dic
[docs]def source_with_ts_sets(src, pairs, sigrows): """ :param src: a source :param pairs: a list of (trt_smr, samples, signature) tuples, one per realization of the source :param sigrows: a list of rows describing the uncertainty signatures, extended with the signatures of the source :returns: a copy of the source with the attributes ts_samples and bysrc_unc set, i.e. the sets of realizations with the same uncertainties are stored in the field ts """ sigdict = {} for trt_smr, samples, sig in pairs: sigdict.setdefault(sig, []).append(trt_smr) # NB: the field ts is the smallest trt_smr of the set of realizations # with the same uncertainties; it is 0 for all the realizations if # the uncertainties are the same everywhere (or there are none), so # the sources without uncertainties keep their ts_samples as it is tsdict = {sig: min(t) for sig, t in sigdict.items()} multi = len(sigdict) > 1 arrays = [(trt_smr, samples, tsdict[sig] if multi else 0) for trt_smr, samples, sig in pairs] new_src = copy.copy(src) new_src.ts_samples = numpy.array( sorted(arrays), ts_samples_dt) # sorted by trt_smr # sigts maps each value of the field ts to a pair (signature, smallest # trt_smr of the realizations with that signature); it is needed to # merge the identical copies of the source, see reduce_sources and # _merge_ts_samples new_src.sigts = {(tsdict[sig] if multi else 0): (sig, tsdict[sig]) for sig in sigdict} # flag the sources which will be modified in the workers: they must # not be split in the preclassical, since the splitting destroys the # geometry (and the MFD of the fault sources) new_src.bysrc_unc = any(sigdict) # NB: () means no uncertainty # the uncertainty signatures of the source, i.e. the sets of # realizations with the same uncertainties, which are the indices of # rate attribution for its rates (see get_ts_sets) for sig, trt_smrs in sigdict.items(): sigrows.append(dict(source_id=src.source_id, realizations=len(arrays), signature=dict(sig), count=len(trt_smrs))) return new_src
[docs]def build_groups(full_lt, rlz_groups, oq, dstore=None): """ Build the source groups without applying the uncertainties, as needed by the workers. The uncertainties to be applied in each realization are not known until the workers, so the realizations with different uncertainties are stored in the field ts of the ts_samples of each source (see get_ts_sets). """ # NB: the uncertainty signatures are stored in the datastore, so that # `oq check_input` can print them, see store_unc_signatures # NB: the same source_id can be used by different sources, i.e. in # different source models, so the sources are keyed by id(src) and not # by source_id; the ids are disambiguated at the end, by adding a # semicolon, see add_semicolons sigrows = [] # dicts with keys source_id, realizations, signature, count dic = collect_sources(full_lt, rlz_groups) out, atomic, acc = [], [], general.AccumDict(accum=[]) for grp, srcs in dic.values(): new_srcs = [source_with_ts_sets(src, pairs, sigrows) for src, pairs in srcs.values()] if grp.atomic: # the atomic groups are never merged with the other groups, # since their sources must be computed together new = copy.copy(grp) new.sources = new_srcs atomic.append(new) else: acc[grp.trt].extend(new_srcs) if atomic: logging.info('Found %d atomic groups', len(atomic)) # NB: the sources are grouped by trt_smrs and TOM, so that there is # one cmaker for each set of realizations, see get_cmakers; the groups # are sorted by TRT index, trt_smrs and TOM, so that their order does # not depend on the order of the sources in the files red_sources = 0 for trt in sorted(acc, key=lambda trt: (full_lt.trti.get(trt, 0), trt)): grps, red = _group_sources(trt, acc[trt]) out.extend(grps) red_sources += red if red_sources: logging.info('reduce_sources was called %d times', red_sources) out.extend(atomic) for grp in out: for src in grp: # sigts is not needed after the reduction vars(src).pop('sigts', None) splitMF(grp.sources, oq.disagg_by_src) add_semicolons(out) # else sources with the same id are lost store_unc_signatures(out, sigrows, full_lt, oq, dstore) return out
[docs]def store_unc_signatures(groups, sigrows, full_lt, oq, dstore=None): """ Store the uncertainty signatures of the sources in the datastore, so that they can be printed by `oq check_input` (see the check_input command) and read back with `oq show unc_signatures`. NB: the signatures are stored as JSON objects, so that the unc_signatures view can display a column for each branchset. :param groups: a list of SourceGroups built without the uncertainties :param sigrows: a list of dicts with keys source_id, realizations, signature (a dictionary branchset_id -> value) and count :param full_lt: a FullLogicTree instance :param oq: an OqParam instance :param dstore: a DataStore instance or None """ if dstore is not None and any(row['signature'] for row in sigrows): # NB: the signatures are stored only if there are uncertainties, # i.e. only if the rates must be split in indices of rate # attribution dt = [('source_id', hdf5.vstr), ('realizations', int), ('signature', hdf5.vstr), ('count', int)] data = [(row['source_id'], row['realizations'], json.dumps( row['signature']), row['count']) for row in sigrows] dstore.create_df('unc_signatures', numpy.array(data, dt)) if oq.calculation_mode not in CLASSICAL_MODES: # the event based calculators attribute the rates to the # realizations of the group, not to the subsets return core_trt_smrs = get_core_trt_smrs(groups) if dstore is not None: dstore.hdf5.save_vlen('core_trt_smrs', core_trt_smrs) Gt = get_core_size(core_trt_smrs, full_lt) if Gt >= TWO32: # NB: the gids are stored as uint32 in the _rates dataset, so # there cannot be more than 2**32 columns in the RateMap raise ValueError( 'The core size Gt=%d is too large (the maximum is %d), ' 'you must reduce the logic tree' % (Gt, TWO32)) # NB: Gt is a sum over the TRTs while R is a product, so Gt can be # larger than R, for instance Gt=5 and R=1 in JPA with 5 TRTs logging.warning('Core size Gt=%d for R=%d realizations', Gt, full_lt.get_num_paths()) if dstore is not None and 'sitecol' in dstore: # NB: the sites are read after the sources, so the size in bytes # is logged only if they are already in the datastore N = len(dstore['sitecol']) imtls = oq.imtls L = imtls.size if hasattr(imtls, 'size') else len(imtls) logging.warning('Global RateMap of %s for %d sites and %d levels', general.humansize(4 * N * L * Gt), N, L)
[docs]def get_core_trt_smrs(groups): """ :param groups: a list of SourceGroups built without applying the uncertainties (i.e. the src_groups of a CompositeSourceModel) :returns: a sorted list of trt_smrs (to be stored as hdf5.vuint32) The uncertainties are applied in the workers, so the realizations with different uncertainties are not separated at build time (see build_groups). The rates are nevertheless computed separately for each set of uncertainties and must be attributed to the right realizations, hence this extra list; a rate is attributed to the unit containing its trt_smrs. """ all_trt_smrs = [trt_smrs for sg in groups for src in sg for trt_smrs in src.get_ts_sets()] unique, _inverse = get_unique_inverse(all_trt_smrs) return [numpy.array(trt_smrs, numpy.uint32) for trt_smrs in unique]
[docs]def get_core_size(core_trt_smrs, full_lt): """ :param core_trt_smrs: the sets of realizations with the same uncertainties, as returned by get_core_trt_smrs :param full_lt: a FullLogicTree instance :returns: the core size Gt = Σ_i G(trt_i) of the logic tree, i.e. the number of distinct rate components, i.e. of gids (see FullLogicTree.get_gids), which is >= len(core_trt_smrs) since each unit contributes G(trt_i) >= 1 gids """ return sum(len(full_lt.gsim_lt.values[full_lt.trts[t[0] // TWO24]]) for t in core_trt_smrs)
[docs]def build_csm(oq, full_lt, smdict, dstore): """ :param oq: OqParam instance :param full_lt: FullLogicTree instance :param smdict: dictionary source_model_path -> SourceModel instance :param dstore: DataStore instance :returns: a CompositeSourceModel instance """ with performance.Monitor('_build_groups', measuremem=True) as mon: rlz_groups = [] for rlz in full_lt.sm_rlzs: rlz_groups.extend((rlz, grp) for grp in gen_groups(full_lt, smdict, rlz)) logging.info(mon) logging.info('Building CompositeSourceModel') groups = build_groups(full_lt, rlz_groups, oq, dstore) csm = CompositeSourceModel(oq, full_lt, groups) store_data(oq, smdict, csm, dstore) return csm
[docs]def store_data(oq, smdict, csm, dstore): """ Create src_mutex, grp_probability in calc_XXX.hdf5 and sources and mf_sections in calc_XXX_tmp.hdf5 """ out = [] probs = [] for sg in csm.src_groups: if sg.src_interdep == 'mutex' and 'src_mutex' not in dstore: segments = [] for src in sg: segments.append(src.source_id.split(':')[1]) t = (src.source_id, src.grp_id, src.num_ruptures, src.mutex_weight, sg.rup_interdep == 'mutex') out.append(t) probs.append((src.grp_id, sg.grp_probability)) assert len(segments) == len(set(segments)), segments if out: dtlist = [('src_id', hdf5.vstr), ('grp_id', int), ('num_ruptures', int), ('mutex_weight', float), ('rup_mutex', bool)] dstore.create_dset('src_mutex', numpy.array(out, dtlist)) lst = [('grp_id', int), ('probability', float)] dstore.create_dset('grp_probability', numpy.array(probs, lst)) # add rupids_by_tag to multifault sources if there is a single site try: sitecol = dstore['sitecol'] except (KeyError, TypeError): # 'NoneType' object is not subscriptable sitecol = None else: # NB: in AELO we can have multiple vs30 on the same location lonlats = set(zip(sitecol.lons, sitecol.lats)) if len(lonlats) > 1: sitecol = None # must be called *after* add_semicolons t0 = time.time() secparams = fix_geometry_sections( smdict, csm.src_groups, dstore.tempname if dstore else '', sitecol if oq.disagg_by_src and oq.use_rates else None, split=not oq.calculation_mode.startswith('event_based')) if secparams is not None and len(secparams): logging.info('Spent %.1f seconds in fix_geometry_sections', time.time()-t0)
# attributes which are not part of the content of a source NOT_CONTENT = {'source_id', 'name', 'ts_samples', 'branch', 'width_of_mfd_bin', 'checksum', 'sigts', 'bysrc_unc'}
[docs]def content_hash(src, exclude=NOT_CONTENT): """ :param src: a source object :param exclude: the attributes to ignore :returns: the md5 hex digest of the other attributes of the source NB: by default the ID and the name are ignored, i.e. two copies of the same source with different IDs or names have the same hash """ dic = {k: v for k, v in vars(src).items() if k not in exclude} return hashlib.md5(pickle.dumps(dic, protocol=4)).hexdigest()
# called by reduce_sources
[docs]def add_checksums(srcs): """ Build and attach a checksum to each source """ # width_of_mfd_bin is set by modify_set_recurrow; the name is not part # of the checksum, i.e. sources with the same content are reduced even # if they have different names for src in srcs: src.checksum = content_hash(src)
# called before add_semicolons
[docs]def add_bangs(smdict): """ Discriminate different sources with same ID (false duplicates) and put an exclamation mark in their source ID """ acc = general.AccumDict(accum=[]) atomic = set() for smodel in smdict.values(): for sgroup in smodel.src_groups: for src in sgroup: src.branch = smodel.branch srcid = (src.source_id if sgroup.atomic else basename(src)) acc[srcid].append(src) if sgroup.atomic: atomic.add(src.source_id) found = [] for srcid, srcs in acc.items(): if len(srcs) > 1: # duplicated ID if any(src.source_id in atomic for src in srcs): raise RuntimeError('Sources in atomic groups cannot be ' 'duplicated: %s', srcid) if any(getattr(src, 'mutex_weight', 0) for src in srcs): raise RuntimeError('Mutually exclusive sources cannot be ' 'duplicated: %s', srcid) gb = general.AccumDict(accum=[]) for src in srcs: gb[checksum(src)].append(src) if len(gb) > 1: for same_checksum in gb.values(): for src in same_checksum: check_branchID(src.branch) src.source_id += '!%s' % src.branch found.append(srcid) return found
def _mfd_variants_ids(pairs): # the sources in pairs [(smfile, src)] with the same geometry, # different IDs and MFDs and a single copy in each file, as lists of # IDs[file]; the original IDs are used, without the bangs bygeom = general.AccumDict(accum=[]) for smfile, src in pairs: bygeom[content_hash(src, NOT_CONTENT | {'mfd'})].append( (smfile, src)) out = [] for same in bygeom.values(): ids = {basename(src).split('!')[0] for _, src in same} # NB: with more copies in the same file the sources are different # (they cannot get the same ID), only their rates would add up if (len({smfile for smfile, _ in same}) == len(same) > 1 and len(ids) > 1 and len({checksum(src) for _, src in same}) > 1): out.append(sorted({'%s[%s]' % ( basename(src).split('!')[0], os.path.basename(smfile)) for smfile, src in same})) return out
[docs]def check_same_sources(smdict, fname): """ Log a warning if the same source (i.e. a source with the same content) appears in different source models with different IDs: then the engine cannot recognize the copies as the same source, i.e. it cannot reduce them (see reduce_sources), and the source is computed once per source model. It is an error in `oq check_input`. A different warning is logged if the sources have the same geometry but different MFDs and different IDs: they could be computed together if they had the same ID and the MFDs were uncertainties of the logic tree, see check_mfd_variants. :param smdict: a dictionary source model file -> SourceModel :param fname: the source model logic tree file, used in the warning """ # NB: the atomic groups are skipped, since their sources cannot be # duplicated anyway (see add_bangs); the sources are compared only if # their code and TRT appear in more than one source model file, to # save time bykey = general.AccumDict(accum=[]) # (code, trt) -> [(smfile, src)] for smfile, smodel in smdict.items(): for sgroup in smodel.src_groups: if sgroup.atomic: continue for src in sgroup: bykey[src.code, src.tectonic_region_type].append( (smfile, src)) errors, variants = [], [] for pairs in bykey.values(): if len({smfile for smfile, _ in pairs}) < 2: continue variants.extend(_mfd_variants_ids(pairs)) byhash = general.AccumDict(accum=[]) for smfile, src in pairs: byhash[checksum(src)].append((smfile, src)) for same in byhash.values(): smfiles = {smfile for smfile, _ in same} # NB: the original IDs, without the bangs added by add_bangs # for sources with the same ID and a different content ids = {basename(src).split('!')[0] for _, src in same} if len(smfiles) > 1 and len(ids) > 1: errors.append(sorted({'%s[%s]' % ( basename(src).split('!')[0], os.path.basename(smfile)) for smfile, src in same})) if variants: shown = [' ' + general.shortlist(v) for v in variants[:3]] logging.warning( '%s:\n%d sources appear in different source models with ' 'different IDs, the same geometry and different MFDs, so each ' 'copy is computed separately. They are likely variants of a ' 'base source: if so, give them the same ID and move the MFDs ' 'in the logic tree (see the warning about MFD variants; ' '`oq check_input --better-smlt` does not change sources with ' 'different IDs):\n%s%s', fname, len(variants), '\n'.join(shown), '\n ...' if len(variants) > 3 else '') if errors: lines = [' ' + general.shortlist(err) for err in errors] msg = ('%s:\n%d sources appear in different source models with ' 'different IDs, use the same ID for each of them:\n%s' % ( fname, len(errors), '\n'.join(lines))) # NB: in oq check_input the models must be fixed, so it is an error if os.environ.get('OQ_CHECK_INPUT'): raise InvalidFile(msg) # Keep normal calculation logs readable when many sources are found. shown = lines[:3] + [' ...'] if len(lines) > 3 else lines logging.warning( '%s:\n%d sources appear in different source models with ' 'different IDs, use the same ID for each of them:\n%s', fname, len(errors), '\n'.join(shown))
# uncertainty types for the MFD variants, by NRML tag: suggested by # check_mfd_variants and used by commonlib.better_smlt to convert them MFD_UNCERTAINTY = { 'incrementalMFD': 'incrementalMFDAbsolute', 'multiMFD': 'multiMFDAbsolute', 'truncGutenbergRichterMFD': 'abMaxMagAbsolute'} # NRML tags of the MFD classes MFD_TAG = {'EvenlyDiscretizedMFD': 'incrementalMFD', 'MultiMFD': 'multiMFD', 'TruncatedGRMFD': 'truncGutenbergRichterMFD'}
[docs]def check_mfd_variants(smdict, fname): """ Log a warning if a source appears in different source model files with the same ID and the same content except the MFD: then the copies are computed separately, once per file, while they could be computed together, sharing the contexts, if the MFDs were given as uncertainties in the source model logic tree (see make_results). For instance in the IND model the HimalayanThrust source is defined in 3 files (himalaya_cut, himalaya_area, himalaya_m55) with different incrementalMFDs and it takes 64% of the time of the calculation. :param smdict: a dictionary source model file -> SourceModel :param fname: the source model logic tree file, used in the warning :returns: the list of the IDs of the sources with MFD variants NB: must be called after add_bangs, which adds the branch to the sources and a bang to the IDs of the sources with the same ID and a different content """ byid = general.AccumDict(accum=[]) # source ID -> [(smfile, src)] for smfile, smodel in smdict.items(): for sgroup in smodel.src_groups: for src in sgroup: if '!' in src.source_id and hasattr(src, 'mfd'): # NB: the fragments 'ID:1', 'ID:2', ... of a split # source are different sources, not MFD variants byid[src.source_id.split('!')[0]].append((smfile, src)) lines = [] for srcid, pairs in byid.items(): # group the copies by content without the MFD, i.e. by geometry bygeom = general.AccumDict(accum=[]) for smfile, src in pairs: bygeom[content_hash(src, NOT_CONTENT | {'mfd'})].append( (smfile, src)) for same in bygeom.values(): # the copies must be in different files: the fragments of a # source split in the same file are not MFD variants if len({smfile for smfile, _ in same}) < 2: continue # the copies with the same MFD were already reduced mfds = {checksum(src) for _, src in same} if len(mfds) < 2: continue mfd = type(same[0][1].mfd).__name__ # NB: the branches are not listed, since a file can be shared # by several branches but src.branch contains only one of them files = sorted({os.path.basename(smfile) for smfile, _ in same}) utype = MFD_UNCERTAINTY.get(MFD_TAG.get(mfd), 'an MFD uncertainty') lines.append(' %s: %d %s variants in %s, use %s' % ( srcid, len(mfds), mfd, ' '.join(files), utype)) if lines: # log only the first 3 sources, then an ellipsis marker shown = lines[:3] + [' ...'] if len(lines) > 3 else lines logging.warning( '%s:\n%d source(s) appear multiple times with ' 'the same geometry and different MFDs,\nso each copy is ' 'computed separately. Keep a single copy in one file and move ' 'the MFDs in the logic tree,\nwith a branchSet with ' 'applyToSources="<source ID>" (and applyToBranches if needed) ' 'containing one branch\nper MFD; then the contexts are computed ' 'once for all the MFDs, which can be much faster. You can ' 'do it automatically with\n`oq check_input job.ini ' '--better-smlt`\nThe sources are:\n%s', fname, len(lines), '\n'.join(shown)) return [line.split(':')[0].strip() for line in lines]
[docs]def fix_geometry_sections(smdict, src_groups, hdf5path='', site1=None, split=True): """ If there are MultiFaultSources, fix the sections according to the GeometryModels (if any). """ gmodels = [] gfiles = [] for fname, mod in smdict.items(): if isinstance(mod, nrml.GeometryModel): gmodels.append(mod) gfiles.append(fname) # merge and reorder the sections sec_ids = [] sections = {} for gmod in gmodels: sec_ids.extend(gmod.sections) sections.update(gmod.sections) check_unique(sec_ids, 'section ID in files ' + ' '.join(gfiles)) if sections: # save in the temporary file sources and sections assert hdf5path, ('You forgot to pass the dstore to ' 'get_composite_source_model') mfsources = [] for sg in src_groups: for src in sg: if src.code == b'F': mfsources.append(src) if mfsources: split_dic, secparams = save_and_split( mfsources, sections, hdf5path, site1, split=split) for sg in src_groups: new = [] for src in sg.sources: tag = src.source_id if tag in split_dic: new.extend(split_dic[tag]) else: new.append(src) sg.sources[:] = new return secparams return None
def _groups_ids(smlt_dir, smdict, fnames): # extract the source groups and ids from a sequence of source files groups = [] for fname in fnames: fullname = os.path.abspath(os.path.join(smlt_dir, fname)) groups.extend(smdict[fullname].src_groups) return groups, set(src.source_id for grp in groups for src in grp) def _add_ts_samples(src, rlz, trti): # associate the source to the ts_samples parameters of the realization; # the same source can appear in multiple realizations, hence the list. # NB: & and it enters # the classical calculations too, via SourceGroup.fix_src_offset and # the source_info rows, so the ts_samples must be always set sampl = ts_samples(rlz.samples, trti * TWO24 + rlz.ordinal) if src.ts_samples is None: # the first time src.ts_samples = [sampl] else: # if the same source belongs to multiple realizations src.ts_samples.append(sampl)
[docs]def gen_groups(full_lt, smdict, rlz): # yield all the possible source groups from the given rlz smlt_file = full_lt.source_model_lt.filename smlt_dir = os.path.dirname(smlt_file) src_groups, source_ids = _groups_ids( smlt_dir, smdict, rlz.value[0].split()) bset_values = full_lt.source_model_lt.bset_values(rlz.lt_path) if rlz.ordinal % 100 == 0: logging.info('Building source groups for rlz' f'#{rlz.ordinal}: {"_".join(rlz.lt_path)}') # NB: the extendModel branchsets are added wherever they are in the # path, since the modifications (e.g. MFD chains) can precede them for _bset, value in bset_values: if _bset.uncertainty_type != 'extendModel': continue extra, extra_ids = _groups_ids(smlt_dir, smdict, value.split()) common = source_ids & extra_ids if common: raise InvalidFile( '%s contains source(s) %s already present in %s' % (value, common, rlz.value)) src_groups.extend(extra) # NB: the uncertainties are not applied here, but in the workers, # one set of realizations at a time, see modified_groups; the # ts_samples info is set anyway, since it determines the multiplicity for src_group in src_groups: trti = full_lt.trti.get(src_group.trt, 0) for src in src_group: # tested in case_83_eb _add_ts_samples(src, rlz, trti) yield src_group # check applyToSources, ignoring the branch added by add_bangs to the # sources with the same ID and a different content, like filter_source sm_branch = rlz.lt_path[0] src_id = full_lt.source_model_lt.info.applytosources[sm_branch] core_ids = {valid.corename(srcid, '!') for srcid in source_ids} for srcid in src_id: if srcid not in core_ids: if full_lt.source_model_lt.branchID: continue raise ValueError( "The source %s is not in the source model," " please fix applyToSources in %s or the " "source model(s) %s" % (srcid, smlt_file, rlz.value[0].split()))
[docs]def reduce_sources(sources_with_same_id): """ :param sources_with_same_id: a list of sources with the same source_id :returns: a list of truly unique sources """ # first reduce identical sources having the same id(src) # tested in LogictreeTestCase.test_case_08, where <PoinstSource 2> # appears 3 times unique = get_unique(sources_with_same_id) out = [] add_checksums(unique) # in LogicTreeCase2ClassicalPSHA there 81 unique sources # grouped in 9 groups of 9 sources each with the same checksum # NB: merging copies of a source can change the order of the sources # and so the numbers of the fragments (see add_semicolons), hence the # random seeds: the event based results can change even if the MFDs # are the same, while the distribution converges to the same MFD for srcs in general.groupby(unique, checksum).values(): # NB: the simplest test featuring the same source in two # different source models is logictree/case_01 src = srcs[0] if len(srcs) > 1: # NB: each copy can have many realizations, since they are # collected before the reduction (see collect_sources), so # the realizations of all the copies must be kept src.ts_samples = numpy.sort( _merge_ts_samples(srcs), order='trt_smr') # the merged source is modified if any copy is modified if any(getattr(s, 'bysrc_unc', False) for s in srcs): src.bysrc_unc = True out.append(src) return out
def _merge_ts_samples(srcs): # concatenate the ts_samples of identical copies of a source in # different files, recomputing the field ts as if the copies were a # single source (see source_with_ts_sets): a copy with a single # signature has ts=0, so copies with different signatures (i.e. with # uncertainties depending on the branch, like the MFD chains) would # be merged in a single set of realizations otherwise if not all(hasattr(src, 'sigts') for src in srcs): # in hazardlib return numpy.concatenate([src.ts_samples for src in srcs]) minimum = {} # signature -> smallest trt_smr for src in srcs: for sig, mint in src.sigts.values(): minimum[sig] = min(minimum.get(sig, mint), mint) multi = len(minimum) > 1 arrays = [] for src in srcs: arr = src.ts_samples.copy() arr['ts'] = [minimum[src.sigts[ts][0]] if multi else 0 for ts in arr['ts']] arrays.append(arr) return numpy.concatenate(arrays)
[docs]def split_by_tom(sources): """ Groups together sources with the same TOM and collect multifault sources """ def key(src): tom = getattr(src, 'temporal_occurrence_model', None) return (tom.__class__.__name__, src.code == b'F') return [srcs for _, srcs in sorted( general.groupby(sources, key).items())]
def _group_sources(trt, sources): """ Reduce identical sources, regroup by trt_smrs and TOM, then return (source_groups, reduction_count). """ key = operator.attrgetter('source_id', 'code') lst = [] red = 0 for srcs in general.groupby(sources, key).values(): if len(srcs) > 1: srcs = reduce_sources(srcs) red += 1 lst.extend(srcs) src_groups = [] for _, sources in sorted(general.groupby(lst, trt_smrs).items()): for grp in split_by_tom(sources): src_groups.append(sourceconverter.SourceGroup(trt, grp)) return src_groups, red