# -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2014-2026 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
import logging
import os.path
import socket
import cProfile
import getpass
import numpy
from openquake.baselib import performance, general, config
from openquake.hazardlib import valid
from openquake.commonlib import logs, readinput, datastore
from openquake.hazardlib.calc.mean_rates import (
get_rmap, get_gweights, calc_mean_rates)
from openquake.hazardlib.source_reader import (
get_bset_values, modified_groups)
from openquake.calculators import base, views
from openquake.commonlib import dbapi
from openquake.engine.engine import create_jobs, run_jobs
from openquake.engine.workflow import run_workflow
from openquake.server import db
calc_path = None # set only when the flag --slowest is given
# print the mean hazard rates of the first site, without parallelization,
# pretty useful when debugging single site calculations
def _mean_rates(job_ini):
_log, dstore = datastore.create_job_dstore(
f'mean rates of {job_ini}')
with dstore:
oq = readinput.get_oqparam(job_ini)
csm = readinput.get_composite_source_model(oq, dstore)
csm.set_msparams() # needed by the multifault sources, if any
# NB: there are no workers here, so the uncertainties are applied
# here, one set of realizations at a time; this way the groups
# returned have the trt_smrs of a single set of uncertainties,
# as expected by get_rmap
src_groups = []
for grp in csm.src_groups:
bset_values = get_bset_values(csm.full_lt, grp)
src_groups.extend(
sg for _trt_smrs, sg in modified_groups(grp, bset_values))
sitecol = readinput.get_site_collection(oq)
assert len(sitecol) <= oq.max_sites_disagg, sitecol
rmap, _cmakers = get_rmap(src_groups, csm.full_lt, sitecol, oq)
gws = get_gweights(src_groups, csm.full_lt)
rates = calc_mean_rates(
rmap, gws, csm.full_lt.gsim_lt.wget, oq.imtls)
N, _M, L1 = rates.shape
mrates = numpy.zeros((N, L1), oq.imt_dt())
for m, imt in enumerate(oq.imtls):
mrates[imt] = rates[:, m]
print('Mean hazard rates for the first site')
print(views.text_table(mrates[0], ext='org'))
# called when profiling
def _run(job_ini, concurrent_tasks, pdb, loglevel, exports,
params, user_name, host=None):
global calc_path
hc_string = 'hazard_calculation_id' in params and isinstance(
params['hazard_calculation_id'], str)
if 'hazard_calculation_id' in params and not hc_string:
hc_id = int(params['hazard_calculation_id'])
if hc_id < 0: # interpret negative calculation ids
calc_ids = logs.get_calc_ids()
try:
params['hazard_calculation_id'] = calc_ids[hc_id]
except IndexError:
raise SystemExit(
'There are %d old calculations, cannot '
'retrieve the %s' % (len(calc_ids), hc_id))
else:
params['hazard_calculation_id'] = hc_id
dic = readinput.get_params(job_ini, params)
if hc_string:
dic['hazard_calculation_id'] = params['hazard_calculation_id']
# set the logs first of all
log = logs.init(dic, log_level=getattr(logging, loglevel.upper()),
user_name=user_name, host=host, pdb=pdb)
logs.dbcmd('update_job', log.calc_id,
{'status': 'executing', 'pid': os.getpid()})
with log, performance.Monitor('total runtime', measuremem=True) as monitor:
calc = base.calculators(log.get_oqparam(), log.calc_id)
calc.run(concurrent_tasks=concurrent_tasks, exports=exports)
logging.info('Total time spent: %s s', monitor.duration)
logging.info('Memory allocated: %s', general.humansize(monitor.mem))
calc_path, _ = os.path.splitext(calc.datastore.filename) # used below
return calc
[docs]def main(job_ini,
pdb=False,
calibrate=False,
mean_rates=False,
*,
slowest: int = None,
hc: valid.calculation = None,
param=None,
concurrent_tasks: int = None,
exports: valid.export_formats = '',
loglevel='info',
nodes: int = 1,
workflow_id=None,
kfilter=''):
"""
Run a set of calculations or a workflow
"""
user_name = getpass.getuser()
dbfile = os.path.expanduser(config.server.dbfile)
if not os.path.exists(dbfile):
db.actions.upgrade_db(dbapi.db)
try:
host = socket.gethostname()
except Exception: # gaierror
host = None
if param:
params = {}
for par in param:
k, v = par.split('=', 1)
params[k] = v
else:
params = {}
if hc:
if hc == -1:
hc = logs.dbcmd('get_job', -1, user_name).id
params['hazard_calculation_id'] = str(hc)
if concurrent_tasks is not None:
params['concurrent_tasks'] = str(concurrent_tasks)
if calibrate: # only the calibration of the preclassical phase
params['calculation_mode'] = 'preclassical'
params['calibration_only'] = 'true'
# calibrate ps_grid_spacing too, overriding the value in the job
params['ps_grid_spacing'] = 'auto'
if mean_rates: # print the mean hazard rates of the first site
return _mean_rates(job_ini)
if slowest:
prof = cProfile.Profile()
prof.runctx('_run(job_ini, None, pdb, loglevel, '
'exports, params, host)', globals(), locals())
pstat = calc_path + '.pstat'
prof.dump_stats(pstat)
print('Saved profiling info in %s' % pstat)
data = performance.get_pstats(pstat, slowest)
print(views.text_table(data, ['ncalls', 'cumtime', 'path'],
ext='org'))
return
if job_ini.endswith(('.ini', '.zip')):
mode = 'ini'
elif job_ini.endswith('.toml'):
mode = 'toml'
else:
raise ValueError(f'Unknown extension in{job_ini}')
if mode == 'ini':
dic = readinput.get_params(job_ini)
dic.update(params)
dic['exports'] = ','.join(exports)
if 'job_id' in dic: # in sensitivity analysis
logs.dbcmd('update_job', dic['job_id'],
{'calculation_mode': dic['calculation_mode']})
jobs = create_jobs([dic], loglevel, hc_id=hc, user_name=user_name,
host=host, pdb=pdb)
run_jobs(jobs, concurrent_jobs=1, nodes=nodes)
return jobs[0].calc_id # used in commands_test
else: # toml
if workflow_id:
params['workflow_id'] = int(workflow_id)
return run_workflow(job_ini, params, nodes=nodes, pdb=pdb,
kfilter=kfilter)
main.job_ini = dict(help='calculation configuration file')
main.pdb = dict(help='enable post mortem debugging', abbrev='-d')
main.calibrate = dict(help='run only the calibration of ps_grid_spacing '
'(set to auto) and pointsource_distance')
main.mean_rates = dict(help='print the mean hazard rates of the first site, '
'computed without parallelization')
main.slowest = dict(help='profile and show the slowest operations')
main.hc = dict(help='previous calculation ID')
main.param = dict(help='override parameters with TOML syntax', nargs='*',
action='extend')
main.concurrent_tasks = dict(help='hint for the number of tasks to spawn',
abbrev='--ct')
main.exports = dict(help='export formats as a comma-separated string')
main.loglevel = dict(help='logging level',
choices='debug info warn error critical'.split())
main.nodes=dict(help='number of worker nodes to start')
main.workflow_id = "ID of a previous workflow or description of a new workflow"
main.kfilter = "Filter workflow jobs with a regex"