Skip to content
Snippets Groups Projects
Commit 16ef93af authored by Malte Nyhuis's avatar Malte Nyhuis
Browse files

implement usage of <PARAM param_name PARAM> and <OPTION option_name OPTION>

PARAMS will be used as wildcards, OPTIONS will only be validated and used in the configuration
parent a9e8e30c
No related branches found
No related tags found
No related merge requests found
Showing
with 115 additions and 55 deletions
......@@ -27,6 +27,10 @@ Installation
install using 'pip install ntrfc'
or install using 'python -m setup install'
when installing from conda, you will run into issues with vtk while using a pip installation of ntrfc.
simply run:
'conda install -c conda-forge vtk'
as a dev use "pip install -e ntrfc" for a linked install.
this way you dont have to reinstall the package after altering the sourcecode.
......
......@@ -4,49 +4,62 @@ description: case configuration file
type: object
properties:
tke_inlet:
description: used in parameterspace
type: number
default: 0.1
omega_inlet:
description: used in parameterspace
type: number
default: 0.1
pressure_inlet:
description: used in parameterspace
type: number
default: 100000
pressure_outlet:
description: used in parameterspace
type: number
default: 130000
temperature_inlet:
description: used in parameterspace
type: number
default: 293.15
velocity_inlet:
description: used in parameterspace
type: string
default: "(40 30 0)"
pitch:
description: used in parameterspace
type: number
default: 0.0765
molarweight:
description: not used in parameterspace
type: number
default: 29
dynvisc:
description: not used in parameterspace
type: number
default: 0.000001
timestep:
description: not used in parameterspace
type: number
default: 0.000001
turbulence_model:
description: not used in parameterspace
type: string
pattern: "sst_turbulence_model"
default: "sst_turbulence_model"
startfrom:
description: not used in parameterspace
type: string
pattern: "latestTime|startTime"
default: "latestTime"
pitch:
type: number
default: 0.0765
zspan:
description: not used in parameterspace
type: number
default: 0.0025
processors:
type: number
default: 1
additionalProperties: false
......@@ -30,7 +30,7 @@ mixture
specie
{
nMoles 1;
molWeight <PARAM molarweight PARAM>; //Air [g/mol]
molWeight <OPTION molarweight OPTION>; //Air [g/mol]
}
thermodynamics
{
......@@ -39,7 +39,7 @@ mixture
}
transport
{
mu <PARAM dynvisc PARAM>; //dynamische viskositaet, aehnlich Luft
mu <OPTION dynvisc OPTION>; //dynamische viskositaet, aehnlich Luft
Pr 0.7; //prandtl-zahl, aehnlich Luft
}
}
......
......@@ -17,7 +17,7 @@ FoamFile
application rhoPimpleFoam;
startFrom <PARAM startfrom PARAM>;
startFrom <OPTION startfrom OPTION>;
startTime 0;
......@@ -25,7 +25,7 @@ stopAt endTime;
endTime 0.6;
deltaT <PARAM timestep PARAM>;
deltaT <OPTION timestep OPTION>;
writeControl timeStep;
......
......@@ -64,7 +64,7 @@ patches
neighbourPatch Z_upper;
transform translational;
separationVector (0 0 <PARAM zspan PARAM>);
separationVector (0 0 <OPTION zspan OPTION>);
matchTolerance 0.02;
}
......@@ -89,7 +89,7 @@ patches
neighbourPatch Z_lower;
transform translational;
separationVector (0 0 -<PARAM zspan PARAM>);
separationVector (0 0 -<OPTION zspan OPTION>);
matchTolerance 0.1;
}
......
......@@ -16,7 +16,7 @@ FoamFile
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
numberOfSubdomains <PARAM processors PARAM>;
numberOfSubdomains <OPTION processors OPTION>;
method hierarchical;
......@@ -27,7 +27,7 @@ simpleCoeffs
hierarchicalCoeffs
{
n (<PARAM processors PARAM> 1 1);
n (<OPTION processors OPTION> 1 1);
delta 0.001;
order yxz;
}
......
......@@ -4,11 +4,11 @@ import re
import shutil
from ntrfc.utils.filehandling.datafiles import inplace_change, get_directory_structure
from ntrfc.utils.dictionaries.dict_utils import nested_dict_pairs_iterator, set_in_dict
from ntrfc.utils.dictionaries.dict_utils import nested_dict_pairs_iterator, set_in_dict, merge
from ntrfc.database.case_templates.case_templates import CASE_TEMPLATES
def search_paras(case_structure, line, pair, siglim, varsignature):
def search_paras(case_structure, line, pair, siglim, varsignature, sign):
"""
"""
......@@ -19,17 +19,18 @@ def search_paras(case_structure, line, pair, siglim, varsignature):
if not lookup_var:
lookforvar = False
filename = os.path.join(*pair[:-1])
assert "PARAM" not in line, f"parameter is not defined correct \n file: {filename}\n line: {line}"
assert sign not in line, f"parameter is not defined correct \n file: {filename}\n line: {line}"
else:
span = lookup_var.span()
parameter = line[span[0] + siglim[0]:span[1] + siglim[1]]
set_in_dict(case_structure, list(pair[:-1]) + [parameter], "PARAM")
# update
set_in_dict(case_structure, list(pair[:-1]) + [parameter], sign)
match = line[span[0]:span[1]]
line = line.replace(match, "")
return case_structure
def check_settings_necessarities(case_structure, settings_dict):
def settings_sanity(case_structure, settings_dict):
"""
: params: case_structure dict
......@@ -38,7 +39,7 @@ def check_settings_necessarities(case_structure, settings_dict):
necessarities = list(nested_dict_pairs_iterator(case_structure))
necessarity_vars = []
for item in necessarities:
if item[-1] == "PARAM":
if item[-1] == "PARAM" or item[-1] == "OPTION":
necessarity_vars.append(item[-2])
defined_variables = list(settings_dict.keys())
......@@ -60,12 +61,12 @@ def check_settings_necessarities(case_structure, settings_dict):
return defined, undefined, used, unused
def create_case(input_files, output_files, template_name, paras):
def create_case(input_files, output_files, template_name, simparams):
"""
:param input_files: list of template-files
:param output_files: list of outputfiles (same as input)
:param template_name: str - template-name
:param paras: dict - dict-settings
:param simparams: dict - dict-settings - passed via filenames
:return:
"""
......@@ -74,23 +75,39 @@ def create_case(input_files, output_files, template_name, paras):
template = CASE_TEMPLATES[template_name]
case_structure = get_directory_structure(template.path)
variables = find_vars_opts(case_structure[template.name], template.path)
param_sign = "PARAM"
option_sign = "OPTION"
parameters = find_vars_opts(case_structure[template.name], template.path, param_sign)
options = find_vars_opts(case_structure[template.name], template.path, option_sign)
case_settings = merge(parameters, options)
defined, undefined, used, unused = check_settings_necessarities(variables, paras)
allparams = merge(parameters,options)
defined, undefined, used, unused = settings_sanity(case_settings, simparams)
print("found ", str(len(defined)), " defined parameters")
print("found ", str(len(undefined)), " undefined parameters")
print("used ", str(len(used)), " parameters")
print("unused ", str(len(unused)), " parameters")
assert len(undefined) == 0, "undefined parameters"
assert len(undefined) == 0, f"undefined parameters: {undefined}"
assert len(unused) == 0, f"unused parameters: {unused}"
necessarities = list(nested_dict_pairs_iterator(case_settings))
paramtypes = {}
for item in necessarities:
if item[-1] == "PARAM":
paramtypes[item[-2]]="PARAM"
elif item[-1] == "OPTION":
paramtypes[item[-2]] = "OPTION"
for templatefile, simfile in zip(input_files, output_files):
shutil.copyfile(templatefile, simfile)
for parameter in used:
inplace_change(simfile, f"<var {parameter} var>", str(paras[parameter]))
sign = paramtypes[parameter]
inplace_change(simfile, f"<{sign} {parameter} {sign}>", str(simparams[parameter]))
def find_vars_opts(case_structure, path_to_sim):
def find_vars_opts(case_structure, path_to_sim, sign):
"""
: param case_structure: dict - case-structure. can carry parameters
: param sign: str - sign of a parameter (Velocity -> U etc.)
......@@ -101,12 +118,12 @@ def find_vars_opts(case_structure, path_to_sim):
# allowing names like JOB_NUMBERS, only capital letters and underlines - no digits, no whitespaces
datadict = copy.deepcopy(case_structure)
all_pairs = list(nested_dict_pairs_iterator(case_structure))
varsignature = r"<PARAM [a-z]{3,}(_{1,1}[a-z]{3,}){,} PARAM>"
varsignature = r"<PLACEHOLDER [a-z]{3,}(_{1,1}[a-z]{3,}){,} PLACEHOLDER>".replace("PLACEHOLDER", sign)
# int
# float
# string
# todo move into param-module
siglim = (len("<PARAM "), -(len(" PARAM>")))
siglim = (len(f"< {sign}"), -(len(f" {sign}>")))
for pair in all_pairs:
# if os.path.isfile(os.path.join(path_to_sim,*pair)):
......@@ -114,5 +131,5 @@ def find_vars_opts(case_structure, path_to_sim):
filepath = os.path.join(*pair[:-1])
with open(os.path.join(path_to_sim, filepath), "r") as fhandle:
for line in fhandle.readlines():
datadict = search_paras(datadict, line, pair, siglim, varsignature)
datadict = search_paras(datadict, line, pair, siglim, varsignature, sign)
return datadict
......@@ -3,6 +3,9 @@ import shutil
def deploy(workflow_name, dest):
"""
copies (part of) a workflow to a destination
"""
import ntrfc
assert os.path.isdir(dest), "destination is not a direcotry"
assert len(os.listdir(dest)) == 0, "destination is not empty"
......
......@@ -2,11 +2,10 @@ pip==21.3.1
pytest==6.2.4
snakemake==7.1.1
matplotlib==3.3.4
numpy==1.20.0
scipy==1.6.1
numpy==1.22.3
scipy==1.8.0
pyvista==0.33.0
PyYaml==6.0
pandas==1.3.3
tqdm==4.59.0
h5py==3.6.0
importlib_resources==5.4.0
pip=21.3.1
pip==21.3.1
pytest==6.2.4
snakemake==7.1.1
matplotlib==3.3.4
numpy==1.20.0
scipy==1.6.1
numpy==1.22.3
scipy==1.8.0
pyvista==0.33.0
PyYaml==6.0
pandas==1.3.3
tqdm==4.59.0
h5py==3.6.0
importlib_resources==5.4.0
......@@ -41,8 +41,8 @@ def test_findvarsopts(tmpdir):
fhandle.write(filecontent)
case_structure = get_directory_structure(tmpdir)
variables = find_vars_opts(case_structure, tmpdir.dirname)
assert (variables[tmpdir.basename][filename][paramnameone] == "PARAM" and variables[tmpdir.basename][filename][
parameters = find_vars_opts(case_structure, tmpdir.dirname,"PARAM")
assert (parameters[tmpdir.basename][filename][paramnameone] == "PARAM" and parameters[tmpdir.basename][filename][
paramnametwo] == "PARAM"), "not all variablees were found in test-run"
......@@ -67,17 +67,21 @@ def test_templates_params():
from ntrfc.database.case_templates.case_templates import CASE_TEMPLATES
from ntrfc.utils.filehandling.datafiles import yaml_dict_read
from ntrfc.utils.filehandling.datafiles import get_directory_structure
from ntrfc.preprocessing.case_creation.create_case import find_vars_opts, check_settings_necessarities
from ntrfc.preprocessing.case_creation.create_case import find_vars_opts, settings_sanity
from ntrfc.utils.dictionaries.dict_utils import merge
for name, template in CASE_TEMPLATES.items():
schema = template.schema
schema_dict = yaml_dict_read(schema)
default_params = {name: {key: value["default"] for (key, value) in schema_dict["properties"].items()}}
default_params = {key: value["default"] for (key, value) in schema_dict["properties"].items()}
path = template.path
tpath = os.path.join(path, "..")
case_structure = get_directory_structure(path)
variables = find_vars_opts(case_structure, tpath)
defined, undefined, used, unused = check_settings_necessarities(variables, default_params[name])
parameters = find_vars_opts(case_structure, tpath, "PARAM")
options = find_vars_opts(case_structure, tpath,"OPTION")
case_settings = merge(parameters,options)
defined, undefined, used, unused = settings_sanity(case_settings, default_params)
assert len(undefined) == 0, f"some parameters have no default: {undefined}"
assert len(unused) == 0, f"some parameters are not used: {unused}"
......@@ -99,10 +103,17 @@ def test_create_case(tmpdir):
input = [f"{template.path}/{file}" for file in templatefiles]
output = [f"{tmpdir}/{template.name}/{file}" for file in templatefiles]
paras = {k: v["default"] for k, v in templateschema["properties"].items()}
default_params = {key: value["default"] for (key, value) in templateschema["properties"].items()}
#sim_params = merge(default_params, default_options)
os.mkdir(os.path.join(tmpdir, template.name))
# it is necessary to create the directory structure before creating the files.
# in snakemake this step can be skipped
create_dirstructure(directories, os.path.join(tmpdir, template.name))
create_case(input, output, template.name, paras)
create_case(input, output, template.name, default_params)
check = [os.path.isfile(fpath) for fpath in output]
assert all(check), "not all files have been created"
......@@ -129,4 +140,4 @@ def test_search_paras(tmpdir):
all_pairs = list(nested_dict_pairs_iterator(case_structure))
for line in filecontent:
for pair in all_pairs:
search_paras(case_structure, line, pair, (len("<PARAM "), len(" PARAM>")), varsignature)
search_paras(case_structure, line, pair, (len("<PARAM "), len(" PARAM>")), varsignature,"PARAM")
......@@ -27,6 +27,7 @@ def test_snakefile(tmpdir: TmpDir) -> None:
# lint=True,
dryrun=True,
quiet=False,
verbose=True,
# log_handler=[logger.log_handler],
ignore_ambiguity=True,
)
......
......@@ -2,16 +2,22 @@ from snakemake.utils import validate
from snakemake.utils import Paramspace
import pandas as pd
from ntrfc.database.case_templates.case_templates import CASE_TEMPLATES
configfile : "casesettings.yaml"
validate(config,"config.schema.yaml")
TEMPLATE = CASE_TEMPLATES[config["case_params"]["case_type"]]
PARAMS=pd.read_csv("caseparams.tsv", sep="\t")
validate(PARAMS, TEMPLATE.schema)
paramspace = Paramspace(PARAMS)
PARAMS = pd.read_csv("caseparams.tsv",sep="\t")
def validate_configuration(config):
validate(config,"config.schema.yaml")
# todo: the next line is not working. currently TEMPLATE.schema does not have the right format
validate(PARAMS, TEMPLATE.schema)
paramspace = Paramspace(PARAMS)
return paramspace, config
paramspace, config = validate_configuration(config)
rule all:
input:
......@@ -31,8 +37,12 @@ rule create_case:
case_type = config["case_params"]["case_type"],
# automatically translate the wildcard values into an instance of the param space
# in the form of a dict (here: {"alpha": ..., "beta": ..., "gamma": ...})
simulation=paramspace.instance
simparams = paramspace.instance
run:
from ntrfc.preprocessing.case_creation.create_case import create_case
from ntrfc.utils.dictionaries.dict_utils import merge
create_case(input,output,params["case_type"],params["simulation"])
print(params["simparams"])
print(config["case_options"])
simconfig = merge(params["simparams"],config["case_options"])
create_case(input,output,params["case_type"],simconfig)
case_params:
case_type: "openfoamCompressorCascadeRas"
paramfile: "caseparams.tsv"
case_options:
processors: 12
zspan: 0.0025
startfrom: latestTime
timestep: 0.000001
......@@ -5,7 +5,6 @@ description: snakemake configuration file
type: object
properties:
case_params:
type: object
properties:
case_type:
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment