implement usage of <PARAM param_name PARAM> and <OPTION option_name OPTION>

PARAMS will be used as wildcards, OPTIONS will only be validated and used in the configuration

README.rst

@@ -27,6 +27,10 @@ Installation
 install using 'pip install ntrfc'
 or install using 'python -m setup install'
 
+when installing from conda, you will run into issues with vtk while using a pip installation of ntrfc.
+simply run:
+    'conda install -c conda-forge vtk'
+
 as a dev use "pip install -e ntrfc" for a linked install.
 this way you dont have to reinstall the package after altering the sourcecode.
 

ntrfc/database/case_templates/openfoamCompressorCascadeRas.schema.yaml

@@ -4,49 +4,62 @@ description: case configuration file
 
 type: object
 properties:
+
     tke_inlet:
+        description: used in parameterspace
         type: number
         default: 0.1
     omega_inlet:
+        description: used in parameterspace
         type: number
         default: 0.1
     pressure_inlet:
+        description: used in parameterspace
         type: number
         default: 100000
     pressure_outlet:
+        description: used in parameterspace
         type: number
         default: 130000
     temperature_inlet:
+        description: used in parameterspace
         type: number
         default: 293.15
     velocity_inlet:
+        description: used in parameterspace
         type: string
         default: "(40 30 0)"
+    pitch:
+        description: used in parameterspace
+        type: number
+        default: 0.0765
+
+
     molarweight:
+        description: not used in parameterspace
         type: number
         default: 29
     dynvisc:
+        description: not used in parameterspace
         type: number
         default: 0.000001
     timestep:
+        description: not used in parameterspace
         type: number
         default: 0.000001
     turbulence_model:
+        description: not used in parameterspace
         type: string
         pattern: "sst_turbulence_model"
         default: "sst_turbulence_model"
     startfrom:
+        description: not used in parameterspace
         type: string
         pattern: "latestTime|startTime"
         default: "latestTime"
-    pitch:
-        type: number
-        default: 0.0765
     zspan:
+        description: not used in parameterspace
         type: number
         default: 0.0025
-    processors:
-        type: number
-        default: 1
 
 additionalProperties: false

ntrfc/database/case_templates/openfoamCompressorCascadeRas/constant/thermophysicalProperties

@@ -30,7 +30,7 @@ mixture
     specie
     {
         nMoles      1;
-        molWeight   <PARAM molarweight PARAM>; 	//Air [g/mol]
+        molWeight   <OPTION molarweight OPTION>; 	//Air [g/mol]
     }
     thermodynamics
     {
@@ -39,7 +39,7 @@ mixture
     }
     transport
     {
-        mu	    <PARAM dynvisc PARAM>;     	//dynamische viskositaet, aehnlich Luft
+        mu	    <OPTION dynvisc OPTION>;     	//dynamische viskositaet, aehnlich Luft
 	    Pr	    0.7;      	//prandtl-zahl, aehnlich Luft
     }
 }

ntrfc/database/case_templates/openfoamCompressorCascadeRas/system/controlDict

@@ -17,7 +17,7 @@ FoamFile
 
 application     rhoPimpleFoam;
 
-startFrom       <PARAM startfrom PARAM>;
+startFrom       <OPTION startfrom OPTION>;
 
 startTime       0;
 
@@ -25,7 +25,7 @@ stopAt		    endTime;
 
 endTime         0.6;
 
-deltaT          <PARAM timestep PARAM>;
+deltaT          <OPTION timestep OPTION>;
 
 writeControl    timeStep;
 

ntrfc/database/case_templates/openfoamCompressorCascadeRas/system/createPatchDict

@@ -64,7 +64,7 @@ patches
             neighbourPatch Z_upper;
 
             transform translational;
-            separationVector (0 0 <PARAM zspan PARAM>);
+            separationVector (0 0 <OPTION zspan OPTION>);
             matchTolerance 0.02;
         }
 
@@ -89,7 +89,7 @@ patches
             neighbourPatch Z_lower;
 
             transform translational;
-            separationVector (0 0 -<PARAM zspan PARAM>);
+            separationVector (0 0 -<OPTION zspan OPTION>);
             matchTolerance 0.1;
         }
 

ntrfc/database/case_templates/openfoamCompressorCascadeRas/system/decomposeParDict

@@ -16,7 +16,7 @@ FoamFile
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
 
-numberOfSubdomains <PARAM processors PARAM>;
+numberOfSubdomains <OPTION processors OPTION>;
 
 method          hierarchical;
 
@@ -27,7 +27,7 @@ simpleCoeffs
 
 hierarchicalCoeffs
 {
-    n               (<PARAM processors PARAM> 1 1);
+    n               (<OPTION processors OPTION> 1 1);
     delta           0.001;
     order           yxz;
 }

ntrfc/preprocessing/case_creation/create_case.py

@@ -4,11 +4,11 @@ import re
 import shutil
 
 from ntrfc.utils.filehandling.datafiles import inplace_change, get_directory_structure
-from ntrfc.utils.dictionaries.dict_utils import nested_dict_pairs_iterator, set_in_dict
+from ntrfc.utils.dictionaries.dict_utils import nested_dict_pairs_iterator, set_in_dict, merge
 from ntrfc.database.case_templates.case_templates import CASE_TEMPLATES
 
 
-def search_paras(case_structure, line, pair, siglim, varsignature):
+def search_paras(case_structure, line, pair, siglim, varsignature, sign):
     """
 
     """
@@ -19,17 +19,18 @@ def search_paras(case_structure, line, pair, siglim, varsignature):
         if not lookup_var:
             lookforvar = False
             filename = os.path.join(*pair[:-1])
-            assert "PARAM" not in line, f"parameter is not defined correct \n file: {filename}\n line: {line}"
+            assert sign not in line, f"parameter is not defined correct \n file: {filename}\n line: {line}"
         else:
             span = lookup_var.span()
             parameter = line[span[0] + siglim[0]:span[1] + siglim[1]]
-            set_in_dict(case_structure, list(pair[:-1]) + [parameter], "PARAM")
+            # update
+            set_in_dict(case_structure, list(pair[:-1]) + [parameter], sign)
             match = line[span[0]:span[1]]
             line = line.replace(match, "")
     return case_structure
 
 
-def check_settings_necessarities(case_structure, settings_dict):
+def settings_sanity(case_structure, settings_dict):
     """
 
     : params: case_structure dict
@@ -38,7 +39,7 @@ def check_settings_necessarities(case_structure, settings_dict):
     necessarities = list(nested_dict_pairs_iterator(case_structure))
     necessarity_vars = []
     for item in necessarities:
-        if item[-1] == "PARAM":
+        if item[-1] == "PARAM" or item[-1] == "OPTION":
             necessarity_vars.append(item[-2])
 
     defined_variables = list(settings_dict.keys())
@@ -60,12 +61,12 @@ def check_settings_necessarities(case_structure, settings_dict):
     return defined, undefined, used, unused
 
 
-def create_case(input_files, output_files, template_name, paras):
+def create_case(input_files, output_files, template_name, simparams):
     """
     :param input_files: list of template-files
     :param output_files: list of outputfiles (same as input)
     :param template_name: str - template-name
-    :param paras: dict - dict-settings
+    :param simparams: dict - dict-settings - passed via filenames
     :return:
     """
 
@@ -74,23 +75,39 @@ def create_case(input_files, output_files, template_name, paras):
     template = CASE_TEMPLATES[template_name]
     case_structure = get_directory_structure(template.path)
 
-    variables = find_vars_opts(case_structure[template.name], template.path)
+    param_sign = "PARAM"
+    option_sign = "OPTION"
+
+    parameters = find_vars_opts(case_structure[template.name], template.path, param_sign)
+    options = find_vars_opts(case_structure[template.name], template.path, option_sign)
+    case_settings = merge(parameters, options)
 
-    defined, undefined, used, unused = check_settings_necessarities(variables, paras)
+    allparams = merge(parameters,options)
+    defined, undefined, used, unused = settings_sanity(case_settings, simparams)
     print("found ", str(len(defined)), " defined parameters")
     print("found ", str(len(undefined)), " undefined parameters")
     print("used ", str(len(used)), " parameters")
     print("unused ", str(len(unused)), " parameters")
 
-    assert len(undefined) == 0, "undefined parameters"
+    assert len(undefined) == 0, f"undefined parameters: {undefined}"
+    assert len(unused) == 0, f"unused parameters: {unused}"
+
+    necessarities = list(nested_dict_pairs_iterator(case_settings))
+    paramtypes = {}
+    for item in necessarities:
+        if item[-1] == "PARAM":
+            paramtypes[item[-2]]="PARAM"
+        elif item[-1] == "OPTION":
+            paramtypes[item[-2]] = "OPTION"
 
     for templatefile, simfile in zip(input_files, output_files):
         shutil.copyfile(templatefile, simfile)
         for parameter in used:
-            inplace_change(simfile, f"<var {parameter} var>", str(paras[parameter]))
+            sign = paramtypes[parameter]
+            inplace_change(simfile, f"<{sign} {parameter} {sign}>", str(simparams[parameter]))
 
 
-def find_vars_opts(case_structure, path_to_sim):
+def find_vars_opts(case_structure, path_to_sim, sign):
     """
     : param case_structure: dict - case-structure. can carry parameters
     : param sign: str - sign of a parameter (Velocity -> U etc.)
@@ -101,12 +118,12 @@ def find_vars_opts(case_structure, path_to_sim):
     # allowing names like JOB_NUMBERS, only capital letters and underlines - no digits, no whitespaces
     datadict = copy.deepcopy(case_structure)
     all_pairs = list(nested_dict_pairs_iterator(case_structure))
-    varsignature = r"<PARAM [a-z]{3,}(_{1,1}[a-z]{3,}){,} PARAM>"
+    varsignature = r"<PLACEHOLDER [a-z]{3,}(_{1,1}[a-z]{3,}){,} PLACEHOLDER>".replace("PLACEHOLDER", sign)
     # int
     # float
     # string
     # todo move into param-module
-    siglim = (len("<PARAM "), -(len(" PARAM>")))
+    siglim = (len(f"< {sign}"), -(len(f" {sign}>")))
 
     for pair in all_pairs:
         # if os.path.isfile(os.path.join(path_to_sim,*pair)):
@@ -114,5 +131,5 @@ def find_vars_opts(case_structure, path_to_sim):
         filepath = os.path.join(*pair[:-1])
         with open(os.path.join(path_to_sim, filepath), "r") as fhandle:
             for line in fhandle.readlines():
-                datadict = search_paras(datadict, line, pair, siglim, varsignature)
+                datadict = search_paras(datadict, line, pair, siglim, varsignature, sign)
     return datadict

ntrfc/utils/snake_utils/deployment.py

@@ -3,6 +3,9 @@ import shutil
 
 
 def deploy(workflow_name, dest):
+    """
+    copies (part of) a workflow to a destination
+    """
     import ntrfc
     assert os.path.isdir(dest), "destination is not a direcotry"
     assert len(os.listdir(dest)) == 0, "destination is not empty"

requirements.txt

@@ -2,11 +2,10 @@ pip==21.3.1
 pytest==6.2.4
 snakemake==7.1.1
 matplotlib==3.3.4
-numpy==1.20.0
-scipy==1.6.1
+numpy==1.22.3
+scipy==1.8.0
 pyvista==0.33.0
 PyYaml==6.0
 pandas==1.3.3
 tqdm==4.59.0
-h5py==3.6.0
 importlib_resources==5.4.0

requirements_dev.txt

-pip=21.3.1
+pip==21.3.1
 pytest==6.2.4
 snakemake==7.1.1
 matplotlib==3.3.4
-numpy==1.20.0
-scipy==1.6.1
+numpy==1.22.3
+scipy==1.8.0
 pyvista==0.33.0
 PyYaml==6.0
 pandas==1.3.3
 tqdm==4.59.0
-h5py==3.6.0
 importlib_resources==5.4.0
-

tests/test_ntrfc_case.py

@@ -41,8 +41,8 @@ def test_findvarsopts(tmpdir):
         fhandle.write(filecontent)
     case_structure = get_directory_structure(tmpdir)
 
-    variables = find_vars_opts(case_structure, tmpdir.dirname)
-    assert (variables[tmpdir.basename][filename][paramnameone] == "PARAM" and variables[tmpdir.basename][filename][
+    parameters = find_vars_opts(case_structure, tmpdir.dirname,"PARAM")
+    assert (parameters[tmpdir.basename][filename][paramnameone] == "PARAM" and parameters[tmpdir.basename][filename][
         paramnametwo] == "PARAM"), "not all variablees were found in test-run"
 
 
@@ -67,17 +67,21 @@ def test_templates_params():
     from ntrfc.database.case_templates.case_templates import CASE_TEMPLATES
     from ntrfc.utils.filehandling.datafiles import yaml_dict_read
     from ntrfc.utils.filehandling.datafiles import get_directory_structure
-    from ntrfc.preprocessing.case_creation.create_case import find_vars_opts, check_settings_necessarities
+    from ntrfc.preprocessing.case_creation.create_case import find_vars_opts, settings_sanity
+    from ntrfc.utils.dictionaries.dict_utils import merge
+
 
     for name, template in CASE_TEMPLATES.items():
         schema = template.schema
         schema_dict = yaml_dict_read(schema)
-        default_params = {name: {key: value["default"] for (key, value) in schema_dict["properties"].items()}}
+        default_params = {key: value["default"] for (key, value) in schema_dict["properties"].items()}
         path = template.path
         tpath = os.path.join(path, "..")
         case_structure = get_directory_structure(path)
-        variables = find_vars_opts(case_structure, tpath)
-        defined, undefined, used, unused = check_settings_necessarities(variables, default_params[name])
+        parameters = find_vars_opts(case_structure, tpath, "PARAM")
+        options = find_vars_opts(case_structure, tpath,"OPTION")
+        case_settings = merge(parameters,options)
+        defined, undefined, used, unused = settings_sanity(case_settings, default_params)
         assert len(undefined) == 0, f"some parameters have no default: {undefined}"
         assert len(unused) == 0, f"some parameters are not used: {unused}"
 
@@ -99,10 +103,17 @@ def test_create_case(tmpdir):
 
     input = [f"{template.path}/{file}" for file in templatefiles]
     output = [f"{tmpdir}/{template.name}/{file}" for file in templatefiles]
-    paras = {k: v["default"] for k, v in templateschema["properties"].items()}
+
+    default_params = {key: value["default"] for (key, value) in templateschema["properties"].items()}
+
+    #sim_params = merge(default_params, default_options)
+
     os.mkdir(os.path.join(tmpdir, template.name))
+    # it is necessary to create the directory structure before creating the files.
+    # in snakemake this step can be skipped
     create_dirstructure(directories, os.path.join(tmpdir, template.name))
-    create_case(input, output, template.name, paras)
+
+    create_case(input, output, template.name, default_params)
     check = [os.path.isfile(fpath) for fpath in output]
     assert all(check), "not all files have been created"
 
@@ -129,4 +140,4 @@ def test_search_paras(tmpdir):
     all_pairs = list(nested_dict_pairs_iterator(case_structure))
     for line in filecontent:
         for pair in all_pairs:
-            search_paras(case_structure, line, pair, (len("<PARAM "), len(" PARAM>")), varsignature)
+            search_paras(case_structure, line, pair, (len("<PARAM "), len(" PARAM>")), varsignature,"PARAM")

tests/test_ntrfc_workflows.py

@@ -27,6 +27,7 @@ def test_snakefile(tmpdir: TmpDir) -> None:
         # lint=True,
         dryrun=True,
         quiet=False,
+        verbose=True,
         # log_handler=[logger.log_handler],
         ignore_ambiguity=True,
     )

workflows/case_creation/Snakefile

@@ -2,16 +2,22 @@ from snakemake.utils import validate
 from snakemake.utils import Paramspace
 import pandas as pd
 
-
 from ntrfc.database.case_templates.case_templates import CASE_TEMPLATES
 
 configfile : "casesettings.yaml"
-validate(config,"config.schema.yaml")
+
 TEMPLATE = CASE_TEMPLATES[config["case_params"]["case_type"]]
-PARAMS=pd.read_csv("caseparams.tsv", sep="\t")
-validate(PARAMS, TEMPLATE.schema)
-paramspace = Paramspace(PARAMS)
+PARAMS = pd.read_csv("caseparams.tsv",sep="\t")
+
+
+def validate_configuration(config):
+    validate(config,"config.schema.yaml")
+    # todo: the next line is not working. currently TEMPLATE.schema does not have the right format
+    validate(PARAMS, TEMPLATE.schema)
+    paramspace = Paramspace(PARAMS)
+    return paramspace, config
 
+paramspace, config = validate_configuration(config)
 
 rule all:
     input:
@@ -31,8 +37,12 @@ rule create_case:
         case_type = config["case_params"]["case_type"],
         # automatically translate the wildcard values into an instance of the param space
         # in the form of a dict (here: {"alpha": ..., "beta": ..., "gamma": ...})
-        simulation=paramspace.instance
+        simparams = paramspace.instance
     run:
         from ntrfc.preprocessing.case_creation.create_case import create_case
+        from ntrfc.utils.dictionaries.dict_utils import merge
 
-        create_case(input,output,params["case_type"],params["simulation"])
+        print(params["simparams"])
+        print(config["case_options"])
+        simconfig = merge(params["simparams"],config["case_options"])
+        create_case(input,output,params["case_type"],simconfig)

workflows/case_creation/casesettings.yaml

 case_params:
     case_type: "openfoamCompressorCascadeRas"
     paramfile: "caseparams.tsv"
+case_options:
+    processors: 12
+    zspan: 0.0025
+    startfrom: latestTime
+    timestep: 0.000001

workflows/case_creation/config.schema.yaml

@@ -5,7 +5,6 @@ description: snakemake configuration file
 type: object
 
 properties:
-  case_params:
     type: object
     properties:
       case_type: