Coverage for src/CSET/_common.py: 100%
115 statements
« prev ^ index » next coverage.py v7.5.4, created at 2024-07-01 15:05 +0000
1# Copyright 2022-2024 Met Office and contributors.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
15"""Common functionality used across CSET."""
17import io
18import json
19import logging
20import re
21from collections.abc import Iterable
22from pathlib import Path
23from typing import Union
25import ruamel.yaml
28class ArgumentError(ValueError):
29 """Provided arguments are not understood."""
32def parse_recipe(recipe_yaml: Union[Path, str], variables: dict = None):
33 """Parse a recipe into a python dictionary.
35 Parameters
36 ----------
37 recipe_yaml: Path | str
38 Path to recipe file, or the recipe YAML directly.
39 variables: dict
40 Dictionary of recipe variables. If None templating is not attempted.
42 Returns
43 -------
44 recipe: dict
45 The recipe as a python dictionary.
47 Raises
48 ------
49 ValueError
50 If the recipe is invalid. E.g. invalid YAML, missing any steps, etc.
51 TypeError
52 If recipe_yaml isn't a Path or string.
53 KeyError
54 If needed recipe variables are not supplied.
56 Examples
57 --------
58 >>> CSET._common.parse_recipe(Path("myrecipe.yaml"))
59 {'parallel': [{'operator': 'misc.noop'}]}
60 """
61 # Ensure recipe_yaml is something the YAML parser can read.
62 if isinstance(recipe_yaml, str):
63 recipe_yaml = io.StringIO(recipe_yaml)
64 elif not isinstance(recipe_yaml, Path):
65 raise TypeError("recipe_yaml must be a str or Path.")
67 # Parse the recipe YAML.
68 with ruamel.yaml.YAML(typ="safe", pure=True) as yaml:
69 try:
70 recipe = yaml.load(recipe_yaml)
71 except ruamel.yaml.parser.ParserError as err:
72 raise ValueError("ParserError: Invalid YAML") from err
74 logging.debug(recipe)
75 check_recipe_has_steps(recipe)
77 if variables is not None:
78 logging.debug("Recipe variables: %s", variables)
79 recipe = template_variables(recipe, variables)
81 return recipe
84def check_recipe_has_steps(recipe: dict):
85 """Check a recipe has the minimum required steps.
87 Checking that the recipe actually has some steps, and providing helpful
88 error messages otherwise. We must have at least a parallel step, as that
89 reads the raw data.
91 Parameters
92 ----------
93 recipe: dict
94 The recipe as a python dictionary.
96 Raises
97 ------
98 ValueError
99 If the recipe is invalid. E.g. invalid YAML, missing any steps, etc.
100 TypeError
101 If recipe isn't a dict.
102 KeyError
103 If needed recipe variables are not supplied.
104 """
105 parallel_steps_key = "parallel"
106 if not isinstance(recipe, dict):
107 raise TypeError("Recipe must contain a mapping.")
108 if "parallel" not in recipe:
109 raise ValueError("Recipe must contain a 'parallel' key.")
110 try:
111 if len(recipe[parallel_steps_key]) < 1:
112 raise ValueError("Recipe must have at least 1 parallel step.")
113 except TypeError as err:
114 raise ValueError("'parallel' key must contain a sequence of steps.") from err
117def slugify(s: str) -> str:
118 """Turn a string into a version that can be used everywhere.
120 The resultant string will only consist of a-z, 0-9, dots, dashes, and
121 underscores.
122 """
123 return re.sub(r"[^a-z0-9\._-]+", "_", s.casefold()).strip("_")
126def get_recipe_metadata() -> dict:
127 """Get the metadata of the running recipe."""
128 try:
129 with open("meta.json", "rt", encoding="UTF-8") as fp:
130 return json.load(fp)
131 except FileNotFoundError:
132 meta = {}
133 with open("meta.json", "wt", encoding="UTF-8") as fp:
134 json.dump(meta, fp)
135 return {}
138def parse_variable_options(arguments: list[str]) -> dict:
139 """Parse a list of arguments into a dictionary of variables.
141 The variable name arguments start with two hyphen-minus (`--`), consisting
142 of only capital letters (`A`-`Z`) and underscores (`_`). While the variable
143 name is restricted, the value of the variable can be any string.
145 Parameters
146 ----------
147 arguments: list[str]
148 List of arguments, e.g: `["--LEVEL", "2", "--STASH=m01s01i001"]`
150 Returns
151 -------
152 recipe_variables: dict
153 Dictionary keyed with the variable names.
155 Raises
156 ------
157 ValueError
158 If any arguments cannot be parsed.
159 """
160 recipe_variables = {}
161 i = 0
162 while i < len(arguments):
163 if re.match(r"^--[A-Z_]+=.*$", arguments[i]):
164 key, value = arguments[i].split("=", 1)
165 elif re.match(r"^--[A-Z_]+$", arguments[i]):
166 try:
167 key = arguments[i].strip("-")
168 value = arguments[i + 1]
169 except IndexError as err:
170 raise ArgumentError(f"No value for variable {arguments[i]}") from err
171 i += 1
172 else:
173 raise ArgumentError(f"Unknown argument: {arguments[i]}")
174 try:
175 recipe_variables[key.strip("-")] = json.loads(value)
176 except json.JSONDecodeError:
177 recipe_variables[key.strip("-")] = value
178 i += 1
179 return recipe_variables
182def template_variables(recipe: Union[dict, list], variables: dict) -> dict:
183 """Insert variables into recipe.
185 Parameters
186 ----------
187 recipe: dict | list
188 The recipe as a python dictionary. It is updated in-place.
189 variables: dict
190 Dictionary of variables for the recipe.
192 Returns
193 -------
194 recipe: dict
195 Filled recipe as a python dictionary.
197 Raises
198 ------
199 KeyError
200 If needed recipe variables are not supplied.
201 """
202 if isinstance(recipe, dict):
203 index = recipe.keys()
204 elif isinstance(recipe, list):
205 # We have to handle lists for when we have one inside a recipe.
206 index = range(len(recipe))
207 else:
208 raise TypeError("recipe must be a dict or list.", recipe)
210 for i in index:
211 if isinstance(recipe[i], (dict, list)):
212 recipe[i] = template_variables(recipe[i], variables)
213 elif isinstance(recipe[i], str):
214 recipe[i] = replace_template_variable(recipe[i], variables)
215 return recipe
218def replace_template_variable(s: str, variables):
219 """Fill all variable placeholders in the string."""
220 for var_name, var_value in variables.items():
221 placeholder = f"${var_name}"
222 # If the value is just the placeholder we directly overwrite it
223 # to keep the value type.
224 if s == placeholder:
225 s = var_value
226 break
227 else:
228 s = s.replace(placeholder, str(var_value))
229 if isinstance(s, str) and re.match(r"^.*\$[A-Z_].*", s):
230 raise KeyError("Variable without a value.", s)
231 return s
234################################################################################
235# Templating code taken from the simple_template package under the 0BSD licence.
236# Original at https://github.com/Fraetor/simple_template
237################################################################################
240class TemplateError(KeyError):
241 """Rendering a template failed due a placeholder without a value."""
244def render(template: str, /, **variables) -> str:
245 """Render the template with the provided variables.
247 The template should contain placeholders that will be replaced. These
248 placeholders consist of the placeholder name within double curly braces. The
249 name of the placeholder should be a valid python identifier. Whitespace
250 between the braces and the name is ignored. E.g.: `{{ placeholder_name }}`
252 An exception will be raised if there are placeholders without corresponding
253 values. It is acceptable to provide unused values; they will be ignored.
255 Parameters
256 ----------
257 template: str
258 Template to fill with variables.
260 **variables: Any
261 Keyword arguments for the placeholder values. The argument name should
262 be the same as the placeholder's name. You can unpack a dictionary of
263 value with `render(template, **my_dict)`.
265 Returns
266 -------
267 rendered_template: str
268 Filled template.
270 Raises
271 ------
272 TemplateError
273 Value not given for a placeholder in the template.
274 TypeError
275 If the template is not a string, or a variable cannot be casted to a
276 string.
278 Examples
279 --------
280 >>> template = "<p>Hello {{myplaceholder}}!</p>"
281 >>> simple_template.render(template, myplaceholder="World")
282 "<p>Hello World!</p>"
283 """
285 def isidentifier(s: str):
286 return s.isidentifier()
288 def extract_placeholders():
289 matches = re.finditer(r"{{\s*([^}]+)\s*}}", template)
290 unique_names = {match.group(1) for match in matches}
291 return filter(isidentifier, unique_names)
293 def substitute_placeholder(name):
294 try:
295 value = str(variables[name])
296 except KeyError as err:
297 raise TemplateError("Placeholder missing value", name) from err
298 pattern = r"{{\s*%s\s*}}" % re.escape(name)
299 return re.sub(pattern, value, template)
301 for name in extract_placeholders():
302 template = substitute_placeholder(name)
303 return template
306def render_file(template_path: str, /, **variables) -> str:
307 """Render a template directly from a file.
309 Otherwise the same as `simple_template.render()`.
311 Examples
312 --------
313 >>> simple_template.render_file("/path/to/template.html", myplaceholder="World")
314 "<p>Hello World!</p>"
315 """
316 with open(template_path, "rt", encoding="UTF-8") as fp:
317 template = fp.read()
318 return render(template, **variables)
321def iter_maybe(thing) -> Iterable:
322 """Ensure thing is Iterable. Strings count as atoms."""
323 if isinstance(thing, Iterable) and not isinstance(thing, str):
324 return thing
325 return (thing,)