Coverage for src/CSET/_common.py: 100%
119 statements
« prev ^ index » next coverage.py v7.5.4, created at 2024-07-01 08:37 +0000
1# Copyright 2022-2024 Met Office and contributors.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
15"""Common functionality used across CSET."""
17import io
18import json
19import logging
20import re
21import warnings
22from collections.abc import Iterable
23from pathlib import Path
24from typing import Union
26import ruamel.yaml
29class ArgumentError(ValueError):
30 """Provided arguments are not understood."""
33def parse_recipe(recipe_yaml: Union[Path, str], variables: dict = None):
34 """Parse a recipe into a python dictionary.
36 Parameters
37 ----------
38 recipe_yaml: Path | str
39 Path to recipe file, or the recipe YAML directly.
40 variables: dict
41 Dictionary of recipe variables. If None templating is not attempted.
43 Returns
44 -------
45 recipe: dict
46 The recipe as a python dictionary.
48 Raises
49 ------
50 ValueError
51 If the recipe is invalid. E.g. invalid YAML, missing any steps, etc.
52 TypeError
53 If recipe_yaml isn't a Path or string.
54 KeyError
55 If needed recipe variables are not supplied.
57 Examples
58 --------
59 >>> CSET._common.parse_recipe(Path("myrecipe.yaml"))
60 {'parallel': [{'operator': 'misc.noop'}]}
61 """
62 # Ensure recipe_yaml is something the YAML parser can read.
63 if isinstance(recipe_yaml, str):
64 recipe_yaml = io.StringIO(recipe_yaml)
65 elif not isinstance(recipe_yaml, Path):
66 raise TypeError("recipe_yaml must be a str or Path.")
68 # Parse the recipe YAML.
69 with ruamel.yaml.YAML(typ="safe", pure=True) as yaml:
70 try:
71 recipe = yaml.load(recipe_yaml)
72 except ruamel.yaml.parser.ParserError as err:
73 raise ValueError("ParserError: Invalid YAML") from err
75 logging.debug(recipe)
76 check_recipe_has_steps(recipe)
78 if variables is not None:
79 logging.debug("Recipe variables: %s", variables)
80 recipe = template_variables(recipe, variables)
82 return recipe
85def check_recipe_has_steps(recipe: dict):
86 """Check a recipe has the minimum required steps.
88 Checking that the recipe actually has some steps, and providing helpful
89 error messages otherwise. We must have at least a parallel step, as that
90 reads the raw data.
92 Parameters
93 ----------
94 recipe: dict
95 The recipe as a python dictionary.
97 Raises
98 ------
99 ValueError
100 If the recipe is invalid. E.g. invalid YAML, missing any steps, etc.
101 TypeError
102 If recipe isn't a dict.
103 KeyError
104 If needed recipe variables are not supplied.
105 """
106 parallel_steps_key = "parallel"
107 if not isinstance(recipe, dict):
108 raise TypeError("Recipe must contain a mapping.")
109 if "parallel" not in recipe:
110 if "steps" in recipe:
111 warnings.warn(
112 "'steps' recipe key is deprecated, use 'parallel' instead.",
113 DeprecationWarning,
114 stacklevel=3,
115 )
116 parallel_steps_key = "steps"
117 else:
118 raise ValueError("Recipe must contain a 'parallel' key.")
119 try:
120 if len(recipe[parallel_steps_key]) < 1:
121 raise ValueError("Recipe must have at least 1 parallel step.")
122 except TypeError as err:
123 raise ValueError("'parallel' key must contain a sequence of steps.") from err
126def slugify(s: str) -> str:
127 """Turn a string into a version that can be used everywhere.
129 The resultant string will only consist of a-z, 0-9, dots, dashes, and
130 underscores.
131 """
132 return re.sub(r"[^a-z0-9\._-]+", "_", s.casefold()).strip("_")
135def get_recipe_metadata() -> dict:
136 """Get the metadata of the running recipe."""
137 try:
138 with open("meta.json", "rt", encoding="UTF-8") as fp:
139 return json.load(fp)
140 except FileNotFoundError:
141 meta = {}
142 with open("meta.json", "wt", encoding="UTF-8") as fp:
143 json.dump(meta, fp)
144 return {}
147def parse_variable_options(arguments: list[str]) -> dict:
148 """Parse a list of arguments into a dictionary of variables.
150 The variable name arguments start with two hyphen-minus (`--`), consisting
151 of only capital letters (`A`-`Z`) and underscores (`_`). While the variable
152 name is restricted, the value of the variable can be any string.
154 Parameters
155 ----------
156 arguments: list[str]
157 List of arguments, e.g: `["--LEVEL", "2", "--STASH=m01s01i001"]`
159 Returns
160 -------
161 recipe_variables: dict
162 Dictionary keyed with the variable names.
164 Raises
165 ------
166 ValueError
167 If any arguments cannot be parsed.
168 """
169 recipe_variables = {}
170 i = 0
171 while i < len(arguments):
172 if re.match(r"^--[A-Z_]+=.*$", arguments[i]):
173 key, value = arguments[i].split("=", 1)
174 elif re.match(r"^--[A-Z_]+$", arguments[i]):
175 try:
176 key = arguments[i].strip("-")
177 value = arguments[i + 1]
178 except IndexError as err:
179 raise ArgumentError(f"No value for variable {arguments[i]}") from err
180 i += 1
181 else:
182 raise ArgumentError(f"Unknown argument: {arguments[i]}")
183 try:
184 recipe_variables[key.strip("-")] = json.loads(value)
185 except json.JSONDecodeError:
186 recipe_variables[key.strip("-")] = value
187 i += 1
188 return recipe_variables
191def template_variables(recipe: Union[dict, list], variables: dict) -> dict:
192 """Insert variables into recipe.
194 Parameters
195 ----------
196 recipe: dict | list
197 The recipe as a python dictionary. It is updated in-place.
198 variables: dict
199 Dictionary of variables for the recipe.
201 Returns
202 -------
203 recipe: dict
204 Filled recipe as a python dictionary.
206 Raises
207 ------
208 KeyError
209 If needed recipe variables are not supplied.
210 """
211 if isinstance(recipe, dict):
212 index = recipe.keys()
213 elif isinstance(recipe, list):
214 # We have to handle lists for when we have one inside a recipe.
215 index = range(len(recipe))
216 else:
217 raise TypeError("recipe must be a dict or list.", recipe)
219 for i in index:
220 if isinstance(recipe[i], (dict, list)):
221 recipe[i] = template_variables(recipe[i], variables)
222 elif isinstance(recipe[i], str):
223 recipe[i] = replace_template_variable(recipe[i], variables)
224 return recipe
227def replace_template_variable(s: str, variables):
228 """Fill all variable placeholders in the string."""
229 for var_name, var_value in variables.items():
230 placeholder = f"${var_name}"
231 # If the value is just the placeholder we directly overwrite it
232 # to keep the value type.
233 if s == placeholder:
234 s = var_value
235 break
236 else:
237 s = s.replace(placeholder, str(var_value))
238 if isinstance(s, str) and re.match(r"^.*\$[A-Z_].*", s):
239 raise KeyError("Variable without a value.", s)
240 return s
243################################################################################
244# Templating code taken from the simple_template package under the 0BSD licence.
245# Original at https://github.com/Fraetor/simple_template
246################################################################################
249class TemplateError(KeyError):
250 """Rendering a template failed due a placeholder without a value."""
253def render(template: str, /, **variables) -> str:
254 """Render the template with the provided variables.
256 The template should contain placeholders that will be replaced. These
257 placeholders consist of the placeholder name within double curly braces. The
258 name of the placeholder should be a valid python identifier. Whitespace
259 between the braces and the name is ignored. E.g.: `{{ placeholder_name }}`
261 An exception will be raised if there are placeholders without corresponding
262 values. It is acceptable to provide unused values; they will be ignored.
264 Parameters
265 ----------
266 template: str
267 Template to fill with variables.
269 **variables: Any
270 Keyword arguments for the placeholder values. The argument name should
271 be the same as the placeholder's name. You can unpack a dictionary of
272 value with `render(template, **my_dict)`.
274 Returns
275 -------
276 rendered_template: str
277 Filled template.
279 Raises
280 ------
281 TemplateError
282 Value not given for a placeholder in the template.
283 TypeError
284 If the template is not a string, or a variable cannot be casted to a
285 string.
287 Examples
288 --------
289 >>> template = "<p>Hello {{myplaceholder}}!</p>"
290 >>> simple_template.render(template, myplaceholder="World")
291 "<p>Hello World!</p>"
292 """
294 def isidentifier(s: str):
295 return s.isidentifier()
297 def extract_placeholders():
298 matches = re.finditer(r"{{\s*([^}]+)\s*}}", template)
299 unique_names = {match.group(1) for match in matches}
300 return filter(isidentifier, unique_names)
302 def substitute_placeholder(name):
303 try:
304 value = str(variables[name])
305 except KeyError as err:
306 raise TemplateError("Placeholder missing value", name) from err
307 pattern = r"{{\s*%s\s*}}" % re.escape(name)
308 return re.sub(pattern, value, template)
310 for name in extract_placeholders():
311 template = substitute_placeholder(name)
312 return template
315def render_file(template_path: str, /, **variables) -> str:
316 """Render a template directly from a file.
318 Otherwise the same as `simple_template.render()`.
320 Examples
321 --------
322 >>> simple_template.render_file("/path/to/template.html", myplaceholder="World")
323 "<p>Hello World!</p>"
324 """
325 with open(template_path, "rt", encoding="UTF-8") as fp:
326 template = fp.read()
327 return render(template, **variables)
330def iter_maybe(thing) -> Iterable:
331 """Ensure thing is Iterable. Strings count as atoms."""
332 if isinstance(thing, Iterable) and not isinstance(thing, str):
333 return thing
334 return (thing,)