Spaces:
Running
Running
File size: 3,837 Bytes
1b7e88c |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 |
import json
import os
import re
from copy import deepcopy
from distutils.util import strtobool
from pathlib import Path
import yaml
def build_from_file(file_path: str):
path = Path(file_path)
workers_path = path / "workers"
if not workers_path.exists():
return []
# init config dict
worker_configs = {}
other_configs = {}
file_names = {}
for file in path.rglob("*"):
if file.suffix not in [".json", ".yaml", ".yml"]:
continue
key = file.name.split(".", 1)[0]
if key in file_names:
raise Exception(
f"Duplicate file name [{key}] found:\n"
f"File 1: {file_names[key]}\n"
f"File 2: {file}"
)
file_names[key] = file
try:
with open(file, "r") as f:
if file.suffix == ".json":
content = json.load(f)
else: # .yaml or .yml
content = yaml.load(f, Loader=yaml.FullLoader)
except Exception as e:
raise Exception(f"Error loading file {file}: {str(e)}")
if workers_path in file.parents:
worker_configs[key] = content
else:
other_configs[key] = content
for conf in worker_configs.values():
prep_config(conf, other_configs, [])
worker_configs_list = []
for worker_config in worker_configs.values():
if isinstance(worker_config, list):
worker_configs_list.extend(worker_config)
else:
worker_configs_list.append(worker_config)
return worker_configs_list
def prep_config(sub_config: dict | list, config: dict, forbid_keys: list):
if isinstance(sub_config, dict):
for key, conf in sub_config.items():
if isinstance(conf, str):
if match := re.search(r"\$\{sub\|([^}]+)\}", conf):
module_key = match.group(1).strip()
if module_key not in config:
raise Exception(
"Incomplete configuration, lack module [{}]".format(
module_key
)
)
elif module_key in forbid_keys:
raise Exception(
"Can't reference submodule recursively. [{}]".format(
module_key
)
)
sub_config[key] = deepcopy(config[module_key])
prep_config(sub_config[key], config, forbid_keys + [module_key])
elif match := re.search(r"\$\{env\|([^,}]+)(?:,([^}]+))?\}", conf):
env_key = match.group(1).strip()
default_value = match.group(2)
env_value = os.getenv(env_key)
if env_value:
sub_config[key] = env_value
elif not env_value and default_value:
sub_config[key] = default_value.strip()
if sub_config[key] == "null" or sub_config[key] == "~":
sub_config[key] = None
else:
raise ValueError(
f"Environmental variable {env_key} need to be set."
)
elif isinstance(conf, dict):
prep_config(sub_config[key], config, forbid_keys)
elif isinstance(conf, list):
for i, item in enumerate(conf):
if isinstance(item, dict):
prep_config(sub_config[key][i], config, forbid_keys)
elif isinstance(sub_config, list):
for item in sub_config:
prep_config(item, config, forbid_keys)
|