This commit is contained in:
Evgeny Zinoviev 2023-06-08 18:06:56 +03:00
parent 27234de929
commit 5b5c433df3
19 changed files with 389 additions and 137 deletions

View File

@ -1,5 +1,7 @@
from .config import (
Config,
ConfigUnit,
AppConfigUnit,
config,
is_development_mode,
setup_logging,
@ -7,5 +9,6 @@ from .config import (
)
from .validators import validate
from ._configs import (
LinuxBoardsConfig
LinuxBoardsConfig,
ServicesListConfig
)

View File

@ -1,5 +1,55 @@
from .config import ConfigUnit
from typing import Optional
class ServicesListConfig(ConfigUnit):
NAME = 'services_list'
@staticmethod
def schema() -> Optional[dict]:
return {
'type': 'list',
'empty': False,
'schema': {
'type': 'string'
}
}
class LinuxBoardsConfig(ConfigUnit):
NAME = 'linux_boards'
@staticmethod
def schema() -> Optional[dict]:
return {
'type': 'dict',
'schema': {
'mdns': {'type': 'string', 'required': True},
'board': {'type': 'string', 'required': True},
'network': {
'type': 'list',
'required': True,
'empty': False,
'allowed': ['wifi', 'ethernet']
},
'ram': {'type': 'integer', 'required': True},
'online': {'type': 'boolean', 'required': True},
# optional
'services': {
'type': 'list',
'empty': False,
'allowed': ServicesListConfig().get()
},
'ext_hdd': {
'type': 'list',
'schema': {
'type': 'dict',
'schema': {
'mountpoint': {'type': 'string', 'required': True},
'size': {'type': 'integer', 'required': True}
}
},
},
}
}

View File

@ -3,13 +3,21 @@ import yaml
import logging
import os
from . import validators
from os.path import join, isdir, isfile
from typing import Optional, Any, MutableMapping
from cerberus import Validator, DocumentError
from typing import Optional, Any, MutableMapping, Union
from argparse import ArgumentParser
from enum import Enum, auto
from os.path import join, isdir, isfile
from . import validators
from ..util import parse_addr
class RootSchemaType(Enum):
DEFAULT = auto()
DICT = auto()
LIST = auto()
_my_validators = {}
@ -28,7 +36,7 @@ def add_validator(name: str, f: callable):
class ConfigUnit:
NAME = 'dumb'
data: MutableMapping[str, Any]
_data: MutableMapping[str, Any]
@classmethod
def get_config_path(cls, name=None) -> str:
@ -49,10 +57,15 @@ class ConfigUnit:
if isfile(filename):
return filename
raise IOError(f'config \'{name}\' not found')
raise IOError(f'config file for \'{name}\' not found')
@staticmethod
def schema() -> Optional[dict]:
return None
def __init__(self, name=None):
self.data = {}
self._data = {}
self._logger = logging.getLogger(self.__class__.__name__)
if self.NAME != 'dumb':
self.load_from(self.get_config_path())
@ -63,26 +76,68 @@ class ConfigUnit:
def load_from(self, path: str):
if path.endswith('.toml'):
self.data = toml.load(path)
self._data = toml.load(path)
elif path.endswith('.yaml'):
with open(path, 'r') as fd:
self.data = yaml.safe_load(fd)
self._data = yaml.safe_load(fd)
def validate(self):
v = _get_validator(self.NAME)
v(self.data)
schema = self.schema()
if not schema:
self._logger.warning('validate: no schema')
return
if isinstance(self, AppConfigUnit):
schema['logging'] = {
'type': 'dict',
'schema': {
'logging': {'type': 'bool'}
}
}
rst = RootSchemaType.DEFAULT
try:
if schema['type'] == 'dict':
rst = RootSchemaType.DICT
elif schema['type'] == 'list':
rst = RootSchemaType.LIST
except KeyError:
pass
if rst == RootSchemaType.DICT:
v = Validator({'document': {
'type': 'dict',
'keysrules': {'type': 'string'},
'valuesrules': schema
}})
result = v.validate({'document': self._data})
elif rst == RootSchemaType.LIST:
v = Validator({'document': schema})
result = v.validate({'document': self._data})
else:
v = Validator(schema)
result = v.validate(self._data)
# pprint(self._data)
if not result:
# pprint(v.errors)
raise DocumentError(f'{self.__class__.__name__}: failed to validate data: {v.errors}')
def __getitem__(self, key):
return self.data[key]
return self._data[key]
def __setitem__(self, key, value):
raise NotImplementedError('overwriting config values is prohibited')
def __contains__(self, key):
return key in self.data
return key in self._data
def get(self, key: str, default=None):
cur = self.data
def get(self,
key: Optional[str] = None,
default=None):
if key is None:
return self._data
cur = self._data
pts = key.split('.')
for i in range(len(pts)):
k = pts[i]
@ -91,39 +146,82 @@ class ConfigUnit:
raise KeyError(f'key {k} not found')
else:
return cur[k] if k in cur else default
cur = self.data[k]
cur = self._data[k]
raise KeyError(f'option {key} not found')
def get_addr(self, key: str):
return parse_addr(self.get(key))
def items(self):
return self.data.items()
return self._data.items()
class AppConfigUnit(ConfigUnit):
_logging_verbose: bool
_logging_fmt: Optional[str]
_logging_file: Optional[str]
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._logging_verbose = False
self._logging_fmt = None
self._logging_file = None
def logging_set_fmt(self, fmt: str) -> None:
self._logging_fmt = fmt
def logging_get_fmt(self) -> Optional[str]:
try:
return self['logging']['default_fmt']
except KeyError:
return self._logging_fmt
def logging_set_file(self, file: str) -> None:
self._logging_file = file
def logging_get_file(self) -> Optional[str]:
try:
return self['logging']['file']
except KeyError:
return self._logging_file
def logging_set_verbose(self):
self._logging_verbose = True
def logging_is_verbose(self) -> bool:
try:
return bool(self['logging']['verbose'])
except KeyError:
return self._logging_verbose
class Config:
app_name: Optional[str]
app_config: ConfigUnit
app_config: AppConfigUnit
def __init__(self):
self.app_name = None
self.app_config = ConfigUnit()
self.app_config = AppConfigUnit()
def load_app(self,
name: Optional[str] = None,
name: Optional[Union[str, ConfigUnit, bool]] = None,
use_cli=True,
parser: ArgumentParser = None):
self.app_name = name
parser: ArgumentParser = None,
no_config=False):
if (name is None) and (not use_cli):
if isinstance(name, ConfigUnit):
self.app_name = name.NAME
self.app_config = name()
else:
self.app_name = name if isinstance(name, str) else None
if self.app_name is None and not use_cli:
raise RuntimeError('either config name must be none or use_cli must be True')
log_default_fmt = False
log_file = None
log_verbose = False
no_config = name is False
no_config = name is False or no_config
path = None
if use_cli:
if parser is None:
parser = ArgumentParser()
@ -139,25 +237,22 @@ class Config:
path = args.config
if args.verbose:
log_verbose = True
self.app_config.logging_set_verbose()
if args.log_file:
log_file = args.log_file
self.app_config.logging_set_file(args.log_file)
if args.log_default_fmt:
log_default_fmt = args.log_default_fmt
self.app_config.logging_set_fmt(args.log_default_fmt)
if not no_config and path is None:
path = ConfigUnit.get_config_path(name=name)
if not isinstance(name, ConfigUnit):
if not no_config and path is None:
path = ConfigUnit.get_config_path(name=name)
if not no_config:
self.app_config.load_from(path)
if not no_config:
self.app_config.load_from(path)
if 'logging' in self.app_config:
if not log_file and 'file' in self.app_config['logging']:
log_file = self.app_config['logging']['file']
if log_default_fmt and 'default_fmt' in self.app_config['logging']:
log_default_fmt = self.app_config['logging']['default_fmt']
setup_logging(log_verbose, log_file, log_default_fmt)
setup_logging(self.app_config.logging_is_verbose(),
self.app_config.logging_get_file(),
self.app_config.logging_get_fmt())
if use_cli:
return args
@ -174,7 +269,7 @@ def is_development_mode() -> bool:
return ('logging' in config.app_config) and ('verbose' in config.app_config['logging']) and (config.app_config['logging']['verbose'] is True)
def setup_logging(verbose=False, log_file=None, default_fmt=False):
def setup_logging(verbose=False, log_file=None, default_fmt=None):
logging_level = logging.INFO
if is_development_mode() or verbose:
logging_level = logging.DEBUG

View File

@ -1,2 +1 @@
from ._validators import *
from ._util import validate

View File

@ -1,32 +0,0 @@
from ._util import validate
__all__ = [
'linux_boards_validator'
]
def linux_boards_validator(data) -> None:
validate({
'type': 'dict',
'valuesrules': {
'type': 'dict',
'schema': {
'mdns': {'type': 'string', 'required': True},
'board': {'type': 'string', 'required': True},
'network': {'type': 'list', 'required': True, 'empty': False},
'ram': {'type': 'integer', 'required': True},
'ext_hdd': {
'type': 'list',
'schema': {
'type': 'dict',
'schema': {
'mountpoint': {'type': 'string', 'required': True},
'size': {'type': 'integer', 'required': True}
}
},
},
'services': {'type': 'list', 'empty': False},
'online': {'type': 'boolean', 'required': True}
}
}
}, data)

View File

@ -0,0 +1,13 @@
from ..config import ConfigUnit
from typing import Optional
class InverterdConfig(ConfigUnit):
NAME = 'inverterd'
@staticmethod
def schema() -> Optional[dict]:
return {
'remote_addr': {'type': 'string'},
'local_addr': {'type': 'string'},
}

View File

@ -1,5 +1,7 @@
from .mqtt import Mqtt, MqttPayload, MqttPayloadCustomField
from ._mqtt import Mqtt
from ._node import MqttNode
from ._module import MqttModule
from ._wrapper import MqttWrapper
from .util import get_modules as get_mqtt_modules
from ._config import MqttConfig, MqttCreds
from ._payload import MqttPayload, MqttPayloadCustomField
from ._util import get_modules as get_mqtt_modules

61
src/home/mqtt/_config.py Normal file
View File

@ -0,0 +1,61 @@
from ..config import ConfigUnit
from typing import Optional
from ..util import Addr
from collections import namedtuple
MqttCreds = namedtuple('MqttCreds', 'username, password')
class MqttConfig(ConfigUnit):
NAME = 'mqtt'
@staticmethod
def schema() -> Optional[dict]:
addr_schema = {
'type': 'dict',
'required': True,
'schema': {
'host': {'type': 'string', 'required': True},
'port': {'type': 'integer', 'required': True}
}
}
schema = {}
for key in ('local', 'remote'):
schema[f'{key}_addr'] = addr_schema
schema['creds'] = {
'type': 'dict',
'required': True,
'keysrules': {'type': 'string'},
'valuesrules': {
'type': 'dict',
'schema': {
'username': {'type': 'string', 'required': True},
'password': {'type': 'string', 'required': True},
}
}
}
for key in ('client', 'server'):
schema[f'default_{key}_creds'] = {'type': 'string', 'required': True}
return schema
def remote_addr(self) -> Addr:
return Addr(host=self['remote_addr']['host'],
port=self['remote_addr']['port'])
def local_addr(self) -> Addr:
return Addr(host=self['local_addr']['host'],
port=self['local_addr']['port'])
def creds_by_name(self, name: str) -> MqttCreds:
return MqttCreds(username=self['creds'][name]['username'],
password=self['creds'][name]['password'])
def creds(self) -> MqttCreds:
return self.creds_by_name(self['default_client_creds'])
def server_creds(self) -> MqttCreds:
return self.creds_by_name(self['default_server_creds'])

View File

@ -3,24 +3,24 @@ import paho.mqtt.client as mqtt
import ssl
import logging
from ..config import config
from ._payload import *
from ._config import MqttCreds, MqttConfig
from typing import Optional
def username_and_password() -> Tuple[str, str]:
username = config['mqtt']['username'] if 'username' in config['mqtt'] else None
password = config['mqtt']['password'] if 'password' in config['mqtt'] else None
return username, password
class Mqtt:
_connected: bool
_is_server: bool
_mqtt_config: MqttConfig
def __init__(self,
clean_session=True,
client_id: Optional[str] = None):
self._client = mqtt.Client(client_id=config['mqtt']['client_id'] if not client_id else client_id,
client_id='',
creds: Optional[MqttCreds] = None,
is_server=False):
if not client_id:
raise ValueError('client_id must not be empty')
self._client = mqtt.Client(client_id=client_id,
protocol=mqtt.MQTTv311,
clean_session=clean_session)
self._client.on_connect = self.on_connect
@ -30,13 +30,14 @@ class Mqtt:
self._client.on_publish = self.on_publish
self._loop_started = False
self._connected = False
self._is_server = is_server
self._mqtt_config = MqttConfig()
self._logger = logging.getLogger(self.__class__.__name__)
username, password = username_and_password()
if username and password:
self._logger.debug(f'username={username} password={password}')
self._client.username_pw_set(username, password)
if not creds:
creds = self._mqtt_config.creds() if not is_server else self._mqtt_config.server_creds()
self._client.username_pw_set(creds.username, creds.password)
def configure_tls(self):
ca_certs = os.path.realpath(os.path.join(
@ -52,10 +53,8 @@ class Mqtt:
tls_version=ssl.PROTOCOL_TLSv1_2)
def connect_and_loop(self, loop_forever=True):
host = config['mqtt']['host']
port = config['mqtt']['port']
self._client.connect(host, port, 60)
addr = self._mqtt_config.local_addr() if self._is_server else self._mqtt_config.remote_addr()
self._client.connect(addr.host, addr.port, 60)
if loop_forever:
self._client.loop_forever()
else:

View File

@ -1,6 +1,6 @@
import paho.mqtt.client as mqtt
from .mqtt import Mqtt
from ._mqtt import Mqtt
from ._node import MqttNode
from ..config import config
from ..util import strgen
@ -10,10 +10,10 @@ class MqttWrapper(Mqtt):
_nodes: list[MqttNode]
def __init__(self,
client_id: str,
topic_prefix='hk',
randomize_client_id=False,
clean_session=True):
client_id = config['mqtt']['client_id']
if randomize_client_id:
client_id += '_'+strgen(6)
super().__init__(clean_session=clean_session,

View File

@ -1,4 +1,4 @@
from ..mqtt import MqttPayload, MqttPayloadCustomField
from .._payload import MqttPayload, MqttPayloadCustomField
from .._node import MqttNode, MqttModule
from typing import Optional

View File

@ -1,7 +1,7 @@
import hashlib
from typing import Optional
from ..mqtt import MqttPayload
from .._payload import MqttPayload
from .._node import MqttModule, MqttNode
MODULE_NAME = 'MqttOtaModule'

View File

@ -1,8 +1,6 @@
# from enum import auto
from .._node import MqttNode
from .._module import MqttModule
from .._payload import MqttPayload
# from ...util import HashableEnum
from typing import Optional
from ...temphum import BaseSensor

View File

@ -7,12 +7,13 @@ import logging
import string
import random
from collections import namedtuple
from enum import Enum
from datetime import datetime
from typing import Tuple, Optional, List
from zlib import adler32
Addr = Tuple[str, int] # network address type (host, port)
Addr = namedtuple('Addr', 'host, port')
logger = logging.getLogger(__name__)

View File

@ -10,7 +10,7 @@ from html import escape
from typing import Optional, Tuple, Union
from home.util import chunks
from home.config import config
from home.config import config, AppConfigUnit
from home.telegram import bot
from home.inverter import (
wrapper_instance as inverter,
@ -24,12 +24,11 @@ from home.inverter.types import (
ACMode,
OutputSourcePriority
)
from home.database.inverter_time_formats import *
from home.database.inverter_time_formats import FormatDate
from home.api.types import BotType
from home.api import WebAPIClient
from telegram import ReplyKeyboardMarkup, InlineKeyboardMarkup, InlineKeyboardButton
monitor: Optional[InverterMonitor] = None
db = None
LT = escape('<=')
flags_map = {
@ -42,9 +41,69 @@ flags_map = {
'alarm_on_on_primary_source_interrupt': 'ALRM',
'fault_code_record': 'FTCR',
}
logger = logging.getLogger(__name__)
config.load_app('inverter_bot')
class InverterBotConfig(AppConfigUnit):
NAME = 'inverter_bot'
@staticmethod
def schema() -> Optional[dict]:
userlist_schema = {
'type': 'list',
'empty': False,
'schema': {'type': 'integer'}
}
acmode_item_schema = {
'thresholds': {
'type': 'list',
'required': True,
'schema': {
'type': 'list',
'min': 40,
'max': 60
},
},
'initial_current': {'type': 'integer'}
}
return {
'bot': {
'type': 'dict',
'required': True,
'schema': {
'token': {'type': 'string'},
'users': userlist_schema,
'notify_users': userlist_schema
}
},
'ac_mode': {
'type': 'dict',
'required': True,
'schema': {
'generator': acmode_item_schema,
'utilities': acmode_item_schema
}
},
'monitor': {
'type': 'dict',
'required': True,
'schema': {
'vlow': {'type': 'integer', 'required': True},
'vcrit': {'type': 'integer', 'required': True},
'gen_currents': {'type': 'list', 'schema': {'type': 'integer'}, 'required': True},
'gen_raise_intervals': {'type': 'list', 'schema': {'type': 'integer'}, 'required': True},
'gen_cur30_v_limit': {'type': 'float', 'required': True},
'gen_cur20_v_limit': {'type': 'float', 'required': True},
'gen_cur10_v_limit': {'type': 'float', 'required': True},
'gen_floating_v': {'type': 'integer', 'required': True},
'gen_floating_time_max': {'type': 'integer', 'required': True}
}
}
}
config.load_app(InverterBotConfig)
bot.initialize()
bot.lang.ru(
@ -863,28 +922,27 @@ class InverterStore(bot.BotDatabase):
self.commit()
if __name__ == '__main__':
inverter.init(host=config['inverter']['ip'], port=config['inverter']['port'])
inverter.init(host=config['inverter']['ip'], port=config['inverter']['port'])
bot.set_database(InverterStore())
bot.enable_logging(BotType.INVERTER)
bot.set_database(InverterStore())
bot.enable_logging(BotType.INVERTER)
bot.add_conversation(SettingsConversation(enable_back=True))
bot.add_conversation(ConsumptionConversation(enable_back=True))
bot.add_conversation(SettingsConversation(enable_back=True))
bot.add_conversation(ConsumptionConversation(enable_back=True))
monitor = InverterMonitor()
monitor.set_charging_event_handler(monitor_charging)
monitor.set_battery_event_handler(monitor_battery)
monitor.set_util_event_handler(monitor_util)
monitor.set_error_handler(monitor_error)
monitor.set_osp_need_change_callback(osp_change_cb)
monitor = InverterMonitor()
monitor.set_charging_event_handler(monitor_charging)
monitor.set_battery_event_handler(monitor_battery)
monitor.set_util_event_handler(monitor_util)
monitor.set_error_handler(monitor_error)
monitor.set_osp_need_change_callback(osp_change_cb)
setacmode(getacmode())
setacmode(getacmode())
if not config.get('monitor.disabled'):
logging.info('starting monitor')
monitor.start()
if not config.get('monitor.disabled'):
logging.info('starting monitor')
monitor.start()
bot.run()
bot.run()
monitor.stop()
monitor.stop()

View File

@ -1,6 +1,6 @@
#!/usr/bin/env python3
from argparse import ArgumentParser
from home.config import config
from home.config import config, app_config
from home.mqtt import MqttWrapper, MqttNode
@ -10,13 +10,15 @@ if __name__ == '__main__':
config.load_app('inverter_mqtt_util', parser=parser)
arg = parser.parse_args()
mode = arg.mode[0]
mqtt = MqttWrapper(clean_session=arg.mode[0] != 'receiver')
mqtt = MqttWrapper(client_id=f'inverter_mqtt_{mode}',
clean_session=arg.mode[0] != 'receiver')
node = MqttNode(node_id='inverter')
module_kwargs = {}
if arg.mode[0] == 'sender':
module_kwargs['status_poll_freq'] = int(config['mqtt']['inverter']['poll_freq'])
module_kwargs['generation_poll_freq'] = int(config['mqtt']['inverter']['generation_poll_freq'])
if mode == 'sender':
module_kwargs['status_poll_freq'] = int(app_config['poll_freq'])
module_kwargs['generation_poll_freq'] = int(app_config['generation_poll_freq'])
node.load_module('inverter', **module_kwargs)
mqtt.add_node(node)

View File

@ -23,13 +23,14 @@ if __name__ == '__main__':
parser.add_argument('--node-secret', type=str,
help='node admin password')
config.load_app('mqtt_util', parser=parser)
config.load_app(parser=parser, no_config=True)
arg = parser.parse_args()
if (arg.switch_relay is not None or arg.node_secret is not None) and 'relay' not in arg.modules:
raise ArgumentError(None, '--relay is only allowed when \'relay\' module included in --modules')
mqtt = MqttWrapper(randomize_client_id=True)
mqtt = MqttWrapper(randomize_client_id=True,
client_id='mqtt_node_util')
mqtt_node = MqttNode(node_id=arg.node_id, node_secret=arg.node_secret)
mqtt.add_node(mqtt_node)

8
src/test_new_config.py Normal file → Executable file
View File

@ -1,9 +1,11 @@
from home.config import config, app_config, LinuxBoardsConfig
#!/usr/bin/env python3
from home.config import config, LinuxBoardsConfig, ServicesListConfig
from home.mqtt import MqttConfig
from pprint import pprint
if __name__ == '__main__':
config.load_app(name=False)
lbc = LinuxBoardsConfig()
pprint(lbc.data)
c = MqttConfig()
print(c.creds())