merge with master

.gitignore

@@ -6,18 +6,19 @@
 config.def.h
 __pycache__
 .DS_Store
-/src/test/test_inverter_monitor.log
+/include/test/test_inverter_monitor.log
 /youtrack-certificate
 /cpp
-/src/test.py
+/test/test.py
-/esp32-cam/CameraWebServer/wifi_password.h
+/bin/test.py
+/arduino/ESP32CameraWebServer/wifi_password.h
 cmake-build-*
 .pio
 platformio.ini
 CMakeListsPrivate.txt
-/platformio/*/CMakeLists.txt
+/pio/*/CMakeLists.txt
-/platformio/*/CMakeListsPrivate.txt
+/pio/*/CMakeListsPrivate.txt
-/platformio/*/.gitignore
+/pio/*/.gitignore
 *.swp
 
 /localwebsite/vendor

README.md

@@ -5,12 +5,6 @@ a country house, solving real life tasks.
 
 Mostly undocumented.
 
-## TODO
-
-esp8266/esp32 code:
-
-- move common stuff to the `commom` directory and use it as a framework
-
 ## License
 
 BSD-3c

bin/__py_include.py

@@ -0,0 +1,9 @@
+import sys
+import os.path
+
+for _name in ('include/py',):
+    sys.path.extend([
+        os.path.realpath(
+            os.path.join(os.path.dirname(os.path.join(__file__)), '..', _name)
+        )
+    ])

bin/camera_node.py

@@ -1,12 +1,13 @@
 #!/usr/bin/env python3
 import asyncio
 import time
+import __py_include
 
-from home.config import config
+from homekit.config import config
-from home.media import MediaNodeServer, ESP32CameraRecordStorage, CameraRecorder
+from homekit.media import MediaNodeServer, ESP32CameraRecordStorage, CameraRecorder
-from home.camera import CameraType, esp32
+from homekit.camera import CameraType, esp32
-from home.util import Addr
+from homekit.util import Addr
-from home import http
+from homekit import http
 
 
 # Implements HTTP API for a camera.

bin/electricity_calc.py

@@ -3,12 +3,12 @@ import logging
 import os
 import sys
 import inspect
-import zoneinfo
+import __py_include
 
-from home.config import config  # do not remove this import!
+from homekit.config import config  # do not remove this import!
 from datetime import datetime, timedelta
 from logging import Logger
-from home.database import InverterDatabase
+from homekit.database import InverterDatabase
 from argparse import ArgumentParser, ArgumentError
 from typing import Optional
 

bin/esp32_capture.py

@@ -2,10 +2,11 @@
 import asyncio
 import logging
 import os.path
+import __py_include
 
 from argparse import ArgumentParser
-from home.camera.esp32 import WebClient
+from homekit.camera.esp32 import WebClient
-from home.util import parse_addr, Addr
+from homekit.util import Addr
 from apscheduler.schedulers.asyncio import AsyncIOScheduler
 from datetime import datetime
 from typing import Optional
@@ -50,7 +51,7 @@ if __name__ == '__main__':
 
     loop = asyncio.get_event_loop()
 
-    ESP32Capture(parse_addr(arg.addr), arg.interval, arg.output_directory)
+    ESP32Capture(Addr.fromstring(arg.addr), arg.interval, arg.output_directory)
     try:
         loop.run_forever()
     except KeyboardInterrupt:

bin/esp32cam_capture_diff_node.py

@@ -3,11 +3,12 @@ import asyncio
 import logging
 import os.path
 import tempfile
-import home.telegram.aio as telegram
+import __py_include
+import homekit.telegram.aio as telegram
 
-from home.config import config
+from homekit.config import config
-from home.camera.esp32 import WebClient
+from homekit.camera.esp32 import WebClient
-from home.util import parse_addr, send_datagram, stringify
+from homekit.util import Addr, send_datagram, stringify
 from apscheduler.schedulers.asyncio import AsyncIOScheduler
 from typing import Optional
 
@@ -34,11 +35,11 @@ async def pyssim(fn1: str, fn2: str) -> float:
 
 class ESP32CamCaptureDiffNode:
     def __init__(self):
-        self.client = WebClient(parse_addr(config['esp32cam_web_addr']))
+        self.client = WebClient(Addr.fromstring(config['esp32cam_web_addr']))
         self.directory = tempfile.gettempdir()
         self.nextpic = 1
         self.first = True
-        self.server_addr = parse_addr(config['node']['server_addr'])
+        self.server_addr = Addr.fromstring(config['node']['server_addr'])
 
         self.scheduler = AsyncIOScheduler()
         self.scheduler.add_job(self.capture, 'interval', seconds=config['node']['interval'])

bin/gpiorelayd.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+import logging
+import os
+import sys
+import __py_include
+
+from argparse import ArgumentParser
+from homekit.util import Addr
+from homekit.config import config
+from homekit.relay.sunxi_h3_server import RelayServer
+
+logger = logging.getLogger(__name__)
+
+
+if __name__ == '__main__':
+    if os.getegid() != 0:
+        sys.exit('Must be run as root.')
+
+    parser = ArgumentParser()
+    parser.add_argument('--pin', type=str, required=True,
+                        help='name of GPIO pin of Allwinner H3 sunxi board')
+    parser.add_argument('--listen', type=str, required=True,
+                        help='address to listen to, in ip:port format')
+
+    arg = config.load_app(no_config=True, parser=parser)
+    listen = Addr.fromstring(arg.listen)
+
+    try:
+        RelayServer(pinname=arg.pin, addr=listen).run()
+    except KeyboardInterrupt:
+        logger.info('Exiting...')

bin/inverter_bot.py

@@ -5,30 +5,31 @@ import datetime
 import json
 import itertools
 import sys
+import asyncio
+import __py_include
 
 from inverterd import Format, InverterError
 from html import escape
 from typing import Optional, Tuple, Union
 
-from home.util import chunks
+from homekit.util import chunks
-from home.config import config, AppConfigUnit
+from homekit.config import config, AppConfigUnit
-from home.telegram import bot
+from homekit.telegram import bot
-from home.telegram.config import TelegramBotConfig, TelegramUserListType
+from homekit.telegram.config import TelegramBotConfig, TelegramUserListType
-from home.inverter import (
+from homekit.inverter import (
     wrapper_instance as inverter,
     beautify_table,
     InverterMonitor,
 )
-from home.inverter.types import (
+from homekit.inverter.types import (
     ChargingEvent,
     ACPresentEvent,
     BatteryState,
     ACMode,
     OutputSourcePriority
 )
-from home.database.inverter_time_formats import FormatDate
+from homekit.database.inverter_time_formats import FormatDate
-from home.api.types import BotType
+from homekit.api import WebApiClient
-from home.api import WebAPIClient
 from telegram import ReplyKeyboardMarkup, InlineKeyboardMarkup, InlineKeyboardButton
 
 
@@ -55,8 +56,8 @@ logger = logging.getLogger(__name__)
 class InverterBotConfig(AppConfigUnit, TelegramBotConfig):
     NAME = 'inverter_bot'
 
-    @staticmethod
+    @classmethod
-    def schema() -> Optional[dict]:
+    def schema(cls) -> Optional[dict]:
         acmode_item_schema = {
             'thresholds': {
                 'type': 'list',
@@ -347,8 +348,11 @@ def monitor_charging(event: ChargingEvent, **kwargs) -> None:
     key = f'chrg_evt_{key}'
     if is_util:
         key = f'util_{key}'
-    bot.notify_all(
+
-        lambda lang: bot.lang.get(key, lang, *args)
+    asyncio.ensure_future(
+        bot.notify_all(
+            lambda lang: bot.lang.get(key, lang, *args)
+        )
     )
 
 
@@ -363,9 +367,11 @@ def monitor_battery(state: BatteryState, v: float, load_watts: int) -> None:
         logger.error('unknown battery state:', state)
         return
 
-    bot.notify_all(
+    asyncio.ensure_future(
-        lambda lang: bot.lang.get('battery_level_changed', lang,
+        bot.notify_all(
-                                  emoji, bot.lang.get(f'bat_state_{state.name.lower()}', lang), v, load_watts)
+            lambda lang: bot.lang.get('battery_level_changed', lang,
+                                      emoji, bot.lang.get(f'bat_state_{state.name.lower()}', lang), v, load_watts)
+        )
     )
 
 
@@ -375,14 +381,18 @@ def monitor_util(event: ACPresentEvent):
     else:
         key = 'disconnected'
     key = f'util_{key}'
-    bot.notify_all(
+    asyncio.ensure_future(
-        lambda lang: bot.lang.get(key, lang)
+        bot.notify_all(
+            lambda lang: bot.lang.get(key, lang)
+        )
     )
 
 
 def monitor_error(error: str) -> None:
-    bot.notify_all(
+    asyncio.ensure_future(
-        lambda lang: bot.lang.get('error_message', lang, error)
+        bot.notify_all(
+            lambda lang: bot.lang.get('error_message', lang, error)
+        )
     )
 
 
@@ -392,35 +402,37 @@ def osp_change_cb(new_osp: OutputSourcePriority,
 
     setosp(new_osp)
 
-    bot.notify_all(
+    asyncio.ensure_future(
-        lambda lang: bot.lang.get('osp_auto_changed_notification', lang,
+        bot.notify_all(
-                                  bot.lang.get(f'settings_osp_{new_osp.value.lower()}', lang), v, solar_input),
+            lambda lang: bot.lang.get('osp_auto_changed_notification', lang,
+                                      bot.lang.get(f'settings_osp_{new_osp.value.lower()}', lang), v, solar_input),
+        )
     )
 
 
 @bot.handler(command='status')
-def full_status(ctx: bot.Context) -> None:
+async def full_status(ctx: bot.Context) -> None:
     status = inverter.exec('get-status', format=Format.TABLE)
-    ctx.reply(beautify_table(status))
+    await ctx.reply(beautify_table(status))
 
 
 @bot.handler(command='config')
-def full_rated(ctx: bot.Context) -> None:
+async def full_rated(ctx: bot.Context) -> None:
     rated = inverter.exec('get-rated', format=Format.TABLE)
-    ctx.reply(beautify_table(rated))
+    await ctx.reply(beautify_table(rated))
 
 
 @bot.handler(command='errors')
-def full_errors(ctx: bot.Context) -> None:
+async def full_errors(ctx: bot.Context) -> None:
     errors = inverter.exec('get-errors', format=Format.TABLE)
-    ctx.reply(beautify_table(errors))
+    await ctx.reply(beautify_table(errors))
 
 
 @bot.handler(command='flags')
-def flags_handler(ctx: bot.Context) -> None:
+async def flags_handler(ctx: bot.Context) -> None:
     flags = inverter.exec('get-flags')['data']
     text, markup = build_flags_keyboard(flags, ctx)
-    ctx.reply(text, markup=markup)
+    await ctx.reply(text, markup=markup)
 
 
 def build_flags_keyboard(flags: dict, ctx: bot.Context) -> Tuple[str, InlineKeyboardMarkup]:
@@ -477,11 +489,11 @@ class SettingsConversation(bot.conversation):
     REDISCHARGE_VOLTAGES = [48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58]
 
     @bot.conventer(START, message='settings')
-    def start_enter(self, ctx: bot.Context):
+    async def start_enter(self, ctx: bot.Context):
         buttons = list(chunks(list(self.START_BUTTONS), 2))
         buttons.reverse()
-        return self.reply(ctx, self.START, ctx.lang('settings_msg'), buttons,
+        return await self.reply(ctx, self.START, ctx.lang('settings_msg'), buttons,
-                          with_cancel=True)
+                                with_cancel=True)
 
     @bot.convinput(START, messages={
         'settings_osp': OSP,
@@ -490,16 +502,16 @@ class SettingsConversation(bot.conversation):
         'settings_bat_cut_off_voltage': BAT_CUT_OFF_VOLTAGE,
         'settings_ac_max_charging_current': AC_MAX_CHARGING_CURRENT
     })
-    def start_input(self, ctx: bot.Context):
+    async def start_input(self, ctx: bot.Context):
         pass
 
     @bot.conventer(OSP)
-    def osp_enter(self, ctx: bot.Context):
+    async def osp_enter(self, ctx: bot.Context):
-        return self.reply(ctx, self.OSP, ctx.lang('settings_osp_msg'), self.OSP_BUTTONS,
+        return await self.reply(ctx, self.OSP, ctx.lang('settings_osp_msg'), self.OSP_BUTTONS,
-                          with_back=True)
+                                with_back=True)
 
     @bot.convinput(OSP, messages=OSP_BUTTONS)
-    def osp_input(self, ctx: bot.Context):
+    async def osp_input(self, ctx: bot.Context):
         selected_sp = None
         for sp in OutputSourcePriority:
             if ctx.text == ctx.lang(f'settings_osp_{sp.value.lower()}'):
@@ -512,25 +524,28 @@ class SettingsConversation(bot.conversation):
         # apply the mode
         setosp(selected_sp)
 
-        # reply to user
+        await asyncio.gather(
-        ctx.reply(ctx.lang('saved'), markup=bot.IgnoreMarkup())
+            # reply to user
+            ctx.reply(ctx.lang('saved'), markup=bot.IgnoreMarkup()),
 
-        # notify other users
+            # notify other users
-        bot.notify_all(
+            bot.notify_all(
-            lambda lang: bot.lang.get('osp_changed_notification', lang,
+                lambda lang: bot.lang.get('osp_changed_notification', lang,
-                                      ctx.user.id, ctx.user.name,
+                                          ctx.user.id, ctx.user.name,
-                                      bot.lang.get(f'settings_osp_{selected_sp.value.lower()}', lang)),
+                                          bot.lang.get(f'settings_osp_{selected_sp.value.lower()}', lang)),
-            exclude=(ctx.user_id,)
+                exclude=(ctx.user_id,)
+            )
         )
+
         return self.END
 
     @bot.conventer(AC_PRESET)
-    def acpreset_enter(self, ctx: bot.Context):
+    async def acpreset_enter(self, ctx: bot.Context):
-        return self.reply(ctx, self.AC_PRESET, ctx.lang('settings_ac_preset_msg'), self.AC_PRESET_BUTTONS,
+        return await self.reply(ctx, self.AC_PRESET, ctx.lang('settings_ac_preset_msg'), self.AC_PRESET_BUTTONS,
-                          with_back=True)
+                                with_back=True)
 
     @bot.convinput(AC_PRESET, messages=AC_PRESET_BUTTONS)
-    def acpreset_input(self, ctx: bot.Context):
+    async def acpreset_input(self, ctx: bot.Context):
         if monitor.active_current is not None:
             raise RuntimeError('generator charging program is active')
 
@@ -547,85 +562,88 @@ class SettingsConversation(bot.conversation):
         # save
         bot.db.set_param('ac_mode', str(newmode.value))
 
-        # reply to user
+        await asyncio.gather(
-        ctx.reply(ctx.lang('saved'), markup=bot.IgnoreMarkup())
+            # reply to user
+            ctx.reply(ctx.lang('saved'), markup=bot.IgnoreMarkup()),
 
-        # notify other users
+            # notify other users
-        bot.notify_all(
+            bot.notify_all(
-            lambda lang: bot.lang.get('ac_mode_changed_notification', lang,
+                lambda lang: bot.lang.get('ac_mode_changed_notification', lang,
-                                      ctx.user.id, ctx.user.name,
+                                          ctx.user.id, ctx.user.name,
-                                      bot.lang.get(str(newmode.value), lang)),
+                                          bot.lang.get(str(newmode.value), lang)),
-            exclude=(ctx.user_id,)
+                exclude=(ctx.user_id,)
+            )
         )
+
         return self.END
 
     @bot.conventer(BAT_THRESHOLDS_1)
-    def thresholds1_enter(self, ctx: bot.Context):
+    async def thresholds1_enter(self, ctx: bot.Context):
         buttons = list(map(lambda v: f'{v} V', self.RECHARGE_VOLTAGES))
         buttons = chunks(buttons, 4)
-        return self.reply(ctx, self.BAT_THRESHOLDS_1, ctx.lang('settings_select_bottom_threshold'), buttons,
+        return await self.reply(ctx, self.BAT_THRESHOLDS_1, ctx.lang('settings_select_bottom_threshold'), buttons,
-                          with_back=True, buttons_lang_completed=True)
+                                with_back=True, buttons_lang_completed=True)
 
     @bot.convinput(BAT_THRESHOLDS_1,
                    messages=list(map(lambda n: f'{n} V', RECHARGE_VOLTAGES)),
                    messages_lang_completed=True)
-    def thresholds1_input(self, ctx: bot.Context):
+    async def thresholds1_input(self, ctx: bot.Context):
         v = self._parse_voltage(ctx.text)
         ctx.user_data['bat_thrsh_v1'] = v
-        return self.invoke(self.BAT_THRESHOLDS_2, ctx)
+        return await self.invoke(self.BAT_THRESHOLDS_2, ctx)
 
     @bot.conventer(BAT_THRESHOLDS_2)
-    def thresholds2_enter(self, ctx: bot.Context):
+    async def thresholds2_enter(self, ctx: bot.Context):
         buttons = list(map(lambda v: f'{v} V', self.REDISCHARGE_VOLTAGES))
         buttons = chunks(buttons, 4)
-        return self.reply(ctx, self.BAT_THRESHOLDS_2, ctx.lang('settings_select_upper_threshold'), buttons,
+        return await self.reply(ctx, self.BAT_THRESHOLDS_2, ctx.lang('settings_select_upper_threshold'), buttons,
-                          with_back=True, buttons_lang_completed=True)
+                                with_back=True, buttons_lang_completed=True)
 
     @bot.convinput(BAT_THRESHOLDS_2,
                    messages=list(map(lambda n: f'{n} V', REDISCHARGE_VOLTAGES)),
                    messages_lang_completed=True)
-    def thresholds2_input(self, ctx: bot.Context):
+    async def thresholds2_input(self, ctx: bot.Context):
         v2 = v = self._parse_voltage(ctx.text)
         v1 = ctx.user_data['bat_thrsh_v1']
         del ctx.user_data['bat_thrsh_v1']
 
         response = inverter.exec('set-charge-thresholds', (v1, v2))
-        ctx.reply(ctx.lang('saved') if response['result'] == 'ok' else 'ERROR',
+        await ctx.reply(ctx.lang('saved') if response['result'] == 'ok' else 'ERROR',
-                  markup=bot.IgnoreMarkup())
+                        markup=bot.IgnoreMarkup())
         return self.END
 
     @bot.conventer(AC_MAX_CHARGING_CURRENT)
-    def ac_max_enter(self, ctx: bot.Context):
+    async def ac_max_enter(self, ctx: bot.Context):
         buttons = self._get_allowed_ac_charge_amps()
         buttons = map(lambda n: f'{n} A', buttons)
         buttons = [list(buttons)]
-        return self.reply(ctx, self.AC_MAX_CHARGING_CURRENT, ctx.lang('settings_select_max_current'), buttons,
+        return await self.reply(ctx, self.AC_MAX_CHARGING_CURRENT, ctx.lang('settings_select_max_current'), buttons,
-                          with_back=True, buttons_lang_completed=True)
+                                with_back=True, buttons_lang_completed=True)
 
     @bot.convinput(AC_MAX_CHARGING_CURRENT, regex=r'^\d+ A$')
-    def ac_max_input(self, ctx: bot.Context):
+    async def ac_max_input(self, ctx: bot.Context):
         a = self._parse_amps(ctx.text)
         allowed = self._get_allowed_ac_charge_amps()
         if a not in allowed:
             raise ValueError('input is not allowed')
 
         response = inverter.exec('set-max-ac-charge-current', (0, a))
-        ctx.reply(ctx.lang('saved') if response['result'] == 'ok' else 'ERROR',
+        await ctx.reply(ctx.lang('saved') if response['result'] == 'ok' else 'ERROR',
-                  markup=bot.IgnoreMarkup())
+                        markup=bot.IgnoreMarkup())
         return self.END
 
     @bot.conventer(BAT_CUT_OFF_VOLTAGE)
-    def cutoff_enter(self, ctx: bot.Context):
+    async def cutoff_enter(self, ctx: bot.Context):
-        return self.reply(ctx, self.BAT_CUT_OFF_VOLTAGE, ctx.lang('settings_enter_cutoff_voltage'), None,
+        return await self.reply(ctx, self.BAT_CUT_OFF_VOLTAGE, ctx.lang('settings_enter_cutoff_voltage'), None,
-                          with_back=True)
+                                with_back=True)
 
     @bot.convinput(BAT_CUT_OFF_VOLTAGE, regex=r'^(\d{2}(\.\d{1})?)$')
-    def cutoff_input(self, ctx: bot.Context):
+    async def cutoff_input(self, ctx: bot.Context):
         v = float(ctx.text)
         if 40.0 <= v <= 48.0:
             response = inverter.exec('set-battery-cutoff-voltage', (v,))
-            ctx.reply(ctx.lang('saved') if response['result'] == 'ok' else 'ERROR',
+            await ctx.reply(ctx.lang('saved') if response['result'] == 'ok' else 'ERROR',
-                      markup=bot.IgnoreMarkup())
+                            markup=bot.IgnoreMarkup())
         else:
             raise ValueError('invalid voltage')
 
@@ -660,38 +678,38 @@ class ConsumptionConversation(bot.conversation):
     INTERVAL_BUTTONS_FLAT = list(itertools.chain.from_iterable(INTERVAL_BUTTONS))
 
     @bot.conventer(START, message='consumption')
-    def start_enter(self, ctx: bot.Context):
+    async def start_enter(self, ctx: bot.Context):
-        return self.reply(ctx, self.START, ctx.lang('consumption_msg'), [self.START_BUTTONS],
+        return await self.reply(ctx, self.START, ctx.lang('consumption_msg'), [self.START_BUTTONS],
-                          with_cancel=True)
+                                with_cancel=True)
 
     @bot.convinput(START, messages={
         'consumption_total': TOTAL,
         'consumption_grid': GRID
     })
-    def start_input(self, ctx: bot.Context):
+    async def start_input(self, ctx: bot.Context):
         pass
 
     @bot.conventer(TOTAL)
-    def total_enter(self, ctx: bot.Context):
+    async def total_enter(self, ctx: bot.Context):
-        return self._render_interval_btns(ctx, self.TOTAL)
+        return await self._render_interval_btns(ctx, self.TOTAL)
 
     @bot.conventer(GRID)
-    def grid_enter(self, ctx: bot.Context):
+    async def grid_enter(self, ctx: bot.Context):
-        return self._render_interval_btns(ctx, self.GRID)
+        return await self._render_interval_btns(ctx, self.GRID)
 
-    def _render_interval_btns(self, ctx: bot.Context, state):
+    async def _render_interval_btns(self, ctx: bot.Context, state):
-        return self.reply(ctx, state, ctx.lang('consumption_select_interval'), self.INTERVAL_BUTTONS,
+        return await self.reply(ctx, state, ctx.lang('consumption_select_interval'), self.INTERVAL_BUTTONS,
-                          with_back=True)
+                                with_back=True)
 
     @bot.convinput(TOTAL, messages=INTERVAL_BUTTONS_FLAT)
-    def total_input(self, ctx: bot.Context):
+    async def total_input(self, ctx: bot.Context):
-        return self._render_interval_results(ctx, self.TOTAL)
+        return await self._render_interval_results(ctx, self.TOTAL)
 
     @bot.convinput(GRID, messages=INTERVAL_BUTTONS_FLAT)
-    def grid_input(self, ctx: bot.Context):
+    async def grid_input(self, ctx: bot.Context):
-        return self._render_interval_results(ctx, self.GRID)
+        return await self._render_interval_results(ctx, self.GRID)
 
-    def _render_interval_results(self, ctx: bot.Context, state):
+    async def _render_interval_results(self, ctx: bot.Context, state):
         # if ctx.text == ctx.lang('to_select_interval'):
         #     TODO
         # pass
@@ -715,41 +733,43 @@ class ConsumptionConversation(bot.conversation):
         #     [InlineKeyboardButton(ctx.lang('please_wait'), callback_data='wait')]
         # ])
 
-        message = ctx.reply(ctx.lang('consumption_request_sent'),
+        message = await ctx.reply(ctx.lang('consumption_request_sent'),
-                  markup=bot.IgnoreMarkup())
+                                  markup=bot.IgnoreMarkup())
 
-        api = WebAPIClient(timeout=60)
+        api = WebApiClient(timeout=60)
         method = 'inverter_get_consumed_energy' if state == self.TOTAL else 'inverter_get_grid_consumed_energy'
 
         try:
             wh = getattr(api, method)(s_from, s_to)
-            bot.delete_message(message.chat_id, message.message_id)
+            await bot.delete_message(message.chat_id, message.message_id)
-            ctx.reply('%.2f Wh' % (wh,),
+            await ctx.reply('%.2f Wh' % (wh,),
-                      markup=bot.IgnoreMarkup())
+                            markup=bot.IgnoreMarkup())
             return self.END
         except Exception as e:
-            bot.delete_message(message.chat_id, message.message_id)
+            await asyncio.gather(
-            ctx.reply_exc(e)
+                bot.delete_message(message.chat_id, message.message_id),
+                ctx.reply_exc(e)
+            )
 
 # other
 # -----
 
 @bot.handler(command='monstatus')
-def monstatus_handler(ctx: bot.Context) -> None:
+async def monstatus_handler(ctx: bot.Context) -> None:
     msg = ''
     st = monitor.dump_status()
     for k, v in st.items():
         msg += k + ': ' + str(v) + '\n'
-    ctx.reply(msg)
+    await ctx.reply(msg)
 
 
 @bot.handler(command='monsetcur')
-def monsetcur_handler(ctx: bot.Context) -> None:
+async def monsetcur_handler(ctx: bot.Context) -> None:
-    ctx.reply('not implemented yet')
+    await ctx.reply('not implemented yet')
 
 
 @bot.callbackhandler
-def button_callback(ctx: bot.Context) -> None:
+async def button_callback(ctx: bot.Context) -> None:
     query = ctx.callback_query
 
     if query.data.startswith('flag_'):
@@ -762,7 +782,7 @@ def button_callback(ctx: bot.Context) -> None:
                 json_key = k
                 break
         if not found:
-            query.answer(ctx.lang('flags_invalid'))
+            await query.answer(ctx.lang('flags_invalid'))
             return
 
         flags = inverter.exec('get-flags')['data']
@@ -773,32 +793,31 @@ def button_callback(ctx: bot.Context) -> None:
         response = inverter.exec('set-flag', (flag, target_flag_value))
 
         # notify user
-        query.answer(ctx.lang('done') if response['result'] == 'ok' else ctx.lang('flags_fail'))
+        await query.answer(ctx.lang('done') if response['result'] == 'ok' else ctx.lang('flags_fail'))
 
         # edit message
         flags[json_key] = not cur_flag_value
         text, markup = build_flags_keyboard(flags, ctx)
-        query.edit_message_text(text, reply_markup=markup)
+        await query.edit_message_text(text, reply_markup=markup)
 
     else:
-        query.answer(ctx.lang('unexpected_callback_data'))
+        await query.answer(ctx.lang('unexpected_callback_data'))
 
 
 @bot.exceptionhandler
-def exception_handler(e: Exception, ctx: bot.Context) -> Optional[bool]:
+async def exception_handler(e: Exception, ctx: bot.Context) -> Optional[bool]:
     if isinstance(e, InverterError):
         try:
             err = json.loads(str(e))['message']
         except json.decoder.JSONDecodeError:
             err = str(e)
         err = re.sub(r'((?:.*)?error:) (.*)', r'<b>\1</b> \2', err)
-        ctx.reply(err,
+        await ctx.reply(err, markup=bot.IgnoreMarkup())
-                  markup=bot.IgnoreMarkup())
         return True
 
 
 @bot.handler(message='status')
-def status_handler(ctx: bot.Context) -> None:
+async def status_handler(ctx: bot.Context) -> None:
     gs = inverter.exec('get-status')['data']
     rated = inverter.exec('get-rated')['data']
 
@@ -842,11 +861,11 @@ def status_handler(ctx: bot.Context) -> None:
     html += f'\n<b>{ctx.lang("priority")}</b>: {rated["output_source_priority"]}'
 
     # send response
-    ctx.reply(html)
+    await ctx.reply(html)
 
 
 @bot.handler(message='generation')
-def generation_handler(ctx: bot.Context) -> None:
+async def generation_handler(ctx: bot.Context) -> None:
     today = datetime.date.today()
     yday = today - datetime.timedelta(days=1)
     yday2 = today - datetime.timedelta(days=2)
@@ -876,7 +895,7 @@ def generation_handler(ctx: bot.Context) -> None:
         html += f'\n<b>{ctx.lang("yday2")}:</b> %s Wh' % (gen_yday2['wh'])
 
     # send response
-    ctx.reply(html)
+    await ctx.reply(html)
 
 
 @bot.defaultreplymarkup
@@ -920,7 +939,7 @@ class InverterStore(bot.BotDatabase):
 inverter.init(host=config['inverter']['ip'], port=config['inverter']['port'])
 
 bot.set_database(InverterStore())
-bot.enable_logging(BotType.INVERTER)
+#bot.enable_logging(BotType.INVERTER)
 
 bot.add_conversation(SettingsConversation(enable_back=True))
 bot.add_conversation(ConsumptionConversation(enable_back=True))

bin/inverter_mqtt_util.py

@@ -1,7 +1,9 @@
 #!/usr/bin/env python3
+import __py_include
+
 from argparse import ArgumentParser
-from home.config import config, app_config
+from homekit.config import config
-from home.mqtt import MqttWrapper, MqttNode
+from homekit.mqtt import MqttWrapper, MqttNode
 
 
 if __name__ == '__main__':
@@ -17,8 +19,8 @@ if __name__ == '__main__':
     node = MqttNode(node_id='inverter')
     module_kwargs = {}
     if mode == 'sender':
-        module_kwargs['status_poll_freq'] = int(app_config['poll_freq'])
+        module_kwargs['status_poll_freq'] = int(config.app_config['poll_freq'])
-        module_kwargs['generation_poll_freq'] = int(app_config['generation_poll_freq'])
+        module_kwargs['generation_poll_freq'] = int(config.app_config['generation_poll_freq'])
     node.load_module('inverter', **module_kwargs)
     mqtt.add_node(node)
 

bin/inverterd_emulator.py

@@ -1,7 +1,8 @@
 #!/usr/bin/env python3
 import logging
+import __py_include
 
-from home.inverter.emulator import InverterEmulator
+from homekit.inverter.emulator import InverterEmulator
 
 
 if __name__ == '__main__':

bin/ipcam_capture.py

@@ -0,0 +1,142 @@
+#!/usr/bin/env python3
+import __py_include
+import sys
+import os
+import subprocess
+import asyncio
+import signal
+
+from typing import TextIO
+from argparse import ArgumentParser
+from socket import gethostname
+from asyncio.streams import StreamReader
+from homekit.config import LinuxBoardsConfig, config as homekit_config
+from homekit.camera import IpcamConfig, CaptureType
+from homekit.camera.util import get_hls_directory, get_hls_channel_name, get_recordings_path
+
+ipcam_config = IpcamConfig()
+lbc_config = LinuxBoardsConfig()
+channels = (1, 2)
+tasks = []
+restart_delay = 3
+lock = asyncio.Lock()
+worker_type: CaptureType
+
+
+async def read_output(stream: StreamReader,
+                      thread_name: str,
+                      output: TextIO):
+    try:
+        while True:
+            line = await stream.readline()
+            if not line:
+                break
+            print(f"[{thread_name}] {line.decode().strip()}", file=output)
+
+    except asyncio.LimitOverrunError:
+        print(f"[{thread_name}] Output limit exceeded.", file=output)
+
+    except Exception as e:
+        print(f"[{thread_name}] Error occurred while reading output: {e}", file=sys.stderr)
+
+
+async def run_ffmpeg(cam: int, channel: int):
+    prefix = get_hls_channel_name(cam, channel)
+
+    if homekit_config.app_config.logging_is_verbose():
+        debug_args = ['-v', '-info']
+    else:
+        debug_args = ['-nostats', '-loglevel', 'error']
+
+    # protocol = 'tcp' if ipcam_config.should_use_tcp_for_rtsp(cam) else 'udp'
+    protocol = 'tcp'
+    user, pw = ipcam_config.get_rtsp_creds()
+    ip = ipcam_config.get_camera_ip(cam)
+    path = ipcam_config.get_camera_type(cam).get_channel_url(channel)
+    ext = ipcam_config.get_camera_container(cam)
+    ffmpeg_command = ['ffmpeg', *debug_args,
+                      '-rtsp_transport', protocol,
+                      '-i', f'rtsp://{user}:{pw}@{ip}:554{path}',
+                      '-c', 'copy',]
+
+    if worker_type == CaptureType.HLS:
+        ffmpeg_command.extend(['-bufsize', '1835k',
+                               '-pix_fmt', 'yuv420p',
+                               '-flags', '-global_header',
+                               '-hls_time', '2',
+                               '-hls_list_size', '3',
+                               '-hls_flags', 'delete_segments',
+                               os.path.join(get_hls_directory(cam, channel), 'live.m3u8')])
+
+    elif worker_type == CaptureType.RECORD:
+        ffmpeg_command.extend(['-f', 'segment',
+                               '-strftime', '1',
+                               '-segment_time', '00:10:00',
+                               '-segment_atclocktime', '1',
+                               os.path.join(get_recordings_path(cam), f'record_%Y-%m-%d-%H.%M.%S.{ext.value}')])
+
+    else:
+        raise ValueError(f'invalid worker type: {worker_type}')
+
+    while True:
+        try:
+            process = await asyncio.create_subprocess_exec(
+                *ffmpeg_command,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE
+            )
+
+            stdout_task = asyncio.create_task(read_output(process.stdout, prefix, sys.stdout))
+            stderr_task = asyncio.create_task(read_output(process.stderr, prefix, sys.stderr))
+
+            await asyncio.gather(stdout_task, stderr_task)
+
+            # check the return code of the process
+            if process.returncode != 0:
+                raise subprocess.CalledProcessError(process.returncode, ffmpeg_command)
+
+        except (FileNotFoundError, PermissionError, subprocess.CalledProcessError) as e:
+            # an error occurred, print the error message
+            error_message = f"Error occurred in {prefix}: {e}"
+            print(error_message, file=sys.stderr)
+
+        # sleep for 5 seconds before restarting the process
+        await asyncio.sleep(restart_delay)
+
+
+async def run():
+    kwargs = {}
+    if worker_type == CaptureType.RECORD:
+        kwargs['filter_by_server'] = gethostname()
+    for cam in ipcam_config.get_all_cam_names(**kwargs):
+        for channel in channels:
+            task = asyncio.create_task(run_ffmpeg(cam, channel))
+            tasks.append(task)
+
+    try:
+        await asyncio.gather(*tasks)
+    except KeyboardInterrupt:
+        print('KeyboardInterrupt: stopping processes...', file=sys.stderr)
+        for task in tasks:
+            task.cancel()
+
+        # wait for subprocesses to terminate
+        await asyncio.gather(*tasks, return_exceptions=True)
+
+        # send termination signal to all subprocesses
+        for task in tasks:
+            process = task.get_stack()
+            if process:
+                process.send_signal(signal.SIGTERM)
+
+
+if __name__ == '__main__':
+    capture_types = [t.value for t in CaptureType]
+    parser = ArgumentParser()
+    parser.add_argument('type', type=str, metavar='CAPTURE_TYPE', choices=tuple(capture_types),
+                        help='capture type (variants: '+', '.join(capture_types)+')')
+
+    arg = homekit_config.load_app(no_config=True, parser=parser)
+    worker_type = CaptureType(arg['type'])
+
+    asyncio.run(run())

bin/ipcam_motion_worker.sh

@@ -5,7 +5,7 @@ set -e
 DIR="$( cd "$( dirname "$(realpath "${BASH_SOURCE[0]}")" )" &>/dev/null && pwd )"
 PROGNAME="$0"
 
-. "$DIR/lib.bash"
+. "$DIR/../include/bash/include.bash"
 
 curl_opts="-s --connect-timeout 10 --retry 5 --max-time 180 --retry-delay 0 --retry-max-time 180"
 allow_multiple=

bin/ipcam_ntp_util.py

@@ -0,0 +1,199 @@
+#!/usr/bin/env python3
+import __py_include
+import requests
+import hashlib
+import xml.etree.ElementTree as ET
+
+from time import time
+from argparse import ArgumentParser, ArgumentError
+from homekit.util import validate_ipv4, validate_ipv4_or_hostname
+from homekit.camera import IpcamConfig
+
+
+def xml_to_dict(xml_data: str) -> dict:
+    # Parse the XML data
+    root = ET.fromstring(xml_data)
+
+    # Function to remove namespace from the tag name
+    def remove_namespace(tag):
+        return tag.split('}')[-1]  # Splits on '}' and returns the last part, the actual tag name without namespace
+
+    # Function to recursively convert XML elements to a dictionary
+    def elem_to_dict(elem):
+        tag = remove_namespace(elem.tag)
+        elem_dict = {tag: {}}
+
+        # If the element has attributes, add them to the dictionary
+        elem_dict[tag].update({'@' + remove_namespace(k): v for k, v in elem.attrib.items()})
+
+        # Handle the element's text content, if present and not just whitespace
+        text = elem.text.strip() if elem.text and elem.text.strip() else None
+        if text:
+            elem_dict[tag]['#text'] = text
+
+        # Process child elements
+        for child in elem:
+            child_dict = elem_to_dict(child)
+            child_tag = remove_namespace(child.tag)
+            if child_tag not in elem_dict[tag]:
+                elem_dict[tag][child_tag] = []
+            elem_dict[tag][child_tag].append(child_dict[child_tag])
+
+        # Simplify structure if there's only text or no children and no attributes
+        if len(elem_dict[tag]) == 1 and '#text' in elem_dict[tag]:
+            return {tag: elem_dict[tag]['#text']}
+        elif not elem_dict[tag]:
+            return {tag: ''}
+
+        return elem_dict
+
+    # Convert the root element to dictionary
+    return elem_to_dict(root)
+
+
+def sha256_hex(input_string: str) -> str:
+    return hashlib.sha256(input_string.encode()).hexdigest()
+
+
+class ResponseError(RuntimeError):
+    pass
+
+
+class AuthError(ResponseError):
+    pass
+
+
+class HikvisionISAPIClient:
+    def __init__(self, host):
+        self.host = host
+        self.cookies = {}
+
+    def auth(self, username: str, password: str):
+        r = requests.get(self.isapi_uri('Security/sessionLogin/capabilities'),
+                         {'username': username},
+                         headers={
+                             'X-Requested-With': 'XMLHttpRequest',
+                         })
+        r.raise_for_status()
+        caps = xml_to_dict(r.text)['SessionLoginCap']
+        is_irreversible = caps['isIrreversible'][0].lower() == 'true'
+
+        # https://github.com/JakeVincet/nvt/blob/master/2018/hikvision/gb_hikvision_ip_camera_default_credentials.nasl
+        # also look into webAuth.js and utils.js
+
+        if 'salt' in caps and is_irreversible:
+            p = sha256_hex(username + caps['salt'][0] + password)
+            p = sha256_hex(p + caps['challenge'][0])
+            for i in range(int(caps['iterations'][0])-2):
+                p = sha256_hex(p)
+        else:
+            p = sha256_hex(password) + caps['challenge'][0]
+            for i in range(int(caps['iterations'][0])-1):
+                p = sha256_hex(p)
+
+        data = '<SessionLogin>'
+        data += f'<userName>{username}</userName>'
+        data += f'<password>{p}</password>'
+        data += f'<sessionID>{caps["sessionID"][0]}</sessionID>'
+        data += '<isSessionIDValidLongTerm>false</isSessionIDValidLongTerm>'
+        data += f'<sessionIDVersion>{caps["sessionIDVersion"][0]}</sessionIDVersion>'
+        data += '</SessionLogin>'
+
+        r = requests.post(self.isapi_uri(f'Security/sessionLogin?timeStamp={int(time())}'), data=data, headers={
+            'Accept-Encoding': 'gzip, deflate',
+            'If-Modified-Since': '0',
+            'X-Requested-With': 'XMLHttpRequest',
+            'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
+        })
+        r.raise_for_status()
+        resp = xml_to_dict(r.text)['SessionLogin']
+        status_value = int(resp['statusValue'][0])
+        status_string = resp['statusString'][0]
+        if status_value != 200:
+            raise AuthError(f'{status_value}: {status_string}')
+
+        self.cookies = r.cookies.get_dict()
+
+    def get_ntp_server(self) -> str:
+        r = requests.get(self.isapi_uri('System/time/ntpServers/capabilities'), cookies=self.cookies)
+        r.raise_for_status()
+        ntp_server = xml_to_dict(r.text)['NTPServerList']['NTPServer'][0]
+
+        if ntp_server['addressingFormatType'][0]['#text'] == 'hostname':
+            ntp_host = ntp_server['hostName'][0]
+        else:
+            ntp_host = ntp_server['ipAddress'][0]
+
+        return ntp_host
+
+    def set_timezone(self):
+        data = '<?xml version="1.0" encoding="UTF-8"?>'
+        data += '<Time><timeMode>NTP</timeMode><timeZone>CST-3:00:00</timeZone></Time>'
+
+        r = requests.put(self.isapi_uri('System/time'), cookies=self.cookies, data=data, headers={
+            'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8'
+        })
+        self.isapi_check_put_response(r)
+
+    def set_ntp_server(self, ntp_host: str, ntp_port: int = 123):
+        format = 'ipaddress' if validate_ipv4(ntp_host) else 'hostname'
+
+        data = '<?xml version="1.0" encoding="UTF-8"?>'
+        data += f'<NTPServer><id>1</id><addressingFormatType>{format}</addressingFormatType><ipAddress>{ntp_host}</ipAddress><portNo>{ntp_port}</portNo><synchronizeInterval>1440</synchronizeInterval></NTPServer>'
+
+        r = requests.put(self.isapi_uri('System/time/ntpServers/1'),
+                         data=data,
+                         cookies=self.cookies,
+                         headers={
+                            'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8'
+                         })
+        self.isapi_check_put_response(r)
+
+    def isapi_uri(self, path: str) -> str:
+        return f'http://{self.host}/ISAPI/{path}'
+
+    def isapi_check_put_response(self, r):
+        r.raise_for_status()
+        resp = xml_to_dict(r.text)['ResponseStatus']
+
+        status_code = int(resp['statusCode'][0])
+        status_string = resp['statusString'][0]
+
+        if status_code != 1 or status_string.lower() != 'ok':
+            raise ResponseError('response status looks bad')
+
+
+def main():
+    parser = ArgumentParser()
+    parser.add_argument('--host', type=str, required=True)
+    parser.add_argument('--get-ntp-server', action='store_true')
+    parser.add_argument('--set-ntp-server', type=str)
+    parser.add_argument('--username', type=str)
+    parser.add_argument('--password', type=str)
+    args = parser.parse_args()
+
+    if not args.get_ntp_server and not args.set_ntp_server:
+        raise ArgumentError(None, 'either --get-ntp-server or --set-ntp-server is required')
+
+    ipcam_config = IpcamConfig()
+    login = args.username if args.username else ipcam_config['web_creds']['login']
+    password = args.password if args.password else ipcam_config['web_creds']['password']
+
+    client = HikvisionISAPIClient(args.host)
+    client.auth(args.username, args.password)
+
+    if args.get_ntp_server:
+        print(client.get_ntp_server())
+        return
+
+    if not args.set_ntp_server:
+        raise ArgumentError(None, '--set-ntp-server is required')
+
+    if not validate_ipv4_or_hostname(args.set_ntp_server):
+        raise ArgumentError(None, 'input ntp server is neither ip address nor a valid hostname')
+
+    client.set_ntp_server(args.set_ntp_server)
+
+
+if __name__ == '__main__':
+    main()

bin/ipcam_server.py

@@ -1,58 +1,52 @@
 #!/usr/bin/env python3
 import logging
 import os
-import re
 import asyncio
 import time
 import shutil
-import home.telegram.aio as telegram
+import __py_include
 
+import homekit.telegram.aio as telegram
+
+from socket import gethostname
+from argparse import ArgumentParser
 from apscheduler.schedulers.asyncio import AsyncIOScheduler
 from asyncio import Lock
 
-from home.config import config
+from homekit.config import config as homekit_config, LinuxBoardsConfig
-from home import http
+from homekit.util import Addr
-from home.database.sqlite import SQLiteBase
+from homekit import http
-from home.camera import util as camutil
+from homekit.database.sqlite import SQLiteBase
+from homekit.camera import util as camutil, IpcamConfig
+from homekit.camera.types import (
+    TimeFilterType,
+    TelegramLinkType,
+    VideoContainerType
+)
+from homekit.camera.util import (
+    get_recordings_path,
+    get_motion_path,
+    is_valid_recording_name,
+    datetime_from_filename
+)
 
-from enum import Enum
 from typing import Optional, Union, List, Tuple
 from datetime import datetime, timedelta
 from functools import cmp_to_key
 
 
-class TimeFilterType(Enum):
+ipcam_config = IpcamConfig()
-    FIX = 'fix'
+lbc_config = LinuxBoardsConfig()
-    MOTION = 'motion'
-    MOTION_START = 'motion_start'
-
-
-class TelegramLinkType(Enum):
-    FRAGMENT = 'fragment'
-    ORIGINAL_FILE = 'original_file'
-
-
-def valid_recording_name(filename: str) -> bool:
-    return filename.startswith('record_') and filename.endswith('.mp4')
-
-
-def filename_to_datetime(filename: str) -> datetime:
-    filename = os.path.basename(filename).replace('record_', '').replace('.mp4', '')
-    return datetime.strptime(filename, datetime_format)
-
-
-def get_all_cams() -> list:
-    return [cam for cam in config['camera'].keys()]
 
 
 # ipcam database
 # --------------
 
-class IPCamServerDatabase(SQLiteBase):
+class IpcamServerDatabase(SQLiteBase):
     SCHEMA = 4
 
-    def __init__(self):
+    def __init__(self, path=None):
-        super().__init__()
+        super().__init__(path=path)
 
     def schema_init(self, version: int) -> None:
         cursor = self.cursor()
@@ -64,7 +58,7 @@ class IPCamServerDatabase(SQLiteBase):
                 fix_time INTEGER NOT NULL,
                 motion_time INTEGER NOT NULL
             )""")
-            for cam in config['camera'].keys():
+            for cam in ipcam_config.get_all_cam_names_for_this_server():
                 self.add_camera(cam)
 
         if version < 2:
@@ -132,7 +126,7 @@ class IPCamServerDatabase(SQLiteBase):
 # ipcam web api
 # -------------
 
-class IPCamWebServer(http.HTTPServer):
+class IpcamWebServer(http.HTTPServer):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
@@ -143,16 +137,16 @@ class IPCamWebServer(http.HTTPServer):
         self.get('/api/timestamp/{name}/{type}', self.get_timestamp)
         self.get('/api/timestamp/all', self.get_all_timestamps)
 
-        self.post('/api/debug/migrate-mtimes', self.debug_migrate_mtimes)
         self.post('/api/debug/fix', self.debug_fix)
         self.post('/api/debug/cleanup', self.debug_cleanup)
+
         self.post('/api/timestamp/{name}/{type}', self.set_timestamp)
 
         self.post('/api/motion/done/{name}', self.submit_motion)
         self.post('/api/motion/fail/{name}', self.submit_motion_failure)
 
-        self.get('/api/motion/params/{name}', self.get_motion_params)
+        # self.get('/api/motion/params/{name}', self.get_motion_params)
-        self.get('/api/motion/params/{name}/roi', self.get_motion_roi_params)
+        # self.get('/api/motion/params/{name}/roi', self.get_motion_roi_params)
 
         self.queue_lock = Lock()
 
@@ -170,7 +164,7 @@ class IPCamWebServer(http.HTTPServer):
 
         files = get_recordings_files(camera, filter, limit)
         if files:
-            time = filename_to_datetime(files[len(files)-1]['name'])
+            time = datetime_from_filename(files[len(files)-1]['name'])
             db.set_timestamp(camera, TimeFilterType.MOTION_START, time)
         return self.ok({'files': files})
 
@@ -185,7 +179,7 @@ class IPCamWebServer(http.HTTPServer):
             if files:
                 times_by_cam = {}
                 for file in files:
-                    time = filename_to_datetime(file['name'])
+                    time = datetime_from_filename(file['name'])
                     if file['cam'] not in times_by_cam or times_by_cam[file['cam']] < time:
                         times_by_cam[file['cam']] = time
                 for cam, time in times_by_cam.items():
@@ -197,14 +191,14 @@ class IPCamWebServer(http.HTTPServer):
         cam = int(req.match_info['name'])
         file = req.match_info['file']
 
-        fullpath = os.path.join(config['camera'][cam]['recordings_path'], file)
+        fullpath = os.path.join(get_recordings_path(cam), file)
         if not os.path.isfile(fullpath):
             raise ValueError(f'file "{fullpath}" does not exists')
 
         return http.FileResponse(fullpath)
 
     async def camlist(self, req: http.Request):
-        return self.ok(config['camera'])
+        return self.ok(ipcam_config.get_all_cam_names_for_this_server())
 
     async def submit_motion(self, req: http.Request):
         data = await req.post()
@@ -213,7 +207,7 @@ class IPCamWebServer(http.HTTPServer):
         timecodes = data['timecodes']
         filename = data['filename']
 
-        time = filename_to_datetime(filename)
+        time = datetime_from_filename(filename)
 
         try:
             if timecodes != '':
@@ -236,27 +230,10 @@ class IPCamWebServer(http.HTTPServer):
         message = data['message']
 
         db.add_motion_failure(camera, filename, message)
-        db.set_timestamp(camera, TimeFilterType.MOTION, filename_to_datetime(filename))
+        db.set_timestamp(camera, TimeFilterType.MOTION, datetime_from_filename(filename))
 
         return self.ok()
 
-    async def debug_migrate_mtimes(self, req: http.Request):
-        written = {}
-        for cam in config['camera'].keys():
-            confdir = os.path.join(os.getenv('HOME'), '.config', f'video-util-{cam}')
-            for time_type in TimeFilterType:
-                txt_file = os.path.join(confdir, f'{time_type.value}_mtime')
-                if os.path.isfile(txt_file):
-                    with open(txt_file, 'r') as fd:
-                        data = fd.read()
-                        db.set_timestamp(cam, time_type, int(data.strip()))
-
-                        if cam not in written:
-                            written[cam] = []
-                        written[cam].append(time_type)
-
-        return self.ok({'written': written})
-
     async def debug_fix(self, req: http.Request):
         asyncio.ensure_future(fix_job())
         return self.ok()
@@ -277,26 +254,26 @@ class IPCamWebServer(http.HTTPServer):
     async def get_all_timestamps(self, req: http.Request):
         return self.ok(db.get_all_timestamps())
 
-    async def get_motion_params(self, req: http.Request):
+    # async def get_motion_params(self, req: http.Request):
-        data = config['motion_params'][int(req.match_info['name'])]
+    #     data = config['motion_params'][int(req.match_info['name'])]
-        lines = [
+    #     lines = [
-            f'threshold={data["threshold"]}',
+    #         f'threshold={data["threshold"]}',
-            f'min_event_length=3s',
+    #         f'min_event_length=3s',
-            f'frame_skip=2',
+    #         f'frame_skip=2',
-            f'downscale_factor=3',
+    #         f'downscale_factor=3',
-        ]
+    #     ]
-        return self.plain('\n'.join(lines)+'\n')
+    #     return self.plain('\n'.join(lines)+'\n')
-
+    #
-    async def get_motion_roi_params(self, req: http.Request):
+    # async def get_motion_roi_params(self, req: http.Request):
-        data = config['motion_params'][int(req.match_info['name'])]
+    #     data = config['motion_params'][int(req.match_info['name'])]
-        return self.plain('\n'.join(data['roi'])+'\n')
+    #     return self.plain('\n'.join(data['roi'])+'\n')
 
     @staticmethod
     def _getset_timestamp_params(req: http.Request, need_time=False):
         values = []
 
         cam = int(req.match_info['name'])
-        assert cam in config['camera'], 'invalid camera'
+        assert cam in ipcam_config.get_all_cam_names_for_this_server(), 'invalid camera'
 
         values.append(cam)
         values.append(TimeFilterType(req.match_info['type']))
@@ -304,7 +281,7 @@ class IPCamWebServer(http.HTTPServer):
         if need_time:
             time = req.query['time']
             if time.startswith('record_'):
-                time = filename_to_datetime(time)
+                time = datetime_from_filename(time)
             elif time.isnumeric():
                 time = int(time)
             else:
@@ -317,32 +294,24 @@ class IPCamWebServer(http.HTTPServer):
 # other global stuff
 # ------------------
 
-def open_database():
+def open_database(database_path: str):
     global db
-    db = IPCamServerDatabase()
+    db = IpcamServerDatabase(database_path)
 
     # update cams list in database, if needed
-    cams = db.get_all_timestamps().keys()
+    stored_cams = db.get_all_timestamps().keys()
-    for cam in config['camera']:
+    for cam in ipcam_config.get_all_cam_names_for_this_server():
-        if cam not in cams:
+        if cam not in stored_cams:
             db.add_camera(cam)
 
 
-def get_recordings_path(cam: int) -> str:
-    return config['camera'][cam]['recordings_path']
-
-
-def get_motion_path(cam: int) -> str:
-    return config['camera'][cam]['motion_path']
-
-
 def get_recordings_files(cam: Optional[int] = None,
                          time_filter_type: Optional[TimeFilterType] = None,
                          limit=0) -> List[dict]:
     from_time = 0
     to_time = int(time.time())
 
-    cams = [cam] if cam is not None else get_all_cams()
+    cams = [cam] if cam is not None else ipcam_config.get_all_cam_names_for_this_server()
     files = []
     for cam in cams:
         if time_filter_type:
@@ -359,7 +328,7 @@ def get_recordings_files(cam: Optional[int] = None,
             'name': file,
             'size': os.path.getsize(os.path.join(recdir, file))}
                  for file in os.listdir(recdir)
-                 if valid_recording_name(file) and from_time < filename_to_datetime(file) <= to_time]
+                 if is_valid_recording_name(file) and from_time < datetime_from_filename(file) <= to_time]
         cam_files.sort(key=lambda file: file['name'])
 
         if cam_files:
@@ -379,7 +348,7 @@ def get_recordings_files(cam: Optional[int] = None,
 async def process_fragments(camera: int,
                             filename: str,
                             fragments: List[Tuple[int, int]]) -> None:
-    time = filename_to_datetime(filename)
+    time = datetime_from_filename(filename)
 
     rec_dir = get_recordings_path(camera)
     motion_dir = get_motion_path(camera)
@@ -389,8 +358,8 @@ async def process_fragments(camera: int,
     for fragment in fragments:
         start, end = fragment
 
-        start -= config['motion']['padding']
+        start -= ipcam_config['motion_padding']
-        end += config['motion']['padding']
+        end += ipcam_config['motion_padding']
 
         if start < 0:
             start = 0
@@ -405,14 +374,14 @@ async def process_fragments(camera: int,
                                  start_pos=start,
                                  duration=duration)
 
-    if fragments and 'telegram' in config['motion'] and config['motion']['telegram']:
+    if fragments and ipcam_config['motion_telegram']:
         asyncio.ensure_future(motion_notify_tg(camera, filename, fragments))
 
 
 async def motion_notify_tg(camera: int,
                            filename: str,
                            fragments: List[Tuple[int, int]]):
-    dt_file = filename_to_datetime(filename)
+    dt_file = datetime_from_filename(filename)
     fmt = '%H:%M:%S'
 
     text = f'Camera: <b>{camera}</b>\n'
@@ -420,8 +389,8 @@ async def motion_notify_tg(camera: int,
     text += _tg_links(TelegramLinkType.ORIGINAL_FILE, camera, filename)
 
     for start, end in fragments:
-        start -= config['motion']['padding']
+        start -= ipcam_config['motion_padding']
-        end += config['motion']['padding']
+        end += ipcam_config['motion_padding']
 
         if start < 0:
             start = 0
@@ -443,7 +412,7 @@ def _tg_links(link_type: TelegramLinkType,
               camera: int,
               file: str) -> str:
     links = []
-    for link_name, link_template in config['telegram'][f'{link_type.value}_url_templates']:
+    for link_name, link_template in ipcam_config[f'{link_type.value}_url_templates']:
         link = link_template.replace('{camera}', str(camera)).replace('{file}', file)
         links.append(f'<a href="{link}">{link_name}</a>')
     return ' '.join(links)
@@ -459,7 +428,7 @@ async def fix_job() -> None:
 
     try:
         fix_job_running = True
-        for cam in config['camera'].keys():
+        for cam in ipcam_config.get_all_cam_names_for_this_server():
             files = get_recordings_files(cam, TimeFilterType.FIX)
             if not files:
                 logger.debug(f'fix_job: no files for camera {cam}')
@@ -470,7 +439,7 @@ async def fix_job() -> None:
             for file in files:
                 fullpath = os.path.join(get_recordings_path(cam), file['name'])
                 await camutil.ffmpeg_recreate(fullpath)
-                timestamp = filename_to_datetime(file['name'])
+                timestamp = datetime_from_filename(file['name'])
                 if timestamp:
                     db.set_timestamp(cam, TimeFilterType.FIX, timestamp)
 
@@ -479,21 +448,9 @@ async def fix_job() -> None:
 
 
 async def cleanup_job() -> None:
-    def fn2dt(name: str) -> datetime:
-        name = os.path.basename(name)
-
-        if name.startswith('record_'):
-            return datetime.strptime(re.match(r'record_(.*?)\.mp4', name).group(1), datetime_format)
-
-        m = re.match(rf'({datetime_format_re})__{datetime_format_re}\.mp4', name)
-        if m:
-            return datetime.strptime(m.group(1), datetime_format)
-
-        raise ValueError(f'unrecognized filename format: {name}')
-
     def compare(i1: str, i2: str) -> int:
-        dt1 = fn2dt(i1)
+        dt1 = datetime_from_filename(i1)
-        dt2 = fn2dt(i2)
+        dt2 = datetime_from_filename(i2)
 
         if dt1 < dt2:
             return -1
@@ -513,18 +470,19 @@ async def cleanup_job() -> None:
         cleanup_job_running = True
 
         gb = float(1 << 30)
-        for storage in config['storages']:
+        disk_number = 0
+        for storage in lbc_config.get_board_disks(gethostname()):
+            disk_number += 1
             if os.path.exists(storage['mountpoint']):
                 total, used, free = shutil.disk_usage(storage['mountpoint'])
                 free_gb = free // gb
-                if free_gb < config['cleanup_min_gb']:
+                if free_gb < ipcam_config['cleanup_min_gb']:
-                    # print(f"{storage['mountpoint']}: free={free}, free_gb={free_gb}")
                     cleaned = 0
                     files = []
-                    for cam in storage['cams']:
+                    for cam in ipcam_config.get_all_cam_names_for_this_server(filter_by_disk=disk_number):
-                        for _dir in (config['camera'][cam]['recordings_path'], config['camera'][cam]['motion_path']):
+                        for _dir in (get_recordings_path(cam), get_motion_path(cam)):
                             files += list(map(lambda file: os.path.join(_dir, file), os.listdir(_dir)))
-                        files = list(filter(lambda path: os.path.isfile(path) and path.endswith('.mp4'), files))
+                        files = list(filter(lambda path: os.path.isfile(path) and path.endswith(tuple([f'.{t.value}' for t in VideoContainerType])), files))
                         files.sort(key=cmp_to_key(compare))
 
                     for file in files:
@@ -534,7 +492,7 @@ async def cleanup_job() -> None:
                             cleaned += size
                         except OSError as e:
                             logger.exception(e)
-                        if (free + cleaned) // gb >= config['cleanup_min_gb']:
+                        if (free + cleaned) // gb >= ipcam_config['cleanup_min_gb']:
                             break
             else:
                 logger.error(f"cleanup_job: {storage['mountpoint']} not found")
@@ -547,8 +505,8 @@ cleanup_job_running = False
 
 datetime_format = '%Y-%m-%d-%H.%M.%S'
 datetime_format_re = r'\d{4}-\d{2}-\d{2}-\d{2}\.\d{2}.\d{2}'
-db: Optional[IPCamServerDatabase] = None
+db: Optional[IpcamServerDatabase] = None
-server: Optional[IPCamWebServer] = None
+server: Optional[IpcamWebServer] = None
 logger = logging.getLogger(__name__)
 
 
@@ -556,18 +514,25 @@ logger = logging.getLogger(__name__)
 # --------------------
 
 if __name__ == '__main__':
-    config.load_app('ipcam_server')
+    parser = ArgumentParser()
+    parser.add_argument('--listen', type=str, required=True)
+    parser.add_argument('--database-path', type=str, required=True)
+    arg = homekit_config.load_app(no_config=True, parser=parser)
 
-    open_database()
+    open_database(arg.database_path)
 
     loop = asyncio.get_event_loop()
 
     try:
         scheduler = AsyncIOScheduler(event_loop=loop)
-        if config['fix_enabled']:
+        if ipcam_config['fix_enabled']:
-            scheduler.add_job(fix_job, 'interval', seconds=config['fix_interval'], misfire_grace_time=None)
+            scheduler.add_job(fix_job, 'interval',
+                              seconds=ipcam_config['fix_interval'],
+                              misfire_grace_time=None)
 
-        scheduler.add_job(cleanup_job, 'interval', seconds=config['cleanup_interval'], misfire_grace_time=None)
+        scheduler.add_job(cleanup_job, 'interval',
+                          seconds=ipcam_config['cleanup_interval'],
+                          misfire_grace_time=None)
         scheduler.start()
     except KeyError:
         pass
@@ -575,5 +540,5 @@ if __name__ == '__main__':
     asyncio.ensure_future(fix_job())
     asyncio.ensure_future(cleanup_job())
 
-    server = IPCamWebServer(config.get_addr('server.listen'))
+    server = IpcamWebServer(Addr.fromstring(arg.listen))
     server.run()

bin/lugovaya_pump_mqtt_bot.py

@@ -0,0 +1,207 @@
+#!/usr/bin/env python3
+import datetime
+import __py_include
+
+from enum import Enum
+from typing import Optional
+from telegram import ReplyKeyboardMarkup, User
+
+from homekit.config import config, AppConfigUnit
+from homekit.telegram import bot
+from homekit.telegram.config import TelegramBotConfig
+from homekit.telegram._botutil import user_any_name
+from homekit.mqtt import MqttNode, MqttPayload, MqttNodesConfig, MqttWrapper
+from homekit.mqtt.module.relay import MqttRelayState, MqttRelayModule
+from homekit.mqtt.module.diagnostics import InitialDiagnosticsPayload, DiagnosticsPayload
+
+
+class LugovayaPumpMqttBotConfig(TelegramBotConfig, AppConfigUnit):
+    NAME = 'lugovaya_pump_mqtt_bot'
+
+    @classmethod
+    def schema(cls) -> Optional[dict]:
+        return {
+            **TelegramBotConfig.schema(),
+            'relay_node_id': {
+                'type': 'string',
+                'required': True
+            },
+        }
+
+    @staticmethod
+    def custom_validator(data):
+        relay_node_names = MqttNodesConfig().get_nodes(filters=('relay',), only_names=True)
+        if data['relay_node_id'] not in relay_node_names:
+            raise ValueError('unknown relay node "%s"' % (data['relay_node_id'],))
+
+
+config.load_app(LugovayaPumpMqttBotConfig)
+
+bot.initialize()
+bot.lang.ru(
+    start_message="Выберите команду на клавиатуре",
+    start_message_no_access="Доступ запрещён. Вы можете отправить заявку на получение доступа.",
+    unknown_command="Неизвестная команда",
+    send_access_request="Отправить заявку",
+    management="Админка",
+
+    enable="Включить",
+    enabled="Включен ✅",
+
+    disable="Выключить",
+    disabled="Выключен ❌",
+
+    status="Статус",
+    status_updated=' (обновлено %s)',
+
+    done="Готово 👌",
+    user_action_notification='Пользователь <a href="tg://user?id=%d">%s</a> <b>%s</b> насос.',
+    user_action_on="включил",
+    user_action_off="выключил",
+    date_yday="вчера",
+    date_yyday="позавчера",
+    date_at="в"
+)
+bot.lang.en(
+    start_message="Select command on the keyboard",
+    start_message_no_access="You have no access.",
+    unknown_command="Unknown command",
+    send_access_request="Send request",
+    management="Admin options",
+
+    enable="Turn ON",
+    enable_silently="Turn ON silently",
+    enabled="Turned ON ✅",
+
+    disable="Turn OFF",
+    disable_silently="Turn OFF silently",
+    disabled="Turned OFF ❌",
+
+    status="Status",
+    status_updated=' (updated %s)',
+
+    done="Done 👌",
+    user_action_notification='User <a href="tg://user?id=%d">%s</a> turned the pump <b>%s</b>.',
+    user_action_on="ON",
+    user_action_off="OFF",
+
+    date_yday="yesterday",
+    date_yyday="the day before yesterday",
+    date_at="at"
+)
+
+
+mqtt: MqttWrapper
+relay_state = MqttRelayState()
+relay_module: MqttRelayModule
+
+
+class UserAction(Enum):
+    ON = 'on'
+    OFF = 'off'
+
+
+# def on_mqtt_message(home_id, message: MqttPayload):
+#     if isinstance(message, InitialDiagnosticsPayload) or isinstance(message, DiagnosticsPayload):
+#         kwargs = dict(rssi=message.rssi, enabled=message.flags.state)
+#         if isinstance(message, InitialDiagnosticsPayload):
+#             kwargs['fw_version'] = message.fw_version
+#         relay_state.update(**kwargs)
+
+
+async def notify(user: User, action: UserAction) -> None:
+    def text_getter(lang: str):
+        action_name = bot.lang.get(f'user_action_{action.value}', lang)
+        user_name = user_any_name(user)
+        return 'ℹ ' + bot.lang.get('user_action_notification', lang,
+                                   user.id, user_name, action_name)
+
+    await bot.notify_all(text_getter, exclude=(user.id,))
+
+
+@bot.handler(message='enable')
+async def enable_handler(ctx: bot.Context) -> None:
+    relay_module.switchpower(True)
+    await ctx.reply(ctx.lang('done'))
+    await notify(ctx.user, UserAction.ON)
+
+
+@bot.handler(message='disable')
+async def disable_handler(ctx: bot.Context) -> None:
+    relay_module.switchpower(False)
+    await ctx.reply(ctx.lang('done'))
+    await notify(ctx.user, UserAction.OFF)
+
+
+@bot.handler(message='status')
+async def status(ctx: bot.Context) -> None:
+    label = ctx.lang('enabled') if relay_state.enabled else ctx.lang('disabled')
+    if relay_state.ever_updated:
+        date_label = ''
+        today = datetime.date.today()
+        if today != relay_state.update_time.date():
+            yday = today - datetime.timedelta(days=1)
+            yyday = today - datetime.timedelta(days=2)
+            if yday == relay_state.update_time.date():
+                date_label = ctx.lang('date_yday')
+            elif yyday == relay_state.update_time.date():
+                date_label = ctx.lang('date_yyday')
+            else:
+                date_label = relay_state.update_time.strftime('%d.%m.%Y')
+            date_label += ' '
+        date_label += ctx.lang('date_at') + ' '
+        date_label += relay_state.update_time.strftime('%H:%M')
+        label += ctx.lang('status_updated', date_label)
+    await ctx.reply(label)
+
+
+async def start(ctx: bot.Context) -> None:
+    if ctx.user_id in config['bot']['users']:
+        await ctx.reply(ctx.lang('start_message'))
+    else:
+        buttons = [
+            [ctx.lang('send_access_request')]
+        ]
+        await ctx.reply(ctx.lang('start_message_no_access'),
+                        markup=ReplyKeyboardMarkup(buttons, one_time_keyboard=False))
+
+
+@bot.exceptionhandler
+def exception_handler(e: Exception, ctx: bot.Context) -> bool:
+    return False
+
+
+@bot.defaultreplymarkup
+def markup(ctx: Optional[bot.Context]) -> Optional[ReplyKeyboardMarkup]:
+    buttons = [
+        [
+            ctx.lang('enable'),
+            ctx.lang('disable')
+        ],
+        # [ctx.lang('status')]
+    ]
+    # if ctx.user_id in config['bot']['admin_users']:
+    #     buttons.append([ctx.lang('management')])
+    return ReplyKeyboardMarkup(buttons, one_time_keyboard=False)
+
+
+node_data = MqttNodesConfig().get_node(config.app_config['relay_node_id'])
+
+mqtt = MqttWrapper(client_id='lugovaya_pump_mqtt_bot')
+mqtt_node = MqttNode(node_id=config.app_config['relay_node_id'],
+                     node_secret=node_data['password'])
+module_kwargs = {}
+try:
+    if node_data['relay']['legacy_topics']:
+        module_kwargs['legacy_topics'] = True
+except KeyError:
+    pass
+relay_module = mqtt_node.load_module('relay', **module_kwargs)
+# mqtt_node.add_payload_callback(on_mqtt_message)
+mqtt.add_node(mqtt_node)
+
+mqtt.connect_and_loop(loop_forever=False)
+
+bot.run(start_handler=start)
+
+mqtt.disconnect()

bin/mqtt_node_util.py

@@ -1,17 +1,43 @@
 #!/usr/bin/env python3
 import os.path
+import __py_include
 
 from time import sleep
 from typing import Optional
 from argparse import ArgumentParser, ArgumentError
 
-from home.config import config
+from homekit.config import config
-from home.mqtt import MqttNode, MqttWrapper, get_mqtt_modules
+from homekit.mqtt import MqttNode, MqttWrapper, get_mqtt_modules, MqttNodesConfig
-from home.mqtt import MqttNodesConfig
+from homekit.mqtt.module.relay import MqttRelayModule
+from homekit.mqtt.module.ota import MqttOtaModule
 
 mqtt_node: Optional[MqttNode] = None
 mqtt: Optional[MqttWrapper] = None
 
+relay_module: Optional[MqttOtaModule] = None
+relay_val = None
+
+ota_module: Optional[MqttRelayModule] = None
+ota_val = False
+
+no_wait = False
+stop_loop = False
+
+
+def on_mqtt_connect():
+    global stop_loop
+
+    if relay_module:
+        relay_module.switchpower(relay_val == 1)
+
+    if ota_val:
+        if not os.path.exists(arg.push_ota):
+            raise OSError(f'--push-ota: file \"{arg.push_ota}\" does not exists')
+        ota_module.push_ota(arg.push_ota, 1)
+
+    if no_wait:
+        stop_loop = True
+
 
 if __name__ == '__main__':
     nodes_config = MqttNodesConfig()
@@ -23,16 +49,22 @@ if __name__ == '__main__':
     parser.add_argument('--switch-relay', choices=[0, 1], type=int,
                         help='send relay state')
     parser.add_argument('--push-ota', type=str, metavar='OTA_FILENAME',
-                        help='push OTA, receives path to firmware.bin')
+                        help='push OTA, receives path to firmware.bin (not .elf!)')
+    parser.add_argument('--no-wait', action='store_true',
+                        help='execute command and exit')
 
     config.load_app(parser=parser, no_config=True)
     arg = parser.parse_args()
 
+    if arg.no_wait:
+        no_wait = True
+
     if arg.switch_relay is not None and 'relay' not in arg.modules:
         raise ArgumentError(None, '--relay is only allowed when \'relay\' module included in --modules')
 
     mqtt = MqttWrapper(randomize_client_id=True,
                        client_id='mqtt_node_util')
+    mqtt.add_connect_callback(on_mqtt_connect)
     mqtt_node = MqttNode(node_id=arg.node_id,
                          node_secret=nodes_config.get_node(arg.node_id)['password'])
 
@@ -40,25 +72,29 @@ if __name__ == '__main__':
 
     # must-have modules
     ota_module = mqtt_node.load_module('ota')
+    ota_val = arg.push_ota
+
     mqtt_node.load_module('diagnostics')
 
     if arg.modules:
         for m in arg.modules:
-            module_instance = mqtt_node.load_module(m)
+            kwargs = {}
+            if m == 'relay' and MqttNodesConfig().node_uses_legacy_relay_power_payload(arg.node_id):
+                kwargs['legacy_topics'] = True
+            if m == 'temphum' and MqttNodesConfig().node_uses_legacy_temphum_data_payload(arg.node_id):
+                kwargs['legacy_payload'] = True
+            module_instance = mqtt_node.load_module(m, **kwargs)
             if m == 'relay' and arg.switch_relay is not None:
-                module_instance.switchpower(arg.switch_relay == 1)
+                relay_module = module_instance
+                relay_val = arg.switch_relay
 
-    mqtt.configure_tls()
     try:
         mqtt.connect_and_loop(loop_forever=False)
-
+        while not stop_loop:
-        if arg.push_ota:
-            if not os.path.exists(arg.push_ota):
-                raise OSError(f'--push-ota: file \"{arg.push_ota}\" does not exists')
-            ota_module.push_ota(arg.push_ota, 1)
-
-        while True:
             sleep(0.1)
 
     except KeyboardInterrupt:
+        pass
+
+    finally:
         mqtt.disconnect()

bin/openwrt_log_analyzer.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+import __py_include
+import homekit.telegram as telegram
+
+from homekit.telegram.config import TelegramChatsConfig
+from homekit.util import validate_mac_address
+from typing import Optional
+from homekit.config import config, AppConfigUnit
+from homekit.database import BotsDatabase, SimpleState
+
+
+class OpenwrtLogAnalyzerConfig(AppConfigUnit):
+    @classmethod
+    def schema(cls) -> Optional[dict]:
+        return {
+            'database_name': {'type': 'string', 'required': True},
+            'devices': {
+                'type': 'dict',
+                'keysrules': {'type': 'string'},
+                'valuesrules': {
+                    'type': 'string',
+                    'check_with': validate_mac_address
+                }
+            },
+            'limit': {'type': 'integer'},
+            'telegram_chat': {'type': 'string'},
+            'aps': {
+                'type': 'list',
+                'schema': {'type': 'integer'}
+            }
+        }
+
+    @staticmethod
+    def custom_validator(data):
+        chats = TelegramChatsConfig()
+        if data['telegram_chat'] not in chats:
+            return ValueError(f'unknown telegram chat {data["telegram_chat"]}')
+
+
+def main(mac: str,
+         title: str,
+         ap: int) -> int:
+    db = BotsDatabase()
+
+    data = db.get_openwrt_logs(filter_text=mac,
+                               min_id=state['last_id'],
+                               access_point=ap,
+                               limit=config['openwrt_log_analyzer']['limit'])
+    if not data:
+        return 0
+
+    max_id = 0
+    for log in data:
+        if log.id > max_id:
+            max_id = log.id
+
+    text = '\n'.join(map(lambda s: str(s), data))
+    telegram.send_message(f'<b>{title} (AP #{ap})</b>\n\n' + text, config.app_config['telegram_chat'])
+
+    return max_id
+
+
+if __name__ == '__main__':
+    config.load_app(OpenwrtLogAnalyzerConfig)
+    for ap in config.app_config['aps']:
+        dbname = config.app_config['database_name']
+        dbname = dbname.replace('.txt', f'-{ap}.txt')
+
+        state = SimpleState(name=dbname,
+                            default={'last_id': 0})
+
+        max_last_id = 0
+        for name, mac in config['devices'].items():
+            last_id = main(mac, title=name, ap=ap)
+            if last_id > max_last_id:
+                max_last_id = last_id
+
+        if max_last_id:
+            state['last_id'] = max_last_id

bin/openwrt_logger.py

@@ -1,30 +1,21 @@
 #!/usr/bin/env python3
 import os
+import __py_include
 
 from datetime import datetime
-from typing import Tuple, List
+from typing import Tuple, List, Optional
 from argparse import ArgumentParser
-from home.config import config
+from homekit.config import config, AppConfigUnit
-from home.database import SimpleState
+from homekit.database import SimpleState
-from home.api import WebAPIClient
+from homekit.api import WebApiClient
 
-f"""
-This script is supposed to be run by cron every 5 minutes or so.
-It looks for new lines in log file and sends them to remote server.
 
-OpenWRT must have remote logging enabled (UDP; IP of host this script is launched on; port 514)
+class OpenwrtLoggerConfig(AppConfigUnit):
-
+    @classmethod
-/etc/rsyslog.conf contains following (assuming 192.168.1.1 is the router IP):
+    def schema(cls) -> Optional[dict]:
-
+        return dict(
-$ModLoad imudp  
+            database_name_template=dict(type='string', required=True)
-$UDPServerRun 514  
+        )
-:fromhost-ip, isequal, "192.168.1.1" /var/log/openwrt.log  
-& ~
-
-Also comment out the following line: 
-$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
- 
-"""
 
 
 def parse_line(line: str) -> Tuple[int, str]:
@@ -46,11 +37,10 @@ if __name__ == '__main__':
     parser.add_argument('--access-point', type=int, required=True,
                         help='access point number')
 
-    arg = config.load_app('openwrt_logger', parser=parser)
+    arg = config.load_app(OpenwrtLoggerConfig, parser=parser)
-
-    state = SimpleState(file=config['simple_state']['file'].replace('{ap}', str(arg.access_point)),
-                        default={'seek': 0, 'size': 0})
 
+    state = SimpleState(name=config.app_config['database_name_template'].replace('{ap}', str(arg.access_point)),
+                        default=dict(seek=0, size=0))
     fsize = os.path.getsize(arg.file)
     if fsize < state['size']:
         state['seek'] = 0
@@ -79,5 +69,5 @@ if __name__ == '__main__':
                 except ValueError:
                     lines.append((0, line))
 
-            api = WebAPIClient()
+            api = WebApiClient()
             api.log_openwrt(lines, arg.access_point)

bin/pio_build.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+import __py_include
 
 if __name__ == '__main__':
     print('TODO')

bin/pio_ini.py

@@ -2,22 +2,24 @@
 import os
 import yaml
 import re
+import __py_include
 
-from pprint import pprint
 from argparse import ArgumentParser, ArgumentError
-from home.pio import get_products, platformio_ini
+from homekit.pio import get_products, platformio_ini
-from home.pio.exceptions import ProductConfigNotFoundError
+from homekit.pio.exceptions import ProductConfigNotFoundError
+from homekit.config import CONFIG_DIRECTORIES
 
 
 def get_config(product: str) -> dict:
-    config_path = os.path.join(
+    path = None
-        os.getenv('HOME'), '.config',
+    for directory in CONFIG_DIRECTORIES:
-        'homekit_pio', f'{product}.yaml'
+        config_path = os.path.join(directory, 'pio', f'{product}.yaml')
-    )
+        if os.path.exists(config_path) and os.path.isfile(config_path):
-    if not os.path.exists(config_path):
+            path = config_path
-        raise ProductConfigNotFoundError(f'{config_path}: product config not found')
+            break
-
+    if not path:
-    with open(config_path, 'r') as f:
+        raise ProductConfigNotFoundError(f'pio/{product}.yaml not found')
+    with open(path, 'r') as f:
         return yaml.safe_load(f)
 
 
@@ -82,7 +84,8 @@ def bsd_get(product_config: dict,
                 defines[f'CONFIG_{define_name}'] = f'HOMEKIT_{attr_value.upper()}'
                 return
             if kwargs['type'] == 'bool':
-                defines[f'CONFIG_{define_name}'] = True
+                if attr_value is True:
+                    defines[f'CONFIG_{define_name}'] = True
                 return
         defines[f'CONFIG_{define_name}'] = str(attr_value)
     bsd_walk(product_config, f)
@@ -106,7 +109,7 @@ if __name__ == '__main__':
 
     product_config = get_config(product)
 
-    # then everythingm else
+    # then everything else
     parser = ArgumentParser(parents=[product_parser])
     parser.add_argument('--target', type=str, required=True, choices=product_config['targets'],
                         help='PIO build target')
@@ -123,6 +126,7 @@ if __name__ == '__main__':
         raise ArgumentError(None, f'target {arg.target} not found for product {product}')
 
     bsd, bsd_enums = bsd_get(product_config, arg)
+
     ini = platformio_ini(product_config=product_config,
                          target=arg.target,
                          build_specific_defines=bsd,

bin/polaris_kettle_bot.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 from __future__ import annotations
 
+import __py_include
 import logging
 import locale
 import queue
@@ -8,11 +9,10 @@ import time
 import threading
 import paho.mqtt.client as mqtt
 
-from home.telegram import bot
+from homekit.telegram import bot
-from home.api.types import BotType
+from homekit.mqtt import Mqtt
-from home.mqtt import Mqtt
+from homekit.config import config
-from home.config import config
+from homekit.util import chunks
-from home.util import chunks
 from syncleo import (
     Kettle,
     PowerType,
@@ -737,9 +737,6 @@ if __name__ == '__main__':
 
     kc = KettleController()
 
-    if 'api' in config:
-        bot.enable_logging(BotType.POLARIS_KETTLE)
-
     bot.run()
 
     # bot library handles signals, so when sigterm or something like that happens, we should stop all other threads here

bin/polaris_kettle_util.py

@@ -4,12 +4,13 @@
 import logging
 import sys
 import paho.mqtt.client as mqtt
+import __py_include
 
 from typing import Optional
 from argparse import ArgumentParser
 from queue import SimpleQueue
-from home.mqtt import Mqtt
+from homekit.mqtt import Mqtt
-from home.config import config
+from homekit.config import config
 from syncleo import (
     Kettle,
     PowerType,

bin/pump_bot.py

@@ -1,26 +1,62 @@
 #!/usr/bin/env python3
+import __py_include
+import sys
+import asyncio
+
 from enum import Enum
-from typing import Optional
+from typing import Optional, Union
 from telegram import ReplyKeyboardMarkup, User
 from time import time
 from datetime import datetime
 
-from home.config import config, is_development_mode
+from homekit.config import config, is_development_mode, AppConfigUnit
-from home.telegram import bot
+from homekit.telegram import bot
-from home.telegram._botutil import user_any_name
+from homekit.telegram.config import TelegramBotConfig, TelegramUserListType
-from home.relay.sunxi_h3_client import RelayClient
+from homekit.telegram._botutil import user_any_name
-from home.api.types import BotType
+from homekit.relay.sunxi_h3_client import RelayClient
-from home.mqtt import MqttNode, MqttWrapper, MqttPayload
+from homekit.mqtt import MqttNode, MqttWrapper, MqttPayload, MqttNodesConfig, MqttModule
-from home.mqtt.module.relay import MqttPowerStatusPayload, MqttRelayModule
+from homekit.mqtt.module.relay import MqttPowerStatusPayload, MqttRelayModule
-from home.mqtt.module.temphum import MqttTemphumDataPayload
+from homekit.mqtt.module.temphum import MqttTemphumDataPayload
-from home.mqtt.module.diagnostics import InitialDiagnosticsPayload, DiagnosticsPayload
+from homekit.mqtt.module.diagnostics import InitialDiagnosticsPayload, DiagnosticsPayload
 
 
-config.load_app('pump_bot')
+if __name__ != '__main__':
+    print(f'this script can not be imported as module', file=sys.stderr)
+    sys.exit(1)
+
+
+mqtt_nodes_config = MqttNodesConfig()
+
+
+class PumpBotUserListType(TelegramUserListType):
+    SILENT = 'silent_users'
+
+
+class PumpBotConfig(AppConfigUnit, TelegramBotConfig):
+    NAME = 'pump_bot'
+
+    @classmethod
+    def schema(cls) -> Optional[dict]:
+        return {
+            **super(TelegramBotConfig).schema(),
+            PumpBotUserListType.SILENT: TelegramBotConfig._userlist_schema(),
+            'watering_relay_node': {'type': 'string'},
+            'pump_relay_addr': cls._addr_schema()
+        }
+
+    @staticmethod
+    def custom_validator(data):
+        relay_node_names = mqtt_nodes_config.get_nodes(filters=('relay',), only_names=True)
+        if data['watering_relay_node'] not in relay_node_names:
+            raise ValueError(f'unknown relay node "{data["watering_relay_node"]}"')
+
+
+config.load_app(PumpBotConfig)
+
+mqtt: MqttWrapper
+mqtt_node: MqttNode
+mqtt_relay_module: Union[MqttRelayModule, MqttModule]
 
-mqtt: Optional[MqttWrapper] = None
-mqtt_node: Optional[MqttNode] = None
-mqtt_relay_module: Optional[MqttRelayModule] = None
 time_format = '%d.%m.%Y, %H:%M:%S'
 
 watering_mcu_status = {
@@ -98,81 +134,89 @@ class UserAction(Enum):
 
 
 def get_relay() -> RelayClient:
-    relay = RelayClient(host=config['relay']['ip'], port=config['relay']['port'])
+    relay = RelayClient(host=config.app_config['pump_relay_addr'].host,
+                        port=config.app_config['pump_relay_addr'].port)
     relay.connect()
     return relay
 
 
-def on(ctx: bot.Context, silent=False) -> None:
+async def on(ctx: bot.Context, silent=False) -> None:
     get_relay().on()
-    ctx.reply(ctx.lang('done'))
+    futures = [ctx.reply(ctx.lang('done'))]
     if not silent:
-        notify(ctx.user, UserAction.ON)
+        futures.append(notify(ctx.user, UserAction.ON))
+    await asyncio.gather(*futures)
 
 
-def off(ctx: bot.Context, silent=False) -> None:
+async def off(ctx: bot.Context, silent=False) -> None:
     get_relay().off()
-    ctx.reply(ctx.lang('done'))
+    futures = [ctx.reply(ctx.lang('done'))]
     if not silent:
-        notify(ctx.user, UserAction.OFF)
+        futures.append(notify(ctx.user, UserAction.OFF))
+    await asyncio.gather(*futures)
 
 
-def watering_on(ctx: bot.Context) -> None:
+async def watering_on(ctx: bot.Context) -> None:
-    mqtt_relay_module.switchpower(True, config.get('mqtt_water_relay.secret'))
+    mqtt_relay_module.switchpower(True)
-    ctx.reply(ctx.lang('sent'))
+    await asyncio.gather(
-    notify(ctx.user, UserAction.WATERING_ON)
+        ctx.reply(ctx.lang('sent')),
+        notify(ctx.user, UserAction.WATERING_ON)
+    )
 
 
-def watering_off(ctx: bot.Context) -> None:
+async def watering_off(ctx: bot.Context) -> None:
-    mqtt_relay_module.switchpower(False, config.get('mqtt_water_relay.secret'))
+    mqtt_relay_module.switchpower(False)
-    ctx.reply(ctx.lang('sent'))
+    await asyncio.gather(
-    notify(ctx.user, UserAction.WATERING_OFF)
+        ctx.reply(ctx.lang('sent')),
+        notify(ctx.user, UserAction.WATERING_OFF)
+    )
 
 
-def notify(user: User, action: UserAction) -> None:
+async def notify(user: User, action: UserAction) -> None:
     notification_key = 'user_watering_notification' if action in (UserAction.WATERING_ON, UserAction.WATERING_OFF) else 'user_action_notification'
+
     def text_getter(lang: str):
         action_name = bot.lang.get(f'user_action_{action.value}', lang)
         user_name = user_any_name(user)
         return 'ℹ ' + bot.lang.get(notification_key, lang,
                                    user.id, user_name, action_name)
 
-    bot.notify_all(text_getter, exclude=(user.id,))
+    await bot.notify_all(text_getter, exclude=(user.id,))
 
 
 @bot.handler(message='enable')
-def enable_handler(ctx: bot.Context) -> None:
+async def enable_handler(ctx: bot.Context) -> None:
-    on(ctx)
+    await on(ctx)
 
 
 @bot.handler(message='enable_silently')
-def enable_s_handler(ctx: bot.Context) -> None:
+async def enable_s_handler(ctx: bot.Context) -> None:
-    on(ctx, True)
+    await on(ctx, True)
 
 
 @bot.handler(message='disable')
-def disable_handler(ctx: bot.Context) -> None:
+async def disable_handler(ctx: bot.Context) -> None:
-    off(ctx)
+    await off(ctx)
 
 
 @bot.handler(message='start_watering')
-def start_watering(ctx: bot.Context) -> None:
+async def start_watering(ctx: bot.Context) -> None:
-    watering_on(ctx)
+    await watering_on(ctx)
 
 
 @bot.handler(message='stop_watering')
-def stop_watering(ctx: bot.Context) -> None:
+async def stop_watering(ctx: bot.Context) -> None:
-    watering_off(ctx)
+    await watering_off(ctx)
 
 
 @bot.handler(message='disable_silently')
-def disable_s_handler(ctx: bot.Context) -> None:
+async def disable_s_handler(ctx: bot.Context) -> None:
-    off(ctx, True)
+    await off(ctx, True)
 
 
 @bot.handler(message='status')
-def status(ctx: bot.Context) -> None:
+async def status(ctx: bot.Context) -> None:
-    ctx.reply(
+    await ctx.reply(
         ctx.lang('enabled') if get_relay().status() == 'on' else ctx.lang('disabled')
     )
 
@@ -185,7 +229,7 @@ def _get_timestamp_as_string(timestamp: int) -> str:
 
 
 @bot.handler(message='watering_status')
-def watering_status(ctx: bot.Context) -> None:
+async def watering_status(ctx: bot.Context) -> None:
     buf = ''
     if 0 < watering_mcu_status["last_time"] < time()-1800:
         buf += '<b>WARNING! long time no reports from mcu! maybe something\'s wrong</b>\n'
@@ -194,13 +238,13 @@ def watering_status(ctx: bot.Context) -> None:
         buf += f'boot time: <b>{_get_timestamp_as_string(watering_mcu_status["last_boot_time"])}</b>\n'
     buf += 'relay opened: <b>' + ('yes' if watering_mcu_status['relay_opened'] else 'no') + '</b>\n'
     buf += f'ambient temp & humidity: <b>{watering_mcu_status["ambient_temp"]} °C, {watering_mcu_status["ambient_rh"]}%</b>'
-    ctx.reply(buf)
+    await ctx.reply(buf)
 
 
 @bot.defaultreplymarkup
 def markup(ctx: Optional[bot.Context]) -> Optional[ReplyKeyboardMarkup]:
     buttons = []
-    if ctx.user_id in config['bot']['silent_users']:
+    if ctx.user_id in config.app_config.get_user_ids(PumpBotUserListType.SILENT):
         buttons.append([ctx.lang('enable_silently'), ctx.lang('disable_silently')])
     buttons.append([ctx.lang('enable'), ctx.lang('disable'), ctx.lang('status')],)
     buttons.append([ctx.lang('start_watering'), ctx.lang('stop_watering'), ctx.lang('watering_status')])
@@ -233,24 +277,21 @@ def mqtt_payload_callback(mqtt_node: MqttNode, payload: MqttPayload):
         watering_mcu_status['relay_opened'] = payload.opened
 
 
-if __name__ == '__main__':
+mqtt = MqttWrapper(client_id='pump_bot')
-    mqtt = MqttWrapper()
+mqtt_node = MqttNode(node_id=config.app_config['watering_relay_node'])
-    mqtt_node = MqttNode(node_id=config.get('mqtt_water_relay.node_id'))
+if is_development_mode():
-    if is_development_mode():
+    mqtt_node.load_module('diagnostics')
-        mqtt_node.load_module('diagnostics')
 
-    mqtt_node.load_module('temphum')
+mqtt_node.load_module('temphum')
-    mqtt_relay_module = mqtt_node.load_module('relay')
+mqtt_relay_module = mqtt_node.load_module('relay')
 
-    mqtt_node.add_payload_callback(mqtt_payload_callback)
+mqtt_node.add_payload_callback(mqtt_payload_callback)
 
-    mqtt.configure_tls()
+mqtt.connect_and_loop(loop_forever=False)
-    mqtt.connect_and_loop(loop_forever=False)
 
-    bot.enable_logging(BotType.PUMP)
+bot.run()
-    bot.run()
 
-    try:
+try:
-        mqtt.disconnect()
+    mqtt.disconnect()
-    except:
+except:
-        pass
+    pass

bin/pump_mqtt_bot.py

@@ -1,16 +1,17 @@
 #!/usr/bin/env python3
 import datetime
+import __py_include
 
 from enum import Enum
 from typing import Optional
 from telegram import ReplyKeyboardMarkup, User
 
-from home.config import config
+from homekit.config import config
-from home.telegram import bot
+from homekit.telegram import bot
-from home.telegram._botutil import user_any_name
+from homekit.telegram._botutil import user_any_name
-from home.mqtt import MqttNode, MqttPayload
+from homekit.mqtt import MqttNode, MqttPayload
-from home.mqtt.module.relay import MqttRelayState
+from homekit.mqtt.module.relay import MqttRelayState
-from home.mqtt.module.diagnostics import InitialDiagnosticsPayload, DiagnosticsPayload
+from homekit.mqtt.module.diagnostics import InitialDiagnosticsPayload, DiagnosticsPayload
 
 
 config.load_app('pump_mqtt_bot')
@@ -159,7 +160,6 @@ if __name__ == '__main__':
     mqtt = MqttRelay(devices=MqttEspDevice(id=config['mqtt']['home_id'],
                                            secret=config['mqtt']['home_secret']))
     mqtt.set_message_callback(on_mqtt_message)
-    mqtt.configure_tls()
     mqtt.connect_and_loop(loop_forever=False)
 
     # bot.enable_logging(BotType.PUMP_MQTT)

bin/relay_mqtt_bot.py

@@ -1,18 +1,18 @@
 #!/usr/bin/env python3
 import sys
+import __py_include
 
 from enum import Enum
 from typing import Optional, Union
 from telegram import ReplyKeyboardMarkup
 from functools import partial
 
-from home.config import config, AppConfigUnit, TranslationsUnit
+from homekit.config import config, AppConfigUnit, Translation
-from home.telegram import bot
+from homekit.telegram import bot
-from home.telegram.config import TelegramBotConfig
+from homekit.telegram.config import TelegramBotConfig
-from home.mqtt import MqttPayload, MqttNode, MqttWrapper, MqttModule
+from homekit.mqtt import MqttPayload, MqttNode, MqttWrapper, MqttModule, MqttNodesConfig
-from home.mqtt import MqttNodesConfig
+from homekit.mqtt.module.relay import MqttRelayModule, MqttRelayState
-from home.mqtt.module.relay import MqttRelayModule, MqttRelayState
+from homekit.mqtt.module.diagnostics import InitialDiagnosticsPayload, DiagnosticsPayload
-from home.mqtt.module.diagnostics import InitialDiagnosticsPayload, DiagnosticsPayload
 
 
 if __name__ != '__main__':
@@ -26,12 +26,14 @@ mqtt_nodes_config = MqttNodesConfig()
 class RelayMqttBotConfig(AppConfigUnit, TelegramBotConfig):
     NAME = 'relay_mqtt_bot'
 
+    _strings: Translation
+
     def __init__(self):
         super().__init__()
-        self._mqtt_nodes_strings = TranslationsUnit('mqtt_nodes')
+        self._strings = Translation('mqtt_nodes')
 
-    @staticmethod
+    @classmethod
-    def schema() -> Optional[dict]:
+    def schema(cls) -> Optional[dict]:
         return {
             **super(TelegramBotConfig).schema(),
             'relay_nodes': {
@@ -51,7 +53,7 @@ class RelayMqttBotConfig(AppConfigUnit, TelegramBotConfig):
                 raise ValueError(f'unknown relay node "{node}"')
 
     def get_relay_name_translated(self, lang: str, relay_name: str) -> str:
-        pass
+        return self._strings.get(lang)[relay_name]['relay']
 
 
 config.load_app(RelayMqttBotConfig)
@@ -78,7 +80,7 @@ status_emoji = {
 }
 
 
-mqtt: Optional[MqttWrapper] = None
+mqtt: MqttWrapper
 relay_nodes: dict[str, Union[MqttRelayModule, MqttModule]] = {}
 relay_states: dict[str, MqttRelayState] = {}
 
@@ -99,32 +101,32 @@ def on_mqtt_message(node: MqttNode,
         relay_states[node.id].update(**kwargs)
 
 
-def enable_handler(node_id: str, ctx: bot.Context) -> None:
+async def enable_handler(node_id: str, ctx: bot.Context) -> None:
     relay_nodes[node_id].switchpower(True)
-    ctx.reply(ctx.lang('done'))
+    await ctx.reply(ctx.lang('done'))
 
 
-def disable_handler(node_id: str, ctx: bot.Context) -> None:
+async def disable_handler(node_id: str, ctx: bot.Context) -> None:
     relay_nodes[node_id].switchpower(False)
-    ctx.reply(ctx.lang('done'))
+    await ctx.reply(ctx.lang('done'))
 
 
-def start(ctx: bot.Context) -> None:
+async def start(ctx: bot.Context) -> None:
-    ctx.reply(ctx.lang('start_message'))
+    await ctx.reply(ctx.lang('start_message'))
 
 
 @bot.exceptionhandler
-def exception_handler(e: Exception, ctx: bot.Context) -> bool:
+async def exception_handler(e: Exception, ctx: bot.Context) -> bool:
     return False
 
 
 @bot.defaultreplymarkup
 def markup(ctx: Optional[bot.Context]) -> Optional[ReplyKeyboardMarkup]:
     buttons = []
-    for device_id, data in config['relays'].items():
+    for node_id in config.app_config['relay_nodes']:
-        labels = data['labels']
+        node_data = mqtt_nodes_config.get_node(node_id)
-        type_emoji = type_emojis[data['type']]
+        type_emoji = type_emojis[node_data['relay']['device_type']]
-        row = [f'{type_emoji}{status_emoji[i.value]} {labels[ctx.user_lang]}'
+        row = [f'{type_emoji}{status_emoji[i.value]} {config.app_config.get_relay_name_translated(ctx.user_lang, node_id)}'
                for i in UserAction]
         buttons.append(row)
     return ReplyKeyboardMarkup(buttons, one_time_keyboard=False)
@@ -132,25 +134,29 @@ def markup(ctx: Optional[bot.Context]) -> Optional[ReplyKeyboardMarkup]:
 
 devices = []
 mqtt = MqttWrapper(client_id='relay_mqtt_bot')
-for device_id, data in config['relays'].items():
+for node_id in config.app_config['relay_nodes']:
-    mqtt_node = MqttNode(node_id=device_id, node_secret=data['secret'])
+    node_data = mqtt_nodes_config.get_node(node_id)
-    relay_nodes[device_id] = mqtt_node.load_module('relay')
+    mqtt_node = MqttNode(node_id=node_id,
+                         node_secret=node_data['password'])
+    module_kwargs = {}
+    try:
+        if node_data['relay']['legacy_topics']:
+            module_kwargs['legacy_topics'] = True
+    except KeyError:
+        pass
+    relay_nodes[node_id] = mqtt_node.load_module('relay',  **module_kwargs)
     mqtt_node.add_payload_callback(on_mqtt_message)
     mqtt.add_node(mqtt_node)
 
-    labels = data['labels']
+    type_emoji = type_emojis[node_data['relay']['device_type']]
-    bot.lang.ru(**{device_id: labels['ru']})
-    bot.lang.en(**{device_id: labels['en']})
-
-    type_emoji = type_emojis[data['type']]
 
     for action in UserAction:
         messages = []
-        for _lang, _label in labels.items():
+        for _lang in Translation.LANGUAGES:
-            messages.append(f'{type_emoji}{status_emoji[action.value]} {labels[_lang]}')
+            _label = config.app_config.get_relay_name_translated(_lang, node_id)
-        bot.handler(texts=messages)(partial(enable_handler if action == UserAction.ON else disable_handler, device_id))
+            messages.append(f'{type_emoji}{status_emoji[action.value]} {_label}')
+        bot.handler(texts=messages)(partial(enable_handler if action == UserAction.ON else disable_handler, node_id))
 
-mqtt.configure_tls()
 mqtt.connect_and_loop(loop_forever=False)
 
 bot.run(start_handler=start)

bin/relay_mqtt_http_proxy.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+import logging
+import __py_include
+
+from homekit import http
+from homekit.config import config, AppConfigUnit
+from homekit.mqtt import MqttPayload, MqttWrapper, MqttNode, MqttModule, MqttNodesConfig
+from homekit.mqtt.module.relay import MqttRelayState, MqttRelayModule, MqttPowerStatusPayload
+from homekit.mqtt.module.diagnostics import InitialDiagnosticsPayload, DiagnosticsPayload
+from typing import Optional, Union
+
+
+logger = logging.getLogger(__name__)
+mqtt: Optional[MqttWrapper] = None
+mqtt_nodes: dict[str, MqttNode] = {}
+relay_modules: dict[str, Union[MqttRelayModule, MqttModule]] = {}
+relay_states: dict[str, MqttRelayState] = {}
+
+mqtt_nodes_config = MqttNodesConfig()
+
+
+class RelayMqttHttpProxyConfig(AppConfigUnit):
+    NAME = 'relay_mqtt_http_proxy'
+
+    @classmethod
+    def schema(cls) -> Optional[dict]:
+        return {
+            'relay_nodes': {
+                'type': 'list',
+                'required': True,
+                'schema': {
+                    'type': 'string'
+                }
+            },
+            'listen_addr': cls._addr_schema(required=True)
+        }
+
+    @staticmethod
+    def custom_validator(data):
+        relay_node_names = mqtt_nodes_config.get_nodes(filters=('relay',), only_names=True)
+        for node in data['relay_nodes']:
+            if node not in relay_node_names:
+                raise ValueError(f'unknown relay node "{node}"')
+
+
+def on_mqtt_message(node: MqttNode,
+                    message: MqttPayload):
+    try:
+        is_legacy = mqtt_nodes_config[node.id]['relay']['legacy_topics']
+        logger.debug(f'on_mqtt_message: relay {node.id} uses legacy topic names')
+    except KeyError:
+        is_legacy = False
+    kwargs = {}
+
+    if isinstance(message, InitialDiagnosticsPayload) or isinstance(message, DiagnosticsPayload):
+        kwargs['rssi'] = message.rssi
+        if is_legacy:
+            kwargs['enabled'] = message.flags.state
+
+    if not is_legacy and isinstance(message, MqttPowerStatusPayload):
+        kwargs['enabled'] = message.opened
+
+    if len(kwargs):
+        logger.debug(f'on_mqtt_message: {node.id}: going to update relay state: {str(kwargs)}')
+        if node.id not in relay_states:
+            relay_states[node.id] = MqttRelayState()
+        relay_states[node.id].update(**kwargs)
+
+
+class RelayMqttHttpProxy(http.HTTPServer):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.get('/relay/{id}/on', self.relay_on)
+        self.get('/relay/{id}/off', self.relay_off)
+        self.get('/relay/{id}/toggle', self.relay_toggle)
+
+    async def _relay_on_off(self,
+                            enable: Optional[bool],
+                            req: http.Request):
+        node_id = req.match_info['id']
+        node_secret = req.query['secret']
+
+        node = mqtt_nodes[node_id]
+        relay_module = relay_modules[node_id]
+
+        if enable is None:
+            if node_id in relay_states and relay_states[node_id].ever_updated:
+                cur_state = relay_states[node_id].enabled
+            else:
+                cur_state = False
+            enable = not cur_state
+
+        node.secret = node_secret
+        relay_module.switchpower(enable)
+        return self.ok()
+
+    async def relay_on(self, req: http.Request):
+        return await self._relay_on_off(True, req)
+
+    async def relay_off(self, req: http.Request):
+        return await self._relay_on_off(False, req)
+
+    async def relay_toggle(self, req: http.Request):
+        return await self._relay_on_off(None, req)
+
+
+if __name__ == '__main__':
+    config.load_app(RelayMqttHttpProxyConfig)
+
+    mqtt = MqttWrapper(client_id='relay_mqtt_http_proxy',
+                       randomize_client_id=True)
+    for node_id in config.app_config['relay_nodes']:
+        node_data = mqtt_nodes_config.get_node(node_id)
+        mqtt_node = MqttNode(node_id=node_id)
+        module_kwargs = {}
+        try:
+            if node_data['relay']['legacy_topics']:
+                module_kwargs['legacy_topics'] = True
+        except KeyError:
+            pass
+        relay_modules[node_id] = mqtt_node.load_module('relay', **module_kwargs)
+        if 'legacy_topics' in module_kwargs:
+            mqtt_node.load_module('diagnostics')
+        mqtt_node.add_payload_callback(on_mqtt_message)
+        mqtt.add_node(mqtt_node)
+        mqtt_nodes[node_id] = mqtt_node
+
+    mqtt.connect_and_loop(loop_forever=False)
+
+    proxy = RelayMqttHttpProxy(config.app_config['listen_addr'])
+    try:
+        proxy.run()
+    except KeyboardInterrupt:
+        mqtt.disconnect()

bin/sensors_bot.py

@@ -4,6 +4,7 @@ import socket
 import logging
 import re
 import gc
+import __py_include
 
 from io import BytesIO
 from typing import Optional
@@ -14,12 +15,11 @@ import matplotlib.ticker as mticker
 
 from telegram import ReplyKeyboardMarkup, InlineKeyboardMarkup, InlineKeyboardButton
 
-from home.config import config
+from homekit.config import config
-from home.telegram import bot
+from homekit.telegram import bot
-from home.util import chunks, MySimpleSocketClient
+from homekit.util import chunks, MySimpleSocketClient
-from home.api import WebAPIClient
+from homekit.api import WebApiClient
-from home.api.types import (
+from homekit.api.types import (
-    BotType,
     TemperatureSensorLocation
 )
 
@@ -111,7 +111,7 @@ def callback_handler(ctx: bot.Context) -> None:
     sensor = TemperatureSensorLocation[match.group(1).upper()]
     hours = int(match.group(2))
 
-    api = WebAPIClient(timeout=20)
+    api = WebApiClient(timeout=20)
     data = api.get_sensors_data(sensor, hours)
 
     title = ctx.lang(sensor.name.lower()) + ' (' + ctx.lang('n_hrs', hours) + ')'
@@ -175,7 +175,4 @@ def markup(ctx: Optional[bot.Context]) -> Optional[ReplyKeyboardMarkup]:
 
 
 if __name__ == '__main__':
-    if 'api' in config:
-        bot.enable_logging(BotType.SENSORS)
-
     bot.run()

bin/sound_bot.py

@@ -2,21 +2,22 @@
 import logging
 import os
 import tempfile
+import __py_include
 
 from enum import Enum
 from datetime import datetime, timedelta
 from html import escape
 from typing import Optional, List, Dict, Tuple
 
-from home.config import config
+from homekit.config import config
-from home.api import WebAPIClient
+from homekit.api import WebApiClient
-from home.api.types import SoundSensorLocation, BotType
+from homekit.api.types import SoundSensorLocation
-from home.api.errors import ApiResponseError
+from homekit.api.errors import ApiResponseError
-from home.media import SoundNodeClient, SoundRecordClient, SoundRecordFile, CameraNodeClient
+from homekit.media import SoundNodeClient, SoundRecordClient, SoundRecordFile, CameraNodeClient
-from home.soundsensor import SoundSensorServerGuardClient
+from homekit.soundsensor import SoundSensorServerGuardClient
-from home.util import parse_addr, chunks, filesize_fmt
+from homekit.util import Addr, chunks, filesize_fmt
 
-from home.telegram import bot
+from homekit.telegram import bot
 
 from telegram.error import TelegramError
 from telegram import ReplyKeyboardMarkup, InlineKeyboardMarkup, InlineKeyboardButton, User
@@ -27,7 +28,7 @@ config.load_app('sound_bot')
 
 nodes = {}
 for nodename, nodecfg in config['nodes'].items():
-    nodes[nodename] = parse_addr(nodecfg['addr'])
+    nodes[nodename] = Addr.fromstring(nodecfg['addr'])
 
 bot.initialize()
 bot.lang.ru(
@@ -142,13 +143,13 @@ cam_client_links: Dict[str, CameraNodeClient] = {}
 
 def node_client(node: str) -> SoundNodeClient:
     if node not in node_client_links:
-        node_client_links[node] = SoundNodeClient(parse_addr(config['nodes'][node]['addr']))
+        node_client_links[node] = SoundNodeClient(Addr.fromstring(config['nodes'][node]['addr']))
     return node_client_links[node]
 
 
 def camera_client(cam: str) -> CameraNodeClient:
     if cam not in node_client_links:
-        cam_client_links[cam] = CameraNodeClient(parse_addr(config['cameras'][cam]['addr']))
+        cam_client_links[cam] = CameraNodeClient(Addr.fromstring(config['cameras'][cam]['addr']))
     return cam_client_links[cam]
 
 
@@ -188,7 +189,7 @@ def manual_recording_allowed(user_id: int) -> bool:
 
 
 def guard_client() -> SoundSensorServerGuardClient:
-    return SoundSensorServerGuardClient(parse_addr(config['bot']['guard_server']))
+    return SoundSensorServerGuardClient(Addr.fromstring(config['bot']['guard_server']))
 
 
 # message renderers
@@ -734,7 +735,7 @@ def sound_sensors_last_24h(ctx: bot.Context):
 
     ctx.answer()
 
-    cl = WebAPIClient()
+    cl = WebApiClient()
     data = cl.get_sound_sensor_hits(location=SoundSensorLocation[node.upper()],
                                     after=datetime.now() - timedelta(hours=24))
 
@@ -757,7 +758,7 @@ def sound_sensors_last_anything(ctx: bot.Context):
 
     ctx.answer()
 
-    cl = WebAPIClient()
+    cl = WebApiClient()
     data = cl.get_last_sound_sensor_hits(location=SoundSensorLocation[node.upper()],
                                          last=20)
 
@@ -883,7 +884,5 @@ if __name__ == '__main__':
                                       finished_handler=record_onfinished,
                                       download_on_finish=True)
 
-    if 'api' in config:
-        bot.enable_logging(BotType.SOUND)
     bot.run()
     record_client.stop()

bin/sound_node.py

@@ -1,12 +1,13 @@
 #!/usr/bin/env python3
 import os
+import __py_include
 
 from typing import Optional
 
-from home.config import config
+from homekit.config import config
-from home.audio import amixer
+from homekit.audio import amixer
-from home.media import MediaNodeServer, SoundRecordStorage, SoundRecorder
+from homekit.media import MediaNodeServer, SoundRecordStorage, SoundRecorder
-from home import http
+from homekit import http
 
 
 # This script must be run as root as it runs arecord.

bin/sound_sensor_node.py

@@ -2,10 +2,11 @@
 import logging
 import os
 import sys
+import __py_include
 
-from home.config import config
+from homekit.config import config
-from home.util import parse_addr
+from homekit.util import Addr
-from home.soundsensor import SoundSensorNode
+from homekit.soundsensor import SoundSensorNode
 
 logger = logging.getLogger(__name__)
 
@@ -21,7 +22,7 @@ if __name__ == '__main__':
         kwargs['delay'] = config['node']['delay']
 
     if 'server_addr' in config['node']:
-        server_addr = parse_addr(config['node']['server_addr'])
+        server_addr = Addr.fromstring(config['node']['server_addr'])
     else:
         server_addr = None
 

bin/sound_sensor_server.py

@@ -1,16 +1,17 @@
 #!/usr/bin/env python3
 import logging
 import threading
+import __py_include
 
 from time import sleep
 from typing import Optional, List, Dict, Tuple
 from functools import partial
-from home.config import config
+from homekit.config import config
-from home.util import parse_addr
+from homekit.util import Addr
-from home.api import WebAPIClient, RequestParams
+from homekit.api import WebApiClient, RequestParams
-from home.api.types import SoundSensorLocation
+from homekit.api.types import SoundSensorLocation
-from home.soundsensor import SoundSensorServer, SoundSensorHitHandler
+from homekit.soundsensor import SoundSensorServer, SoundSensorHitHandler
-from home.media import MediaNodeType, SoundRecordClient, CameraRecordClient, RecordClient
+from homekit.media import MediaNodeType, SoundRecordClient, CameraRecordClient, RecordClient
 
 interrupted = False
 logger = logging.getLogger(__name__)
@@ -120,7 +121,7 @@ def hits_sender():
         sleep(5)
 
 
-api: Optional[WebAPIClient] = None
+api: Optional[WebApiClient] = None
 hc: Optional[HitCounter] = None
 record_clients: Dict[MediaNodeType, RecordClient] = {}
 
@@ -162,7 +163,7 @@ if __name__ == '__main__':
     config.load_app('sound_sensor_server')
 
     hc = HitCounter()
-    api = WebAPIClient(timeout=(10, 60))
+    api = WebApiClient(timeout=(10, 60))
     api.enable_async(error_handler=api_error_handler)
 
     t = threading.Thread(target=hits_sender)
@@ -172,12 +173,12 @@ if __name__ == '__main__':
     sound_nodes = {}
     if 'sound_nodes' in config:
         for nodename, nodecfg in config['sound_nodes'].items():
-            sound_nodes[nodename] = parse_addr(nodecfg['addr'])
+            sound_nodes[nodename] = Addr.fromstring(nodecfg['addr'])
 
     camera_nodes = {}
     if 'camera_nodes' in config:
         for nodename, nodecfg in config['camera_nodes'].items():
-            camera_nodes[nodename] = parse_addr(nodecfg['addr'])
+            camera_nodes[nodename] = Addr.fromstring(nodecfg['addr'])
 
     if sound_nodes:
         record_clients[MediaNodeType.SOUND] = SoundRecordClient(sound_nodes,

bin/ssh_tunnels_config_util.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
-
+import __py_include
-from home.config import config
+from homekit.config import config
 
 if __name__ == '__main__':
     config.load_app('ssh_tunnels_config_util')
@@ -8,7 +8,7 @@ if __name__ == '__main__':
     network_prefix = config['network']
     hostnames = []
 
-    for k, v in config.items():
+    for k, v in config.app_config.get().items():
         if type(v) is not dict:
             continue
         hostnames.append(k)

bin/temphum_mqtt_node.py

@@ -2,12 +2,13 @@
 import asyncio
 import json
 import logging
+import __py_include
 
 from typing import Optional
 
-from home.config import config
+from homekit.config import config
-from home.temphum import SensorType, BaseSensor
+from homekit.temphum import SensorType, BaseSensor
-from home.temphum.i2c import create_sensor
+from homekit.temphum.i2c import create_sensor
 
 logger = logging.getLogger(__name__)
 sensor: Optional[BaseSensor] = None

bin/temphum_mqtt_receiver.py

@@ -1,9 +1,10 @@
 #!/usr/bin/env python3
 import paho.mqtt.client as mqtt
 import re
+import __py_include
 
-from home.config import config
+from homekit.config import config
-from home.mqtt import MqttWrapper, MqttNode
+from homekit.mqtt import MqttWrapper, MqttNode
 
 
 class MqttServer(Mqtt):
@@ -44,5 +45,4 @@ if __name__ == '__main__':
     node.load_module('temphum', write_to_database=True)
     mqtt.add_node(node)
 
-    mqtt.configure_tls()
     mqtt.connect_and_loop()

bin/temphum_nodes_util.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python3
-from home.mqtt.temphum import MqttTempHumNodes
+import __py_include
+
+from homekit.mqtt.temphum import MqttTempHumNodes
 
 if __name__ == '__main__':
     max_name_len = 0

bin/temphum_smbus_util.py

@@ -1,7 +1,9 @@
 #!/usr/bin/env python3
+import __py_include
+
 from argparse import ArgumentParser
-from home.temphum import SensorType
+from homekit.temphum import SensorType
-from home.temphum.i2c import create_sensor
+from homekit.temphum.i2c import create_sensor
 
 
 if __name__ == '__main__':

bin/temphumd.py

@@ -2,12 +2,13 @@
 import asyncio
 import json
 import logging
+import __py_include
 
 from typing import Optional
 
-from home.config import config
+from homekit.config import config
-from home.temphum import SensorType, BaseSensor
+from homekit.temphum import SensorType, BaseSensor
-from home.temphum.i2c import create_sensor
+from homekit.temphum.i2c import create_sensor
 
 logger = logging.getLogger(__name__)
 sensor: Optional[BaseSensor] = None

bin/web_api.py

@@ -2,16 +2,17 @@
 import asyncio
 import json
 import os
+import __py_include
 
 from datetime import datetime, timedelta
 
 from aiohttp import web
-from home import http
+from homekit import http
-from home.config import config, is_development_mode
+from homekit.config import config, is_development_mode
-from home.database import BotsDatabase, SensorsDatabase, InverterDatabase
+from homekit.database import BotsDatabase, SensorsDatabase, InverterDatabase
-from home.database.inverter_time_formats import *
+from homekit.database.inverter_time_formats import *
-from home.api.types import BotType, TemperatureSensorLocation, SoundSensorLocation
+from homekit.api.types import TemperatureSensorLocation, SoundSensorLocation
-from home.media import SoundRecordStorage
+from homekit.media import SoundRecordStorage
 
 
 def strptime_auto(s: str) -> datetime:
@@ -41,7 +42,6 @@ class WebAPIServer(http.HTTPServer):
         self.get('/sound_sensors/hits/', self.GET_sound_sensors_hits)
         self.post('/sound_sensors/hits/', self.POST_sound_sensors_hits)
 
-        self.post('/log/bot_request/', self.POST_bot_request_log)
         self.post('/log/openwrt/', self.POST_openwrt_log)
 
         self.get('/inverter/consumed_energy/', self.GET_consumed_energy)
@@ -125,30 +125,6 @@ class WebAPIServer(http.HTTPServer):
         BotsDatabase().add_sound_hits(hits, datetime.now())
         return self.ok()
 
-    async def POST_bot_request_log(self, req: http.Request):
-        data = await req.post()
-
-        try:
-            user_id = int(data['user_id'])
-        except KeyError:
-            user_id = 0
-
-        try:
-            message = data['message']
-        except KeyError:
-            message = ''
-
-        bot = BotType(int(data['bot']))
-
-        # validate message
-        if message.strip() == '':
-            raise ValueError('message can\'t be empty')
-
-        # add record to the database
-        BotsDatabase().add_request(bot, user_id, message)
-
-        return self.ok()
-
     async def POST_openwrt_log(self, req: http.Request):
         data = await req.post()
 

bin/web_kbn.py

@@ -0,0 +1,354 @@
+#!/usr/bin/env python3
+import asyncio
+import jinja2
+import aiohttp_jinja2
+import json
+import re
+import inverterd
+import phonenumbers
+import __py_include
+
+from io import StringIO
+from aiohttp.web import HTTPFound
+from typing import Optional, Union
+from homekit.config import config, AppConfigUnit
+from homekit.util import homekit_path, filesize_fmt, seconds_to_human_readable_string
+from homekit.modem import E3372, ModemsConfig, MacroNetWorkType
+from homekit.inverter.config import InverterdConfig
+from homekit.relay.sunxi_h3_client import RelayClient
+from homekit import http
+
+
+class WebKbnConfig(AppConfigUnit):
+    NAME = 'web_kbn'
+
+    @classmethod
+    def schema(cls) -> Optional[dict]:
+        return {
+            'listen_addr': cls._addr_schema(required=True),
+            'assets_public_path': {'type': 'string'},
+            'pump_addr': cls._addr_schema(required=True),
+            'inverter_grafana_url': {'type': 'string'},
+            'sensors_grafana_url': {'type': 'string'},
+        }
+
+
+STATIC_FILES = [
+    'bootstrap.min.css',
+    'bootstrap.min.js',
+    'polyfills.js',
+    'app.js',
+    'app.css'
+]
+
+
+def get_js_link(file, version) -> str:
+    if version:
+        file += f'?version={version}'
+    return f'<script src="{config.app_config["assets_public_path"]}/{file}" type="text/javascript"></script>'
+
+
+def get_css_link(file, version) -> str:
+    if version:
+        file += f'?version={version}'
+    return f'<link rel="stylesheet" type="text/css" href="{config.app_config["assets_public_path"]}/{file}">'
+
+
+def get_head_static() -> str:
+    buf = StringIO()
+    for file in STATIC_FILES:
+        v = 2
+        try:
+            q_ind = file.index('?')
+            v = file[q_ind+1:]
+            file = file[:file.index('?')]
+        except ValueError:
+            pass
+
+        if file.endswith('.js'):
+            buf.write(get_js_link(file, v))
+        else:
+            buf.write(get_css_link(file, v))
+    return buf.getvalue()
+
+
+def get_modem_client(modem_cfg: dict) -> E3372:
+    return E3372(modem_cfg['ip'], legacy_token_auth=modem_cfg['legacy_auth'])
+
+
+def get_modem_data(modem_cfg: dict, get_raw=False) -> Union[dict, tuple]:
+    cl = get_modem_client(modem_cfg)
+
+    signal = cl.device_signal
+    status = cl.monitoring_status
+    traffic = cl.traffic_stats
+
+    if get_raw:
+        device_info = cl.device_information
+        dialup_conn = cl.dialup_connection
+        return signal, status, traffic, device_info, dialup_conn
+    else:
+        network_type_label = re.sub('^MACRO_NET_WORK_TYPE(_EX)?_', '', MacroNetWorkType(int(status['CurrentNetworkType'])).name)
+        return {
+            'type': network_type_label,
+            'level': int(status['SignalIcon']) if 'SignalIcon' in status else 0,
+            'rssi': signal['rssi'],
+            'sinr': signal['sinr'],
+            'connected_time': seconds_to_human_readable_string(int(traffic['CurrentConnectTime'])),
+            'downloaded': filesize_fmt(int(traffic['CurrentDownload'])),
+            'uploaded': filesize_fmt(int(traffic['CurrentUpload']))
+        }
+
+
+def get_pump_client() -> RelayClient:
+    addr = config.app_config['pump_addr']
+    cl = RelayClient(host=addr.host, port=addr.port)
+    cl.connect()
+    return cl
+
+
+def get_inverter_client() -> inverterd.Client:
+    cl = inverterd.Client(host=InverterdConfig()['remote_addr'].host)
+    cl.connect()
+    cl.format(inverterd.Format.JSON)
+    return cl
+
+
+def get_inverter_data() -> tuple:
+    cl = get_inverter_client()
+
+    status = json.loads(cl.exec('get-status'))['data']
+    rated = json.loads(cl.exec('get-rated'))['data']
+
+    power_direction = status['battery_power_direction'].lower()
+    power_direction = re.sub('ge$', 'ging', power_direction)
+
+    charging_rate = ''
+    if power_direction == 'charging':
+        charging_rate = ' @ %s %s' % (
+            status['battery_charge_current']['value'],
+            status['battery_charge_current']['unit'])
+    elif power_direction == 'discharging':
+        charging_rate = ' @ %s %s' % (
+            status['battery_discharge_current']['value'],
+            status['battery_discharge_current']['unit'])
+
+    html = '<b>Battery:</b> %s %s' % (
+        status['battery_voltage']['value'],
+        status['battery_voltage']['unit'])
+    html += ' (%s%s, ' % (
+        status['battery_capacity']['value'],
+        status['battery_capacity']['unit'])
+    html += '%s%s)' % (power_direction, charging_rate)
+
+    html += "\n"
+    html += '<b>Load:</b> %s %s' % (
+        status['ac_output_active_power']['value'],
+        status['ac_output_active_power']['unit'])
+    html += ' (%s%%)' % (status['output_load_percent']['value'],)
+
+    if status['pv1_input_power']['value'] > 0:
+        html += "\n"
+        html += '<b>Input power:</b> %s %s' % (
+            status['pv1_input_power']['value'],
+            status['pv1_input_power']['unit'])
+
+    if status['grid_voltage']['value'] > 0 or status['grid_freq']['value'] > 0:
+        html += "\n"
+        html += '<b>AC input:</b> %s %s' % (
+            status['grid_voltage']['value'],
+            status['grid_voltage']['unit'])
+        html += ', %s %s' % (
+            status['grid_freq']['value'],
+            status['grid_freq']['unit'])
+
+    html += "\n"
+    html += '<b>Priority:</b> %s' % (rated['output_source_priority'],)
+
+    html = html.replace("\n", '<br>')
+
+    return status, rated, html
+
+
+class WebSite(http.HTTPServer):
+    _modems_config: ModemsConfig
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self._modems_config = ModemsConfig()
+
+        aiohttp_jinja2.setup(
+            self.app,
+            loader=jinja2.FileSystemLoader(homekit_path('web', 'kbn_templates')),
+            autoescape=jinja2.select_autoescape(['html', 'xml']),
+        )
+        env = aiohttp_jinja2.get_env(self.app)
+        env.filters['tojson'] = lambda obj: json.dumps(obj, separators=(',', ':'))
+
+        self.app.router.add_static('/assets/', path=homekit_path('web', 'kbn_assets'))
+
+        self.get('/main.cgi', self.index)
+
+        self.get('/modems.cgi', self.modems)
+        self.get('/modems/info.ajx', self.modems_ajx)
+        self.get('/modems/verbose.cgi', self.modems_verbose)
+
+        self.get('/inverter.cgi', self.inverter)
+        self.get('/inverter.ajx', self.inverter_ajx)
+        self.get('/pump.cgi', self.pump)
+        self.get('/sms.cgi', self.sms)
+        self.post('/sms.cgi', self.sms_post)
+
+    async def render_page(self,
+                          req: http.Request,
+                          template_name: str,
+                          title: Optional[str] = None,
+                          context: Optional[dict] = None):
+        if context is None:
+            context = {}
+        context = {
+            **context,
+            'head_static': get_head_static()
+        }
+        if title is not None:
+            context['title'] = title
+        response = aiohttp_jinja2.render_template(template_name+'.j2', req, context=context)
+        return response
+
+    async def index(self, req: http.Request):
+        ctx = {}
+        for k in 'inverter', 'sensors':
+            ctx[f'{k}_grafana_url'] = config.app_config[f'{k}_grafana_url']
+        return await self.render_page(req, 'index',
+                                      title="Home web site",
+                                      context=ctx)
+
+    async def modems(self, req: http.Request):
+        return await self.render_page(req, 'modems',
+                                      title='Состояние модемов',
+                                      context=dict(modems=self._modems_config))
+
+    async def modems_ajx(self, req: http.Request):
+        modem = req.query.get('id', None)
+        if modem not in self._modems_config.keys():
+            raise ValueError('invalid modem id')
+
+        modem_cfg = self._modems_config.get(modem)
+        loop = asyncio.get_event_loop()
+        modem_data = await loop.run_in_executor(None, lambda: get_modem_data(modem_cfg))
+
+        html = aiohttp_jinja2.render_string('modem_data.j2', req, context=dict(
+            modem_data=modem_data,
+            modem=modem
+        ))
+
+        return self.ok({'html': html})
+
+    async def modems_verbose(self, req: http.Request):
+        modem = req.query.get('id', None)
+        if modem not in self._modems_config.keys():
+            raise ValueError('invalid modem id')
+
+        modem_cfg = self._modems_config.get(modem)
+        loop = asyncio.get_event_loop()
+        signal, status, traffic, device, dialup_conn = await loop.run_in_executor(None, lambda: get_modem_data(modem_cfg, True))
+        data = [
+            ['Signal', signal],
+            ['Connection', status],
+            ['Traffic', traffic],
+            ['Device info', device],
+            ['Dialup connection', dialup_conn]
+        ]
+
+        modem_name = self._modems_config.getfullname(modem)
+        return await self.render_page(req, 'modem_verbose',
+                                      title=f'Подробная информация о модеме "{modem_name}"',
+                                      context=dict(data=data, modem_name=modem_name))
+
+    async def sms(self, req: http.Request):
+        modem = req.query.get('id', list(self._modems_config.keys())[0])
+        is_outbox = int(req.query.get('outbox', 0)) == 1
+        error = req.query.get('error', None)
+        sent = int(req.query.get('sent', 0)) == 1
+
+        cl = get_modem_client(self._modems_config[modem])
+        messages = cl.sms_list(1, 20, is_outbox)
+        return await self.render_page(req, 'sms',
+                                      title=f"SMS-сообщения ({'исходящие' if is_outbox else 'входящие'}, {modem})",
+                                      context=dict(
+                                          modems=self._modems_config,
+                                          selected_modem=modem,
+                                          is_outbox=is_outbox,
+                                          error=error,
+                                          is_sent=sent,
+                                          messages=messages
+                                      ))
+
+    async def sms_post(self, req: http.Request):
+        modem = req.query.get('id', list(self._modems_config.keys())[0])
+        is_outbox = int(req.query.get('outbox', 0)) == 1
+
+        fd = await req.post()
+        phone = fd.get('phone', None)
+        text = fd.get('text', None)
+
+        return_url = f'/sms.cgi?id={modem}&outbox={int(is_outbox)}'
+        phone = re.sub('\s+', '', phone)
+
+        if len(phone) > 4:
+            country = None
+            if not phone.startswith('+'):
+                country = 'RU'
+            number = phonenumbers.parse(phone, country)
+            if not phonenumbers.is_valid_number(number):
+                raise HTTPFound(f'{return_url}&error=Неверный+номер')
+            phone = phonenumbers.format_number(number, phonenumbers.PhoneNumberFormat.E164)
+
+        cl = get_modem_client(self._modems_config[modem])
+        cl.sms_send(phone, text)
+        raise HTTPFound(return_url)
+
+    async def inverter(self, req: http.Request):
+        action = req.query.get('do', None)
+        if action == 'set-osp':
+            val = req.query.get('value')
+            if val not in ('sub', 'sbu'):
+                raise ValueError('invalid osp value')
+            cl = get_inverter_client()
+            cl.exec('set-output-source-priority',
+                    arguments=(val.upper(),))
+            raise HTTPFound('/inverter.cgi')
+
+        status, rated, html = await asyncio.get_event_loop().run_in_executor(None, get_inverter_data)
+        return await self.render_page(req, 'inverter',
+                                      title='Инвертор',
+                                      context=dict(status=status, rated=rated, html=html))
+
+    async def inverter_ajx(self, req: http.Request):
+        status, rated, html = await asyncio.get_event_loop().run_in_executor(None, get_inverter_data)
+        return self.ok({'html': html})
+
+    async def pump(self, req: http.Request):
+        # TODO
+        # these are blocking calls
+        # should be rewritten using aio
+
+        cl = get_pump_client()
+
+        action = req.query.get('set', None)
+        if action in ('on', 'off'):
+            getattr(cl, action)()
+            raise HTTPFound('/pump.cgi')
+
+        status = cl.status()
+        return await self.render_page(req, 'pump',
+                                      title='Насос',
+                                      context=dict(status=status))
+
+
+if __name__ == '__main__':
+    config.load_app(WebKbnConfig)
+
+    server = WebSite(config.app_config['listen_addr'])
+    server.run()

doc/openwrt_logger.md

@@ -0,0 +1,28 @@
+# openwrt_logger.py
+
+This script is supposed to be run by cron every 5 minutes or so.
+It looks for new lines in log file and sends them to remote server.
+
+OpenWRT must have remote logging enabled (UDP; IP of host this script is launched on; port 514)
+
+`/etc/rsyslog.conf` contains following (assuming `192.168.1.1` is the router IP):
+
+```
+$ModLoad imudp  
+$UDPServerRun 514  
+:fromhost-ip, isequal, "192.168.1.1" /var/log/openwrt.log  
+& ~
+```
+
+Also comment out the following line:
+```
+$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
+```
+
+Cron line example:
+```
+* * * * *  /home/user/homekit/src/openwrt_logger.py --access-point 1 --file /var/wrtlogfs/openwrt-5.log  >/dev/null
+```
+
+`/var/wrtlogfs` is recommended to be tmpfs, to avoid writes on mmc card, in case
+you use arm sbcs as I do.

include/pio/libs/main/library.json

@@ -0,0 +1,12 @@
+{
+  "name": "homekit_main",
+  "version": "1.0.11",
+  "build": {
+    "flags": "-I../../include"
+  },
+  "dependencies": {
+    "homekit_mqtt_module_ota": "file://../../include/pio/libs/mqtt_module_ota",
+    "homekit_mqtt_module_diagnostics": "file://../../include/pio/libs/mqtt_module_diagnostics"
+  }
+}
+

include/pio/libs/mqtt/homekit/mqtt/mqtt.cpp

@@ -119,7 +119,7 @@ void Mqtt::reconnect() {
 void Mqtt::disconnect() {
     // TODO test how this works???
     reconnectTimer.detach();
-    client.disconnect();
+    client.disconnect(true);
 }
 
 void Mqtt::loop() {

include/pio/libs/mqtt/library.json

@@ -1,6 +1,6 @@
 {
   "name": "homekit_mqtt",
-  "version": "1.0.11",
+  "version": "1.0.12",
   "build": {
     "flags": "-I../../include"
   }

include/pio/libs/mqtt_module_diagnostics/library.json

@@ -1,10 +1,10 @@
 {
   "name": "homekit_mqtt_module_diagnostics",
-  "version": "1.0.2",
+  "version": "1.0.3",
   "build": {
     "flags": "-I../../include"
   },
   "dependencies": {
-    "homekit_mqtt": "file://../common/libs/mqtt"
+    "homekit_mqtt": "file://../../include/pio/libs/mqtt"
   }
 }

include/pio/libs/mqtt_module_ota/library.json

@@ -0,0 +1,11 @@
+{
+  "name": "homekit_mqtt_module_ota",
+  "version": "1.0.6",
+  "build": {
+    "flags": "-I../../include"
+  },
+  "dependencies": {
+    "homekit_led": "file://../../include/pio/libs/led",
+    "homekit_mqtt": "file://../../include/pio/libs/mqtt"
+  }
+}

include/pio/libs/mqtt_module_relay/library.json

@@ -0,0 +1,11 @@
+{
+  "name": "homekit_mqtt_module_relay",
+  "version": "1.0.6",
+  "build": {
+    "flags": "-I../../include"
+  },
+  "dependencies": {
+    "homekit_mqtt": "file://../../include/pio/libs/mqtt",
+    "homekit_relay": "file://../../include/pio/libs/relay"
+  }
+}

include/pio/libs/mqtt_module_temphum/library.json

@@ -5,7 +5,7 @@
     "flags": "-I../../include"
   },
   "dependencies": {
-    "homekit_mqtt": "file://../common/libs/mqtt",
+    "homekit_mqtt": "file://../../include/pio/libs/mqtt",
-    "homekit_temphum": "file://../common/libs/temphum"
+    "homekit_temphum": "file://../../include/pio/libs/temphum"
   }
 }

include/pio/libs/temphum/library.json

@@ -1,6 +1,6 @@
 {
   "name": "homekit_temphum",
-  "version": "1.0.3",
+  "version": "1.0.4",
   "build": {
     "flags": "-I../../include"
   }