sms: hide pronet

This reverts commit 3da04de6fd83bca19447a865bf84b3403a14e0c1.

.clang-format

@@ -0,0 +1,66 @@
+# Generated from CLion C/C++ Code Style settings
+BasedOnStyle: LLVM
+AccessModifierOffset: -4
+AlignAfterOpenBracket: Align
+AlignConsecutiveAssignments: None
+AlignOperands: Align
+AllowAllArgumentsOnNextLine: false
+AllowAllConstructorInitializersOnNextLine: false
+AllowAllParametersOfDeclarationOnNextLine: false
+AllowShortBlocksOnASingleLine: Always
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: All
+AllowShortIfStatementsOnASingleLine: Always
+AllowShortLambdasOnASingleLine: All
+AllowShortLoopsOnASingleLine: true
+AlwaysBreakAfterReturnType: None
+AlwaysBreakTemplateDeclarations: Yes
+BreakBeforeBraces: Custom
+BraceWrapping:
+  AfterCaseLabel: false
+  AfterClass: false
+  AfterControlStatement: Never
+  AfterEnum: false
+  AfterFunction: false
+  AfterNamespace: false
+  AfterUnion: false
+  BeforeCatch: false
+  BeforeElse: false
+  IndentBraces: false
+  SplitEmptyFunction: false
+  SplitEmptyRecord: true
+BreakBeforeBinaryOperators: None
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializers: BeforeColon
+BreakInheritanceList: BeforeColon
+ColumnLimit: 0
+CompactNamespaces: false
+ContinuationIndentWidth: 8
+IndentCaseLabels: true
+IndentPPDirectives: None
+IndentWidth: 4
+KeepEmptyLinesAtTheStartOfBlocks: true
+MaxEmptyLinesToKeep: 2
+NamespaceIndentation: All
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: true
+PointerAlignment: Left
+ReflowComments: false
+SpaceAfterCStyleCast: true
+SpaceAfterLogicalNot: false
+SpaceAfterTemplateKeyword: false
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeCpp11BracedList: false
+SpaceBeforeCtorInitializerColon: true
+SpaceBeforeInheritanceColon: true
+SpaceBeforeParens: ControlStatements
+SpaceBeforeRangeBasedForLoopColon: false
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 0
+SpacesInAngles: false
+SpacesInCStyleCastParentheses: false
+SpacesInContainerLiterals: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+TabWidth: 4
+UseTab: Never

.gitignore

@@ -1,19 +1,24 @@
 .idea
+.vscode
 /venv
 /node_modules
 *.pyc
 config.def.h
 __pycache__
 .DS_Store
-/src/test/test_inverter_monitor.log
+/include/test/test_inverter_monitor.log
 /youtrack-certificate
 /cpp
-/src/test.py
-/esp32-cam/CameraWebServer/wifi_password.h
+/test/test.py
+/bin/test.py
+/arduino/ESP32CameraWebServer/wifi_password.h
 cmake-build-*
 .pio
+platformio.ini
 CMakeListsPrivate.txt
-
+/pio/*/CMakeLists.txt
+/pio/*/CMakeListsPrivate.txt
+/pio/*/.gitignore
 *.swp
 
 /localwebsite/vendor

IDEAInspectionsProfile.xml

@@ -1,85 +0,0 @@
-<profile version="1.0">
-  <option name="myName" value="IDEAInspectionsProfile" />
-  <inspection_tool class="CheckTagEmptyBody" enabled="false" level="WARNING" enabled_by_default="false" />
-  <inspection_tool class="CssUnknownProperty" enabled="true" level="WARNING" enabled_by_default="true">
-    <option name="myCustomPropertiesEnabled" value="true" />
-    <option name="myIgnoreVendorSpecificProperties" value="false" />
-    <option name="myCustomPropertiesList">
-      <value>
-        <list size="2">
-          <item index="0" class="java.lang.String" itemvalue="line-clamp" />
-          <item index="1" class="java.lang.String" itemvalue="animation" />
-        </list>
-      </value>
-    </option>
-  </inspection_tool>
-  <inspection_tool class="ES6ModulesDependencies" enabled="true" level="WARNING" enabled_by_default="true" />
-  <inspection_tool class="HtmlUnknownTarget" enabled="false" level="WARNING" enabled_by_default="false" />
-  <inspection_tool class="HttpUrlsUsage" enabled="true" level="WEAK WARNING" enabled_by_default="true">
-    <option name="ignoredUrls">
-      <list>
-        <option value="http://localhost" />
-        <option value="http://127.0.0.1" />
-        <option value="http://0.0.0.0" />
-        <option value="http://www.w3.org/" />
-        <option value="http://json-schema.org/draft" />
-        <option value="http://java.sun.com/" />
-        <option value="http://xmlns.jcp.org/" />
-        <option value="http://javafx.com/javafx/" />
-        <option value="http://javafx.com/fxml" />
-        <option value="http://maven.apache.org/xsd/" />
-        <option value="http://maven.apache.org/POM/" />
-        <option value="http://www.springframework.org/schema/" />
-        <option value="http://www.springframework.org/tags" />
-        <option value="http://www.springframework.org/security/tags" />
-        <option value="http://www.thymeleaf.org" />
-        <option value="http://www.jboss.org/j2ee/schema/" />
-        <option value="http://www.jboss.com/xml/ns/" />
-        <option value="http://www.ibm.com/webservices/xsd" />
-        <option value="http://activemq.apache.org/schema/" />
-        <option value="http://schema.cloudfoundry.org/spring/" />
-        <option value="http://schemas.xmlsoap.org/" />
-        <option value="http://cxf.apache.org/schemas/" />
-        <option value="http://primefaces.org/ui" />
-        <option value="http://tiles.apache.org/" />
-        <option value="http://{{upstream}};" />
-      </list>
-    </option>
-  </inspection_tool>
-  <inspection_tool class="JSUnfilteredForInLoop" enabled="true" level="WEAK WARNING" enabled_by_default="true" />
-  <inspection_tool class="JSUnresolvedFunction" enabled="true" level="ERROR" enabled_by_default="true" />
-  <inspection_tool class="JSUnresolvedLibraryURL" enabled="false" level="WARNING" enabled_by_default="false" />
-  <inspection_tool class="JSUnresolvedVariable" enabled="true" level="ERROR" enabled_by_default="true">
-    <option name="myStrictlyCheckGlobalDefinitions" value="true" />
-    <option name="myStrictlyCheckProperties" value="false" />
-  </inspection_tool>
-  <inspection_tool class="JSXNamespaceValidation" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
-  <inspection_tool class="LessResolvedByNameOnly" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
-  <inspection_tool class="PackageJsonMismatchedDependency" enabled="false" level="WARNING" enabled_by_default="false" />
-  <inspection_tool class="PhpDefineCanBeReplacedWithConstInspection" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
-  <inspection_tool class="PhpIllegalPsrClassPathInspection" enabled="false" level="WARNING" enabled_by_default="false" />
-  <inspection_tool class="PhpUndefinedClassInspection" enabled="true" level="ERROR" enabled_by_default="true" />
-  <inspection_tool class="PhpUndefinedConstantInspection" enabled="true" level="ERROR" enabled_by_default="true" />
-  <inspection_tool class="PhpUndefinedFieldInspection" enabled="true" level="ERROR" enabled_by_default="true" />
-  <inspection_tool class="PhpUndefinedFunctionInspection" enabled="true" level="ERROR" enabled_by_default="true" />
-  <inspection_tool class="PhpUndefinedGotoLabelInspection" enabled="true" level="ERROR" enabled_by_default="true" />
-  <inspection_tool class="PhpUndefinedMethodInspection" enabled="true" level="ERROR" enabled_by_default="true" />
-  <inspection_tool class="PhpUndefinedNamespaceInspection" enabled="true" level="ERROR" enabled_by_default="true" />
-  <inspection_tool class="PyPep8Inspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
-    <option name="ignoredErrors">
-      <list>
-        <option value="E731" />
-      </list>
-    </option>
-  </inspection_tool>
-  <inspection_tool class="PyPep8NamingInspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
-    <option name="ignoredErrors">
-      <list>
-        <option value="N802" />
-      </list>
-    </option>
-  </inspection_tool>
-  <inspection_tool class="SqlNoDataSourceInspection" enabled="false" level="WARNING" enabled_by_default="false" />
-  <inspection_tool class="SqlResolveInspection" enabled="false" level="ERROR" enabled_by_default="false" />
-  <inspection_tool class="bashproGoogleFileNameStyle" enabled="false" level="WARNING" enabled_by_default="false" />
-</profile>

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2022, Evgeny Zinoviev
+Copyright 2022-2024, Evgeny Zinoviev
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification,

README.md

@@ -5,12 +5,6 @@ a country house, solving real life tasks.
 
 Mostly undocumented.
 
-## TODO
-
-esp8266/esp32 code:
-
-- move common stuff to the `commom` directory and use it as a framework
-
 ## License
 
 BSD-3c

bin/camera_node.py

@@ -1,12 +1,13 @@
 #!/usr/bin/env python3
 import asyncio
 import time
+import include_homekit
 
-from home.config import config
-from home.media import MediaNodeServer, ESP32CameraRecordStorage, CameraRecorder
-from home.camera import CameraType, esp32
-from home.util import Addr
-from home import http
+from homekit.config import config
+from homekit.media import MediaNodeServer, ESP32CameraRecordStorage, CameraRecorder
+from homekit.camera import CameraType, esp32
+from homekit.util import Addr
+from homekit import http
 
 
 # Implements HTTP API for a camera.
@@ -65,7 +66,7 @@ class ESP32CameraNodeServer(MediaNodeServer):
 
 
 if __name__ == '__main__':
-    config.load('camera_node')
+    config.load_app('camera_node')
 
     recorder_kwargs = {}
     camera_type = CameraType(config['camera']['type'])

bin/electricity_calc.py

@@ -3,12 +3,12 @@ import logging
 import os
 import sys
 import inspect
-import zoneinfo
+import include_homekit
 
-from home.config import config  # do not remove this import!
+from homekit.config import config  # do not remove this import!
 from datetime import datetime, timedelta
 from logging import Logger
-from home.database import InverterDatabase
+from homekit.database import InverterDatabase
 from argparse import ArgumentParser, ArgumentError
 from typing import Optional
 

bin/esp32_capture.py

@@ -2,10 +2,11 @@
 import asyncio
 import logging
 import os.path
+import include_homekit
 
 from argparse import ArgumentParser
-from home.camera.esp32 import WebClient
-from home.util import parse_addr, Addr
+from homekit.camera.esp32 import WebClient
+from homekit.util import Addr
 from apscheduler.schedulers.asyncio import AsyncIOScheduler
 from datetime import datetime
 from typing import Optional
@@ -50,7 +51,7 @@ if __name__ == '__main__':
 
     loop = asyncio.get_event_loop()
 
-    ESP32Capture(parse_addr(arg.addr), arg.interval, arg.output_directory)
+    ESP32Capture(Addr.fromstring(arg.addr), arg.interval, arg.output_directory)
     try:
         loop.run_forever()
     except KeyboardInterrupt:

bin/esp32cam_capture_diff_node.py

@@ -3,11 +3,12 @@ import asyncio
 import logging
 import os.path
 import tempfile
-import home.telegram.aio as telegram
+import include_homekit
+import homekit.telegram.aio as telegram
 
-from home.config import config
-from home.camera.esp32 import WebClient
-from home.util import parse_addr, send_datagram, stringify
+from homekit.config import config
+from homekit.camera.esp32 import WebClient
+from homekit.util import Addr, send_datagram, stringify
 from apscheduler.schedulers.asyncio import AsyncIOScheduler
 from typing import Optional
 
@@ -34,11 +35,11 @@ async def pyssim(fn1: str, fn2: str) -> float:
 
 class ESP32CamCaptureDiffNode:
     def __init__(self):
-        self.client = WebClient(parse_addr(config['esp32cam_web_addr']))
+        self.client = WebClient(Addr.fromstring(config['esp32cam_web_addr']))
         self.directory = tempfile.gettempdir()
         self.nextpic = 1
         self.first = True
-        self.server_addr = parse_addr(config['node']['server_addr'])
+        self.server_addr = Addr.fromstring(config['node']['server_addr'])
 
         self.scheduler = AsyncIOScheduler()
         self.scheduler.add_job(self.capture, 'interval', seconds=config['node']['interval'])
@@ -76,7 +77,7 @@ class ESP32CamCaptureDiffNode:
 
 
 if __name__ == '__main__':
-    config.load('esp32cam_capture_diff_node')
+    config.load_app('esp32cam_capture_diff_node')
 
     loop = asyncio.get_event_loop()
     ESP32CamCaptureDiffNode()

bin/gpiorelayd.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+import logging
+import os
+import sys
+import include_homekit
+
+from argparse import ArgumentParser
+from homekit.util import Addr
+from homekit.config import config
+from homekit.relay.sunxi_h3_server import RelayServer
+
+logger = logging.getLogger(__name__)
+
+
+if __name__ == '__main__':
+    if os.getegid() != 0:
+        sys.exit('Must be run as root.')
+
+    parser = ArgumentParser()
+    parser.add_argument('--pin', type=str, required=True,
+                        help='name of GPIO pin of Allwinner H3 sunxi board')
+    parser.add_argument('--listen', type=str, required=True,
+                        help='address to listen to, in ip:port format')
+
+    arg = config.load_app(no_config=True, parser=parser)
+    listen = Addr.fromstring(arg.listen)
+
+    try:
+        RelayServer(pinname=arg.pin, addr=listen).run()
+    except KeyboardInterrupt:
+        logger.info('Exiting...')

bin/include_homekit.py

@@ -0,0 +1 @@
+../include_homekit.py

bin/inverter_bot.py

@@ -4,32 +4,40 @@ import re
 import datetime
 import json
 import itertools
+import sys
+import asyncio
+import include_homekit
 
 from inverterd import Format, InverterError
 from html import escape
 from typing import Optional, Tuple, Union
 
-from home.util import chunks
-from home.config import config
-from home.telegram import bot
-from home.inverter import (
+from homekit.util import chunks
+from homekit.config import config, AppConfigUnit
+from homekit.telegram import bot
+from homekit.telegram.config import TelegramBotConfig, TelegramUserListType
+from homekit.inverter import (
     wrapper_instance as inverter,
     beautify_table,
     InverterMonitor,
 )
-from home.inverter.types import (
+from homekit.inverter.types import (
     ChargingEvent,
     ACPresentEvent,
     BatteryState,
     ACMode,
     OutputSourcePriority
 )
-from home.database.inverter_time_formats import *
-from home.api.types import BotType
-from home.api import WebAPIClient
+from homekit.database.inverter_time_formats import FormatDate
+from homekit.api import WebApiClient
 from telegram import ReplyKeyboardMarkup, InlineKeyboardMarkup, InlineKeyboardButton
 
-monitor: Optional[InverterMonitor] = None
+
+if __name__ != '__main__':
+    print(f'this script can not be imported as module', file=sys.stderr)
+    sys.exit(1)
+
+
 db = None
 LT = escape('<=')
 flags_map = {
@@ -42,9 +50,56 @@ flags_map = {
     'alarm_on_on_primary_source_interrupt': 'ALRM',
     'fault_code_record': 'FTCR',
 }
-
 logger = logging.getLogger(__name__)
-config.load('inverter_bot')
+
+
+class InverterBotConfig(AppConfigUnit, TelegramBotConfig):
+    NAME = 'inverter_bot'
+
+    @classmethod
+    def schema(cls) -> Optional[dict]:
+        acmode_item_schema = {
+            'thresholds': {
+                'type': 'list',
+                'required': True,
+                'schema': {
+                    'type': 'list',
+                    'min': 40,
+                    'max': 60
+                },
+            },
+            'initial_current': {'type': 'integer'}
+        }
+
+        return {
+            **super(TelegramBotConfig).schema(),
+            'ac_mode': {
+                'type': 'dict',
+                'required': True,
+                'schema': {
+                    'generator': acmode_item_schema,
+                    'utilities': acmode_item_schema
+                }
+            },
+            'monitor': {
+                'type': 'dict',
+                'required': True,
+                'schema': {
+                    'vlow': {'type': 'integer', 'required': True},
+                    'vcrit': {'type': 'integer', 'required': True},
+                    'gen_currents': {'type': 'list', 'schema': {'type': 'integer'}, 'required': True},
+                    'gen_raise_intervals': {'type': 'list', 'schema': {'type': 'integer'}, 'required': True},
+                    'gen_cur30_v_limit': {'type': 'float', 'required': True},
+                    'gen_cur20_v_limit': {'type': 'float', 'required': True},
+                    'gen_cur10_v_limit': {'type': 'float', 'required': True},
+                    'gen_floating_v': {'type': 'integer', 'required': True},
+                    'gen_floating_time_max': {'type': 'integer', 'required': True}
+                }
+            }
+        }
+
+
+config.load_app(InverterBotConfig)
 
 bot.initialize()
 bot.lang.ru(
@@ -293,9 +348,12 @@ def monitor_charging(event: ChargingEvent, **kwargs) -> None:
     key = f'chrg_evt_{key}'
     if is_util:
         key = f'util_{key}'
+
+    asyncio.ensure_future(
         bot.notify_all(
             lambda lang: bot.lang.get(key, lang, *args)
         )
+    )
 
 
 def monitor_battery(state: BatteryState, v: float, load_watts: int) -> None:
@@ -309,10 +367,12 @@ def monitor_battery(state: BatteryState, v: float, load_watts: int) -> None:
         logger.error('unknown battery state:', state)
         return
 
+    asyncio.ensure_future(
         bot.notify_all(
             lambda lang: bot.lang.get('battery_level_changed', lang,
                                       emoji, bot.lang.get(f'bat_state_{state.name.lower()}', lang), v, load_watts)
         )
+    )
 
 
 def monitor_util(event: ACPresentEvent):
@@ -321,15 +381,19 @@ def monitor_util(event: ACPresentEvent):
     else:
         key = 'disconnected'
     key = f'util_{key}'
+    asyncio.ensure_future(
         bot.notify_all(
             lambda lang: bot.lang.get(key, lang)
         )
+    )
 
 
 def monitor_error(error: str) -> None:
+    asyncio.ensure_future(
         bot.notify_all(
             lambda lang: bot.lang.get('error_message', lang, error)
         )
+    )
 
 
 def osp_change_cb(new_osp: OutputSourcePriority,
@@ -338,35 +402,37 @@ def osp_change_cb(new_osp: OutputSourcePriority,
 
     setosp(new_osp)
 
+    asyncio.ensure_future(
         bot.notify_all(
             lambda lang: bot.lang.get('osp_auto_changed_notification', lang,
                                       bot.lang.get(f'settings_osp_{new_osp.value.lower()}', lang), v, solar_input),
         )
+    )
 
 
 @bot.handler(command='status')
-def full_status(ctx: bot.Context) -> None:
+async def full_status(ctx: bot.Context) -> None:
     status = inverter.exec('get-status', format=Format.TABLE)
-    ctx.reply(beautify_table(status))
+    await ctx.reply(beautify_table(status))
 
 
 @bot.handler(command='config')
-def full_rated(ctx: bot.Context) -> None:
+async def full_rated(ctx: bot.Context) -> None:
     rated = inverter.exec('get-rated', format=Format.TABLE)
-    ctx.reply(beautify_table(rated))
+    await ctx.reply(beautify_table(rated))
 
 
 @bot.handler(command='errors')
-def full_errors(ctx: bot.Context) -> None:
+async def full_errors(ctx: bot.Context) -> None:
     errors = inverter.exec('get-errors', format=Format.TABLE)
-    ctx.reply(beautify_table(errors))
+    await ctx.reply(beautify_table(errors))
 
 
 @bot.handler(command='flags')
-def flags_handler(ctx: bot.Context) -> None:
+async def flags_handler(ctx: bot.Context) -> None:
     flags = inverter.exec('get-flags')['data']
     text, markup = build_flags_keyboard(flags, ctx)
-    ctx.reply(text, markup=markup)
+    await ctx.reply(text, markup=markup)
 
 
 def build_flags_keyboard(flags: dict, ctx: bot.Context) -> Tuple[str, InlineKeyboardMarkup]:
@@ -423,10 +489,10 @@ class SettingsConversation(bot.conversation):
     REDISCHARGE_VOLTAGES = [48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58]
 
     @bot.conventer(START, message='settings')
-    def start_enter(self, ctx: bot.Context):
+    async def start_enter(self, ctx: bot.Context):
         buttons = list(chunks(list(self.START_BUTTONS), 2))
         buttons.reverse()
-        return self.reply(ctx, self.START, ctx.lang('settings_msg'), buttons,
+        return await self.reply(ctx, self.START, ctx.lang('settings_msg'), buttons,
                                 with_cancel=True)
 
     @bot.convinput(START, messages={
@@ -436,16 +502,16 @@ class SettingsConversation(bot.conversation):
         'settings_bat_cut_off_voltage': BAT_CUT_OFF_VOLTAGE,
         'settings_ac_max_charging_current': AC_MAX_CHARGING_CURRENT
     })
-    def start_input(self, ctx: bot.Context):
+    async def start_input(self, ctx: bot.Context):
         pass
 
     @bot.conventer(OSP)
-    def osp_enter(self, ctx: bot.Context):
-        return self.reply(ctx, self.OSP, ctx.lang('settings_osp_msg'), self.OSP_BUTTONS,
+    async def osp_enter(self, ctx: bot.Context):
+        return await self.reply(ctx, self.OSP, ctx.lang('settings_osp_msg'), self.OSP_BUTTONS,
                                 with_back=True)
 
     @bot.convinput(OSP, messages=OSP_BUTTONS)
-    def osp_input(self, ctx: bot.Context):
+    async def osp_input(self, ctx: bot.Context):
         selected_sp = None
         for sp in OutputSourcePriority:
             if ctx.text == ctx.lang(f'settings_osp_{sp.value.lower()}'):
@@ -458,8 +524,9 @@ class SettingsConversation(bot.conversation):
         # apply the mode
         setosp(selected_sp)
 
+        await asyncio.gather(
             # reply to user
-        ctx.reply(ctx.lang('saved'), markup=bot.IgnoreMarkup())
+            ctx.reply(ctx.lang('saved'), markup=bot.IgnoreMarkup()),
 
             # notify other users
             bot.notify_all(
@@ -468,15 +535,17 @@ class SettingsConversation(bot.conversation):
                                           bot.lang.get(f'settings_osp_{selected_sp.value.lower()}', lang)),
                 exclude=(ctx.user_id,)
             )
+        )
+
         return self.END
 
     @bot.conventer(AC_PRESET)
-    def acpreset_enter(self, ctx: bot.Context):
-        return self.reply(ctx, self.AC_PRESET, ctx.lang('settings_ac_preset_msg'), self.AC_PRESET_BUTTONS,
+    async def acpreset_enter(self, ctx: bot.Context):
+        return await self.reply(ctx, self.AC_PRESET, ctx.lang('settings_ac_preset_msg'), self.AC_PRESET_BUTTONS,
                                 with_back=True)
 
     @bot.convinput(AC_PRESET, messages=AC_PRESET_BUTTONS)
-    def acpreset_input(self, ctx: bot.Context):
+    async def acpreset_input(self, ctx: bot.Context):
         if monitor.active_current is not None:
             raise RuntimeError('generator charging program is active')
 
@@ -493,8 +562,9 @@ class SettingsConversation(bot.conversation):
         # save
         bot.db.set_param('ac_mode', str(newmode.value))
 
+        await asyncio.gather(
             # reply to user
-        ctx.reply(ctx.lang('saved'), markup=bot.IgnoreMarkup())
+            ctx.reply(ctx.lang('saved'), markup=bot.IgnoreMarkup()),
 
             # notify other users
             bot.notify_all(
@@ -503,74 +573,76 @@ class SettingsConversation(bot.conversation):
                                           bot.lang.get(str(newmode.value), lang)),
                 exclude=(ctx.user_id,)
             )
+        )
+
         return self.END
 
     @bot.conventer(BAT_THRESHOLDS_1)
-    def thresholds1_enter(self, ctx: bot.Context):
+    async def thresholds1_enter(self, ctx: bot.Context):
         buttons = list(map(lambda v: f'{v} V', self.RECHARGE_VOLTAGES))
         buttons = chunks(buttons, 4)
-        return self.reply(ctx, self.BAT_THRESHOLDS_1, ctx.lang('settings_select_bottom_threshold'), buttons,
+        return await self.reply(ctx, self.BAT_THRESHOLDS_1, ctx.lang('settings_select_bottom_threshold'), buttons,
                                 with_back=True, buttons_lang_completed=True)
 
     @bot.convinput(BAT_THRESHOLDS_1,
                    messages=list(map(lambda n: f'{n} V', RECHARGE_VOLTAGES)),
                    messages_lang_completed=True)
-    def thresholds1_input(self, ctx: bot.Context):
+    async def thresholds1_input(self, ctx: bot.Context):
         v = self._parse_voltage(ctx.text)
         ctx.user_data['bat_thrsh_v1'] = v
-        return self.invoke(self.BAT_THRESHOLDS_2, ctx)
+        return await self.invoke(self.BAT_THRESHOLDS_2, ctx)
 
     @bot.conventer(BAT_THRESHOLDS_2)
-    def thresholds2_enter(self, ctx: bot.Context):
+    async def thresholds2_enter(self, ctx: bot.Context):
         buttons = list(map(lambda v: f'{v} V', self.REDISCHARGE_VOLTAGES))
         buttons = chunks(buttons, 4)
-        return self.reply(ctx, self.BAT_THRESHOLDS_2, ctx.lang('settings_select_upper_threshold'), buttons,
+        return await self.reply(ctx, self.BAT_THRESHOLDS_2, ctx.lang('settings_select_upper_threshold'), buttons,
                                 with_back=True, buttons_lang_completed=True)
 
     @bot.convinput(BAT_THRESHOLDS_2,
                    messages=list(map(lambda n: f'{n} V', REDISCHARGE_VOLTAGES)),
                    messages_lang_completed=True)
-    def thresholds2_input(self, ctx: bot.Context):
+    async def thresholds2_input(self, ctx: bot.Context):
         v2 = v = self._parse_voltage(ctx.text)
         v1 = ctx.user_data['bat_thrsh_v1']
         del ctx.user_data['bat_thrsh_v1']
 
         response = inverter.exec('set-charge-thresholds', (v1, v2))
-        ctx.reply(ctx.lang('saved') if response['result'] == 'ok' else 'ERROR',
+        await ctx.reply(ctx.lang('saved') if response['result'] == 'ok' else 'ERROR',
                         markup=bot.IgnoreMarkup())
         return self.END
 
     @bot.conventer(AC_MAX_CHARGING_CURRENT)
-    def ac_max_enter(self, ctx: bot.Context):
+    async def ac_max_enter(self, ctx: bot.Context):
         buttons = self._get_allowed_ac_charge_amps()
         buttons = map(lambda n: f'{n} A', buttons)
         buttons = [list(buttons)]
-        return self.reply(ctx, self.AC_MAX_CHARGING_CURRENT, ctx.lang('settings_select_max_current'), buttons,
+        return await self.reply(ctx, self.AC_MAX_CHARGING_CURRENT, ctx.lang('settings_select_max_current'), buttons,
                                 with_back=True, buttons_lang_completed=True)
 
     @bot.convinput(AC_MAX_CHARGING_CURRENT, regex=r'^\d+ A$')
-    def ac_max_input(self, ctx: bot.Context):
+    async def ac_max_input(self, ctx: bot.Context):
         a = self._parse_amps(ctx.text)
         allowed = self._get_allowed_ac_charge_amps()
         if a not in allowed:
             raise ValueError('input is not allowed')
 
         response = inverter.exec('set-max-ac-charge-current', (0, a))
-        ctx.reply(ctx.lang('saved') if response['result'] == 'ok' else 'ERROR',
+        await ctx.reply(ctx.lang('saved') if response['result'] == 'ok' else 'ERROR',
                         markup=bot.IgnoreMarkup())
         return self.END
 
     @bot.conventer(BAT_CUT_OFF_VOLTAGE)
-    def cutoff_enter(self, ctx: bot.Context):
-        return self.reply(ctx, self.BAT_CUT_OFF_VOLTAGE, ctx.lang('settings_enter_cutoff_voltage'), None,
+    async def cutoff_enter(self, ctx: bot.Context):
+        return await self.reply(ctx, self.BAT_CUT_OFF_VOLTAGE, ctx.lang('settings_enter_cutoff_voltage'), None,
                                 with_back=True)
 
     @bot.convinput(BAT_CUT_OFF_VOLTAGE, regex=r'^(\d{2}(\.\d{1})?)$')
-    def cutoff_input(self, ctx: bot.Context):
+    async def cutoff_input(self, ctx: bot.Context):
         v = float(ctx.text)
         if 40.0 <= v <= 48.0:
             response = inverter.exec('set-battery-cutoff-voltage', (v,))
-            ctx.reply(ctx.lang('saved') if response['result'] == 'ok' else 'ERROR',
+            await ctx.reply(ctx.lang('saved') if response['result'] == 'ok' else 'ERROR',
                             markup=bot.IgnoreMarkup())
         else:
             raise ValueError('invalid voltage')
@@ -606,38 +678,38 @@ class ConsumptionConversation(bot.conversation):
     INTERVAL_BUTTONS_FLAT = list(itertools.chain.from_iterable(INTERVAL_BUTTONS))
 
     @bot.conventer(START, message='consumption')
-    def start_enter(self, ctx: bot.Context):
-        return self.reply(ctx, self.START, ctx.lang('consumption_msg'), [self.START_BUTTONS],
+    async def start_enter(self, ctx: bot.Context):
+        return await self.reply(ctx, self.START, ctx.lang('consumption_msg'), [self.START_BUTTONS],
                                 with_cancel=True)
 
     @bot.convinput(START, messages={
         'consumption_total': TOTAL,
         'consumption_grid': GRID
     })
-    def start_input(self, ctx: bot.Context):
+    async def start_input(self, ctx: bot.Context):
         pass
 
     @bot.conventer(TOTAL)
-    def total_enter(self, ctx: bot.Context):
-        return self._render_interval_btns(ctx, self.TOTAL)
+    async def total_enter(self, ctx: bot.Context):
+        return await self._render_interval_btns(ctx, self.TOTAL)
 
     @bot.conventer(GRID)
-    def grid_enter(self, ctx: bot.Context):
-        return self._render_interval_btns(ctx, self.GRID)
+    async def grid_enter(self, ctx: bot.Context):
+        return await self._render_interval_btns(ctx, self.GRID)
 
-    def _render_interval_btns(self, ctx: bot.Context, state):
-        return self.reply(ctx, state, ctx.lang('consumption_select_interval'), self.INTERVAL_BUTTONS,
+    async def _render_interval_btns(self, ctx: bot.Context, state):
+        return await self.reply(ctx, state, ctx.lang('consumption_select_interval'), self.INTERVAL_BUTTONS,
                                 with_back=True)
 
     @bot.convinput(TOTAL, messages=INTERVAL_BUTTONS_FLAT)
-    def total_input(self, ctx: bot.Context):
-        return self._render_interval_results(ctx, self.TOTAL)
+    async def total_input(self, ctx: bot.Context):
+        return await self._render_interval_results(ctx, self.TOTAL)
 
     @bot.convinput(GRID, messages=INTERVAL_BUTTONS_FLAT)
-    def grid_input(self, ctx: bot.Context):
-        return self._render_interval_results(ctx, self.GRID)
+    async def grid_input(self, ctx: bot.Context):
+        return await self._render_interval_results(ctx, self.GRID)
 
-    def _render_interval_results(self, ctx: bot.Context, state):
+    async def _render_interval_results(self, ctx: bot.Context, state):
         # if ctx.text == ctx.lang('to_select_interval'):
         #     TODO
         # pass
@@ -661,41 +733,43 @@ class ConsumptionConversation(bot.conversation):
         #     [InlineKeyboardButton(ctx.lang('please_wait'), callback_data='wait')]
         # ])
 
-        message = ctx.reply(ctx.lang('consumption_request_sent'),
+        message = await ctx.reply(ctx.lang('consumption_request_sent'),
                                   markup=bot.IgnoreMarkup())
 
-        api = WebAPIClient(timeout=60)
+        api = WebApiClient(timeout=60)
         method = 'inverter_get_consumed_energy' if state == self.TOTAL else 'inverter_get_grid_consumed_energy'
 
         try:
             wh = getattr(api, method)(s_from, s_to)
-            bot.delete_message(message.chat_id, message.message_id)
-            ctx.reply('%.2f Wh' % (wh,),
+            await bot.delete_message(message.chat_id, message.message_id)
+            await ctx.reply('%.2f Wh' % (wh,),
                             markup=bot.IgnoreMarkup())
             return self.END
         except Exception as e:
-            bot.delete_message(message.chat_id, message.message_id)
+            await asyncio.gather(
+                bot.delete_message(message.chat_id, message.message_id),
                 ctx.reply_exc(e)
+            )
 
 # other
 # -----
 
 @bot.handler(command='monstatus')
-def monstatus_handler(ctx: bot.Context) -> None:
+async def monstatus_handler(ctx: bot.Context) -> None:
     msg = ''
     st = monitor.dump_status()
     for k, v in st.items():
         msg += k + ': ' + str(v) + '\n'
-    ctx.reply(msg)
+    await ctx.reply(msg)
 
 
 @bot.handler(command='monsetcur')
-def monsetcur_handler(ctx: bot.Context) -> None:
-    ctx.reply('not implemented yet')
+async def monsetcur_handler(ctx: bot.Context) -> None:
+    await ctx.reply('not implemented yet')
 
 
 @bot.callbackhandler
-def button_callback(ctx: bot.Context) -> None:
+async def button_callback(ctx: bot.Context) -> None:
     query = ctx.callback_query
 
     if query.data.startswith('flag_'):
@@ -708,7 +782,7 @@ def button_callback(ctx: bot.Context) -> None:
                 json_key = k
                 break
         if not found:
-            query.answer(ctx.lang('flags_invalid'))
+            await query.answer(ctx.lang('flags_invalid'))
             return
 
         flags = inverter.exec('get-flags')['data']
@@ -719,32 +793,31 @@ def button_callback(ctx: bot.Context) -> None:
         response = inverter.exec('set-flag', (flag, target_flag_value))
 
         # notify user
-        query.answer(ctx.lang('done') if response['result'] == 'ok' else ctx.lang('flags_fail'))
+        await query.answer(ctx.lang('done') if response['result'] == 'ok' else ctx.lang('flags_fail'))
 
         # edit message
         flags[json_key] = not cur_flag_value
         text, markup = build_flags_keyboard(flags, ctx)
-        query.edit_message_text(text, reply_markup=markup)
+        await query.edit_message_text(text, reply_markup=markup)
 
     else:
-        query.answer(ctx.lang('unexpected_callback_data'))
+        await query.answer(ctx.lang('unexpected_callback_data'))
 
 
 @bot.exceptionhandler
-def exception_handler(e: Exception, ctx: bot.Context) -> Optional[bool]:
+async def exception_handler(e: Exception, ctx: bot.Context) -> Optional[bool]:
     if isinstance(e, InverterError):
         try:
             err = json.loads(str(e))['message']
         except json.decoder.JSONDecodeError:
             err = str(e)
         err = re.sub(r'((?:.*)?error:) (.*)', r'<b>\1</b> \2', err)
-        ctx.reply(err,
-                  markup=bot.IgnoreMarkup())
+        await ctx.reply(err, markup=bot.IgnoreMarkup())
         return True
 
 
 @bot.handler(message='status')
-def status_handler(ctx: bot.Context) -> None:
+async def status_handler(ctx: bot.Context) -> None:
     gs = inverter.exec('get-status')['data']
     rated = inverter.exec('get-rated')['data']
 
@@ -788,11 +861,11 @@ def status_handler(ctx: bot.Context) -> None:
     html += f'\n<b>{ctx.lang("priority")}</b>: {rated["output_source_priority"]}'
 
     # send response
-    ctx.reply(html)
+    await ctx.reply(html)
 
 
 @bot.handler(message='generation')
-def generation_handler(ctx: bot.Context) -> None:
+async def generation_handler(ctx: bot.Context) -> None:
     today = datetime.date.today()
     yday = today - datetime.timedelta(days=1)
     yday2 = today - datetime.timedelta(days=2)
@@ -822,7 +895,7 @@ def generation_handler(ctx: bot.Context) -> None:
         html += f'\n<b>{ctx.lang("yday2")}:</b> %s Wh' % (gen_yday2['wh'])
 
     # send response
-    ctx.reply(html)
+    await ctx.reply(html)
 
 
 @bot.defaultreplymarkup
@@ -863,28 +936,27 @@ class InverterStore(bot.BotDatabase):
         self.commit()
 
 
-if __name__ == '__main__':
-    inverter.init(host=config['inverter']['ip'], port=config['inverter']['port'])
+inverter.init(host=config['inverter']['ip'], port=config['inverter']['port'])
 
-    bot.set_database(InverterStore())
-    bot.enable_logging(BotType.INVERTER)
+bot.set_database(InverterStore())
+#bot.enable_logging(BotType.INVERTER)
 
-    bot.add_conversation(SettingsConversation(enable_back=True))
-    bot.add_conversation(ConsumptionConversation(enable_back=True))
+bot.add_conversation(SettingsConversation(enable_back=True))
+bot.add_conversation(ConsumptionConversation(enable_back=True))
 
-    monitor = InverterMonitor()
-    monitor.set_charging_event_handler(monitor_charging)
-    monitor.set_battery_event_handler(monitor_battery)
-    monitor.set_util_event_handler(monitor_util)
-    monitor.set_error_handler(monitor_error)
-    monitor.set_osp_need_change_callback(osp_change_cb)
+monitor = InverterMonitor()
+monitor.set_charging_event_handler(monitor_charging)
+monitor.set_battery_event_handler(monitor_battery)
+monitor.set_util_event_handler(monitor_util)
+monitor.set_error_handler(monitor_error)
+monitor.set_osp_need_change_callback(osp_change_cb)
 
-    setacmode(getacmode())
+setacmode(getacmode())
 
-    if not config.get('monitor.disabled'):
+if not config.get('monitor.disabled'):
     logging.info('starting monitor')
     monitor.start()
 
-    bot.run()
+bot.run()
 
-    monitor.stop()
+monitor.stop()

bin/inverter_mqtt_util.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+import include_homekit
+
+from argparse import ArgumentParser
+from homekit.config import config
+from homekit.mqtt import MqttWrapper, MqttNode
+
+
+if __name__ == '__main__':
+    parser = ArgumentParser()
+    parser.add_argument('mode', type=str, choices=('sender', 'receiver'), nargs=1)
+
+    config.load_app('inverter_mqtt_util', parser=parser)
+    arg = parser.parse_args()
+    mode = arg.mode[0]
+
+    mqtt = MqttWrapper(client_id=f'inverter_mqtt_{mode}',
+                       clean_session=mode != 'receiver')
+    node = MqttNode(node_id='inverter')
+    module_kwargs = {}
+    if mode == 'sender':
+        module_kwargs['status_poll_freq'] = int(config.app_config['poll_freq'])
+        module_kwargs['generation_poll_freq'] = int(config.app_config['generation_poll_freq'])
+    node.load_module('inverter', **module_kwargs)
+    mqtt.add_node(node)
+
+    mqtt.connect_and_loop()

bin/inverterd_emulator.py

@@ -1,7 +1,8 @@
 #!/usr/bin/env python3
 import logging
+import include_homekit
 
-from home.inverter.emulator import InverterEmulator
+from homekit.inverter.emulator import InverterEmulator
 
 
 if __name__ == '__main__':

bin/ipcam_capture.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+import include_homekit
+import sys
+import os
+import subprocess
+import asyncio
+import signal
+
+from typing import TextIO
+from argparse import ArgumentParser
+from socket import gethostname
+from asyncio.streams import StreamReader
+from homekit.config import config as homekit_config
+from homekit.linux import LinuxBoardsConfig
+from homekit.camera import IpcamConfig, CaptureType
+from homekit.camera.util import get_hls_directory, get_hls_channel_name, get_recordings_path
+
+ipcam_config = IpcamConfig()
+lbc_config = LinuxBoardsConfig()
+channels = (1, 2)
+tasks = []
+restart_delay = 3
+lock = asyncio.Lock()
+worker_type: CaptureType
+
+
+async def read_output(stream: StreamReader,
+                      thread_name: str,
+                      output: TextIO):
+    try:
+        while True:
+            line = await stream.readline()
+            if not line:
+                break
+            print(f"[{thread_name}] {line.decode().strip()}", file=output)
+
+    except asyncio.LimitOverrunError:
+        print(f"[{thread_name}] Output limit exceeded.", file=output)
+
+    except Exception as e:
+        print(f"[{thread_name}] Error occurred while reading output: {e}", file=sys.stderr)
+
+
+async def run_ffmpeg(cam: int, channel: int):
+    prefix = get_hls_channel_name(cam, channel)
+
+    if homekit_config.app_config.logging_is_verbose():
+        debug_args = ['-v', '-info']
+    else:
+        debug_args = ['-nostats', '-loglevel', 'error']
+
+    # protocol = 'tcp' if ipcam_config.should_use_tcp_for_rtsp(cam) else 'udp'
+    protocol = 'tcp'
+    user, pw = ipcam_config.get_rtsp_creds()
+    ip = ipcam_config.get_camera_ip(cam)
+    path = ipcam_config.get_camera_type(cam).get_channel_url(channel)
+    ext = ipcam_config.get_camera_container(cam)
+    ffmpeg_command = ['ffmpeg', *debug_args,
+                      '-rtsp_transport', protocol,
+                      '-i', f'rtsp://{user}:{pw}@{ip}:554{path}',
+                      '-c', 'copy',]
+
+    if worker_type == CaptureType.HLS:
+        ffmpeg_command.extend(['-bufsize', '1835k',
+                               '-pix_fmt', 'yuv420p',
+                               '-flags', '-global_header',
+                               '-hls_time', '2',
+                               '-hls_list_size', '3',
+                               '-hls_flags', 'delete_segments',
+                               os.path.join(get_hls_directory(cam, channel), 'live.m3u8')])
+
+    elif worker_type == CaptureType.RECORD:
+        ffmpeg_command.extend(['-f', 'segment',
+                               '-strftime', '1',
+                               '-segment_time', '00:10:00',
+                               '-segment_atclocktime', '1',
+                               os.path.join(get_recordings_path(cam), f'record_%Y-%m-%d-%H.%M.%S.{ext.value}')])
+
+    else:
+        raise ValueError(f'invalid worker type: {worker_type}')
+
+    while True:
+        try:
+            process = await asyncio.create_subprocess_exec(
+                *ffmpeg_command,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE
+            )
+
+            stdout_task = asyncio.create_task(read_output(process.stdout, prefix, sys.stdout))
+            stderr_task = asyncio.create_task(read_output(process.stderr, prefix, sys.stderr))
+
+            await asyncio.gather(stdout_task, stderr_task)
+
+            # check the return code of the process
+            if process.returncode != 0:
+                raise subprocess.CalledProcessError(process.returncode, ffmpeg_command)
+
+        except (FileNotFoundError, PermissionError, subprocess.CalledProcessError) as e:
+            # an error occurred, print the error message
+            error_message = f"Error occurred in {prefix}: {e}"
+            print(error_message, file=sys.stderr)
+
+        # sleep for 5 seconds before restarting the process
+        await asyncio.sleep(restart_delay)
+
+
+async def run():
+    kwargs = {}
+    if worker_type == CaptureType.RECORD:
+        kwargs['filter_by_server'] = gethostname()
+    for cam in ipcam_config.get_all_cam_names(**kwargs):
+        for channel in channels:
+            task = asyncio.create_task(run_ffmpeg(cam, channel))
+            tasks.append(task)
+
+    try:
+        await asyncio.gather(*tasks)
+    except KeyboardInterrupt:
+        print('KeyboardInterrupt: stopping processes...', file=sys.stderr)
+        for task in tasks:
+            task.cancel()
+
+        # wait for subprocesses to terminate
+        await asyncio.gather(*tasks, return_exceptions=True)
+
+        # send termination signal to all subprocesses
+        for task in tasks:
+            process = task.get_stack()
+            if process:
+                process.send_signal(signal.SIGTERM)
+
+
+if __name__ == '__main__':
+    capture_types = [t.value for t in CaptureType]
+    parser = ArgumentParser()
+    parser.add_argument('type', type=str, metavar='CAPTURE_TYPE', choices=tuple(capture_types),
+                        help='capture type (variants: '+', '.join(capture_types)+')')
+
+    arg = homekit_config.load_app(no_config=True, parser=parser)
+    worker_type = CaptureType(arg['type'])
+
+    asyncio.run(run())

bin/ipcam_motion_worker.sh

@@ -5,7 +5,7 @@ set -e
 DIR="$( cd "$( dirname "$(realpath "${BASH_SOURCE[0]}")" )" &>/dev/null && pwd )"
 PROGNAME="$0"
 
-. "$DIR/lib.bash"
+. "$DIR/../include/bash/include.bash"
 
 curl_opts="-s --connect-timeout 10 --retry 5 --max-time 180 --retry-delay 0 --retry-max-time 180"
 allow_multiple=

bin/ipcam_ntp_util.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+import include_homekit
+import hikvision, xmeye
+
+from enum import Enum, auto
+from typing import Optional
+from argparse import ArgumentParser, ArgumentError
+from homekit.util import validate_ipv4_or_hostname
+from homekit.camera import IpcamConfig, CameraType
+
+ipcam_config = IpcamConfig()
+
+
+class Action(Enum):
+    GET_NTP = auto()
+    SET_NTP = auto()
+
+
+def process_camera(host: str,
+                   action: Action,
+                   login: str,
+                   password: str,
+                   camera_type: CameraType,
+                   ntp_server: Optional[str] = None):
+    if camera_type.is_hikvision():
+        client = hikvision.ISAPIClient(host)
+        try:
+            client.auth(login, password)
+            if action == Action.GET_NTP:
+                print(f'[{host}] {client.get_ntp_server()}')
+                return
+
+            client.set_ntp_server(ntp_server)
+            print(f'[{host}] done')
+
+        except hikvision.AuthError as e:
+            print(f'[{host}] ({str(e)})')
+
+        except hikvision.ResponseError as e:
+            print(f'[{host}] ({str(e)})')
+
+    elif camera_type.is_xmeye():
+        try:
+            client = xmeye.XMEyeCamera(hostname=host, username=login, password=password)
+            client.login()
+
+            if action == Action.GET_NTP:
+                print(f'[{host}] {client.get_ntp_server()}')
+                return
+
+            client.set_ntp_server(ntp_server)
+            print(f'[{host}] done')
+
+        except OSError as e:
+            print(f'[{host}] ({str(e)})')
+
+
+def main():
+    camera_types = ['hikvision', 'xmeye']
+    parser = ArgumentParser()
+    parser.add_argument('--camera', type=str)
+    parser.add_argument('--camera-type', type=str, choices=camera_types)
+    parser.add_argument('--all', action='store_true')
+    parser.add_argument('--all-of-type', type=str, choices=camera_types)
+    parser.add_argument('--get-ntp-server', action='store_true')
+    parser.add_argument('--set-ntp-server', type=str)
+    parser.add_argument('--username', type=str)
+    parser.add_argument('--password', type=str)
+    args = parser.parse_args()
+
+    if args.all and args.all_of_type:
+        raise ArgumentError(None, 'you can\'t pass both --all and --all-of-type')
+
+    if not args.camera and not args.all and not args.all_of_type:
+        raise ArgumentError(None, 'either --all, --all-of-type <TYPE> or --camera <NUM> is required')
+
+    if not args.get_ntp_server and not args.set_ntp_server:
+        raise ArgumentError(None, 'either --get-ntp-server or --set-ntp-server is required')
+
+    action = Action.GET_NTP if args.get_ntp_server else Action.SET_NTP
+    login = args.username if args.username else ipcam_config['web_creds']['login']
+    password = args.password if args.password else ipcam_config['web_creds']['password']
+
+    if action == Action.SET_NTP:
+        if not args.set_ntp_server:
+            raise ArgumentError(None, '--set-ntp-server is required')
+        if not validate_ipv4_or_hostname(args.set_ntp_server):
+            raise ArgumentError(None, 'input ntp server is neither ip address nor a valid hostname')
+
+    kwargs = {}
+    if args.set_ntp_server:
+        kwargs['ntp_server'] = args.set_ntp_server
+    if not args.all and not args.all_of_type:
+        if not args.camera_type:
+            raise ArgumentError(None, '--camera-type is required')
+
+        if not ipcam_config.has_camera(int(args.camera)):
+            raise ArgumentError(None, f'invalid camera {args.camera}')
+        camera_host = ipcam_config.get_camera_ip(args.camera)
+
+        if args.camera_type == 'hikvision':
+            camera_type = CameraType.HIKVISION_264
+        elif args.camera_type == 'xmeye':
+            camera_type = CameraType.XMEYE
+        else:
+            raise ValueError('invalid --camera-type')
+        process_camera(camera_host, action, login, password, camera_type, **kwargs)
+    else:
+        for cam in ipcam_config.get_all_cam_names():
+            if not ipcam_config.is_camera_enabled(cam):
+                continue
+
+            cam_type = ipcam_config.get_camera_type(cam)
+            if args.all_of_type == 'hikvision' and not cam_type.is_hikvision():
+                continue
+            if args.all_of_type == 'xmeye' and not ipcam_config.get_camera_type(cam).is_xmeye():
+                continue
+            process_camera(ipcam_config.get_camera_ip(cam), action, login, password, cam_type, **kwargs)
+
+
+if __name__ == '__main__':
+    main()

bin/ipcam_server.py

@@ -1,58 +1,53 @@
 #!/usr/bin/env python3
 import logging
 import os
-import re
 import asyncio
 import time
 import shutil
-import home.telegram.aio as telegram
+import include_homekit
 
+import homekit.telegram.aio as telegram
+
+from socket import gethostname
+from argparse import ArgumentParser
 from apscheduler.schedulers.asyncio import AsyncIOScheduler
 from asyncio import Lock
 
-from home.config import config
-from home import http
-from home.database.sqlite import SQLiteBase
-from home.camera import util as camutil
+from homekit.config import config as homekit_config
+from homekit.linux import LinuxBoardsConfig
+from homekit.util import Addr
+from homekit import http
+from homekit.database.sqlite import SQLiteBase
+from homekit.camera import util as camutil, IpcamConfig
+from homekit.camera.types import (
+    TimeFilterType,
+    TelegramLinkType,
+    VideoContainerType
+)
+from homekit.camera.util import (
+    get_recordings_path,
+    get_motion_path,
+    is_valid_recording_name,
+    datetime_from_filename
+)
 
-from enum import Enum
 from typing import Optional, Union, List, Tuple
 from datetime import datetime, timedelta
 from functools import cmp_to_key
 
 
-class TimeFilterType(Enum):
-    FIX = 'fix'
-    MOTION = 'motion'
-    MOTION_START = 'motion_start'
-
-
-class TelegramLinkType(Enum):
-    FRAGMENT = 'fragment'
-    ORIGINAL_FILE = 'original_file'
-
-
-def valid_recording_name(filename: str) -> bool:
-    return filename.startswith('record_') and filename.endswith('.mp4')
-
-
-def filename_to_datetime(filename: str) -> datetime:
-    filename = os.path.basename(filename).replace('record_', '').replace('.mp4', '')
-    return datetime.strptime(filename, datetime_format)
-
-
-def get_all_cams() -> list:
-    return [cam for cam in config['camera'].keys()]
+ipcam_config = IpcamConfig()
+lbc_config = LinuxBoardsConfig()
 
 
 # ipcam database
 # --------------
 
-class IPCamServerDatabase(SQLiteBase):
+class IpcamServerDatabase(SQLiteBase):
     SCHEMA = 4
 
-    def __init__(self):
-        super().__init__()
+    def __init__(self, path=None):
+        super().__init__(path=path)
 
     def schema_init(self, version: int) -> None:
         cursor = self.cursor()
@@ -64,7 +59,7 @@ class IPCamServerDatabase(SQLiteBase):
                 fix_time INTEGER NOT NULL,
                 motion_time INTEGER NOT NULL
             )""")
-            for cam in config['camera'].keys():
+            for cam in ipcam_config.get_all_cam_names_for_this_server():
                 self.add_camera(cam)
 
         if version < 2:
@@ -132,7 +127,7 @@ class IPCamServerDatabase(SQLiteBase):
 # ipcam web api
 # -------------
 
-class IPCamWebServer(http.HTTPServer):
+class IpcamWebServer(http.HTTPServer):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
@@ -143,16 +138,16 @@ class IPCamWebServer(http.HTTPServer):
         self.get('/api/timestamp/{name}/{type}', self.get_timestamp)
         self.get('/api/timestamp/all', self.get_all_timestamps)
 
-        self.post('/api/debug/migrate-mtimes', self.debug_migrate_mtimes)
         self.post('/api/debug/fix', self.debug_fix)
         self.post('/api/debug/cleanup', self.debug_cleanup)
+
         self.post('/api/timestamp/{name}/{type}', self.set_timestamp)
 
         self.post('/api/motion/done/{name}', self.submit_motion)
         self.post('/api/motion/fail/{name}', self.submit_motion_failure)
 
-        self.get('/api/motion/params/{name}', self.get_motion_params)
-        self.get('/api/motion/params/{name}/roi', self.get_motion_roi_params)
+        # self.get('/api/motion/params/{name}', self.get_motion_params)
+        # self.get('/api/motion/params/{name}/roi', self.get_motion_roi_params)
 
         self.queue_lock = Lock()
 
@@ -170,7 +165,7 @@ class IPCamWebServer(http.HTTPServer):
 
         files = get_recordings_files(camera, filter, limit)
         if files:
-            time = filename_to_datetime(files[len(files)-1]['name'])
+            time = datetime_from_filename(files[len(files)-1]['name'])
             db.set_timestamp(camera, TimeFilterType.MOTION_START, time)
         return self.ok({'files': files})
 
@@ -185,7 +180,7 @@ class IPCamWebServer(http.HTTPServer):
             if files:
                 times_by_cam = {}
                 for file in files:
-                    time = filename_to_datetime(file['name'])
+                    time = datetime_from_filename(file['name'])
                     if file['cam'] not in times_by_cam or times_by_cam[file['cam']] < time:
                         times_by_cam[file['cam']] = time
                 for cam, time in times_by_cam.items():
@@ -197,14 +192,14 @@ class IPCamWebServer(http.HTTPServer):
         cam = int(req.match_info['name'])
         file = req.match_info['file']
 
-        fullpath = os.path.join(config['camera'][cam]['recordings_path'], file)
+        fullpath = os.path.join(get_recordings_path(cam), file)
         if not os.path.isfile(fullpath):
             raise ValueError(f'file "{fullpath}" does not exists')
 
         return http.FileResponse(fullpath)
 
     async def camlist(self, req: http.Request):
-        return self.ok(config['camera'])
+        return self.ok(ipcam_config.get_all_cam_names_for_this_server())
 
     async def submit_motion(self, req: http.Request):
         data = await req.post()
@@ -213,7 +208,7 @@ class IPCamWebServer(http.HTTPServer):
         timecodes = data['timecodes']
         filename = data['filename']
 
-        time = filename_to_datetime(filename)
+        time = datetime_from_filename(filename)
 
         try:
             if timecodes != '':
@@ -236,27 +231,10 @@ class IPCamWebServer(http.HTTPServer):
         message = data['message']
 
         db.add_motion_failure(camera, filename, message)
-        db.set_timestamp(camera, TimeFilterType.MOTION, filename_to_datetime(filename))
+        db.set_timestamp(camera, TimeFilterType.MOTION, datetime_from_filename(filename))
 
         return self.ok()
 
-    async def debug_migrate_mtimes(self, req: http.Request):
-        written = {}
-        for cam in config['camera'].keys():
-            confdir = os.path.join(os.getenv('HOME'), '.config', f'video-util-{cam}')
-            for time_type in TimeFilterType:
-                txt_file = os.path.join(confdir, f'{time_type.value}_mtime')
-                if os.path.isfile(txt_file):
-                    with open(txt_file, 'r') as fd:
-                        data = fd.read()
-                        db.set_timestamp(cam, time_type, int(data.strip()))
-
-                        if cam not in written:
-                            written[cam] = []
-                        written[cam].append(time_type)
-
-        return self.ok({'written': written})
-
     async def debug_fix(self, req: http.Request):
         asyncio.ensure_future(fix_job())
         return self.ok()
@@ -277,26 +255,26 @@ class IPCamWebServer(http.HTTPServer):
     async def get_all_timestamps(self, req: http.Request):
         return self.ok(db.get_all_timestamps())
 
-    async def get_motion_params(self, req: http.Request):
-        data = config['motion_params'][int(req.match_info['name'])]
-        lines = [
-            f'threshold={data["threshold"]}',
-            f'min_event_length=3s',
-            f'frame_skip=2',
-            f'downscale_factor=3',
-        ]
-        return self.plain('\n'.join(lines)+'\n')
-
-    async def get_motion_roi_params(self, req: http.Request):
-        data = config['motion_params'][int(req.match_info['name'])]
-        return self.plain('\n'.join(data['roi'])+'\n')
+    # async def get_motion_params(self, req: http.Request):
+    #     data = config['motion_params'][int(req.match_info['name'])]
+    #     lines = [
+    #         f'threshold={data["threshold"]}',
+    #         f'min_event_length=3s',
+    #         f'frame_skip=2',
+    #         f'downscale_factor=3',
+    #     ]
+    #     return self.plain('\n'.join(lines)+'\n')
+    #
+    # async def get_motion_roi_params(self, req: http.Request):
+    #     data = config['motion_params'][int(req.match_info['name'])]
+    #     return self.plain('\n'.join(data['roi'])+'\n')
 
     @staticmethod
     def _getset_timestamp_params(req: http.Request, need_time=False):
         values = []
 
         cam = int(req.match_info['name'])
-        assert cam in config['camera'], 'invalid camera'
+        assert cam in ipcam_config.get_all_cam_names_for_this_server(), 'invalid camera'
 
         values.append(cam)
         values.append(TimeFilterType(req.match_info['type']))
@@ -304,7 +282,7 @@ class IPCamWebServer(http.HTTPServer):
         if need_time:
             time = req.query['time']
             if time.startswith('record_'):
-                time = filename_to_datetime(time)
+                time = datetime_from_filename(time)
             elif time.isnumeric():
                 time = int(time)
             else:
@@ -317,32 +295,24 @@ class IPCamWebServer(http.HTTPServer):
 # other global stuff
 # ------------------
 
-def open_database():
+def open_database(database_path: str):
     global db
-    db = IPCamServerDatabase()
+    db = IpcamServerDatabase(database_path)
 
     # update cams list in database, if needed
-    cams = db.get_all_timestamps().keys()
-    for cam in config['camera']:
-        if cam not in cams:
+    stored_cams = db.get_all_timestamps().keys()
+    for cam in ipcam_config.get_all_cam_names_for_this_server():
+        if cam not in stored_cams:
             db.add_camera(cam)
 
 
-def get_recordings_path(cam: int) -> str:
-    return config['camera'][cam]['recordings_path']
-
-
-def get_motion_path(cam: int) -> str:
-    return config['camera'][cam]['motion_path']
-
-
 def get_recordings_files(cam: Optional[int] = None,
                          time_filter_type: Optional[TimeFilterType] = None,
                          limit=0) -> List[dict]:
     from_time = 0
     to_time = int(time.time())
 
-    cams = [cam] if cam is not None else get_all_cams()
+    cams = [cam] if cam is not None else ipcam_config.get_all_cam_names_for_this_server()
     files = []
     for cam in cams:
         if time_filter_type:
@@ -359,7 +329,7 @@ def get_recordings_files(cam: Optional[int] = None,
             'name': file,
             'size': os.path.getsize(os.path.join(recdir, file))}
                  for file in os.listdir(recdir)
-                 if valid_recording_name(file) and from_time < filename_to_datetime(file) <= to_time]
+                 if is_valid_recording_name(file) and from_time < datetime_from_filename(file) <= to_time]
         cam_files.sort(key=lambda file: file['name'])
 
         if cam_files:
@@ -379,7 +349,7 @@ def get_recordings_files(cam: Optional[int] = None,
 async def process_fragments(camera: int,
                             filename: str,
                             fragments: List[Tuple[int, int]]) -> None:
-    time = filename_to_datetime(filename)
+    time = datetime_from_filename(filename)
 
     rec_dir = get_recordings_path(camera)
     motion_dir = get_motion_path(camera)
@@ -389,8 +359,8 @@ async def process_fragments(camera: int,
     for fragment in fragments:
         start, end = fragment
 
-        start -= config['motion']['padding']
-        end += config['motion']['padding']
+        start -= ipcam_config['motion_padding']
+        end += ipcam_config['motion_padding']
 
         if start < 0:
             start = 0
@@ -405,14 +375,14 @@ async def process_fragments(camera: int,
                                  start_pos=start,
                                  duration=duration)
 
-    if fragments and 'telegram' in config['motion'] and config['motion']['telegram']:
+    if fragments and ipcam_config['motion_telegram']:
         asyncio.ensure_future(motion_notify_tg(camera, filename, fragments))
 
 
 async def motion_notify_tg(camera: int,
                            filename: str,
                            fragments: List[Tuple[int, int]]):
-    dt_file = filename_to_datetime(filename)
+    dt_file = datetime_from_filename(filename)
     fmt = '%H:%M:%S'
 
     text = f'Camera: <b>{camera}</b>\n'
@@ -420,8 +390,8 @@ async def motion_notify_tg(camera: int,
     text += _tg_links(TelegramLinkType.ORIGINAL_FILE, camera, filename)
 
     for start, end in fragments:
-        start -= config['motion']['padding']
-        end += config['motion']['padding']
+        start -= ipcam_config['motion_padding']
+        end += ipcam_config['motion_padding']
 
         if start < 0:
             start = 0
@@ -443,7 +413,7 @@ def _tg_links(link_type: TelegramLinkType,
               camera: int,
               file: str) -> str:
     links = []
-    for link_name, link_template in config['telegram'][f'{link_type.value}_url_templates']:
+    for link_name, link_template in ipcam_config[f'{link_type.value}_url_templates']:
         link = link_template.replace('{camera}', str(camera)).replace('{file}', file)
         links.append(f'<a href="{link}">{link_name}</a>')
     return ' '.join(links)
@@ -459,7 +429,7 @@ async def fix_job() -> None:
 
     try:
         fix_job_running = True
-        for cam in config['camera'].keys():
+        for cam in ipcam_config.get_all_cam_names_for_this_server():
             files = get_recordings_files(cam, TimeFilterType.FIX)
             if not files:
                 logger.debug(f'fix_job: no files for camera {cam}')
@@ -470,7 +440,7 @@ async def fix_job() -> None:
             for file in files:
                 fullpath = os.path.join(get_recordings_path(cam), file['name'])
                 await camutil.ffmpeg_recreate(fullpath)
-                timestamp = filename_to_datetime(file['name'])
+                timestamp = datetime_from_filename(file['name'])
                 if timestamp:
                     db.set_timestamp(cam, TimeFilterType.FIX, timestamp)
 
@@ -479,21 +449,9 @@ async def fix_job() -> None:
 
 
 async def cleanup_job() -> None:
-    def fn2dt(name: str) -> datetime:
-        name = os.path.basename(name)
-
-        if name.startswith('record_'):
-            return datetime.strptime(re.match(r'record_(.*?)\.mp4', name).group(1), datetime_format)
-
-        m = re.match(rf'({datetime_format_re})__{datetime_format_re}\.mp4', name)
-        if m:
-            return datetime.strptime(m.group(1), datetime_format)
-
-        raise ValueError(f'unrecognized filename format: {name}')
-
     def compare(i1: str, i2: str) -> int:
-        dt1 = fn2dt(i1)
-        dt2 = fn2dt(i2)
+        dt1 = datetime_from_filename(i1)
+        dt2 = datetime_from_filename(i2)
 
         if dt1 < dt2:
             return -1
@@ -513,18 +471,19 @@ async def cleanup_job() -> None:
         cleanup_job_running = True
 
         gb = float(1 << 30)
-        for storage in config['storages']:
+        disk_number = 0
+        for storage in lbc_config.get_board_disks(gethostname()):
+            disk_number += 1
             if os.path.exists(storage['mountpoint']):
                 total, used, free = shutil.disk_usage(storage['mountpoint'])
                 free_gb = free // gb
-                if free_gb < config['cleanup_min_gb']:
-                    # print(f"{storage['mountpoint']}: free={free}, free_gb={free_gb}")
+                if free_gb < ipcam_config['cleanup_min_gb']:
                     cleaned = 0
                     files = []
-                    for cam in storage['cams']:
-                        for _dir in (config['camera'][cam]['recordings_path'], config['camera'][cam]['motion_path']):
+                    for cam in ipcam_config.get_all_cam_names_for_this_server(filter_by_disk=disk_number):
+                        for _dir in (get_recordings_path(cam), get_motion_path(cam)):
                             files += list(map(lambda file: os.path.join(_dir, file), os.listdir(_dir)))
-                        files = list(filter(lambda path: os.path.isfile(path) and path.endswith('.mp4'), files))
+                        files = list(filter(lambda path: os.path.isfile(path) and path.endswith(tuple([f'.{t.value}' for t in VideoContainerType])), files))
                         files.sort(key=cmp_to_key(compare))
 
                     for file in files:
@@ -534,7 +493,7 @@ async def cleanup_job() -> None:
                             cleaned += size
                         except OSError as e:
                             logger.exception(e)
-                        if (free + cleaned) // gb >= config['cleanup_min_gb']:
+                        if (free + cleaned) // gb >= ipcam_config['cleanup_min_gb']:
                             break
             else:
                 logger.error(f"cleanup_job: {storage['mountpoint']} not found")
@@ -547,8 +506,8 @@ cleanup_job_running = False
 
 datetime_format = '%Y-%m-%d-%H.%M.%S'
 datetime_format_re = r'\d{4}-\d{2}-\d{2}-\d{2}\.\d{2}.\d{2}'
-db: Optional[IPCamServerDatabase] = None
-server: Optional[IPCamWebServer] = None
+db: Optional[IpcamServerDatabase] = None
+server: Optional[IpcamWebServer] = None
 logger = logging.getLogger(__name__)
 
 
@@ -556,18 +515,25 @@ logger = logging.getLogger(__name__)
 # --------------------
 
 if __name__ == '__main__':
-    config.load('ipcam_server')
+    parser = ArgumentParser()
+    parser.add_argument('--listen', type=str, required=True)
+    parser.add_argument('--database-path', type=str, required=True)
+    arg = homekit_config.load_app(no_config=True, parser=parser)
 
-    open_database()
+    open_database(arg.database_path)
 
     loop = asyncio.get_event_loop()
 
     try:
         scheduler = AsyncIOScheduler(event_loop=loop)
-        if config['fix_enabled']:
-            scheduler.add_job(fix_job, 'interval', seconds=config['fix_interval'], misfire_grace_time=None)
+        if ipcam_config['fix_enabled']:
+            scheduler.add_job(fix_job, 'interval',
+                              seconds=ipcam_config['fix_interval'],
+                              misfire_grace_time=None)
 
-        scheduler.add_job(cleanup_job, 'interval', seconds=config['cleanup_interval'], misfire_grace_time=None)
+        scheduler.add_job(cleanup_job, 'interval',
+                          seconds=ipcam_config['cleanup_interval'],
+                          misfire_grace_time=None)
         scheduler.start()
     except KeyError:
         pass
@@ -575,5 +541,5 @@ if __name__ == '__main__':
     asyncio.ensure_future(fix_job())
     asyncio.ensure_future(cleanup_job())
 
-    server = IPCamWebServer(config.get_addr('server.listen'))
+    server = IpcamWebServer(Addr.fromstring(arg.listen))
     server.run()

bin/lugovaya_pump_mqtt_bot.py

@@ -0,0 +1,207 @@
+#!/usr/bin/env python3
+import datetime
+import include_homekit
+
+from enum import Enum
+from typing import Optional
+from telegram import ReplyKeyboardMarkup, User
+
+from homekit.config import config, AppConfigUnit
+from homekit.telegram import bot
+from homekit.telegram.config import TelegramBotConfig
+from homekit.telegram._botutil import user_any_name
+from homekit.mqtt import MqttNode, MqttPayload, MqttNodesConfig, MqttWrapper
+from homekit.mqtt.module.relay import MqttRelayState, MqttRelayModule
+from homekit.mqtt.module.diagnostics import InitialDiagnosticsPayload, DiagnosticsPayload
+
+
+class LugovayaPumpMqttBotConfig(TelegramBotConfig, AppConfigUnit):
+    NAME = 'lugovaya_pump_mqtt_bot'
+
+    @classmethod
+    def schema(cls) -> Optional[dict]:
+        return {
+            **TelegramBotConfig.schema(),
+            'relay_node_id': {
+                'type': 'string',
+                'required': True
+            },
+        }
+
+    @staticmethod
+    def custom_validator(data):
+        relay_node_names = MqttNodesConfig().get_nodes(filters=('relay',), only_names=True)
+        if data['relay_node_id'] not in relay_node_names:
+            raise ValueError('unknown relay node "%s"' % (data['relay_node_id'],))
+
+
+config.load_app(LugovayaPumpMqttBotConfig)
+
+bot.initialize()
+bot.lang.ru(
+    start_message="Выберите команду на клавиатуре",
+    start_message_no_access="Доступ запрещён. Вы можете отправить заявку на получение доступа.",
+    unknown_command="Неизвестная команда",
+    send_access_request="Отправить заявку",
+    management="Админка",
+
+    enable="Включить",
+    enabled="Включен ✅",
+
+    disable="Выключить",
+    disabled="Выключен ❌",
+
+    status="Статус",
+    status_updated=' (обновлено %s)',
+
+    done="Готово 👌",
+    user_action_notification='Пользователь <a href="tg://user?id=%d">%s</a> <b>%s</b> насос.',
+    user_action_on="включил",
+    user_action_off="выключил",
+    date_yday="вчера",
+    date_yyday="позавчера",
+    date_at="в"
+)
+bot.lang.en(
+    start_message="Select command on the keyboard",
+    start_message_no_access="You have no access.",
+    unknown_command="Unknown command",
+    send_access_request="Send request",
+    management="Admin options",
+
+    enable="Turn ON",
+    enable_silently="Turn ON silently",
+    enabled="Turned ON ✅",
+
+    disable="Turn OFF",
+    disable_silently="Turn OFF silently",
+    disabled="Turned OFF ❌",
+
+    status="Status",
+    status_updated=' (updated %s)',
+
+    done="Done 👌",
+    user_action_notification='User <a href="tg://user?id=%d">%s</a> turned the pump <b>%s</b>.',
+    user_action_on="ON",
+    user_action_off="OFF",
+
+    date_yday="yesterday",
+    date_yyday="the day before yesterday",
+    date_at="at"
+)
+
+
+mqtt: MqttWrapper
+relay_state = MqttRelayState()
+relay_module: MqttRelayModule
+
+
+class UserAction(Enum):
+    ON = 'on'
+    OFF = 'off'
+
+
+# def on_mqtt_message(home_id, message: MqttPayload):
+#     if isinstance(message, InitialDiagnosticsPayload) or isinstance(message, DiagnosticsPayload):
+#         kwargs = dict(rssi=message.rssi, enabled=message.flags.state)
+#         if isinstance(message, InitialDiagnosticsPayload):
+#             kwargs['fw_version'] = message.fw_version
+#         relay_state.update(**kwargs)
+
+
+async def notify(user: User, action: UserAction) -> None:
+    def text_getter(lang: str):
+        action_name = bot.lang.get(f'user_action_{action.value}', lang)
+        user_name = user_any_name(user)
+        return 'ℹ ' + bot.lang.get('user_action_notification', lang,
+                                   user.id, user_name, action_name)
+
+    await bot.notify_all(text_getter, exclude=(user.id,))
+
+
+@bot.handler(message='enable')
+async def enable_handler(ctx: bot.Context) -> None:
+    relay_module.switchpower(True)
+    await ctx.reply(ctx.lang('done'))
+    await notify(ctx.user, UserAction.ON)
+
+
+@bot.handler(message='disable')
+async def disable_handler(ctx: bot.Context) -> None:
+    relay_module.switchpower(False)
+    await ctx.reply(ctx.lang('done'))
+    await notify(ctx.user, UserAction.OFF)
+
+
+@bot.handler(message='status')
+async def status(ctx: bot.Context) -> None:
+    label = ctx.lang('enabled') if relay_state.enabled else ctx.lang('disabled')
+    if relay_state.ever_updated:
+        date_label = ''
+        today = datetime.date.today()
+        if today != relay_state.update_time.date():
+            yday = today - datetime.timedelta(days=1)
+            yyday = today - datetime.timedelta(days=2)
+            if yday == relay_state.update_time.date():
+                date_label = ctx.lang('date_yday')
+            elif yyday == relay_state.update_time.date():
+                date_label = ctx.lang('date_yyday')
+            else:
+                date_label = relay_state.update_time.strftime('%d.%m.%Y')
+            date_label += ' '
+        date_label += ctx.lang('date_at') + ' '
+        date_label += relay_state.update_time.strftime('%H:%M')
+        label += ctx.lang('status_updated', date_label)
+    await ctx.reply(label)
+
+
+async def start(ctx: bot.Context) -> None:
+    if ctx.user_id in config['bot']['users']:
+        await ctx.reply(ctx.lang('start_message'))
+    else:
+        buttons = [
+            [ctx.lang('send_access_request')]
+        ]
+        await ctx.reply(ctx.lang('start_message_no_access'),
+                        markup=ReplyKeyboardMarkup(buttons, one_time_keyboard=False))
+
+
+@bot.exceptionhandler
+def exception_handler(e: Exception, ctx: bot.Context) -> bool:
+    return False
+
+
+@bot.defaultreplymarkup
+def markup(ctx: Optional[bot.Context]) -> Optional[ReplyKeyboardMarkup]:
+    buttons = [
+        [
+            ctx.lang('enable'),
+            ctx.lang('disable')
+        ],
+        # [ctx.lang('status')]
+    ]
+    # if ctx.user_id in config['bot']['admin_users']:
+    #     buttons.append([ctx.lang('management')])
+    return ReplyKeyboardMarkup(buttons, one_time_keyboard=False)
+
+
+node_data = MqttNodesConfig().get_node(config.app_config['relay_node_id'])
+
+mqtt = MqttWrapper(client_id='lugovaya_pump_mqtt_bot')
+mqtt_node = MqttNode(node_id=config.app_config['relay_node_id'],
+                     node_secret=node_data['password'])
+module_kwargs = {}
+try:
+    if node_data['relay']['legacy_topics']:
+        module_kwargs['legacy_topics'] = True
+except KeyError:
+    pass
+relay_module = mqtt_node.load_module('relay', **module_kwargs)
+# mqtt_node.add_payload_callback(on_mqtt_message)
+mqtt.add_node(mqtt_node)
+
+mqtt.connect_and_loop(loop_forever=False)
+
+bot.run(start_handler=start)
+
+mqtt.disconnect()

bin/mqtt_node_util.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+import os.path
+import include_homekit
+
+from time import sleep
+from typing import Optional
+from argparse import ArgumentParser, ArgumentError
+
+from homekit.config import config
+from homekit.mqtt import MqttNode, MqttWrapper, get_mqtt_modules, MqttNodesConfig
+from homekit.mqtt.module.relay import MqttRelayModule
+from homekit.mqtt.module.ota import MqttOtaModule
+
+mqtt_node: Optional[MqttNode] = None
+mqtt: Optional[MqttWrapper] = None
+
+relay_module: Optional[MqttOtaModule] = None
+relay_val = None
+
+ota_module: Optional[MqttRelayModule] = None
+ota_val = False
+
+no_wait = False
+stop_loop = False
+
+
+def on_mqtt_connect():
+    global stop_loop
+
+    if relay_module:
+        relay_module.switchpower(relay_val == 1)
+
+    if ota_val:
+        if not os.path.exists(arg.push_ota):
+            raise OSError(f'--push-ota: file \"{arg.push_ota}\" does not exists')
+        ota_module.push_ota(arg.push_ota, 1)
+
+    if no_wait:
+        stop_loop = True
+
+
+if __name__ == '__main__':
+    nodes_config = MqttNodesConfig()
+    node_names = nodes_config.get_nodes(only_names=True)
+
+    parser = ArgumentParser()
+    parser.add_argument('--node-id', type=str, required=True,
+                        help='one of: '+', '.join(node_names))
+    parser.add_argument('--node-id-no-check', action='store_true',
+                        help='when enabled, the script will not check for definition of the node in the mqtt_nodes.yaml config and will use the default password')
+    parser.add_argument('--modules', type=str, choices=get_mqtt_modules(), nargs='*',
+                        help='mqtt modules to include')
+    parser.add_argument('--switch-relay', choices=[0, 1], type=int,
+                        help='send relay state')
+    parser.add_argument('--push-ota', type=str, metavar='OTA_FILENAME',
+                        help='push OTA, receives path to firmware.bin (not .elf!)')
+    parser.add_argument('--custom-ota-topic', type=str,
+                        help='only needed for update very old devices')
+    parser.add_argument('--no-wait', action='store_true',
+                        help='execute command and exit')
+
+    config.load_app(parser=parser, no_config=True)
+    arg = parser.parse_args()
+
+    if not arg.node_id_no_check and arg.node_id not in node_names:
+        raise ArgumentError(None, f'invalid node_id {arg.node_id}')
+
+    if arg.no_wait:
+        no_wait = True
+
+    if arg.switch_relay is not None and 'relay' not in arg.modules:
+        raise ArgumentError(None, '--relay is only allowed when \'relay\' module included in --modules')
+
+    mqtt = MqttWrapper(randomize_client_id=True,
+                       client_id='mqtt_node_util')
+    mqtt.add_connect_callback(on_mqtt_connect)
+
+    try:
+        node_password = nodes_config.get_node(arg.node_id)['password']
+    except KeyError as e:
+        if arg.node_id_no_check:
+            node_password = nodes_config['common']['password']
+        else:
+            raise e
+    mqtt_node = MqttNode(node_id=arg.node_id,
+                         node_secret=node_password)
+
+    mqtt.add_node(mqtt_node)
+
+    # must-have modules
+    ota_kwargs = {}
+    if arg.custom_ota_topic:
+        ota_kwargs['custom_ota_topic'] = arg.custom_ota_topic
+    ota_module = mqtt_node.load_module('ota', **ota_kwargs)
+    ota_val = arg.push_ota
+
+    mqtt_node.load_module('diagnostics')
+
+    if arg.modules:
+        for m in arg.modules:
+            kwargs = {}
+            if m == 'relay' and MqttNodesConfig().node_uses_legacy_relay_power_payload(arg.node_id):
+                kwargs['legacy_topics'] = True
+            if m == 'temphum' and MqttNodesConfig().node_uses_legacy_temphum_data_payload(arg.node_id):
+                kwargs['legacy_payload'] = True
+            module_instance = mqtt_node.load_module(m, **kwargs)
+            if m == 'relay' and arg.switch_relay is not None:
+                relay_module = module_instance
+                relay_val = arg.switch_relay
+
+    try:
+        mqtt.connect_and_loop(loop_forever=False)
+        while not stop_loop:
+            sleep(0.1)
+
+    except KeyboardInterrupt:
+        pass
+
+    finally:
+        mqtt.disconnect()

bin/mqtt_sensors_listener.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+import include_homekit
+import sys
+import asyncio
+import logging
+
+from enum import Enum
+from typing import Optional, Union
+from telegram import ReplyKeyboardMarkup, User
+from time import time
+from datetime import datetime
+from aiohttp import web
+
+from homekit.config import config, is_development_mode, AppConfigUnit
+from homekit.telegram import bot
+from homekit.telegram.config import TelegramBotConfig, TelegramUserListType
+from homekit.telegram._botutil import user_any_name
+from homekit.relay.sunxi_h3_client import RelayClient
+from homekit import http
+from homekit.mqtt import MqttNode, MqttWrapper, MqttPayload, MqttNodesConfig, MqttModule
+from homekit.mqtt.module.relay import MqttPowerStatusPayload, MqttRelayModule
+from homekit.mqtt.module.temphum import MqttTemphumDataPayload
+from homekit.mqtt.module.diagnostics import InitialDiagnosticsPayload, DiagnosticsPayload
+
+
+class MqttSensorsListenerConfig(AppConfigUnit):
+    NAME = 'mqtt_sensors_listeners'
+
+    @classmethod
+    def schema(cls) -> Optional[dict]:
+        return {
+            'http_listen': cls._addr_schema(required=True),
+            'nodes': {
+                'type': 'list',
+                'required': True,
+                'empty': False,
+                'schema': {
+                    'type': 'string',
+                    'allowed': MqttNodesConfig.get_nodes(only_names=True)
+                }
+            }
+        }
+
+
+# config.load_app(MqttSensorsListenerConfig)
+
+routes = web.RouteTableDef()
+logger = logging.getLogger(__name__)
+
+mqtt: MqttWrapper
+mqtt_node: MqttNode
+mqtt_relay_module: Union[MqttRelayModule, MqttModule]
+
+time_format = '%d.%m.%Y, %H:%M:%S'
+
+
+class SensorStatus:
+    last_time: int = 0
+    last_boot_time: int = 0
+
+
+class TemphumStatus(SensorStatus):
+    temp: float = 0.0
+    rh: float = 0.0
+
+
+class RelayStatus(SensorStatus):
+    opened = False
+
+
+class WateringMcuStatus(RelayStatus):
+    ambient_temp: float = 0.0
+    ambient_rh: float = 0.0
+
+
+@routes.get('/sensors')
+async def http_sensors_get(req: web.Request):
+    return await http.ajax_ok({
+        'hello': 'world'
+    })
+
+
+if __name__ == '__main__':
+    mqtt = MqttWrapper(client_id='mqtt_sensors_listener',
+                       randomize_client_id=is_development_mode())
+    http.serve(addr=config.app_config['http_addr'],
+               routes=routes)

bin/openwrt_log_analyzer.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+import include_homekit
+import homekit.telegram as telegram
+
+from homekit.telegram.config import TelegramChatsConfig
+from homekit.util import validate_mac_address
+from typing import Optional
+from homekit.config import config, AppConfigUnit
+from homekit.database import BotsDatabase, SimpleState
+
+
+class OpenwrtLogAnalyzerConfig(AppConfigUnit):
+    @classmethod
+    def schema(cls) -> Optional[dict]:
+        return {
+            'database_name': {'type': 'string', 'required': True},
+            'devices': {
+                'type': 'dict',
+                'keysrules': {'type': 'string'},
+                'valuesrules': {
+                    'type': 'string',
+                    'check_with': validate_mac_address
+                }
+            },
+            'limit': {'type': 'integer'},
+            'telegram_chat': {'type': 'string'},
+            'aps': {
+                'type': 'list',
+                'schema': {'type': 'integer'}
+            }
+        }
+
+    @staticmethod
+    def custom_validator(data):
+        chats = TelegramChatsConfig()
+        if data['telegram_chat'] not in chats:
+            return ValueError(f'unknown telegram chat {data["telegram_chat"]}')
+
+
+def main(mac: str,
+         title: str,
+         ap: int) -> int:
+    db = BotsDatabase()
+
+    data = db.get_openwrt_logs(filter_text=mac,
+                               min_id=state['last_id'],
+                               access_point=ap,
+                               limit=config['openwrt_log_analyzer']['limit'])
+    if not data:
+        return 0
+
+    max_id = 0
+    for log in data:
+        if log.id > max_id:
+            max_id = log.id
+
+    text = '\n'.join(map(lambda s: str(s), data))
+    telegram.send_message(f'<b>{title} (AP #{ap})</b>\n\n' + text, config.app_config['telegram_chat'])
+
+    return max_id
+
+
+if __name__ == '__main__':
+    config.load_app(OpenwrtLogAnalyzerConfig)
+    for ap in config.app_config['aps']:
+        dbname = config.app_config['database_name']
+        dbname = dbname.replace('.txt', f'-{ap}.txt')
+
+        state = SimpleState(name=dbname,
+                            default={'last_id': 0})
+
+        max_last_id = 0
+        for name, mac in config['devices'].items():
+            last_id = main(mac, title=name, ap=ap)
+            if last_id > max_last_id:
+                max_last_id = last_id
+
+        if max_last_id:
+            state['last_id'] = max_last_id

bin/openwrt_logger.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+import os
+import include_homekit
+
+from datetime import datetime
+from typing import Tuple, List, Optional
+from argparse import ArgumentParser
+from homekit.config import config, AppConfigUnit
+from homekit.database import SimpleState
+from homekit.api import WebApiClient
+
+
+class OpenwrtLoggerConfig(AppConfigUnit):
+    @classmethod
+    def schema(cls) -> Optional[dict]:
+        return dict(
+            database_name_template=dict(type='string', required=True)
+        )
+
+
+def parse_line(line: str) -> Tuple[int, str]:
+    space_pos = line.index(' ')
+
+    date = line[:space_pos]
+    rest = line[space_pos+1:]
+
+    return (
+        int(datetime.strptime(date, "%Y-%m-%dT%H:%M:%S%z").timestamp()),
+        rest
+    )
+
+
+if __name__ == '__main__':
+    parser = ArgumentParser()
+    parser.add_argument('--file', type=str, required=True,
+                        help='openwrt log file')
+    parser.add_argument('--access-point', type=int, required=True,
+                        help='access point number')
+
+    arg = config.load_app(OpenwrtLoggerConfig, parser=parser)
+
+    state = SimpleState(name=config.app_config['database_name_template'].replace('{ap}', str(arg.access_point)),
+                        default=dict(seek=0, size=0))
+    fsize = os.path.getsize(arg.file)
+    if fsize < state['size']:
+        state['seek'] = 0
+
+    with open(arg.file, 'r') as f:
+        if state['seek']:
+            # jump to the latest read position
+            f.seek(state['seek'])
+
+        # read till the end of the file
+        content = f.read()
+
+        # save new position
+        state['seek'] = f.tell()
+        state['size'] = fsize
+
+        lines: List[Tuple[int, str]] = []
+
+        if content != '':
+            for line in content.strip().split('\n'):
+                if not line:
+                    continue
+
+                try:
+                    lines.append(parse_line(line))
+                except ValueError:
+                    lines.append((0, line))
+
+            api = WebApiClient()
+            api.log_openwrt(lines, arg.access_point)

bin/pio_build.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python3
+import include_homekit
+
+if __name__ == '__main__':
+    print('TODO')

bin/pio_ini.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+import os
+import yaml
+import re
+import include_homekit
+
+from argparse import ArgumentParser, ArgumentError
+from homekit.pio import get_products, platformio_ini
+from homekit.pio.exceptions import ProductConfigNotFoundError
+from homekit.config import CONFIG_DIRECTORIES
+
+
+def get_config(product: str) -> dict:
+    path = None
+    for directory in CONFIG_DIRECTORIES:
+        config_path = os.path.join(directory, 'pio', f'{product}.yaml')
+        if os.path.exists(config_path) and os.path.isfile(config_path):
+            path = config_path
+            break
+    if not path:
+        raise ProductConfigNotFoundError(f'pio/{product}.yaml not found')
+    with open(path, 'r') as f:
+        return yaml.safe_load(f)
+
+
+def bsd_walk(product_config: dict,
+             f: callable):
+    try:
+        for define_name, define_extra_params in product_config['build_specific_defines'].items():
+            define_name = re.sub(r'^CONFIG_', '', define_name)
+            kwargs = {}
+            if isinstance(define_extra_params, dict):
+                kwargs = define_extra_params
+            f(define_name, **kwargs)
+    except KeyError:
+        pass
+
+
+# 'bsd' means 'build_specific_defines'
+def bsd_parser(product_config: dict,
+               parser: ArgumentParser):
+    def f(define_name, **kwargs):
+        arg_kwargs = {}
+        define_name = define_name.lower().replace('_', '-')
+
+        if 'type' in kwargs:
+            if kwargs['type'] in ('str', 'enum'):
+                arg_kwargs['type'] = str
+                if kwargs['type'] == 'enum' and 'list_config_key' in kwargs:
+                    if not isinstance(product_config[kwargs['list_config_key']], list):
+                        raise TypeError(f'product_config[{kwargs["list_config_key"]}] enum is not list')
+                    if not product_config[kwargs['list_config_key']]:
+                        raise ValueError(f'product_config[{kwargs["list_config_key"]}] enum cannot be empty')
+                    arg_kwargs['choices'] = product_config[kwargs['list_config_key']]
+                    if isinstance(product_config[kwargs['list_config_key']][0], int):
+                        arg_kwargs['type'] = int
+            elif kwargs['type'] == 'int':
+                arg_kwargs['type'] = int
+            elif kwargs['type'] == 'bool':
+                arg_kwargs['action'] = 'store_true'
+                arg_kwargs['required'] = False
+            else:
+                raise TypeError(f'unsupported type {kwargs["type"]} for define {define_name}')
+        else:
+            arg_kwargs['action'] = 'store_true'
+
+        if 'required' not in arg_kwargs:
+            arg_kwargs['required'] = True
+        parser.add_argument(f'--{define_name}', **arg_kwargs)
+
+    bsd_walk(product_config, f)
+
+
+def bsd_get(product_config: dict,
+            arg: object):
+    defines = {}
+    enums = []
+    def f(define_name, **kwargs):
+        attr_name = define_name.lower()
+        attr_value = getattr(arg, attr_name)
+        if 'type' in kwargs:
+            if kwargs['type'] == 'enum':
+                enums.append(f'CONFIG_{define_name}')
+                defines[f'CONFIG_{define_name}'] = f'HOMEKIT_{attr_value.upper()}'
+                return
+            if kwargs['type'] == 'bool':
+                if attr_value is True:
+                    defines[f'CONFIG_{define_name}'] = True
+                return
+        defines[f'CONFIG_{define_name}'] = str(attr_value)
+    bsd_walk(product_config, f)
+    return defines, enums
+
+
+if __name__ == '__main__':
+    products = get_products()
+
+    # first, get the product
+    product_parser = ArgumentParser(add_help=False)
+    product_parser.add_argument('--product', type=str, choices=products, required=True,
+                                help='PIO product name')
+    arg, _ = product_parser.parse_known_args()
+    if not arg.product:
+        product = os.path.basename(os.path.realpath(os.getcwd()))
+        if product not in products:
+            raise ArgumentError(None, 'invalid product')
+    else:
+        product = arg.product
+
+    product_config = get_config(product)
+
+    # then everything else
+    parser = ArgumentParser(parents=[product_parser])
+    parser.add_argument('--target', type=str, required=True, choices=product_config['targets'],
+                        help='PIO build target')
+    parser.add_argument('--platform', default='espressif8266', type=str)
+    parser.add_argument('--framework', default='arduino', type=str)
+    parser.add_argument('--upload-port', default='/dev/ttyUSB0', type=str)
+    parser.add_argument('--monitor-speed', default=115200)
+    parser.add_argument('--debug', action='store_true')
+    parser.add_argument('--debug-network', action='store_true')
+    bsd_parser(product_config, parser)
+    arg = parser.parse_args()
+
+    if arg.target not in product_config['targets']:
+        raise ArgumentError(None, f'target {arg.target} not found for product {product}')
+
+    bsd, bsd_enums = bsd_get(product_config, arg)
+
+    ini = platformio_ini(product_config=product_config,
+                         target=arg.target,
+                         build_specific_defines=bsd,
+                         build_specific_defines_enums=bsd_enums,
+                         platform=arg.platform,
+                         framework=arg.framework,
+                         upload_port=arg.upload_port,
+                         monitor_speed=arg.monitor_speed,
+                         debug=arg.debug,
+                         debug_network=arg.debug_network)
+    print(ini)

bin/polaris_kettle_bot.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 from __future__ import annotations
 
+import include_homekit
 import logging
 import locale
 import queue
@@ -8,11 +9,10 @@ import time
 import threading
 import paho.mqtt.client as mqtt
 
-from home.telegram import bot
-from home.api.types import BotType
-from home.mqtt import MqttBase
-from home.config import config
-from home.util import chunks
+from homekit.telegram import bot
+from homekit.mqtt import Mqtt
+from homekit.config import config
+from homekit.util import chunks
 from syncleo import (
     Kettle,
     PowerType,
@@ -41,7 +41,7 @@ from telegram.ext import (
 )
 
 logger = logging.getLogger(__name__)
-config.load('polaris_kettle_bot')
+config.load_app('polaris_kettle_bot')
 
 primary_choices = (70, 80, 90, 100)
 all_choices = range(
@@ -204,7 +204,7 @@ class KettleInfo:
 
 
 class KettleController(threading.Thread,
-                       MqttBase,
+                       Mqtt,
                        DeviceListener,
                        IncomingMessageListener,
                        KettleInfoListener,
@@ -224,7 +224,7 @@ class KettleController(threading.Thread,
 
     def __init__(self):
         # basic setup
-        MqttBase.__init__(self, clean_session=False)
+        Mqtt.__init__(self, clean_session=False)
         threading.Thread.__init__(self)
 
         self._logger = logging.getLogger(self.__class__.__name__)
@@ -737,9 +737,6 @@ if __name__ == '__main__':
 
     kc = KettleController()
 
-    if 'api' in config:
-        bot.enable_logging(BotType.POLARIS_KETTLE)
-
     bot.run()
 
     # bot library handles signals, so when sigterm or something like that happens, we should stop all other threads here

bin/polaris_kettle_util.py

@@ -4,12 +4,13 @@
 import logging
 import sys
 import paho.mqtt.client as mqtt
+import include_homekit
 
 from typing import Optional
 from argparse import ArgumentParser
 from queue import SimpleQueue
-from home.mqtt import MqttBase
-from home.config import config
+from homekit.mqtt import Mqtt
+from homekit.config import config
 from syncleo import (
     Kettle,
     PowerType,
@@ -21,7 +22,7 @@ logger = logging.getLogger(__name__)
 control_tasks = SimpleQueue()
 
 
-class MqttServer(MqttBase):
+class MqttServer(Mqtt):
     def __init__(self):
         super().__init__(clean_session=False)
 
@@ -75,7 +76,7 @@ def main():
     parser.add_argument('-t', '--temperature', dest='temp', type=int, default=tempmax,
                         choices=range(tempmin, tempmax+tempstep, tempstep))
 
-    arg = config.load('polaris_kettle_util', use_cli=True, parser=parser)
+    arg = config.load_app('polaris_kettle_util', use_cli=True, parser=parser)
 
     if arg.mode == 'mqtt':
         server = MqttServer()

bin/pump_bot.py

@@ -0,0 +1,297 @@
+#!/usr/bin/env python3
+import include_homekit
+import sys
+import asyncio
+
+from enum import Enum
+from typing import Optional, Union
+from telegram import ReplyKeyboardMarkup, User
+from time import time
+from datetime import datetime
+
+from homekit.config import config, is_development_mode, AppConfigUnit
+from homekit.telegram import bot
+from homekit.telegram.config import TelegramBotConfig, TelegramUserListType
+from homekit.telegram._botutil import user_any_name
+from homekit.relay.sunxi_h3_client import RelayClient
+from homekit.mqtt import MqttNode, MqttWrapper, MqttPayload, MqttNodesConfig, MqttModule
+from homekit.mqtt.module.relay import MqttPowerStatusPayload, MqttRelayModule
+from homekit.mqtt.module.temphum import MqttTemphumDataPayload
+from homekit.mqtt.module.diagnostics import InitialDiagnosticsPayload, DiagnosticsPayload
+
+
+if __name__ != '__main__':
+    print(f'this script can not be imported as module', file=sys.stderr)
+    sys.exit(1)
+
+
+mqtt_nodes_config = MqttNodesConfig()
+
+
+class PumpBotUserListType(TelegramUserListType):
+    SILENT = 'silent_users'
+
+
+class PumpBotConfig(AppConfigUnit, TelegramBotConfig):
+    NAME = 'pump_bot'
+
+    @classmethod
+    def schema(cls) -> Optional[dict]:
+        return {
+            **super(TelegramBotConfig).schema(),
+            PumpBotUserListType.SILENT: TelegramBotConfig._userlist_schema(),
+            'watering_relay_node': {'type': 'string'},
+            'pump_relay_addr': cls._addr_schema()
+        }
+
+    @staticmethod
+    def custom_validator(data):
+        relay_node_names = mqtt_nodes_config.get_nodes(filters=('relay',), only_names=True)
+        if data['watering_relay_node'] not in relay_node_names:
+            raise ValueError(f'unknown relay node "{data["watering_relay_node"]}"')
+
+
+config.load_app(PumpBotConfig)
+
+mqtt: MqttWrapper
+mqtt_node: MqttNode
+mqtt_relay_module: Union[MqttRelayModule, MqttModule]
+
+time_format = '%d.%m.%Y, %H:%M:%S'
+
+watering_mcu_status = {
+    'last_time': 0,
+    'last_boot_time': 0,
+    'relay_opened': False,
+    'ambient_temp': 0.0,
+    'ambient_rh': 0.0,
+}
+
+bot.initialize()
+bot.lang.ru(
+    start_message="Выберите команду на клавиатуре",
+    unknown_command="Неизвестная команда",
+
+    enable="Включить",
+    enable_silently="Включить тихо",
+    enabled="Насос включен ✅",
+
+    disable="Выключить",
+    disable_silently="Выключить тихо",
+    disabled="Насос выключен ❌",
+
+    start_watering="Включить полив",
+    stop_watering="Отключить полив",
+
+    status="Статус насоса",
+    watering_status="Статус полива",
+
+    done="Готово 👌",
+    sent="Команда отправлена",
+
+    user_action_notification='Пользователь <a href="tg://user?id=%d">%s</a> <b>%s</b> насос.',
+    user_watering_notification='Пользователь <a href="tg://user?id=%d">%s</a> <b>%s</b> полив.',
+    user_action_on="включил",
+    user_action_off="выключил",
+    user_action_watering_on="включил",
+    user_action_watering_off="выключил",
+)
+bot.lang.en(
+    start_message="Select command on the keyboard",
+    unknown_command="Unknown command",
+
+    enable="Turn ON",
+    enable_silently="Turn ON silently",
+    enabled="The pump is turned ON ✅",
+
+    disable="Turn OFF",
+    disable_silently="Turn OFF silently",
+    disabled="The pump is turned OFF ❌",
+
+    start_watering="Start watering",
+    stop_watering="Stop watering",
+
+    status="Pump status",
+    watering_status="Watering status",
+
+    done="Done 👌",
+    sent="Request sent",
+
+    user_action_notification='User <a href="tg://user?id=%d">%s</a> turned the pump <b>%s</b>.',
+    user_watering_notification='User <a href="tg://user?id=%d">%s</a> <b>%s</b> the watering.',
+    user_action_on="ON",
+    user_action_off="OFF",
+    user_action_watering_on="started",
+    user_action_watering_off="stopped",
+)
+
+
+class UserAction(Enum):
+    ON = 'on'
+    OFF = 'off'
+    WATERING_ON = 'watering_on'
+    WATERING_OFF = 'watering_off'
+
+
+def get_relay() -> RelayClient:
+    relay = RelayClient(host=config.app_config['pump_relay_addr'].host,
+                        port=config.app_config['pump_relay_addr'].port)
+    relay.connect()
+    return relay
+
+
+async def on(ctx: bot.Context, silent=False) -> None:
+    get_relay().on()
+    futures = [ctx.reply(ctx.lang('done'))]
+    if not silent:
+        futures.append(notify(ctx.user, UserAction.ON))
+    await asyncio.gather(*futures)
+
+
+async def off(ctx: bot.Context, silent=False) -> None:
+    get_relay().off()
+    futures = [ctx.reply(ctx.lang('done'))]
+    if not silent:
+        futures.append(notify(ctx.user, UserAction.OFF))
+    await asyncio.gather(*futures)
+
+
+async def watering_on(ctx: bot.Context) -> None:
+    mqtt_relay_module.switchpower(True)
+    await asyncio.gather(
+        ctx.reply(ctx.lang('sent')),
+        notify(ctx.user, UserAction.WATERING_ON)
+    )
+
+
+async def watering_off(ctx: bot.Context) -> None:
+    mqtt_relay_module.switchpower(False)
+    await asyncio.gather(
+        ctx.reply(ctx.lang('sent')),
+        notify(ctx.user, UserAction.WATERING_OFF)
+    )
+
+
+async def notify(user: User, action: UserAction) -> None:
+    notification_key = 'user_watering_notification' if action in (UserAction.WATERING_ON, UserAction.WATERING_OFF) else 'user_action_notification'
+
+    def text_getter(lang: str):
+        action_name = bot.lang.get(f'user_action_{action.value}', lang)
+        user_name = user_any_name(user)
+        return 'ℹ ' + bot.lang.get(notification_key, lang,
+                                   user.id, user_name, action_name)
+
+    await bot.notify_all(text_getter, exclude=(user.id,))
+
+
+@bot.handler(message='enable')
+async def enable_handler(ctx: bot.Context) -> None:
+    await on(ctx)
+
+
+@bot.handler(message='enable_silently')
+async def enable_s_handler(ctx: bot.Context) -> None:
+    await on(ctx, True)
+
+
+@bot.handler(message='disable')
+async def disable_handler(ctx: bot.Context) -> None:
+    await off(ctx)
+
+
+@bot.handler(message='start_watering')
+async def start_watering(ctx: bot.Context) -> None:
+    await watering_on(ctx)
+
+
+@bot.handler(message='stop_watering')
+async def stop_watering(ctx: bot.Context) -> None:
+    await watering_off(ctx)
+
+
+@bot.handler(message='disable_silently')
+async def disable_s_handler(ctx: bot.Context) -> None:
+    await off(ctx, True)
+
+
+@bot.handler(message='status')
+async def status(ctx: bot.Context) -> None:
+    await ctx.reply(
+        ctx.lang('enabled') if get_relay().status() == 'on' else ctx.lang('disabled')
+    )
+
+
+def _get_timestamp_as_string(timestamp: int) -> str:
+    if timestamp != 0:
+        return datetime.fromtimestamp(timestamp).strftime(time_format)
+    else:
+        return 'unknown'
+
+
+@bot.handler(message='watering_status')
+async def watering_status(ctx: bot.Context) -> None:
+    buf = ''
+    if 0 < watering_mcu_status["last_time"] < time()-1800:
+        buf += '<b>WARNING! long time no reports from mcu! maybe something\'s wrong</b>\n'
+    buf += f'last report time: <b>{_get_timestamp_as_string(watering_mcu_status["last_time"])}</b>\n'
+    if watering_mcu_status["last_boot_time"] != 0:
+        buf += f'boot time: <b>{_get_timestamp_as_string(watering_mcu_status["last_boot_time"])}</b>\n'
+    buf += 'relay opened: <b>' + ('yes' if watering_mcu_status['relay_opened'] else 'no') + '</b>\n'
+    buf += f'ambient temp & humidity: <b>{watering_mcu_status["ambient_temp"]} °C, {watering_mcu_status["ambient_rh"]}%</b>'
+    await ctx.reply(buf)
+
+
+@bot.defaultreplymarkup
+def markup(ctx: Optional[bot.Context]) -> Optional[ReplyKeyboardMarkup]:
+    buttons = []
+    if ctx.user_id in config.app_config.get_user_ids(PumpBotUserListType.SILENT):
+        buttons.append([ctx.lang('enable_silently'), ctx.lang('disable_silently')])
+    buttons.append([ctx.lang('enable'), ctx.lang('disable'), ctx.lang('status')],)
+    buttons.append([ctx.lang('start_watering'), ctx.lang('stop_watering'), ctx.lang('watering_status')])
+
+    return ReplyKeyboardMarkup(buttons, one_time_keyboard=False)
+
+
+def mqtt_payload_callback(mqtt_node: MqttNode, payload: MqttPayload):
+    global watering_mcu_status
+
+    types_the_node_can_send = (
+        InitialDiagnosticsPayload,
+        DiagnosticsPayload,
+        MqttTemphumDataPayload,
+        MqttPowerStatusPayload
+    )
+    for cl in types_the_node_can_send:
+        if isinstance(payload, cl):
+            watering_mcu_status['last_time'] = int(time())
+            break
+
+    if isinstance(payload, InitialDiagnosticsPayload):
+        watering_mcu_status['last_boot_time'] = int(time())
+
+    elif isinstance(payload, MqttTemphumDataPayload):
+        watering_mcu_status['ambient_temp'] = payload.temp
+        watering_mcu_status['ambient_rh'] = payload.rh
+
+    elif isinstance(payload, MqttPowerStatusPayload):
+        watering_mcu_status['relay_opened'] = payload.opened
+
+
+mqtt = MqttWrapper(client_id='pump_bot')
+mqtt_node = MqttNode(node_id=config.app_config['watering_relay_node'])
+if is_development_mode():
+    mqtt_node.load_module('diagnostics')
+
+mqtt_node.load_module('temphum')
+mqtt_relay_module = mqtt_node.load_module('relay')
+
+mqtt_node.add_payload_callback(mqtt_payload_callback)
+
+mqtt.connect_and_loop(loop_forever=False)
+
+bot.run()
+
+try:
+    mqtt.disconnect()
+except:
+    pass

bin/pump_mqtt_bot.py

@@ -1,20 +1,20 @@
 #!/usr/bin/env python3
 import datetime
+import include_homekit
 
 from enum import Enum
 from typing import Optional
 from telegram import ReplyKeyboardMarkup, User
 
-from home.config import config
-from home.telegram import bot
-from home.telegram._botutil import user_any_name
-from home.mqtt.esp import MqttEspDevice
-from home.mqtt import MqttRelay, MqttRelayState
-from home.mqtt.payload import MqttPayload
-from home.mqtt.payload.relay import InitialDiagnosticsPayload, DiagnosticsPayload
+from homekit.config import config
+from homekit.telegram import bot
+from homekit.telegram._botutil import user_any_name
+from homekit.mqtt import MqttNode, MqttPayload
+from homekit.mqtt.module.relay import MqttRelayState
+from homekit.mqtt.module.diagnostics import InitialDiagnosticsPayload, DiagnosticsPayload
 
 
-config.load('pump_mqtt_bot')
+config.load_app('pump_mqtt_bot')
 
 bot.initialize()
 bot.lang.ru(
@@ -70,7 +70,7 @@ bot.lang.en(
 )
 
 
-mqtt_relay: Optional[MqttRelay] = None
+mqtt: Optional[MqttNode] = None
 relay_state = MqttRelayState()
 
 
@@ -99,14 +99,14 @@ def notify(user: User, action: UserAction) -> None:
 
 @bot.handler(message='enable')
 def enable_handler(ctx: bot.Context) -> None:
-    mqtt_relay.set_power(config['mqtt']['home_id'], True)
+    mqtt.set_power(config['mqtt']['home_id'], True)
     ctx.reply(ctx.lang('done'))
     notify(ctx.user, UserAction.ON)
 
 
 @bot.handler(message='disable')
 def disable_handler(ctx: bot.Context) -> None:
-    mqtt_relay.set_power(config['mqtt']['home_id'], False)
+    mqtt.set_power(config['mqtt']['home_id'], False)
     ctx.reply(ctx.lang('done'))
     notify(ctx.user, UserAction.OFF)
 
@@ -157,13 +157,12 @@ def markup(ctx: Optional[bot.Context]) -> Optional[ReplyKeyboardMarkup]:
 
 
 if __name__ == '__main__':
-    mqtt_relay = MqttRelay(devices=MqttEspDevice(id=config['mqtt']['home_id'],
+    mqtt = MqttRelay(devices=MqttEspDevice(id=config['mqtt']['home_id'],
                                            secret=config['mqtt']['home_secret']))
-    mqtt_relay.set_message_callback(on_mqtt_message)
-    mqtt_relay.configure_tls()
-    mqtt_relay.connect_and_loop(loop_forever=False)
+    mqtt.set_message_callback(on_mqtt_message)
+    mqtt.connect_and_loop(loop_forever=False)
 
     # bot.enable_logging(BotType.PUMP_MQTT)
     bot.run(start_handler=start)
 
-    mqtt_relay.disconnect()
+    mqtt.disconnect()

bin/relay_mqtt_bot.py

@@ -0,0 +1,164 @@
+#!/usr/bin/env python3
+import sys
+import include_homekit
+
+from enum import Enum
+from typing import Optional, Union
+from telegram import ReplyKeyboardMarkup
+from functools import partial
+
+from homekit.config import config, AppConfigUnit, Translation
+from homekit.telegram import bot
+from homekit.telegram.config import TelegramBotConfig
+from homekit.mqtt import MqttPayload, MqttNode, MqttWrapper, MqttModule, MqttNodesConfig
+from homekit.mqtt.module.relay import MqttRelayModule, MqttRelayState
+from homekit.mqtt.module.diagnostics import InitialDiagnosticsPayload, DiagnosticsPayload
+
+
+if __name__ != '__main__':
+    print(f'this script can not be imported as module', file=sys.stderr)
+    sys.exit(1)
+
+
+mqtt_nodes_config = MqttNodesConfig()
+
+
+class RelayMqttBotConfig(AppConfigUnit, TelegramBotConfig):
+    NAME = 'relay_mqtt_bot'
+
+    _strings: Translation
+
+    def __init__(self):
+        super().__init__()
+        self._strings = Translation('mqtt_nodes')
+
+    @classmethod
+    def schema(cls) -> Optional[dict]:
+        return {
+            **super(TelegramBotConfig).schema(),
+            'relay_nodes': {
+                'type': 'list',
+                'required': True,
+                'schema': {
+                    'type': 'string'
+                }
+            },
+        }
+
+    @staticmethod
+    def custom_validator(data):
+        relay_node_names = mqtt_nodes_config.get_nodes(filters=('relay',), only_names=True)
+        for node in data['relay_nodes']:
+            if node not in relay_node_names:
+                raise ValueError(f'unknown relay node "{node}"')
+
+    def get_relay_name_translated(self, lang: str, relay_name: str) -> str:
+        return self._strings.get(lang)[relay_name]['relay']
+
+
+config.load_app(RelayMqttBotConfig)
+
+bot.initialize()
+bot.lang.ru(
+    start_message="Выберите команду на клавиатуре",
+    unknown_command="Неизвестная команда",
+    done="Готово 👌",
+)
+bot.lang.en(
+    start_message="Select command on the keyboard",
+    unknown_command="Unknown command",
+    done="Done 👌",
+)
+
+
+type_emojis = {
+    'lamp': '💡'
+}
+status_emoji = {
+    'on': '✅',
+    'off': '❌'
+}
+
+
+mqtt: MqttWrapper
+relay_nodes: dict[str, Union[MqttRelayModule, MqttModule]] = {}
+relay_states: dict[str, MqttRelayState] = {}
+
+
+class UserAction(Enum):
+    ON = 'on'
+    OFF = 'off'
+
+
+def on_mqtt_message(node: MqttNode,
+                    message: MqttPayload):
+    if isinstance(message, InitialDiagnosticsPayload) or isinstance(message, DiagnosticsPayload):
+        kwargs = dict(rssi=message.rssi, enabled=message.flags.state)
+        if isinstance(message, InitialDiagnosticsPayload):
+            kwargs['fw_version'] = message.fw_version
+        if node.id not in relay_states:
+            relay_states[node.id] = MqttRelayState()
+        relay_states[node.id].update(**kwargs)
+
+
+async def enable_handler(node_id: str, ctx: bot.Context) -> None:
+    relay_nodes[node_id].switchpower(True)
+    await ctx.reply(ctx.lang('done'))
+
+
+async def disable_handler(node_id: str, ctx: bot.Context) -> None:
+    relay_nodes[node_id].switchpower(False)
+    await ctx.reply(ctx.lang('done'))
+
+
+async def start(ctx: bot.Context) -> None:
+    await ctx.reply(ctx.lang('start_message'))
+
+
+@bot.exceptionhandler
+async def exception_handler(e: Exception, ctx: bot.Context) -> bool:
+    return False
+
+
+@bot.defaultreplymarkup
+def markup(ctx: Optional[bot.Context]) -> Optional[ReplyKeyboardMarkup]:
+    buttons = []
+    for node_id in config.app_config['relay_nodes']:
+        node_data = mqtt_nodes_config.get_node(node_id)
+        type_emoji = type_emojis[node_data['relay']['device_type']]
+        row = [f'{type_emoji}{status_emoji[i.value]} {config.app_config.get_relay_name_translated(ctx.user_lang, node_id)}'
+               for i in UserAction]
+        buttons.append(row)
+    return ReplyKeyboardMarkup(buttons, one_time_keyboard=False)
+
+
+devices = []
+mqtt = MqttWrapper(client_id='relay_mqtt_bot')
+for node_id in config.app_config['relay_nodes']:
+    node_data = mqtt_nodes_config.get_node(node_id)
+    mqtt_node = MqttNode(node_id=node_id,
+                         node_secret=node_data['password'])
+    module_kwargs = {}
+    try:
+        if node_data['relay']['legacy_topics']:
+            module_kwargs['legacy_topics'] = True
+    except KeyError:
+        pass
+    relay_nodes[node_id] = mqtt_node.load_module('relay',  **module_kwargs)
+    mqtt_node.add_payload_callback(on_mqtt_message)
+    mqtt.add_node(mqtt_node)
+
+    type_emoji = type_emojis[node_data['relay']['device_type']]
+
+    for action in UserAction:
+        messages = []
+        for _lang in Translation.LANGUAGES:
+            _label = config.app_config.get_relay_name_translated(_lang, node_id)
+            messages.append(f'{type_emoji}{status_emoji[action.value]} {_label}')
+        bot.handler(texts=messages)(partial(enable_handler if action == UserAction.ON else disable_handler, node_id))
+
+mqtt.connect_and_loop(loop_forever=False)
+
+bot.run(start_handler=start)
+
+mqtt.disconnect()

bin/relay_mqtt_http_proxy.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+import logging
+import include_homekit
+
+from aiohttp import web
+from homekit import http
+from homekit.config import config, AppConfigUnit
+from homekit.mqtt import MqttPayload, MqttWrapper, MqttNode, MqttModule, MqttNodesConfig
+from homekit.mqtt.module.relay import MqttRelayState, MqttRelayModule, MqttPowerStatusPayload
+from homekit.mqtt.module.diagnostics import InitialDiagnosticsPayload, DiagnosticsPayload
+from typing import Optional, Union
+
+
+logger = logging.getLogger(__name__)
+mqtt: Optional[MqttWrapper] = None
+mqtt_nodes: dict[str, MqttNode] = {}
+relay_modules: dict[str, Union[MqttRelayModule, MqttModule]] = {}
+relay_states: dict[str, MqttRelayState] = {}
+mqtt_nodes_config = MqttNodesConfig()
+
+
+class RelayMqttHttpProxyConfig(AppConfigUnit):
+    NAME = 'relay_mqtt_http_proxy'
+
+    @classmethod
+    def schema(cls) -> Optional[dict]:
+        return {
+            'relay_nodes': {
+                'type': 'list',
+                'required': True,
+                'schema': {
+                    'type': 'string'
+                }
+            },
+            'listen_addr': cls._addr_schema(required=True)
+        }
+
+    @staticmethod
+    def custom_validator(data):
+        relay_node_names = mqtt_nodes_config.get_nodes(filters=('relay',), only_names=True)
+        for node in data['relay_nodes']:
+            if node not in relay_node_names:
+                raise ValueError(f'unknown relay node "{node}"')
+
+
+def on_mqtt_message(node: MqttNode,
+                    message: MqttPayload):
+    try:
+        is_legacy = mqtt_nodes_config[node.id]['relay']['legacy_topics']
+        logger.debug(f'on_mqtt_message: relay {node.id} uses legacy topic names')
+    except KeyError:
+        is_legacy = False
+    kwargs = {}
+
+    if isinstance(message, InitialDiagnosticsPayload) or isinstance(message, DiagnosticsPayload):
+        kwargs['rssi'] = message.rssi
+        if is_legacy:
+            kwargs['enabled'] = message.flags.state
+
+    if not is_legacy and isinstance(message, MqttPowerStatusPayload):
+        kwargs['enabled'] = message.opened
+
+    if len(kwargs):
+        logger.debug(f'on_mqtt_message: {node.id}: going to update relay state: {str(kwargs)}')
+        if node.id not in relay_states:
+            relay_states[node.id] = MqttRelayState()
+        relay_states[node.id].update(**kwargs)
+
+
+# -=-=-=-=-=-=- #
+# Web interface #
+# -=-=-=-=-=-=- #
+
+routes = web.RouteTableDef()
+
+
+async def _relay_on_off(self,
+                        enable: Optional[bool],
+                        req: web.Request):
+    node_id = req.match_info['id']
+    node_secret = req.query['secret']
+
+    node = mqtt_nodes[node_id]
+    relay_module = relay_modules[node_id]
+
+    if enable is None:
+        if node_id in relay_states and relay_states[node_id].ever_updated:
+            cur_state = relay_states[node_id].enabled
+        else:
+            cur_state = False
+        enable = not cur_state
+
+    node.secret = node_secret
+    relay_module.switchpower(enable)
+    return self.ok()
+
+
+@routes.get('/relay/{id}/on')
+async def relay_on(self, req: web.Request):
+    return await self._relay_on_off(True, req)
+
+
+@routes.get('/relay/{id}/off')
+async def relay_off(self, req: web.Request):
+    return await self._relay_on_off(False, req)
+
+
+@routes.get('/relay/{id}/toggle')
+async def relay_toggle(self, req: web.Request):
+    return await self._relay_on_off(None, req)
+
+
+if __name__ == '__main__':
+    config.load_app(RelayMqttHttpProxyConfig)
+
+    mqtt = MqttWrapper(client_id='relay_mqtt_http_proxy',
+                       randomize_client_id=True)
+    for node_id in config.app_config['relay_nodes']:
+        node_data = mqtt_nodes_config.get_node(node_id)
+        mqtt_node = MqttNode(node_id=node_id)
+        module_kwargs = {}
+        try:
+            if node_data['relay']['legacy_topics']:
+                module_kwargs['legacy_topics'] = True
+        except KeyError:
+            pass
+        relay_modules[node_id] = mqtt_node.load_module('relay', **module_kwargs)
+        if 'legacy_topics' in module_kwargs:
+            mqtt_node.load_module('diagnostics')
+        mqtt_node.add_payload_callback(on_mqtt_message)
+        mqtt.add_node(mqtt_node)
+        mqtt_nodes[node_id] = mqtt_node
+
+    mqtt.connect_and_loop(loop_forever=False)
+
+    try:
+        http.serve(config.app_config['listen_addr'], routes=routes)
+    except KeyboardInterrupt:
+        mqtt.disconnect()

bin/sensors_bot.py

@@ -4,6 +4,7 @@ import socket
 import logging
 import re
 import gc
+import include_homekit
 
 from io import BytesIO
 from typing import Optional
@@ -14,16 +15,15 @@ import matplotlib.ticker as mticker
 
 from telegram import ReplyKeyboardMarkup, InlineKeyboardMarkup, InlineKeyboardButton
 
-from home.config import config
-from home.telegram import bot
-from home.util import chunks, MySimpleSocketClient
-from home.api import WebAPIClient
-from home.api.types import (
-    BotType,
+from homekit.config import config
+from homekit.telegram import bot
+from homekit.util import chunks, MySimpleSocketClient
+from homekit.api import WebApiClient
+from homekit.api.types import (
     TemperatureSensorLocation
 )
 
-config.load('sensors_bot')
+config.load_app('sensors_bot')
 bot.initialize()
 
 bot.lang.ru(
@@ -111,7 +111,7 @@ def callback_handler(ctx: bot.Context) -> None:
     sensor = TemperatureSensorLocation[match.group(1).upper()]
     hours = int(match.group(2))
 
-    api = WebAPIClient(timeout=20)
+    api = WebApiClient(timeout=20)
     data = api.get_sensors_data(sensor, hours)
 
     title = ctx.lang(sensor.name.lower()) + ' (' + ctx.lang('n_hrs', hours) + ')'
@@ -175,7 +175,4 @@ def markup(ctx: Optional[bot.Context]) -> Optional[ReplyKeyboardMarkup]:
 
 
 if __name__ == '__main__':
-    if 'api' in config:
-        bot.enable_logging(BotType.SENSORS)
-
     bot.run()

bin/sound_bot.py

@@ -2,32 +2,33 @@
 import logging
 import os
 import tempfile
+import include_homekit
 
 from enum import Enum
 from datetime import datetime, timedelta
 from html import escape
 from typing import Optional, List, Dict, Tuple
 
-from home.config import config
-from home.api import WebAPIClient
-from home.api.types import SoundSensorLocation, BotType
-from home.api.errors import ApiResponseError
-from home.media import SoundNodeClient, SoundRecordClient, SoundRecordFile, CameraNodeClient
-from home.soundsensor import SoundSensorServerGuardClient
-from home.util import parse_addr, chunks, filesize_fmt
+from homekit.config import config
+from homekit.api import WebApiClient
+from homekit.api.types import SoundSensorLocation
+from homekit.api.errors import ApiResponseError
+from homekit.media import SoundNodeClient, SoundRecordClient, SoundRecordFile, CameraNodeClient
+from homekit.soundsensor import SoundSensorServerGuardClient
+from homekit.util import Addr, chunks, filesize_fmt
 
-from home.telegram import bot
+from homekit.telegram import bot
 
 from telegram.error import TelegramError
 from telegram import ReplyKeyboardMarkup, InlineKeyboardMarkup, InlineKeyboardButton, User
 
 from PIL import Image
 
-config.load('sound_bot')
+config.load_app('sound_bot')
 
 nodes = {}
 for nodename, nodecfg in config['nodes'].items():
-    nodes[nodename] = parse_addr(nodecfg['addr'])
+    nodes[nodename] = Addr.fromstring(nodecfg['addr'])
 
 bot.initialize()
 bot.lang.ru(
@@ -142,13 +143,13 @@ cam_client_links: Dict[str, CameraNodeClient] = {}
 
 def node_client(node: str) -> SoundNodeClient:
     if node not in node_client_links:
-        node_client_links[node] = SoundNodeClient(parse_addr(config['nodes'][node]['addr']))
+        node_client_links[node] = SoundNodeClient(Addr.fromstring(config['nodes'][node]['addr']))
     return node_client_links[node]
 
 
 def camera_client(cam: str) -> CameraNodeClient:
     if cam not in node_client_links:
-        cam_client_links[cam] = CameraNodeClient(parse_addr(config['cameras'][cam]['addr']))
+        cam_client_links[cam] = CameraNodeClient(Addr.fromstring(config['cameras'][cam]['addr']))
     return cam_client_links[cam]
 
 
@@ -188,7 +189,7 @@ def manual_recording_allowed(user_id: int) -> bool:
 
 
 def guard_client() -> SoundSensorServerGuardClient:
-    return SoundSensorServerGuardClient(parse_addr(config['bot']['guard_server']))
+    return SoundSensorServerGuardClient(Addr.fromstring(config['bot']['guard_server']))
 
 
 # message renderers
@@ -734,7 +735,7 @@ def sound_sensors_last_24h(ctx: bot.Context):
 
     ctx.answer()
 
-    cl = WebAPIClient()
+    cl = WebApiClient()
     data = cl.get_sound_sensor_hits(location=SoundSensorLocation[node.upper()],
                                     after=datetime.now() - timedelta(hours=24))
 
@@ -757,7 +758,7 @@ def sound_sensors_last_anything(ctx: bot.Context):
 
     ctx.answer()
 
-    cl = WebAPIClient()
+    cl = WebApiClient()
     data = cl.get_last_sound_sensor_hits(location=SoundSensorLocation[node.upper()],
                                          last=20)
 
@@ -883,7 +884,5 @@ if __name__ == '__main__':
                                       finished_handler=record_onfinished,
                                       download_on_finish=True)
 
-    if 'api' in config:
-        bot.enable_logging(BotType.SOUND)
     bot.run()
     record_client.stop()

bin/sound_node.py

@@ -1,12 +1,13 @@
 #!/usr/bin/env python3
 import os
+import include_homekit
 
 from typing import Optional
 
-from home.config import config
-from home.audio import amixer
-from home.media import MediaNodeServer, SoundRecordStorage, SoundRecorder
-from home import http
+from homekit.config import config
+from homekit.audio import amixer
+from homekit.media import MediaNodeServer, SoundRecordStorage, SoundRecorder
+from homekit import http
 
 
 # This script must be run as root as it runs arecord.
@@ -16,7 +17,7 @@ from home import http
 def _amixer_control_response(control):
     info = amixer.get(control)
     caps = amixer.get_caps(control)
-    return http.ok({
+    return http.ajax_ok({
         'caps': caps,
         'info': info
     })
@@ -77,7 +78,7 @@ if __name__ == '__main__':
     if not os.getegid() == 0:
         raise RuntimeError("Must be run as root.")
 
-    config.load('sound_node')
+    config.load_app('sound_node')
 
     storage = SoundRecordStorage(config['node']['storage'])
 

bin/sound_sensor_node.py

@@ -2,10 +2,11 @@
 import logging
 import os
 import sys
+import include_homekit
 
-from home.config import config
-from home.util import parse_addr
-from home.soundsensor import SoundSensorNode
+from homekit.config import config
+from homekit.util import Addr
+from homekit.soundsensor import SoundSensorNode
 
 logger = logging.getLogger(__name__)
 
@@ -14,14 +15,14 @@ if __name__ == '__main__':
     if not os.getegid() == 0:
         sys.exit('Must be run as root.')
 
-    config.load('sound_sensor_node')
+    config.load_app('sound_sensor_node')
 
     kwargs = {}
     if 'delay' in config['node']:
         kwargs['delay'] = config['node']['delay']
 
     if 'server_addr' in config['node']:
-        server_addr = parse_addr(config['node']['server_addr'])
+        server_addr = Addr.fromstring(config['node']['server_addr'])
     else:
         server_addr = None
 

bin/sound_sensor_server.py

@@ -1,16 +1,17 @@
 #!/usr/bin/env python3
 import logging
 import threading
+import include_homekit
 
 from time import sleep
 from typing import Optional, List, Dict, Tuple
 from functools import partial
-from home.config import config
-from home.util import parse_addr
-from home.api import WebAPIClient, RequestParams
-from home.api.types import SoundSensorLocation
-from home.soundsensor import SoundSensorServer, SoundSensorHitHandler
-from home.media import MediaNodeType, SoundRecordClient, CameraRecordClient, RecordClient
+from homekit.config import config
+from homekit.util import Addr
+from homekit.api import WebApiClient, RequestParams
+from homekit.api.types import SoundSensorLocation
+from homekit.soundsensor import SoundSensorServer, SoundSensorHitHandler
+from homekit.media import MediaNodeType, SoundRecordClient, CameraRecordClient, RecordClient
 
 interrupted = False
 logger = logging.getLogger(__name__)
@@ -120,7 +121,7 @@ def hits_sender():
         sleep(5)
 
 
-api: Optional[WebAPIClient] = None
+api: Optional[WebApiClient] = None
 hc: Optional[HitCounter] = None
 record_clients: Dict[MediaNodeType, RecordClient] = {}
 
@@ -159,10 +160,10 @@ def api_error_handler(exc, name, req: RequestParams):
 
 
 if __name__ == '__main__':
-    config.load('sound_sensor_server')
+    config.load_app('sound_sensor_server')
 
     hc = HitCounter()
-    api = WebAPIClient(timeout=(10, 60))
+    api = WebApiClient(timeout=(10, 60))
     api.enable_async(error_handler=api_error_handler)
 
     t = threading.Thread(target=hits_sender)
@@ -172,12 +173,12 @@ if __name__ == '__main__':
     sound_nodes = {}
     if 'sound_nodes' in config:
         for nodename, nodecfg in config['sound_nodes'].items():
-            sound_nodes[nodename] = parse_addr(nodecfg['addr'])
+            sound_nodes[nodename] = Addr.fromstring(nodecfg['addr'])
 
     camera_nodes = {}
     if 'camera_nodes' in config:
         for nodename, nodecfg in config['camera_nodes'].items():
-            camera_nodes[nodename] = parse_addr(nodecfg['addr'])
+            camera_nodes[nodename] = Addr.fromstring(nodecfg['addr'])
 
     if sound_nodes:
         record_clients[MediaNodeType.SOUND] = SoundRecordClient(sound_nodes,

bin/ssh_tunnels_config_util.py

@@ -1,14 +1,14 @@
 #!/usr/bin/env python3
-
-from home.config import config
+import include_homekit
+from homekit.config import config
 
 if __name__ == '__main__':
-    config.load('ssh_tunnels_config_util')
+    config.load_app('ssh_tunnels_config_util')
 
     network_prefix = config['network']
     hostnames = []
 
-    for k, v in config.items():
+    for k, v in config.app_config.get().items():
         if type(v) is not dict:
             continue
         hostnames.append(k)

bin/temphum_mqtt_node.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+import include_homekit
+import asyncio
+import logging
+
+from datetime import datetime
+from apscheduler.schedulers.asyncio import AsyncIOScheduler
+from typing import Optional
+from argparse import ArgumentParser
+from homekit.config import config
+from homekit.mqtt import MqttNodesConfig, MqttNode, MqttWrapper
+from homekit.mqtt.module.temphum import MqttTempHumModule, MqttTemphumDataPayload, DATA_TOPIC
+from homekit.temphum import SensorType, BaseSensor
+from homekit.temphum.i2c import create_sensor
+
+_logger = logging.getLogger(__name__)
+_sensor: Optional[BaseSensor] = None
+_lock = asyncio.Lock()
+_mqtt: MqttWrapper
+_mqtt_ndoe: MqttNode
+_mqtt_temphum: MqttTempHumModule
+_stopped = True
+_scheduler = AsyncIOScheduler()
+_sched_task_added = False
+
+
+async def get_measurements():
+    async with _lock:
+        temp = _sensor.temperature()
+        rh = _sensor.humidity()
+
+        return rh, temp
+
+
+def on_mqtt_connect():
+    global _stopped, _sched_task_added
+    _stopped = False
+
+    if not _sched_task_added:
+        _scheduler.add_job(on_sched_task, 'interval', seconds=60, next_run_time=datetime.now())
+        _scheduler.start()
+        _sched_task_added = True
+    elif _scheduler:
+        _scheduler.resume()
+
+
+def on_mqtt_disconnect():
+    global _stopped
+    _stopped = True
+
+    if _scheduler:
+        _scheduler.pause()
+
+
+async def on_sched_task():
+    if _stopped:
+        return
+
+    rh, temp = await get_measurements()
+    payload = MqttTemphumDataPayload(temp=temp, rh=rh)
+    _mqtt_node.publish(DATA_TOPIC, payload.pack())
+
+
+if __name__ == '__main__':
+    parser = ArgumentParser()
+    parser.add_argument('--node-id',
+                        type=str,
+                        required=True,
+                        choices=MqttNodesConfig().get_nodes(only_names=True),
+                        help='node id must be defined in the config')
+    args = config.load_app(parser=parser)
+
+    node_cfg = MqttNodesConfig()[args.node_id]
+    _sensor = create_sensor(SensorType(node_cfg['temphum']['module']),
+                            int(node_cfg['temphum']['i2c_bus']))
+
+    _mqtt = MqttWrapper(client_id=args.node_id)
+    _mqtt.add_connect_callback(on_mqtt_connect)
+    _mqtt.add_disconnect_callback(on_mqtt_disconnect)
+
+    _mqtt_node = MqttNode(node_id=args.node_id,
+                          node_secret=MqttNodesConfig.get_node(args.node_id)['password'])
+    _mqtt.add_node(_mqtt_node)
+
+    _mqtt_temphum = _mqtt_node.load_module('temphum')
+
+    try:
+        _mqtt.connect_and_loop(loop_forever=True)
+
+    except (KeyboardInterrupt, SystemExit):
+        if _scheduler:
+            _scheduler.shutdown()
+        _logger.info('Exiting...')
+
+    finally:
+        _mqtt.disconnect()

bin/temphum_mqtt_receiver.py

@@ -1,22 +1,13 @@
 #!/usr/bin/env python3
 import paho.mqtt.client as mqtt
 import re
+import include_homekit
 
-from home.mqtt import MqttBase
-from home.config import config
-from home.mqtt.payload.sensors import Temperature
-from home.api.types import TemperatureSensorLocation
-from home.database import SensorsDatabase
+from homekit.config import config
+from homekit.mqtt import MqttWrapper, MqttNode
 
 
-def get_sensor_type(sensor: str) -> TemperatureSensorLocation:
-    for item in TemperatureSensorLocation:
-        if sensor == item.name.lower():
-            return item
-    raise ValueError(f'unexpected sensor value: {sensor}')
-
-
-class MqttServer(MqttBase):
+class MqttServer(Mqtt):
     def __init__(self):
         super().__init__(clean_session=False)
         self.database = SensorsDatabase()
@@ -47,7 +38,11 @@ class MqttServer(MqttBase):
 
 
 if __name__ == '__main__':
-    config.load('sensors_mqtt_receiver')
+    config.load_app('temphum_mqtt_receiver')
 
-    server = MqttServer()
-    server.connect_and_loop()
+    mqtt = MqttWrapper(clean_session=False)
+    node = MqttNode(node_id='+')
+    node.load_module('temphum', write_to_database=True)
+    mqtt.add_node(node)
+
+    mqtt.connect_and_loop()

bin/temphum_nodes_util.py

@@ -1,14 +1,16 @@
 #!/usr/bin/env python3
-from home.temphum import TempHumNodes
+import include_homekit
+
+from homekit.mqtt.temphum import MqttTempHumNodes
 
 if __name__ == '__main__':
     max_name_len = 0
-    for node in TempHumNodes:
+    for node in MqttTempHumNodes:
         if len(node.name) > max_name_len:
             max_name_len = len(node.name)
 
     values = []
-    for node in TempHumNodes:
+    for node in MqttTempHumNodes:
         hash = node.hash()
         if hash in values:
             raise ValueError(f'collision detected: {hash}')

bin/temphum_smbus_util.py

@@ -1,6 +1,9 @@
 #!/usr/bin/env python3
+import include_homekit
+
 from argparse import ArgumentParser
-from home.temphum import SensorType, create_sensor
+from homekit.temphum import SensorType
+from homekit.temphum.i2c import create_sensor
 
 
 if __name__ == '__main__':

bin/temphumd.py

@@ -2,14 +2,16 @@
 import asyncio
 import json
 import logging
+import include_homekit
 
 from typing import Optional
 
-from home.config import config
-from home.temphum import SensorType, create_sensor, TempHumSensor
+from homekit.config import config
+from homekit.temphum import SensorType, BaseSensor
+from homekit.temphum.i2c import create_sensor
 
 logger = logging.getLogger(__name__)
-sensor: Optional[TempHumSensor] = None
+sensor: Optional[BaseSensor] = None
 lock = asyncio.Lock()
 delay = 0.01
 
@@ -62,7 +64,7 @@ async def run_server(host, port):
 
 
 if __name__ == '__main__':
-    config.load()
+    config.load_app()
 
     if 'measure_delay' in config['sensor']:
         delay = float(config['sensor']['measure_delay'])

bin/vk_sms_checker.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+import include_homekit
+import re
+
+from html import escape
+from typing import Optional
+from homekit.config import AppConfigUnit, config
+from homekit.modem import ModemsConfig, E3372
+from homekit.database import MySQLHomeDatabase
+from homekit.telegram import send_message
+
+db: Optional[MySQLHomeDatabase] = None
+
+
+class VkSmsCheckerConfig(AppConfigUnit):
+    NAME = 'vk_sms_checker'
+
+    @classmethod
+    def schema(cls) -> Optional[dict]:
+        return {
+            'modem': {'type': 'string', 'required': True}
+        }
+
+    @staticmethod
+    def custom_validator(data):
+        if data['modem'] not in ModemsConfig():
+            raise ValueError('invalid modem')
+
+
+def get_last_time() -> int:
+    cur = db.cursor()
+    cur.execute("SELECT last_message_time FROM vk_sms LIMIT 1")
+    return int(cur.fetchone()[0])
+
+
+def set_last_time(timestamp: int) -> None:
+    cur = db.cursor()
+    cur.execute("UPDATE vk_sms SET last_message_time=%s", (timestamp,))
+    db.commit()
+
+
+def check_sms():
+    modem = ModemsConfig()[config.app_config['modem']]
+    cl = E3372(modem['ip'], legacy_token_auth=modem['legacy_auth'])
+    messages = cl.sms_list()
+    messages.reverse()
+
+    last_time = get_last_time()
+    new_last_time = None
+    results = []
+
+    if not messages:
+        return
+
+    for m in messages:
+        if m['UnixTime'] <= last_time:
+            continue
+        new_last_time = m['UnixTime']
+        if re.match(r'^vk', m['Phone'], flags=re.IGNORECASE) or re.match(r'vk', m['Content'], flags=re.IGNORECASE):
+            results.append(m)
+
+    if results:
+        for m in results:
+            text = '<b>'+escape(m['Phone'])+'</b> ('+m['Date']+')'
+            text += "\n"+escape(m['Content'])
+            send_message(text=text, chat='vk_sms_checker')
+
+    if new_last_time:
+        set_last_time(new_last_time)
+
+
+if __name__ == '__main__':
+    db = MySQLHomeDatabase()
+    config.load_app(VkSmsCheckerConfig)
+    check_sms()

bin/web_api.py

@@ -2,16 +2,17 @@
 import asyncio
 import json
 import os
+import include_homekit
 
 from datetime import datetime, timedelta
 
 from aiohttp import web
-from home import http
-from home.config import config, is_development_mode
-from home.database import BotsDatabase, SensorsDatabase, InverterDatabase
-from home.database.inverter_time_formats import *
-from home.api.types import BotType, TemperatureSensorLocation, SoundSensorLocation
-from home.media import SoundRecordStorage
+from homekit import http
+from homekit.config import config, is_development_mode
+from homekit.database import BotsDatabase, SensorsDatabase, InverterDatabase
+from homekit.database.inverter_time_formats import *
+from homekit.api.types import TemperatureSensorLocation, SoundSensorLocation
+from homekit.media import SoundRecordStorage
 
 
 def strptime_auto(s: str) -> datetime:
@@ -41,7 +42,6 @@ class WebAPIServer(http.HTTPServer):
         self.get('/sound_sensors/hits/', self.GET_sound_sensors_hits)
         self.post('/sound_sensors/hits/', self.POST_sound_sensors_hits)
 
-        self.post('/log/bot_request/', self.POST_bot_request_log)
         self.post('/log/openwrt/', self.POST_openwrt_log)
 
         self.get('/inverter/consumed_energy/', self.GET_consumed_energy)
@@ -51,7 +51,7 @@ class WebAPIServer(http.HTTPServer):
 
     @staticmethod
     @web.middleware
-    async def validate_auth(req: http.Request, handler):
+    async def validate_auth(req: web.Request, handler):
         def get_token() -> str:
             name = 'X-Token'
             if name in req.headers:
@@ -70,13 +70,13 @@ class WebAPIServer(http.HTTPServer):
         return await handler(req)
 
     @staticmethod
-    async def get_index(req: http.Request):
+    async def get_index(req: web.Request):
         message = "nothing here, keep lurking"
         if is_development_mode():
             message += ' (dev mode)'
-        return http.Response(text=message, content_type='text/plain')
+        return web.Response(text=message, content_type='text/plain')
 
-    async def GET_sensors_data(self, req: http.Request):
+    async def GET_sensors_data(self, req: web.Request):
         try:
             hours = int(req.query['hours'])
             if hours < 1 or hours > 24:
@@ -93,7 +93,7 @@ class WebAPIServer(http.HTTPServer):
         data = db.get_temperature_recordings(sensor, (dt_from, dt_to))
         return self.ok(data)
 
-    async def GET_sound_sensors_hits(self, req: http.Request):
+    async def GET_sound_sensors_hits(self, req: web.Request):
         location = SoundSensorLocation(int(req.query['location']))
 
         after = int(req.query['after'])
@@ -112,7 +112,7 @@ class WebAPIServer(http.HTTPServer):
         data = BotsDatabase().get_sound_hits(location, **kwargs)
         return self.ok(data)
 
-    async def POST_sound_sensors_hits(self, req: http.Request):
+    async def POST_sound_sensors_hits(self, req: web.Request):
         hits = []
         data = await req.post()
         for hit, count in json.loads(data['hits']):
@@ -125,37 +125,15 @@ class WebAPIServer(http.HTTPServer):
         BotsDatabase().add_sound_hits(hits, datetime.now())
         return self.ok()
 
-    async def POST_bot_request_log(self, req: http.Request):
-        data = await req.post()
-
-        try:
-            user_id = int(data['user_id'])
-        except KeyError:
-            user_id = 0
-
-        try:
-            message = data['message']
-        except KeyError:
-            message = ''
-
-        bot = BotType(int(data['bot']))
-
-        # validate message
-        if message.strip() == '':
-            raise ValueError('message can\'t be empty')
-
-        # add record to the database
-        BotsDatabase().add_request(bot, user_id, message)
-
-        return self.ok()
-
-    async def POST_openwrt_log(self, req: http.Request):
+    async def POST_openwrt_log(self, req: web.Request):
         data = await req.post()
 
         try:
             logs = data['logs']
+            ap = int(data['ap'])
         except KeyError:
             logs = ''
+            ap = 0
 
         # validate it
         logs = json.loads(logs)
@@ -173,10 +151,10 @@ class WebAPIServer(http.HTTPServer):
                 line[1]
             ))
 
-        BotsDatabase().add_openwrt_logs(lines)
+        BotsDatabase().add_openwrt_logs(lines, ap)
         return self.ok()
 
-    async def GET_recordings_list(self, req: http.Request):
+    async def GET_recordings_list(self, req: web.Request):
         data = await req.post()
 
         try:
@@ -198,7 +176,7 @@ class WebAPIServer(http.HTTPServer):
         return self.ok(files)
 
     @staticmethod
-    def _get_inverter_from_to(req: http.Request):
+    def _get_inverter_from_to(req: web.Request):
         s_from = req.query['from']
         s_to = req.query['to']
 
@@ -211,12 +189,12 @@ class WebAPIServer(http.HTTPServer):
 
         return dt_from, dt_to
 
-    async def GET_consumed_energy(self, req: http.Request):
+    async def GET_consumed_energy(self, req: web.Request):
         dt_from, dt_to = self._get_inverter_from_to(req)
         wh = InverterDatabase().get_consumed_energy(dt_from, dt_to)
         return self.ok(wh)
 
-    async def GET_grid_consumed_energy(self, req: http.Request):
+    async def GET_grid_consumed_energy(self, req: web.Request):
         dt_from, dt_to = self._get_inverter_from_to(req)
         wh = InverterDatabase().get_grid_consumed_energy(dt_from, dt_to)
         return self.ok(wh)
@@ -229,7 +207,7 @@ if __name__ == '__main__':
     _app_name = 'web_api'
     if is_development_mode():
         _app_name += '_dev'
-    config.load(_app_name)
+    config.load_app(_app_name)
 
     loop = asyncio.get_event_loop()
 

bin/web_kbn.py

@@ -0,0 +1,610 @@
+#!/usr/bin/env python3
+import include_homekit
+import asyncio
+import logging
+import jinja2
+import aiohttp_jinja2
+import json
+import re
+import inverterd
+import phonenumbers
+import time
+import os.path
+
+from io import StringIO
+from aiohttp import web
+from typing import Optional, Union
+from urllib.parse import quote_plus
+from contextvars import ContextVar
+from homekit.config import config, AppConfigUnit, is_development_mode, Translation, Language
+from homekit.camera import IpcamConfig
+from homekit.util import homekit_path, filesize_fmt, seconds_to_human_readable_string, json_serial, validate_ipv4
+from homekit.modem import E3372, ModemsConfig, MacroNetWorkType
+from homekit.inverter.config import InverterdConfig
+from homekit.relay.sunxi_h3_client import RelayClient
+from homekit import openwrt, http
+
+
+class WebKbnConfig(AppConfigUnit):
+    NAME = 'web_kbn'
+
+    @classmethod
+    def schema(cls) -> Optional[dict]:
+        return {
+            'listen_addr': cls._addr_schema(required=True),
+            'assets_public_path': {'type': 'string'},
+            'pump_addr': cls._addr_schema(required=True),
+            'hls_local_host': cls._addr_schema(required=True, only_ip=True),
+            'inverter_grafana_url': {'type': 'string'},
+            'sensors_grafana_url': {'type': 'string'},
+        }
+
+
+# files marked with + at the beginning are included by default
+common_static_files = {
+    '+bootstrap.min.css': 1,
+    '+bootstrap.bundle.min.js': 1,
+    '+polyfills.js': 1,
+    '+app.js': 10,
+    '+app.css': 6,
+    'hls.js': 1
+}
+routes = web.RouteTableDef()
+logger = logging.getLogger(__name__)
+lang_context_var = ContextVar('lang', default=Translation.DEFAULT_LANGUAGE)
+
+
+def get_js_link(file, version) -> str:
+    if is_development_mode():
+        version = int(time.time())
+    file += f'?version={version}'
+    return f'<script src="{config.app_config["assets_public_path"]}/{file}" type="text/javascript"></script>'
+
+
+def get_css_link(file, version) -> str:
+    if is_development_mode():
+        version = int(time.time())
+    file += f'?version={version}'
+    return f'<link rel="stylesheet" type="text/css" href="{config.app_config["assets_public_path"]}/{file}">'
+
+
+def get_head_static(additional_files=None) -> str:
+    buf = StringIO()
+    if additional_files is None:
+        additional_files = []
+
+    for file, version in common_static_files.items():
+        enabled_by_default = file.startswith('+')
+        if not enabled_by_default and file not in additional_files:
+            continue
+
+        if enabled_by_default:
+            file = file[1:]
+
+        if file.endswith('.js'):
+            buf.write(get_js_link(file, version))
+        else:
+            buf.write(get_css_link(file, version))
+
+    return buf.getvalue()
+
+
+def get_modem_client(modem_cfg: dict) -> E3372:
+    return E3372(modem_cfg['ip'], legacy_token_auth=modem_cfg['legacy_auth'])
+
+
+def get_modem_data(modem_cfg: dict, get_raw=False) -> Union[dict, tuple]:
+    cl = get_modem_client(modem_cfg)
+
+    signal = cl.device_signal
+    status = cl.monitoring_status
+    traffic = cl.traffic_stats
+
+    if get_raw:
+        device_info = cl.device_information
+        dialup_conn = cl.dialup_connection
+        return signal, status, traffic, device_info, dialup_conn
+    else:
+        network_type_label = re.sub('^MACRO_NET_WORK_TYPE(_EX)?_', '', MacroNetWorkType(int(status['CurrentNetworkType'])).name)
+        return {
+            'type': network_type_label,
+            'level': int(status['SignalIcon']) if 'SignalIcon' in status else 0,
+            'rssi': signal['rssi'],
+            'sinr': signal['sinr'],
+            'connected_time': seconds_to_human_readable_string(int(traffic['CurrentConnectTime'])),
+            'downloaded': filesize_fmt(int(traffic['CurrentDownload'])),
+            'uploaded': filesize_fmt(int(traffic['CurrentUpload']))
+        }
+
+
+def get_pump_client() -> RelayClient:
+    addr = config.app_config['pump_addr']
+    cl = RelayClient(host=addr.host, port=addr.port)
+    cl.connect()
+    return cl
+
+
+def get_inverter_client() -> inverterd.Client:
+    cl = inverterd.Client(host=InverterdConfig()['remote_addr'].host)
+    cl.connect()
+    cl.format(inverterd.Format.JSON)
+    return cl
+
+
+def get_inverter_data() -> tuple:
+    cl = get_inverter_client()
+
+    status = json.loads(cl.exec('get-status'))['data']
+    rated = json.loads(cl.exec('get-rated'))['data']
+
+    power_direction = status['battery_power_direction'].lower()
+    power_direction = re.sub('ge$', 'ging', power_direction)
+
+    charging_rate = ''
+    if power_direction == 'charging':
+        charging_rate = ' @ %s %s' % (
+            status['battery_charge_current']['value'],
+            status['battery_charge_current']['unit'])
+    elif power_direction == 'discharging':
+        charging_rate = ' @ %s %s' % (
+            status['battery_discharge_current']['value'],
+            status['battery_discharge_current']['unit'])
+
+    html = '<b>Battery:</b> %s %s' % (
+        status['battery_voltage']['value'],
+        status['battery_voltage']['unit'])
+    html += ' (%s%s, ' % (
+        status['battery_capacity']['value'],
+        status['battery_capacity']['unit'])
+    html += '%s%s)' % (power_direction, charging_rate)
+
+    html += "\n"
+    html += '<b>Load:</b> %s %s' % (
+        status['ac_output_active_power']['value'],
+        status['ac_output_active_power']['unit'])
+    html += ' (%s%%)' % (status['output_load_percent']['value'],)
+
+    if status['pv1_input_power']['value'] > 0:
+        html += "\n"
+        html += '<b>Input power:</b> %s %s' % (
+            status['pv1_input_power']['value'],
+            status['pv1_input_power']['unit'])
+
+    if status['grid_voltage']['value'] > 0 or status['grid_freq']['value'] > 0:
+        html += "\n"
+        html += '<b>AC input:</b> %s %s' % (
+            status['grid_voltage']['value'],
+            status['grid_voltage']['unit'])
+        html += ', %s %s' % (
+            status['grid_freq']['value'],
+            status['grid_freq']['unit'])
+
+    html += "\n"
+    html += '<b>Priority:</b> %s' % (rated['output_source_priority'],)
+
+    html = html.replace("\n", '<br>')
+
+    return status, rated, html
+
+
+def get_current_upstream() -> str:
+    r = openwrt.get_default_route()
+    logger.info(f'default route: {r}')
+    mc = ModemsConfig()
+    for k, v in mc.items():
+        if 'gateway_ip' in v and v['gateway_ip'] == r:
+            r = v['ip']
+            break
+    upstream = None
+    for k, v in mc.items():
+        if r == v['ip']:
+            upstream = k
+    if not upstream:
+        raise RuntimeError('failed to determine current upstream!')
+    return upstream
+
+
+def get_preferred_lang(req: web.Request) -> Language:
+    lang_cookie = req.cookies.get('lang', None)
+    if lang_cookie is None:
+        return Translation.DEFAULT_LANGUAGE
+    try:
+        return Language(lang_cookie)
+    except ValueError:
+        logger.debug(f"unsupported lang_cookie value: {lang_cookie}")
+        return Translation.DEFAULT_LANGUAGE
+
+
+@web.middleware
+async def language_middleware(request, handler):
+    lang_context_var.set(get_preferred_lang(request))
+    return await handler(request)
+
+
+def lang(key, unit='web_kbn'):
+    strings = Translation(unit)
+    if isinstance(key, str) and '.' in key:
+        return strings.get(lang_context_var.get()).get(key)
+    else:
+        return strings.get(lang_context_var.get())[key]
+
+
+async def render(req: web.Request,
+                 template_name: str,
+                 title: Optional[str] = None,
+                 context: Optional[dict] = None,
+                 assets: Optional[list] = None):
+    if context is None:
+        context = {}
+    context = {
+        **context,
+        'head_static': get_head_static(assets),
+        'user_lang': lang_context_var.get().value
+    }
+    if title is not None:
+        context['title'] = title
+    response = aiohttp_jinja2.render_template(template_name+'.j2', req, context=context)
+    return response
+
+
+@routes.get('/')
+async def index0(req: web.Request):
+    raise web.HTTPFound('main.cgi')
+
+
+@routes.get('/main.cgi')
+async def index(req: web.Request):
+    tabs = ['zones', 'list']
+    tab = req.query.get('tab', None)
+    if tab and (tab not in tabs or tab == tabs[0]):
+        raise web.HTTPFound('main.cgi')
+    if tab is None:
+        tab = tabs[0]
+
+    ctx = {}
+    for k in 'inverter', 'sensors':
+        ctx[f'{k}_grafana_url'] = config.app_config[f'{k}_grafana_url']
+
+    cc = IpcamConfig()
+    ctx['camzones'] = cc['zones'].keys()
+    ctx['allcams'] = cc.get_all_cam_names()
+    ctx['lang_enum'] = Language
+    ctx['lang_selected'] = lang_context_var.get()
+    ctx['tab_selected'] = tab
+    ctx['tabs'] = tabs
+
+    return await render(req, 'index',
+                        title=lang('sitename'),
+                        context=ctx)
+
+
+@routes.get('/modems.cgi')
+async def modems(req: web.Request):
+    return await render(req, 'modems',
+                        title=lang('modem_statuses'),
+                        context=dict(modems=ModemsConfig(),
+                                     modems_js_list=[key for key, value in ModemsConfig().items() if value['type'] == 'e3372']))
+
+
+@routes.get('/modems_info.ajx')
+async def modems_ajx(req: web.Request):
+    mc = ModemsConfig()
+    modem = req.query.get('id', None)
+    if modem not in mc.keys():
+        raise ValueError('invalid modem id')
+
+    modem_cfg = mc.get(modem)
+    if modem_cfg['type'] != 'e3372':
+        raise ValueError('invalid modem type')
+
+    loop = asyncio.get_event_loop()
+    modem_data = await loop.run_in_executor(None, lambda: get_modem_data(modem_cfg))
+
+    html = aiohttp_jinja2.render_string('modem_data.j2', req, context=dict(
+        modem_data=modem_data,
+        modem=modem
+    ))
+
+    return http.ajax_ok({'html': html})
+
+
+@routes.get('/modems_verbose.cgi')
+async def modems_verbose(req: web.Request):
+    modem = req.query.get('id', None)
+    if modem not in ModemsConfig().keys():
+        raise ValueError('invalid modem id')
+
+    modem_cfg = ModemsConfig().get(modem)
+    if modem_cfg['type'] != 'e3372':
+        raise ValueError('invalid modem type')
+
+    loop = asyncio.get_event_loop()
+    signal, status, traffic, device, dialup_conn = await loop.run_in_executor(None, lambda: get_modem_data(modem_cfg, True))
+    data = [
+        ['Signal', signal],
+        ['Connection', status],
+        ['Traffic', traffic],
+        ['Device info', device],
+        ['Dialup connection', dialup_conn]
+    ]
+
+    modem_name = Translation('modems').get(lang_context_var.get())[modem]['full']
+    return await render(req, 'modem_verbose',
+                        title=lang('modem_verbose_info_about_modem') % (modem_name,),
+                        context=dict(data=data, modem_name=modem_name))
+
+
+@routes.get('/sms.cgi')
+async def sms(req: web.Request):
+    modem = req.query.get('id', list(ModemsConfig().keys())[0])
+    is_outbox = int(req.query.get('outbox', 0)) == 1
+    error = req.query.get('error', None)
+    sent = int(req.query.get('sent', 0)) == 1
+
+    input_modem = ModemsConfig()[modem]
+    if input_modem['type'] != 'e3372':
+        raise ValueError('invalid modem')
+
+    cl = get_modem_client(input_modem)
+    messages = cl.sms_list(1, 20, is_outbox)
+    return await render(req, 'sms',
+                        title=lang('sms_page_title') % (lang('sms_outbox') if is_outbox else lang('sms_inbox'), modem),
+                        context=dict(
+                            modems=ModemsConfig(),
+                            selected_modem=modem,
+                            is_outbox=is_outbox,
+                            error=error,
+                            is_sent=sent,
+                            messages=messages
+                        ))
+
+
+@routes.post('/sms.cgi')
+async def sms_post(req: web.Request):
+    modem = req.query.get('id', list(ModemsConfig().keys())[0])
+    is_outbox = int(req.query.get('outbox', 0)) == 1
+
+    fd = await req.post()
+    phone = fd.get('phone', None)
+    text = fd.get('text', None)
+
+    return_url = f'sms.cgi?id={modem}&outbox={int(is_outbox)}'
+    phone = re.sub(r'\s+', '', phone)
+
+    if len(phone) > 4:
+        country = None
+        if not phone.startswith('+'):
+            country = 'RU'
+        number = phonenumbers.parse(phone, country)
+        if not phonenumbers.is_valid_number(number):
+            raise web.HTTPFound(f'{return_url}&error=Неверный+номер')
+        phone = phonenumbers.format_number(number, phonenumbers.PhoneNumberFormat.E164)
+
+    cl = get_modem_client(ModemsConfig()[modem])
+    cl.sms_send(phone, text)
+    raise web.HTTPFound(return_url)
+
+
+@routes.get('/inverter.cgi')
+async def inverter(req: web.Request):
+    action = req.query.get('do', None)
+    if action == 'set-osp':
+        val = req.query.get('value')
+        if val not in ('sub', 'sbu'):
+            raise ValueError('invalid osp value')
+        cl = get_inverter_client()
+        cl.exec('set-output-source-priority',
+                arguments=(val.upper(),))
+        raise web.HTTPFound('inverter.cgi')
+
+    status, rated, html = await asyncio.get_event_loop().run_in_executor(None, get_inverter_data)
+    return await render(req, 'inverter',
+                        title=lang('inverter'),
+                        context=dict(status=status, rated=rated, html=html))
+
+
+@routes.get('/inverter.ajx')
+async def inverter_ajx(req: web.Request):
+    status, rated, html = await asyncio.get_event_loop().run_in_executor(None, get_inverter_data)
+    return http.ajax_ok({'html': html})
+
+
+@routes.get('/pump.cgi')
+async def pump(req: web.Request):
+    # TODO
+    # these are blocking calls
+    # should be rewritten using aio
+
+    cl = get_pump_client()
+
+    action = req.query.get('set', None)
+    if action in ('on', 'off'):
+        getattr(cl, action)()
+        raise web.HTTPFound('pump.cgi')
+
+    status = cl.status()
+    return await render(req, 'pump',
+                        title=lang('pump'),
+                        context=dict(status=status))
+
+
+@routes.get('/cams.cgi')
+async def cams(req: web.Request):
+    cc = IpcamConfig()
+
+    cam = req.query.get('id', None)
+    zone = req.query.get('zone', None)
+    debug_hls = bool(req.query.get('debug_hls', False))
+    debug_video_events = bool(req.query.get('debug_video_events', False))
+
+    if cam is not None:
+        if not cc.has_camera(int(cam)):
+            raise ValueError('invalid camera id')
+        cams = [int(cam)]
+        mode = {'type': 'single', 'cam': int(cam)}
+
+    elif zone is not None:
+        if not cc.has_zone(zone):
+            raise ValueError('invalid zone')
+        cams = cc['zones'][zone]
+        mode = {'type': 'zone', 'zone': zone}
+
+    else:
+        cams = cc.get_all_cam_names()
+        mode = {'type': 'all'}
+
+    if req.headers.get('Host').endswith('.manor.id'):
+        hls_pfx = 'https://'+req.headers.get('Host')
+        hls_pfx += re.sub(r'/home/?$', '/ipcam/', os.path.dirname(req.headers.get('X-Real-URI')))
+    else:
+        hls_pfx = 'http://'+str(config.app_config['hls_local_host'])+'/ipcam/'
+
+    js_config = {
+        'pfx': hls_pfx,
+        # 'host': config.app_config['hls_local_host'],
+        # 'proto': 'http',
+        'cams': cams,
+        'hlsConfig': {
+            'opts': {
+                'startPosition': -1,
+                # https://github.com/video-dev/hls.js/issues/3884#issuecomment-842380784
+                'liveSyncDuration': 2,
+                'liveMaxLatencyDuration': 3,
+                'maxLiveSyncPlaybackRate': 2,
+                'liveDurationInfinity': True
+            },
+            'debugVideoEvents': debug_video_events,
+            'debug': debug_hls
+        }
+    }
+
+    return await render(req, 'cams',
+                        title=lang('cams'),
+                        assets=['hls.js'],
+                        context=dict(
+                            mode=mode,
+                            js_config=js_config,
+                        ))
+
+
+@routes.get('/routing_main.cgi')
+async def routing_main(req: web.Request):
+    upstream = get_current_upstream()
+
+    set_upstream_to = req.query.get('set-upstream-to', None)
+    mc = ModemsConfig()
+
+    if set_upstream_to and set_upstream_to in mc and set_upstream_to != upstream:
+        modem = mc[set_upstream_to]
+        new_upstream = str(modem['gateway_ip'] if 'gateway_ip' in modem else modem['ip'])
+        openwrt.set_upstream(new_upstream)
+        raise web.HTTPFound('routing_main.cgi')
+
+    context = dict(
+        upstream=upstream,
+        selected_tab='main',
+        modems=mc.keys()
+    )
+    return await render(req, 'routing_main', title=lang('routing'), context=context)
+
+
+@routes.get('/routing_rules.cgi')
+async def routing_rules(req: web.Request):
+    mc = ModemsConfig()
+
+    action = req.query.get('action', None)
+    error = req.query.get('error', None)
+    set_name = req.query.get('set', None)
+    ip = req.query.get('ip', None)
+
+    def validate_input():
+        # validate set
+        if not set_name or set_name not in mc:
+            raise ValueError(f'invalid set \'{set_name}\'')
+
+        # validate ip
+        if not isinstance(ip, str):
+            raise ValueError('invalid ip')
+
+        slash_pos = None
+        try:
+            slash_pos = ip.index('/')
+        except ValueError:
+            pass
+
+        if slash_pos is not None:
+            ip_without_mask = ip[0:slash_pos]
+        else:
+            ip_without_mask = ip
+        if not validate_ipv4(ip_without_mask):
+            raise ValueError(f'invalid ip \'{ip}\'')
+
+    base_url = 'routing_rules.cgi'
+    if action in ('add', 'del'):
+        try:
+            validate_input()
+        except ValueError as e:
+            raise web.HTTPFound(f'{base_url}?error='+quote_plus(str(e)))
+        f = getattr(openwrt, f'ipset_{action}')
+        output = f(set_name, ip)
+        url = base_url
+        if output != '':
+            url += '?error='+quote_plus(output)
+        raise web.HTTPFound(url)
+
+    ipsets = openwrt.ipset_list_all()
+    context = dict(
+        sets=ipsets,
+        selected_tab='rules',
+        error=error
+    )
+    return await render(req, 'routing_rules',
+                        title=lang('routing') + ' // ' + lang('routing_rules'),
+                        context=context)
+
+
+@routes.get('/routing_dhcp.cgi')
+async def routing_dhcp(req: web.Request):
+    leases = openwrt.get_dhcp_leases()
+    return await render(req, 'routing_dhcp',
+                        title=lang('routing') + ' // DHCP',
+                        context=dict(leases=leases, selected_tab='dhcp'))
+
+
+@routes.get('/debug.cgi')
+async def debug(req: web.Request):
+    info = dict(
+        headers=dict(req.headers),
+        host=req.headers.get('Host'),
+        url=str(req.url),
+        method=req.method,
+    )
+    return http.ajax_ok(info)
+
+
+def init_web_app(app: web.Application):
+    app.middlewares.append(language_middleware)
+    aiohttp_jinja2.setup(
+        app,
+        loader=jinja2.FileSystemLoader(homekit_path('web', 'kbn_templates')),
+        autoescape=jinja2.select_autoescape(['html', 'xml']),
+    )
+    env = aiohttp_jinja2.get_env(app)
+
+    # @pass_context is used only to prevent jinja2 from caching the result of lang filter results of constant values.
+    # as of now i don't know a better way of doing it
+    @jinja2.pass_context
+    def filter_lang(ctx, key, unit='web_kbn'):
+        return lang(key, unit)
+
+    env.filters['tojson'] = lambda obj: json.dumps(obj, separators=(',', ':'), default=json_serial)
+    env.filters['lang'] = filter_lang
+
+    app.router.add_static('/assets/', path=homekit_path('web', 'kbn_assets'))
+
+
+if __name__ == '__main__':
+    config.load_app(WebKbnConfig)
+    http.serve(addr=config.app_config['listen_addr'],
+               routes=routes,
+               before_start=init_web_app)

doc/common_requirements.md

@@ -1,4 +1,4 @@
 Debian packages:
 ```
-apt-get install git cmake build-essential python3-dev python3-wheel python3-pip python3-build python3-yaml python3-toml python3-psutil python3-aiohttp python3-requests python3-apscheduler python3-smbus
+apt-get install git cmake build-essential python3-dev python3-wheel python3-pip python3-build python3-yaml python3-toml python3-psutil python3-aiohttp python3-requests python3-apscheduler python3-smbus traceroute tcpdump
 ```

doc/localwebsite.md

@@ -1,7 +0,0 @@
-## Dependencies
-
-```
-apt install nginx-extras php-fpm php-mbstring php-sqlite3 php-curl php-simplexml php-gmp composer 
-```
-
-

doc/openwrt_logger.md

@@ -0,0 +1,28 @@
+# openwrt_logger.py
+
+This script is supposed to be run by cron every 5 minutes or so.
+It looks for new lines in log file and sends them to remote server.
+
+OpenWRT must have remote logging enabled (UDP; IP of host this script is launched on; port 514)
+
+`/etc/rsyslog.conf` contains following (assuming `192.168.1.1` is the router IP):
+
+```
+$ModLoad imudp  
+$UDPServerRun 514  
+:fromhost-ip, isequal, "192.168.1.1" /var/log/openwrt.log  
+& ~
+```
+
+Also comment out the following line:
+```
+$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
+```
+
+Cron line example:
+```
+* * * * *  /home/user/homekit/src/openwrt_logger.py --access-point 1 --file /var/wrtlogfs/openwrt-5.log  >/dev/null
+```
+
+`/var/wrtlogfs` is recommended to be tmpfs, to avoid writes on mmc card, in case
+you use arm sbcs as I do.

include/pio/include/homekit/logging.h

@@ -1,7 +1,7 @@
-#pragma once
+#ifndef COMMON_HOMEKIT_LOGGING_H
+#define COMMON_HOMEKIT_LOGGING_H
 
 #include <stdlib.h>
-#include "config.def.h"
 
 #ifdef DEBUG
 
@@ -16,3 +16,5 @@
 #define PRINTF(...)
 
 #endif
+
+#endif //COMMON_HOMEKIT_LOGGING_H

include/pio/include/homekit/macros.h

@@ -0,0 +1 @@
+#define ARRAY_SIZE(X) sizeof((X))/sizeof((X)[0])

include/pio/libs/config/homekit/config.cpp

@@ -1,7 +1,7 @@
 #include <EEPROM.h>
 #include <strings.h>
 #include "config.h"
-#include "logging.h"
+#include <homekit/logging.h>
 
 #define GET_DATA_CRC(data) \
     eeprom_crc(reinterpret_cast<uint8_t*>(&(data))+4, sizeof(ConfigData)-4)

include/pio/libs/config/homekit/config.h

@@ -1,4 +1,5 @@
-#pragma once
+#ifndef COMMON_HOMEKIT_CONFIG_H
+#define COMMON_HOMEKIT_CONFIG_H
 
 #include <Arduino.h>
 
@@ -32,3 +33,5 @@ bool isValid(ConfigData& data);
 bool isDirty(ConfigData& data);
 
 }
+
+#endif //COMMON_HOMEKIT_CONFIG_H

include/pio/libs/config/library.json

@@ -0,0 +1,8 @@
+{
+  "name": "homekit_config",
+  "version": "1.0.2",
+  "build": {
+    "flags": "-I../../include"
+  }
+}
+

include/pio/libs/http_server/homekit/http_server.cpp

@@ -1,13 +1,13 @@
+#include "http_server.h"
+
 #include <Arduino.h>
 #include <string.h>
 
-#include "static.h"
-#include "http_server.h"
-#include "config.h"
-#include "config.def.h"
-#include "logging.h"
-#include "util.h"
-#include "led.h"
+#include <homekit/static.h>
+#include <homekit/config.h>
+#include <homekit/logging.h>
+#include <homekit/macros.h>
+#include <homekit/util.h>
 
 namespace homekit {
 
@@ -64,7 +64,7 @@ void HttpServer::start() {
 
         if (!isValid(cfg) || !cfg.flags.node_configured) {
             sprintf_P(json_buf, JSON_STATUS_FMT
-                    , DEFAULT_NODE_ID
+                    , CONFIG_NODE_ID
 #ifdef DEBUG
                     , 0
                     , cfg.crc
@@ -215,7 +215,8 @@ void HttpServer::start() {
                 return;
 
             PRINTF("http/ota: writing %ul\n", upload.currentSize);
-            esp_led.blink(1, 1);
+            ota_led();
+
             if (Update.write(upload.buf, upload.currentSize) != upload.currentSize) {
 #ifdef DEBUG
                 Update.printError(Serial);
@@ -276,4 +277,6 @@ bool HttpServer::getInputParam(const char *field_name,
     return true;
 }
 
+void HttpServer::ota_led() const {}
+
 }

include/pio/libs/http_server/homekit/http_server.h

@@ -1,12 +1,15 @@
-#pragma once
+#ifndef COMMON_HOMEKIT_HTTP_SERVER_H
+#define COMMON_HOMEKIT_HTTP_SERVER_H
+
 #include <ESP8266WebServer.h>
 #include <Ticker.h>
 #include <memory>
 #include <list>
 #include <utility>
-#include "config.h"
-#include "wifi.h"
-#include "static.h"
+
+#include <homekit/config.h>
+#include <homekit/wifi.h>
+#include <homekit/static.h>
 
 namespace homekit {
 
@@ -36,6 +39,7 @@ private:
     void sendError(const String& message);
 
     bool getInputParam(const char* field_name, size_t max_len, String& dst);
+    virtual void ota_led() const;
 
 public:
     explicit HttpServer(std::shared_ptr<std::list<wifi::ScanResult>> scanResults)
@@ -54,3 +58,5 @@ public:
 };
 
 }
+
+#endif //COMMON_HOMEKIT_HTTP_SERVER_H

include/pio/libs/http_server/library.json

@@ -0,0 +1,8 @@
+{
+  "name": "homekit_http_server",
+  "version": "1.0.3",
+  "build": {
+    "flags": "-I../../include"
+  }
+}
+

include/pio/libs/led/homekit/led.cpp

@@ -0,0 +1,27 @@
+#include "led.h"
+
+namespace homekit::led {
+
+void Led::on_off(uint16_t delay_ms, bool last_delay) const {
+    on();
+    delay(delay_ms);
+
+    off();
+    if (last_delay)
+        delay(delay_ms);
+}
+
+void Led::blink(uint8_t count, uint16_t delay_ms) const {
+    for (uint8_t i = 0; i < count; i++) {
+        on_off(delay_ms, i < count-1);
+    }
+}
+
+
+#ifdef CONFIG_TARGET_NODEMCU
+const Led* board_led = new Led(CONFIG_BOARD_LED_GPIO);
+#endif
+const Led* mcu_led = new Led(CONFIG_MCU_LED_GPIO);
+
+
+}

include/pio/libs/led/homekit/led.h

@@ -0,0 +1,33 @@
+#ifndef HOMEKIT_LIB_LED_H
+#define HOMEKIT_LIB_LED_H
+
+#include <Arduino.h>
+#include <stdint.h>
+
+namespace homekit::led {
+
+class Led {
+private:
+    uint8_t _pin;
+
+public:
+    explicit Led(uint8_t pin) : _pin(pin) {
+        pinMode(_pin, OUTPUT);
+        off();
+    }
+
+    inline void off() const { digitalWrite(_pin, HIGH); }
+    inline void on() const { digitalWrite(_pin, LOW); }
+
+    void on_off(uint16_t delay_ms, bool last_delay = false) const;
+    void blink(uint8_t count, uint16_t delay_ms) const;
+};
+
+#ifdef CONFIG_TARGET_NODEMCU
+extern const Led* board_led;
+#endif
+extern const Led* mcu_led;
+
+}
+
+#endif //HOMEKIT_LIB_LED_H

include/pio/libs/led/library.json

@@ -0,0 +1,8 @@
+{
+  "name": "homekit_led",
+  "version": "1.0.8",
+  "build": {
+    "flags": "-I../../include"
+  }
+}
+

include/pio/libs/main/homekit/main.cpp

@@ -1,32 +1,16 @@
-#include <Arduino.h>
-#include <ESP8266WiFi.h>
-#include <DNSServer.h>
-#include <Ticker.h>
-#include <Wire.h>
+#include "./main.h"
+#include <homekit/led.h>
+#include <homekit/mqtt/mqtt.h>
+#include <homekit/mqtt/module/diagnostics.h>
+#include <homekit/mqtt/module/ota.h>
 
-#include "mqtt.h"
-#include "config.h"
-#include "logging.h"
-#include "http_server.h"
-#include "led.h"
-#include "config.def.h"
-#include "wifi.h"
-#include "temphum.h"
-#include "stopwatch.h"
+namespace homekit::main {
 
-using namespace homekit;
-
-enum class WorkingMode {
-    RECOVERY, // AP mode, http server with configuration
-    NORMAL,   // MQTT client
-};
-static enum WorkingMode working_mode = WorkingMode::NORMAL;
-
-enum class WiFiConnectionState {
-    WAITING = 0,
-    JUST_CONNECTED = 1,
-    CONNECTED = 2
-};
+#ifndef CONFIG_TARGET_ESP01
+#ifndef CONFIG_NO_RECOVERY
+enum WorkingMode working_mode = WorkingMode::NORMAL;
+#endif
+#endif
 
 static const uint16_t recovery_boot_detection_ms = 2000;
 static const uint8_t recovery_boot_delay_ms = 100;
@@ -35,11 +19,18 @@ static volatile enum WiFiConnectionState wifi_state = WiFiConnectionState::WAITI
 static void* service = nullptr;
 static WiFiEventHandler wifiConnectHandler, wifiDisconnectHandler;
 static Ticker wifiTimer;
+static mqtt::MqttDiagnosticsModule* mqttDiagModule;
+static mqtt::MqttOtaModule* mqttOtaModule;
+
 #if MQTT_BLINK
 static StopWatch blinkStopWatch;
 #endif
 
+#ifndef CONFIG_TARGET_ESP01
+#ifndef CONFIG_NO_RECOVERY
 static DNSServer* dnsServer = nullptr;
+#endif
+#endif
 
 static void onWifiConnected(const WiFiEventStationModeGotIP& event);
 static void onWifiDisconnected(const WiFiEventStationModeDisconnected& event);
@@ -60,8 +51,10 @@ static void wifiConnect() {
     PRINT("connecting to wifi..");
 }
 
+#ifndef CONFIG_TARGET_ESP01
+#ifndef CONFIG_NO_RECOVERY
 static void wifiHotspot() {
-    esp_led.on();
+    led::mcu_led->on();
 
     auto scanResults = wifi::scan();
 
@@ -76,21 +69,26 @@ static void wifiHotspot() {
 }
 
 static void waitForRecoveryPress() {
-    pinMode(FLASH_BUTTON_PIN, INPUT_PULLUP);
+    pinMode(CONFIG_FLASH_GPIO, INPUT_PULLUP);
     for (uint16_t i = 0; i < recovery_boot_detection_ms; i += recovery_boot_delay_ms) {
         delay(recovery_boot_delay_ms);
-        if (digitalRead(FLASH_BUTTON_PIN) == LOW) {
+        if (digitalRead(CONFIG_FLASH_GPIO) == LOW) {
             working_mode = WorkingMode::RECOVERY;
             break;
         }
     }
 }
+#endif
+#endif
+
 
 void setup() {
     WiFi.disconnect();
-    waitForRecoveryPress();
-
-    temphum::setup();
+#ifndef CONFIG_NO_RECOVERY
+#ifndef CONFIG_TARGET_ESP01
+    homekit::main::waitForRecoveryPress();
+#endif
+#endif
 
 #ifdef DEBUG
     Serial.begin(115200);
@@ -100,65 +98,93 @@ void setup() {
     if (config::isDirty(cfg)) {
         PRINTLN("config is dirty, erasing...");
         config::erase(cfg);
-        board_led.blink(10, 50);
+#ifdef CONFIG_TARGET_NODEMCU
+        led::board_led->blink(10, 50);
+#else
+        led::mcu_led->blink(10, 50);
+#endif
     }
 
+#ifndef CONFIG_TARGET_ESP01
+#ifndef CONFIG_NO_RECOVERY
     switch (working_mode) {
     case WorkingMode::RECOVERY:
         wifiHotspot();
         break;
 
     case WorkingMode::NORMAL:
+#endif
+#endif
         wifiConnectHandler = WiFi.onStationModeGotIP(onWifiConnected);
         wifiDisconnectHandler = WiFi.onStationModeDisconnected(onWifiDisconnected);
         wifiConnect();
+#ifndef CONFIG_NO_RECOVERY
+#ifndef CONFIG_TARGET_ESP01
         break;
     }
+#endif
+#endif
 }
 
-void loop() {
+void loop(LoopConfig* config) {
+#ifndef CONFIG_NO_RECOVERY
+#ifndef CONFIG_TARGET_ESP01
     if (working_mode == WorkingMode::NORMAL) {
+#endif
+#endif
         if (wifi_state == WiFiConnectionState::WAITING) {
             PRINT(".");
-            esp_led.blink(2, 50);
+            led::mcu_led->blink(2, 50);
             delay(1000);
             return;
         }
 
         if (wifi_state == WiFiConnectionState::JUST_CONNECTED) {
-            board_led.blink(3, 300);
+#ifdef CONFIG_TARGET_NODEMCU
+            led::board_led->blink(3, 300);
+#else
+            led::mcu_led->blink(3, 300);
+#endif
             wifi_state = WiFiConnectionState::CONNECTED;
 
-            if (service == nullptr)
-                service = new mqtt::MQTT();
+            if (service == nullptr) {
+                service = new mqtt::Mqtt();
+                mqttDiagModule = new mqtt::MqttDiagnosticsModule();
+                mqttOtaModule = new mqtt::MqttOtaModule();
 
-            ((mqtt::MQTT*)service)->connect();
+                ((mqtt::Mqtt*)service)->addModule(mqttDiagModule);
+                ((mqtt::Mqtt*)service)->addModule(mqttOtaModule);
+
+                if (config != nullptr)
+                    config->onMqttCreated(*(mqtt::Mqtt*)service);
+            }
+
+            ((mqtt::Mqtt*)service)->connect();
 #if MQTT_BLINK
             blinkStopWatch.save();
 #endif
         }
 
-        auto mqtt = (mqtt::MQTT*)service;
+        auto mqtt = (mqtt::Mqtt*)service;
         if (static_cast<int>(wifi_state) >= 1 && mqtt != nullptr) {
             mqtt->loop();
 
-            if (mqtt->ota.readyToRestart) {
+            if (mqttOtaModule != nullptr && mqttOtaModule->isReadyToRestart()) {
                 mqtt->disconnect();
-            } else if (mqtt->diagnosticsStopWatch.elapsed(10000)) {
-                mqtt->sendDiagnostics();
-
-                auto data = temphum::read();
-                mqtt->sendTempHumData(data.temp, data.rh);
             }
 
 #if MQTT_BLINK
             // periodically blink board led
             if (blinkStopWatch.elapsed(5000)) {
-                board_led.blink(1, 10);
+#ifdef CONFIG_TARGET_NODEMCU
+                board_led->blink(1, 10);
+#endif
                 blinkStopWatch.save();
             }
 #endif
         }
+#ifndef CONFIG_NO_RECOVERY
+#ifndef CONFIG_TARGET_ESP01
     } else {
         if (dnsServer != nullptr)
             dnsServer->processNextRequest();
@@ -167,6 +193,8 @@ void loop() {
         if (httpServer != nullptr)
             httpServer->loop();
     }
+#endif
+#endif
 }
 
 static void onWifiConnected(const WiFiEventStationModeGotIP& event) {
@@ -178,6 +206,8 @@ static void onWifiDisconnected(const WiFiEventStationModeDisconnected& event) {
     PRINTLN("disconnected from wi-fi");
     wifi_state = WiFiConnectionState::WAITING;
     if (service != nullptr)
-        ((mqtt::MQTT*)service)->disconnect();
+        ((mqtt::Mqtt*)service)->disconnect();
     wifiTimer.once(2, wifiConnect);
 }
+
+}

include/pio/libs/main/homekit/main.h

@@ -0,0 +1,52 @@
+#ifndef HOMEKIT_LIB_MAIN_H
+#define HOMEKIT_LIB_MAIN_H
+
+#include <Arduino.h>
+#include <ESP8266WiFi.h>
+#include <DNSServer.h>
+#include <Ticker.h>
+#include <Wire.h>
+
+#include <homekit/config.h>
+#include <homekit/logging.h>
+#ifndef CONFIG_TARGET_ESP01
+#ifndef CONFIG_NO_RECOVERY
+#include <homekit/http_server.h>
+#endif
+#endif
+#include <homekit/wifi.h>
+#include <homekit/mqtt/mqtt.h>
+
+#include <functional>
+
+namespace homekit::main {
+
+#ifndef CONFIG_TARGET_ESP01
+#ifndef CONFIG_NO_RECOVERY
+enum class WorkingMode {
+    RECOVERY, // AP mode, http server with configuration
+    NORMAL,   // MQTT client
+};
+
+extern enum WorkingMode working_mode;
+#endif
+#endif
+
+enum class WiFiConnectionState {
+    WAITING = 0,
+    JUST_CONNECTED = 1,
+    CONNECTED = 2
+};
+
+
+struct LoopConfig {
+    std::function<void(mqtt::Mqtt&)> onMqttCreated;
+};
+
+
+void setup();
+void loop(LoopConfig* config);
+
+}
+
+#endif //HOMEKIT_LIB_MAIN_H

include/pio/libs/main/library.json

@@ -0,0 +1,12 @@
+{
+  "name": "homekit_main",
+  "version": "1.0.11",
+  "build": {
+    "flags": "-I../../include"
+  },
+  "dependencies": {
+    "homekit_mqtt_module_ota": "file://../../include/pio/libs/mqtt_module_ota",
+    "homekit_mqtt_module_diagnostics": "file://../../include/pio/libs/mqtt_module_diagnostics"
+  }
+}
+

include/pio/libs/mqtt/homekit/mqtt/module.cpp

@@ -0,0 +1,26 @@
+#include "./module.h"
+#include <homekit/logging.h>
+
+namespace homekit::mqtt {
+
+bool MqttModule::tickElapsed() {
+    if (!tickSw.elapsed(tickInterval*1000))
+        return false;
+
+    tickSw.save();
+    return true;
+}
+
+void MqttModule::handlePayload(Mqtt& mqtt, String& topic, uint16_t packetId, const uint8_t* payload, size_t length,
+                               size_t index, size_t total) {
+    if (length != total)
+        PRINTLN("mqtt: received partial message, not supported");
+
+    // TODO
+}
+
+void MqttModule::handleOnPublish(uint16_t packetId) {}
+
+void MqttModule::onDisconnect(Mqtt& mqtt, espMqttClientTypes::DisconnectReason reason) {}
+
+}

include/pio/libs/mqtt/homekit/mqtt/module.h

@@ -0,0 +1,56 @@
+#ifndef HOMEKIT_LIB_MQTT_MODULE_H
+#define HOMEKIT_LIB_MQTT_MODULE_H
+
+#include "./mqtt.h"
+#include "./payload.h"
+#include <homekit/stopwatch.h>
+
+
+namespace homekit::mqtt {
+
+class Mqtt;
+
+class MqttModule {
+protected:
+    bool initialized;
+    StopWatch tickSw;
+    short tickInterval;
+
+    bool receiveOnPublish;
+    bool receiveOnDisconnect;
+
+    bool tickElapsed();
+
+public:
+    MqttModule(short _tickInterval, bool _receiveOnPublish = false, bool _receiveOnDisconnect = false)
+        : initialized(false)
+        , tickInterval(_tickInterval)
+        , receiveOnPublish(_receiveOnPublish)
+        , receiveOnDisconnect(_receiveOnDisconnect) {}
+
+    virtual void tick(Mqtt& mqtt) = 0;
+
+    virtual void onConnect(Mqtt& mqtt) = 0;
+    virtual void onDisconnect(Mqtt& mqtt, espMqttClientTypes::DisconnectReason reason);
+
+    virtual void handlePayload(Mqtt& mqtt, String& topic, uint16_t packetId, const uint8_t *payload, size_t length, size_t index, size_t total);
+    virtual void handleOnPublish(uint16_t packetId);
+
+    inline void setInitialized() {
+        initialized = true;
+    }
+
+    inline void unsetInitialized() {
+        initialized = false;
+    }
+
+    inline short getTickInterval() const {
+        return tickInterval;
+    }
+
+    friend class Mqtt;
+};
+
+}
+
+#endif //HOMEKIT_LIB_MQTT_MODULE_H

include/pio/libs/mqtt/homekit/mqtt/mqtt.cpp

@@ -0,0 +1,162 @@
+#include "./mqtt.h"
+
+#include <homekit/config.h>
+#include <homekit/wifi.h>
+#include <homekit/logging.h>
+
+namespace homekit::mqtt {
+  
+const uint8_t MQTT_CA_FINGERPRINT[] = { \
+    0x0e, 0xb6, 0x3a, 0x02, 0x1f,     \
+    0x4e, 0x1e, 0xe1, 0x6a, 0x67,     \
+    0x62, 0xec, 0x64, 0xd4, 0x84,     \
+    0x8a, 0xb0, 0xc9, 0x9c, 0xbb      \
+};;
+const char MQTT_SERVER[] = "mqtt.solarmon.ru";
+const uint16_t MQTT_PORT = 8883;
+const char MQTT_USERNAME[] = CONFIG_MQTT_USERNAME;
+const char MQTT_PASSWORD[] = CONFIG_MQTT_PASSWORD;
+const char MQTT_CLIENT_ID[] = CONFIG_MQTT_CLIENT_ID;
+const char MQTT_SECRET[CONFIG_NODE_SECRET_SIZE+1] = CONFIG_NODE_SECRET;
+
+static const uint16_t MQTT_KEEPALIVE = 30;
+
+using namespace espMqttClientTypes;
+
+Mqtt::Mqtt() {
+    auto cfg = config::read();
+    nodeId = String(cfg.flags.node_configured ? cfg.node_id : wifi::NODE_ID);
+
+    randomSeed(micros());
+
+    client.onConnect([&](bool sessionPresent) {
+        PRINTLN("mqtt: connected");
+
+        for (auto* module: modules) {
+            if (!module->initialized) {
+                module->onConnect(*this);
+                module->setInitialized();
+            }
+        }
+
+        connected = true;
+    });
+
+    client.onDisconnect([&](DisconnectReason reason) {
+        PRINTF("mqtt: disconnected, reason=%d\n", static_cast<int>(reason));
+#ifdef DEBUG
+        if (reason == DisconnectReason::TLS_BAD_FINGERPRINT)
+            PRINTLN("reason: bad fingerprint");
+#endif
+
+        for (auto* module: modules) {
+            module->onDisconnect(*this, reason);
+            module->unsetInitialized();
+        }
+
+        reconnectTimer.once(2, [&]() {
+            reconnect();
+        });
+    });
+
+    client.onSubscribe([&](uint16_t packetId, const SubscribeReturncode* returncodes, size_t len) {
+        PRINTF("mqtt: subscribe ack, packet_id=%d\n", packetId);
+        for (size_t i = 0; i < len; i++) {
+            PRINTF("    return code: %u\n", static_cast<unsigned int>(*(returncodes+i)));
+        }
+    });
+
+    client.onUnsubscribe([&](uint16_t packetId) {
+        PRINTF("mqtt: unsubscribe ack, packet_id=%d\n", packetId);
+    });
+
+    client.onMessage([&](const MessageProperties& properties, const char* topic, const uint8_t* payload, size_t len, size_t index, size_t total) {
+        PRINTF("mqtt: message received, topic=%s, qos=%d, dup=%d, retain=%d, len=%ul, index=%ul, total=%ul\n",
+               topic, properties.qos, (int)properties.dup, (int)properties.retain, len, index, total);
+
+        const char *ptr = topic + nodeId.length() + 4;
+        String relevantTopic(ptr);
+
+        auto it = moduleSubscriptions.find(relevantTopic);
+        if (it != moduleSubscriptions.end()) {
+            auto module = it->second;
+            module->handlePayload(*this, relevantTopic, properties.packetId, payload, len, index, total);
+        } else {
+            PRINTF("error: module subscription for topic %s not found\n", relevantTopic.c_str());
+        }
+    });
+
+    client.onPublish([&](uint16_t packetId) {
+        PRINTF("mqtt: publish ack, packet_id=%d\n", packetId);
+
+        for (auto* module: modules) {
+            if (module->receiveOnPublish) {
+                module->handleOnPublish(packetId);
+            }
+        }
+    });
+
+    client.setServer(MQTT_SERVER, MQTT_PORT);
+    client.setClientId(MQTT_CLIENT_ID);
+    client.setCredentials(MQTT_USERNAME, MQTT_PASSWORD);
+    client.setCleanSession(true);
+    client.setFingerprint(MQTT_CA_FINGERPRINT);
+    client.setKeepAlive(MQTT_KEEPALIVE);
+}
+
+void Mqtt::connect() {
+    reconnect();
+}
+
+void Mqtt::reconnect() {
+    if (client.connected()) {
+        PRINTLN("warning: already connected");
+        return;
+    }
+    client.connect();
+}
+
+void Mqtt::disconnect() {
+    // TODO test how this works???
+    reconnectTimer.detach();
+    client.disconnect(true);
+}
+
+void Mqtt::loop() {
+    client.loop();
+    for (auto& module: modules) {
+        if (module->getTickInterval() != 0)
+            module->tick(*this);
+    }
+}
+
+uint16_t Mqtt::publish(const String& topic, uint8_t* payload, size_t length) {
+    String fullTopic = "hk/" + nodeId + "/" + topic;
+    return client.publish(fullTopic.c_str(), 1, false, payload, length);
+}
+
+uint16_t Mqtt::subscribe(const String& topic, uint8_t qos) {
+    String fullTopic = "hk/" + nodeId + "/" + topic;
+    PRINTF("mqtt: subscribing to %s...\n", fullTopic.c_str());
+
+    uint16_t packetId = client.subscribe(fullTopic.c_str(), qos);
+    if (!packetId)
+        PRINTF("error: failed to subscribe to %s\n", fullTopic.c_str());
+
+    return packetId;
+}
+
+void Mqtt::addModule(MqttModule* module) {
+    modules.emplace_back(module);
+    if (connected) {
+        module->onConnect(*this);
+        module->setInitialized();
+    }
+}
+
+void Mqtt::subscribeModule(String& topic, MqttModule* module, uint8_t qos) {
+    moduleSubscriptions[topic] = module;
+    subscribe(topic, qos);
+}
+
+}

include/pio/libs/mqtt/homekit/mqtt/mqtt.h

@@ -0,0 +1,48 @@
+#ifndef HOMEKIT_LIB_MQTT_H
+#define HOMEKIT_LIB_MQTT_H
+
+#include <vector>
+#include <map>
+#include <cstdint>
+#include <espMqttClient.h>
+#include <Ticker.h>
+#include "./module.h"
+
+namespace homekit::mqtt {
+    
+extern const uint8_t MQTT_CA_FINGERPRINT[];
+extern const char MQTT_SERVER[];
+extern const uint16_t MQTT_PORT;
+extern const char MQTT_USERNAME[];
+extern const char MQTT_PASSWORD[];
+extern const char MQTT_CLIENT_ID[];
+extern const char MQTT_SECRET[CONFIG_NODE_SECRET_SIZE+1];
+
+class MqttModule;
+
+class Mqtt {
+private:
+    String nodeId;
+    WiFiClientSecure httpsSecureClient;
+    espMqttClientSecure client;
+    Ticker reconnectTimer;
+    std::vector<MqttModule*> modules;
+    std::map<String, MqttModule*> moduleSubscriptions;
+    bool connected;
+
+    uint16_t subscribe(const String& topic, uint8_t qos = 0);
+
+public:
+    Mqtt();
+    void connect();
+    void disconnect();
+    void reconnect();
+    void loop();
+    void addModule(MqttModule* module);
+    void subscribeModule(String& topic, MqttModule* module, uint8_t qos = 0);
+    uint16_t publish(const String& topic, uint8_t* payload, size_t length);
+};
+
+}
+
+#endif //HOMEKIT_LIB_MQTT_H

include/pio/libs/mqtt/homekit/mqtt/payload.h

@@ -0,0 +1,15 @@
+#ifndef HOMEKIT_MQTT_PAYLOAD_H
+#define HOMEKIT_MQTT_PAYLOAD_H
+
+#include <unistd.h>
+
+namespace homekit::mqtt {
+
+struct MqttPayload {
+    virtual ~MqttPayload() = default;
+    virtual size_t size() const = 0;
+};
+
+}
+
+#endif

include/pio/libs/mqtt/library.json

@@ -0,0 +1,7 @@
+{
+  "name": "homekit_mqtt",
+  "version": "1.0.12",
+  "build": {
+    "flags": "-I../../include"
+  }
+}

include/pio/libs/mqtt_module_diagnostics/homekit/mqtt/module/diagnostics.cpp

@@ -0,0 +1,56 @@
+#include "./diagnostics.h"
+#include <homekit/wifi.h>
+#include <ESP8266WiFi.h>
+
+namespace homekit::mqtt {
+
+static const char TOPIC_DIAGNOSTICS[] = "diag";
+static const char TOPIC_INITIAL_DIAGNOSTICS[] = "d1ag";
+
+void MqttDiagnosticsModule::onConnect(Mqtt &mqtt) {
+    sendDiagnostics(mqtt);
+}
+
+void MqttDiagnosticsModule::onDisconnect(Mqtt &mqtt, espMqttClientTypes::DisconnectReason reason) {
+    initialSent = false;
+}
+
+void MqttDiagnosticsModule::tick(Mqtt& mqtt) {
+    if (!tickElapsed())
+        return;
+    sendDiagnostics(mqtt);
+}
+
+void MqttDiagnosticsModule::sendDiagnostics(Mqtt& mqtt) {
+    auto cfg = config::read();
+
+    if (!initialSent) {
+        MqttInitialDiagnosticsPayload stat{
+                .ip = wifi::getIPAsInteger(),
+                .fw_version = CONFIG_FW_VERSION,
+                .rssi = wifi::getRSSI(),
+                .free_heap = ESP.getFreeHeap(),
+                .flags = DiagnosticsFlags{
+                        .state = 1,
+                        .config_changed_value_present = 1,
+                        .config_changed = static_cast<uint8_t>(cfg.flags.node_configured ||
+                                                               cfg.flags.wifi_configured ? 1 : 0)
+                }
+        };
+        mqtt.publish(TOPIC_INITIAL_DIAGNOSTICS, reinterpret_cast<uint8_t*>(&stat), sizeof(stat));
+        initialSent = true;
+    } else {
+        MqttDiagnosticsPayload stat{
+                .rssi = wifi::getRSSI(),
+                .free_heap = ESP.getFreeHeap(),
+                .flags = DiagnosticsFlags{
+                        .state = 1,
+                        .config_changed_value_present = 0,
+                        .config_changed = 0
+                }
+        };
+        mqtt.publish(TOPIC_DIAGNOSTICS, reinterpret_cast<uint8_t*>(&stat), sizeof(stat));
+    }
+}
+
+}

include/pio/libs/mqtt_module_diagnostics/homekit/mqtt/module/diagnostics.h

@@ -0,0 +1,49 @@
+#ifndef HOMEKIT_LIB_MQTT_MODULE_DIAGNOSTICS_H
+#define HOMEKIT_LIB_MQTT_MODULE_DIAGNOSTICS_H
+
+#include <stdint.h>
+#include <homekit/mqtt/module.h>
+
+namespace homekit::mqtt {
+
+struct DiagnosticsFlags {
+    uint8_t state: 1;
+    uint8_t config_changed_value_present: 1;
+    uint8_t config_changed: 1;
+    uint8_t reserved: 5;
+} __attribute__((packed));
+
+struct MqttInitialDiagnosticsPayload {
+    uint32_t ip;
+    uint8_t fw_version;
+    int8_t rssi;
+    uint32_t free_heap;
+    DiagnosticsFlags flags;
+} __attribute__((packed));
+
+struct MqttDiagnosticsPayload {
+    int8_t rssi;
+    uint32_t free_heap;
+    DiagnosticsFlags flags;
+} __attribute__((packed));
+
+
+class MqttDiagnosticsModule: public MqttModule {
+private:
+    bool initialSent;
+
+    void sendDiagnostics(Mqtt& mqtt);
+
+public:
+    MqttDiagnosticsModule()
+        : MqttModule(30)
+        , initialSent(false) {}
+
+    void onConnect(Mqtt& mqtt) override;
+    void onDisconnect(Mqtt& mqtt, espMqttClientTypes::DisconnectReason reason) override;
+    void tick(Mqtt& mqtt) override;
+};
+
+}
+
+#endif //HOMEKIT_LIB_MQTT_MODULE_DIAGNOSTICS_H

include/pio/libs/mqtt_module_diagnostics/library.json

@@ -0,0 +1,10 @@
+{
+  "name": "homekit_mqtt_module_diagnostics",
+  "version": "1.0.3",
+  "build": {
+    "flags": "-I../../include"
+  },
+  "dependencies": {
+    "homekit_mqtt": "file://../../include/pio/libs/mqtt"
+  }
+}

include/pio/libs/mqtt_module_ota/homekit/mqtt/module/ota.cpp

@@ -0,0 +1,160 @@
+#include "./ota.h"
+#include <homekit/logging.h>
+#include <homekit/util.h>
+#include <homekit/led.h>
+
+namespace homekit::mqtt {
+
+using homekit::led::mcu_led;
+
+#define MD5_SIZE 16
+
+static const char TOPIC_OTA[] = "ota";
+static const char TOPIC_OTA_RESPONSE[] = "otares";
+
+void MqttOtaModule::onConnect(Mqtt& mqtt) {
+    String topic(TOPIC_OTA);
+    mqtt.subscribeModule(topic, this);
+}
+
+void MqttOtaModule::tick(Mqtt& mqtt) {
+    if (!tickElapsed())
+        return;
+}
+
+void MqttOtaModule::handlePayload(Mqtt& mqtt, String& topic, uint16_t packetId, const uint8_t *payload, size_t length, size_t index, size_t total) {
+    char md5[33];
+    char* md5Ptr = md5;
+
+    if (index != 0 && ota.dataPacketId != packetId) {
+        PRINTLN("mqtt/ota: non-matching packet id");
+        return;
+    }
+
+    Update.runAsync(true);
+
+    if (index == 0) {
+        if (length < CONFIG_NODE_SECRET_SIZE + MD5_SIZE) {
+            PRINTLN("mqtt/ota: failed to check secret, first packet size is too small");
+            return;
+        }
+
+        if (memcmp((const char*)payload, CONFIG_NODE_SECRET, CONFIG_NODE_SECRET_SIZE) != 0) {
+            PRINTLN("mqtt/ota: invalid secret");
+            return;
+        }
+
+        PRINTF("mqtt/ota: starting update, total=%ul\n", total-CONFIG_NODE_SECRET_SIZE);
+        for (int i = 0; i < MD5_SIZE; i++) {
+            md5Ptr += sprintf(md5Ptr, "%02x", *((unsigned char*)(payload+CONFIG_NODE_SECRET_SIZE+i)));
+        }
+        md5[32] = '\0';
+        PRINTF("mqtt/ota: md5 is %s\n", md5);
+        PRINTF("mqtt/ota: first packet is %ul bytes length\n", length);
+
+        md5[32] = '\0';
+
+        if (Update.isRunning()) {
+            Update.end();
+            Update.clearError();
+        }
+
+        if (!Update.setMD5(md5)) {
+            PRINTLN("mqtt/ota: setMD5 failed");
+            return;
+        }
+
+        ota.dataPacketId = packetId;
+
+        if (!Update.begin(total - CONFIG_NODE_SECRET_SIZE - MD5_SIZE)) {
+            ota.clean();
+#ifdef DEBUG
+            Update.printError(Serial);
+#endif
+            sendResponse(mqtt, OtaResult::UPDATE_ERROR, Update.getError());
+        }
+
+        ota.written = Update.write(const_cast<uint8_t*>(payload)+CONFIG_NODE_SECRET_SIZE + MD5_SIZE, length-CONFIG_NODE_SECRET_SIZE - MD5_SIZE);
+        ota.written += CONFIG_NODE_SECRET_SIZE + MD5_SIZE;
+
+        mcu_led->blink(1, 1);
+        PRINTF("mqtt/ota: updating %u/%u\n", ota.written, Update.size());
+
+    } else {
+        if (!Update.isRunning()) {
+            PRINTLN("mqtt/ota: update is not running");
+            return;
+        }
+
+        if (index == ota.written) {
+            size_t written;
+            if ((written = Update.write(const_cast<uint8_t*>(payload), length)) != length) {
+                PRINTF("mqtt/ota: error: tried to write %ul bytes, write() returned %ul\n",
+                       length, written);
+                ota.clean();
+                Update.end();
+                Update.clearError();
+                sendResponse(mqtt, OtaResult::WRITE_ERROR);
+                return;
+            }
+            ota.written += length;
+
+            mcu_led->blink(1, 1);
+            PRINTF("mqtt/ota: updating %u/%u\n",
+                   ota.written - CONFIG_NODE_SECRET_SIZE - MD5_SIZE,
+                   Update.size());
+        } else {
+            PRINTF("mqtt/ota: position is invalid, expected %ul, got %ul\n", ota.written, index);
+            ota.clean();
+            Update.end();
+            Update.clearError();
+        }
+    }
+
+    if (Update.isFinished()) {
+        ota.dataPacketId = 0;
+
+        if (Update.end()) {
+            ota.finished = true;
+            ota.publishResultPacketId = sendResponse(mqtt, OtaResult::OK);
+            PRINTF("mqtt/ota: ok, otares packet_id=%d\n", ota.publishResultPacketId);
+        } else {
+            ota.clean();
+
+            PRINTF("mqtt/ota: error: %u\n", Update.getError());
+#ifdef DEBUG
+            Update.printError(Serial);
+#endif
+            Update.clearError();
+
+            sendResponse(mqtt, OtaResult::UPDATE_ERROR, Update.getError());
+        }
+    }
+}
+
+uint16_t MqttOtaModule::sendResponse(Mqtt& mqtt, OtaResult status, uint8_t error_code) const {
+    MqttOtaResponsePayload resp{
+        .status = status,
+        .error_code = error_code
+    };
+    return mqtt.publish(TOPIC_OTA_RESPONSE, reinterpret_cast<uint8_t*>(&resp), sizeof(resp));
+}
+
+void MqttOtaModule::onDisconnect(Mqtt& mqtt, espMqttClientTypes::DisconnectReason reason) {
+    if (ota.readyToRestart) {
+        restartTimer.once(1, restart);
+    } else if (ota.started()) {
+        PRINTLN("mqtt: update was in progress, canceling..");
+        ota.clean();
+        Update.end();
+        Update.clearError();
+    }
+}
+
+void MqttOtaModule::handleOnPublish(uint16_t packetId) {
+    if (ota.finished && packetId == ota.publishResultPacketId) {
+        ota.readyToRestart = true;
+    }
+}
+
+}

include/pio/libs/mqtt_module_ota/homekit/mqtt/module/ota.h

@@ -0,0 +1,75 @@
+#ifndef HOMEKIT_LIB_MQTT_MODULE_OTA_H
+#define HOMEKIT_LIB_MQTT_MODULE_OTA_H
+
+#include <stdint.h>
+#include <Ticker.h>
+#include <homekit/mqtt/module.h>
+
+namespace homekit::mqtt {
+
+enum class OtaResult: uint8_t {
+    OK = 0,
+    UPDATE_ERROR = 1,
+    WRITE_ERROR = 2,
+};
+
+struct OtaStatus {
+    uint16_t dataPacketId;
+    uint16_t publishResultPacketId;
+    bool finished;
+    bool readyToRestart;
+    size_t written;
+
+    OtaStatus()
+            : dataPacketId(0)
+            , publishResultPacketId(0)
+            , finished(false)
+            , readyToRestart(false)
+            , written(0)
+    {}
+
+    inline void clean() {
+        dataPacketId = 0;
+        publishResultPacketId = 0;
+        finished = false;
+        readyToRestart = false;
+        written = 0;
+    }
+
+    inline bool started() const {
+        return dataPacketId != 0;
+    }
+};
+
+struct MqttOtaResponsePayload {
+    OtaResult status;
+    uint8_t error_code;
+} __attribute__((packed));
+
+
+class MqttOtaModule: public MqttModule {
+private:
+    OtaStatus ota;
+    Ticker restartTimer;
+
+    uint16_t sendResponse(Mqtt& mqtt, OtaResult status, uint8_t error_code = 0) const;
+
+public:
+    MqttOtaModule() : MqttModule(0, true, true) {}
+
+    void onConnect(Mqtt& mqtt) override;
+	void onDisconnect(Mqtt& mqtt, espMqttClientTypes::DisconnectReason reason) override;
+
+    void tick(Mqtt& mqtt) override;
+	
+    void handlePayload(Mqtt& mqtt, String& topic, uint16_t packetId, const uint8_t *payload, size_t length, size_t index, size_t total) override;
+    void handleOnPublish(uint16_t packetId) override;
+
+    inline bool isReadyToRestart() const {
+        return ota.readyToRestart;
+    }
+};
+
+}
+
+#endif //HOMEKIT_LIB_MQTT_MODULE_OTA_H

include/pio/libs/mqtt_module_ota/library.json

@@ -0,0 +1,11 @@
+{
+  "name": "homekit_mqtt_module_ota",
+  "version": "1.0.6",
+  "build": {
+    "flags": "-I../../include"
+  },
+  "dependencies": {
+    "homekit_led": "file://../../include/pio/libs/led",
+    "homekit_mqtt": "file://../../include/pio/libs/mqtt"
+  }
+}

include/pio/libs/mqtt_module_relay/homekit/mqtt/module/relay.cpp

@@ -0,0 +1,58 @@
+#include "./relay.h"
+#include <homekit/relay.h>
+#include <homekit/logging.h>
+
+namespace homekit::mqtt {
+
+static const char TOPIC_RELAY_SWITCH[] = "relay/switch";
+static const char TOPIC_RELAY_STATUS[] = "relay/status";
+
+void MqttRelayModule::onConnect(Mqtt &mqtt) {
+    String topic(TOPIC_RELAY_SWITCH);
+    mqtt.subscribeModule(topic, this, 1);
+}
+
+void MqttRelayModule::onDisconnect(Mqtt &mqtt, espMqttClientTypes::DisconnectReason reason) {
+#ifdef CONFIG_RELAY_OFF_ON_DISCONNECT
+    if (relay::state()) {
+        relay::off();
+    }
+#endif
+}
+
+void MqttRelayModule::tick(homekit::mqtt::Mqtt& mqtt) {}
+
+void MqttRelayModule::handlePayload(Mqtt& mqtt, String& topic, uint16_t packetId, const uint8_t *payload, size_t length, size_t index, size_t total) {
+    if (topic != TOPIC_RELAY_SWITCH)
+        return;
+
+    if (length != sizeof(MqttRelaySwitchPayload)) {
+        PRINTF("error: size of payload (%ul) does not match expected (%ul)\n",
+               length, sizeof(MqttRelaySwitchPayload));
+        return;
+    }
+
+    auto pd = reinterpret_cast<const struct MqttRelaySwitchPayload*>(payload);
+    if (strncmp(pd->secret, MQTT_SECRET, sizeof(pd->secret)) != 0) {
+        PRINTLN("error: invalid secret");
+        return;
+    }
+
+    MqttRelayStatusPayload resp{};
+
+    if (pd->state == 1) {
+        PRINTLN("mqtt: turning relay on");
+        relay::on();
+    } else if (pd->state == 0) {
+        PRINTLN("mqtt: turning relay off");
+        relay::off();
+    } else {
+        PRINTLN("error: unexpected state value");
+    }
+
+    resp.opened = relay::state();
+    mqtt.publish(TOPIC_RELAY_STATUS, reinterpret_cast<uint8_t*>(&resp), sizeof(resp));
+}
+
+}
+

include/pio/libs/mqtt_module_relay/homekit/mqtt/module/relay.h

@@ -0,0 +1,29 @@
+#ifndef HOMEKIT_LIB_MQTT_MODULE_RELAY_H
+#define HOMEKIT_LIB_MQTT_MODULE_RELAY_H
+
+#include <homekit/mqtt/module.h>
+
+namespace homekit::mqtt {
+
+struct MqttRelaySwitchPayload {
+    char secret[12];
+    uint8_t state;
+} __attribute__((packed));
+
+struct MqttRelayStatusPayload {
+    uint8_t opened;
+} __attribute__((packed));
+
+class MqttRelayModule : public MqttModule {
+public:
+    MqttRelayModule() : MqttModule(0) {}
+    void onConnect(Mqtt& mqtt) override;
+    void onDisconnect(Mqtt& mqtt, espMqttClientTypes::DisconnectReason reason) override;
+    void tick(Mqtt& mqtt) override;
+    void handlePayload(Mqtt& mqtt, String& topic, uint16_t packetId, const uint8_t *payload, size_t length, size_t index, size_t total) override;
+};
+
+}
+
+#endif //HOMEKIT_LIB_MQTT_MODULE_RELAY_H
+

include/pio/libs/mqtt_module_relay/library.json

@@ -0,0 +1,11 @@
+{
+  "name": "homekit_mqtt_module_relay",
+  "version": "1.0.6",
+  "build": {
+    "flags": "-I../../include"
+  },
+  "dependencies": {
+    "homekit_mqtt": "file://../../include/pio/libs/mqtt",
+    "homekit_relay": "file://../../include/pio/libs/relay"
+  }
+}

include/pio/libs/mqtt_module_temphum/homekit/mqtt/module/temphum.cpp

@@ -0,0 +1,23 @@
+#include "temphum.h"
+
+namespace homekit::mqtt {
+
+static const char TOPIC_TEMPHUM_DATA[] = "temphum/data";
+
+void MqttTemphumModule::onConnect(Mqtt &mqtt) {}
+
+void MqttTemphumModule::tick(homekit::mqtt::Mqtt& mqtt) {
+    if (!tickElapsed())
+        return;
+
+    temphum::SensorData sd = sensor->read();
+    MqttTemphumPayload payload {
+        .temp = sd.temp,
+        .rh = sd.rh,
+        .error = sd.error
+    };
+
+    mqtt.publish(TOPIC_TEMPHUM_DATA, reinterpret_cast<uint8_t*>(&payload), sizeof(payload));
+}
+
+}

include/pio/libs/mqtt_module_temphum/homekit/mqtt/module/temphum.h

@@ -0,0 +1,28 @@
+#ifndef HOMEKIT_LIB_MQTT_MODULE_TEMPHUM_H
+#define HOMEKIT_LIB_MQTT_MODULE_TEMPHUM_H
+
+#include <homekit/mqtt/module.h>
+#include <homekit/temphum.h>
+
+namespace homekit::mqtt {
+
+struct MqttTemphumPayload {
+    double temp = 0;
+    double rh = 0;
+    uint8_t error = 0;
+} __attribute__((packed));
+
+
+class MqttTemphumModule : public MqttModule {
+private:
+    temphum::Sensor* sensor;
+
+public:
+    MqttTemphumModule(temphum::Sensor* _sensor) : MqttModule(10), sensor(_sensor) {}
+    void onConnect(Mqtt& mqtt) override;
+    void tick(Mqtt& mqtt) override;
+};
+
+}
+
+#endif //HOMEKIT_LIB_MQTT_MODULE_TEMPHUM_H

include/pio/libs/mqtt_module_temphum/library.json

@@ -0,0 +1,11 @@
+{
+  "name": "homekit_mqtt_module_temphum",
+  "version": "1.0.10",
+  "build": {
+    "flags": "-I../../include"
+  },
+  "dependencies": {
+    "homekit_mqtt": "file://../../include/pio/libs/mqtt",
+    "homekit_temphum": "file://../../include/pio/libs/temphum"
+  }
+}

include/pio/libs/relay/homekit/relay.cpp

@@ -0,0 +1,22 @@
+#include <Arduino.h>
+#include "./relay.h"
+
+namespace homekit::relay {
+	
+void init() {
+    pinMode(CONFIG_RELAY_GPIO, OUTPUT);
+}
+
+bool state() {
+    return digitalRead(CONFIG_RELAY_GPIO) == HIGH;
+}
+
+void on() {
+    digitalWrite(CONFIG_RELAY_GPIO, HIGH);
+}
+
+void off() {
+    digitalWrite(CONFIG_RELAY_GPIO, LOW);
+}
+
+}

include/pio/libs/relay/homekit/relay.h

@@ -0,0 +1,13 @@
+#ifndef HOMEKIT_LIB_RELAY_H
+#define HOMEKIT_LIB_RELAY_H
+
+namespace homekit::relay {
+
+void init();
+bool state();
+void on();
+void off();
+
+}
+
+#endif //HOMEKIT_LIB_RELAY_H

include/pio/libs/relay/library.json

@@ -0,0 +1,8 @@
+{
+  "name": "homekit_relay",
+  "version": "1.0.0",
+  "build": {
+    "flags": "-I../../include"
+  }
+}
+

include/pio/libs/static/homekit/static.h

@@ -2,7 +2,8 @@
  * This file is autogenerated with make_static.sh script
  */
 
-#pragma once
+#ifndef COMMON_HOMEKIT_STATIC_H
+#define COMMON_HOMEKIT_STATIC_H
 
 #include <stdlib.h>
 
@@ -20,3 +21,5 @@ extern const StaticFile style_css;
 extern const StaticFile favicon_ico;
 
 }
+
+#endif //COMMON_HOMEKIT_STATIC_H

include/pio/libs/static/library.json

@@ -0,0 +1,8 @@
+{
+  "name": "homekit_static",
+  "version": "1.0.1",
+  "build": {
+    "flags": "-I../../include"
+  }
+}
+

include/pio/libs/temphum/homekit/temphum.cpp

@@ -0,0 +1,89 @@
+#ifndef CONFIG_TARGET_ESP01
+#include <Arduino.h>
+#endif
+#include <homekit/logging.h>
+#include "temphum.h"
+
+namespace homekit::temphum {
+
+void Sensor::setup() const {
+#ifndef CONFIG_TARGET_ESP01
+    pinMode(CONFIG_SDA_GPIO, OUTPUT);
+    pinMode(CONFIG_SCL_GPIO, OUTPUT);
+
+    Wire.begin(CONFIG_SDA_GPIO, CONFIG_SCL_GPIO);
+#else
+    Wire.begin();
+#endif
+}
+
+void Sensor::writeCommand(int reg) const {
+    Wire.beginTransmission(dev_addr);
+    Wire.write(reg);
+    Wire.endTransmission();
+    delay(500); // wait for the measurement to be ready
+}
+
+SensorData Si7021::read() {
+    uint8_t error = 0;
+    writeCommand(0xf3); // command to measure temperature
+    Wire.requestFrom(dev_addr, 2);
+    if (Wire.available() < 2) {
+        PRINTLN("Si7021: 0xf3: could not read 2 bytes");
+        error = 1;
+    }
+    uint16_t temp_raw = Wire.read() << 8 | Wire.read();
+    double temperature = ((175.72 * temp_raw) / 65536.0) - 46.85;
+
+    writeCommand(0xf5); // command to measure humidity
+    Wire.requestFrom(dev_addr, 2);
+    if (Wire.available() < 2) {
+        PRINTLN("Si7021: 0xf5: could not read 2 bytes");
+        error = 1;
+    }
+    uint16_t hum_raw = Wire.read() << 8 | Wire.read();
+    double humidity = ((125.0 * hum_raw) / 65536.0) - 6.0;
+
+    return {
+        .error = error,
+        .temp = temperature,
+        .rh = humidity
+    };
+}
+
+SensorData DHT12::read() {
+    SensorData sd;
+    byte raw[5];
+    sd.error = 1;
+
+    writeCommand(0);
+    Wire.requestFrom(dev_addr, 5);
+
+    if (Wire.available() < 5) {
+        PRINTLN("DHT12: could not read 5 bytes");
+        goto end;
+    }
+
+    // Parse the received data
+    for (uint8_t i = 0; i < 5; i++)
+        raw[i] = Wire.read();
+
+    if (((raw[0] + raw[1] + raw[2] + raw[3]) & 0xff) != raw[4]) {
+        PRINTLN("DHT12: checksum error");
+        goto end;
+    }
+
+    // Calculate temperature and humidity values
+    sd.temp = raw[2] + (raw[3] & 0x7f) * 0.1;
+    if (raw[3] & 0x80)
+        sd.temp *= -1;
+
+    sd.rh = raw[0] + raw[1] * 0.1;
+
+    sd.error = 0;
+
+end:
+    return sd;
+}
+
+}

include/pio/libs/temphum/homekit/temphum.h

@@ -0,0 +1,38 @@
+#pragma once
+
+#include <Wire.h>
+
+namespace homekit::temphum {
+
+struct SensorData {
+    uint8_t error = 0;
+    double temp = 0; // celsius
+    double rh = 0; // relative humidity percentage
+};
+
+
+class Sensor {
+protected:
+    int dev_addr;
+public:
+    explicit Sensor(int dev) : dev_addr(dev) {}
+    void setup() const;
+    void writeCommand(int reg) const;
+    virtual SensorData read() = 0;
+};
+
+
+class Si7021 : public Sensor {
+public:
+    SensorData read() override;
+    Si7021() : Sensor(0x40) {}
+};
+
+
+class DHT12 : public Sensor {
+public:
+    SensorData read() override;
+    DHT12() : Sensor(0x5c) {}
+};
+
+}

include/pio/libs/temphum/library.json

@@ -0,0 +1,8 @@
+{
+  "name": "homekit_temphum",
+  "version": "1.0.4",
+  "build": {
+    "flags": "-I../../include"
+  }
+}
+

include/pio/libs/wifi/homekit/wifi.cpp

@@ -1,18 +1,17 @@
 #include <pgmspace.h>
-#include "config.def.h"
 #include "wifi.h"
-#include "config.h"
-#include "logging.h"
+#include <homekit/config.h>
+#include <homekit/logging.h>
 
 namespace homekit::wifi {
 
 using namespace homekit;
 using homekit::config::ConfigData;
 
-const char NODE_ID[] = DEFAULT_NODE_ID;
-const char AP_SSID[] = DEFAULT_WIFI_AP_SSID;
-const char STA_SSID[] = DEFAULT_WIFI_STA_SSID;
-const char STA_PSK[] = DEFAULT_WIFI_STA_PSK;
+const char NODE_ID[] = CONFIG_NODE_ID;
+const char AP_SSID[] = CONFIG_WIFI_AP_SSID;
+const char STA_SSID[] = CONFIG_WIFI_STA_SSID;
+const char STA_PSK[] = CONFIG_WIFI_STA_PSK;
 
 void getConfig(ConfigData &cfg, const char** ssid, const char** psk, const char** hostname) {
     if (cfg.flags.wifi_configured) {

include/pio/libs/wifi/homekit/wifi.h

@@ -1,9 +1,11 @@
-#pragma once
+#ifndef HOMEKIT_TEPMHUM_WIFI_H
+#define HOMEKIT_TEPMHUM_WIFI_H
 
 #include <ESP8266WiFi.h>
 #include <list>
 #include <memory>
-#include "config.h"
+
+#include <homekit/config.h>
 
 namespace homekit::wifi {
 
@@ -34,3 +36,5 @@ extern const char STA_PSK[];
 extern const char NODE_ID[];
 
 }
+
+#endif //HOMEKIT_TEPMHUM_WIFI_H

include/pio/libs/wifi/library.json

@@ -0,0 +1,8 @@
+{
+  "name": "homekit_wifi",
+  "version": "1.0.1",
+  "build": {
+    "flags": "-I../../include"
+  }
+}
+