new config: port openwrt_logger and webapiclient

doc/openwrt_logger.md

@@ -0,0 +1,28 @@
+# openwrt_logger.py
+
+This script is supposed to be run by cron every 5 minutes or so.
+It looks for new lines in log file and sends them to remote server.
+
+OpenWRT must have remote logging enabled (UDP; IP of host this script is launched on; port 514)
+
+`/etc/rsyslog.conf` contains following (assuming `192.168.1.1` is the router IP):
+
+```
+$ModLoad imudp  
+$UDPServerRun 514  
+:fromhost-ip, isequal, "192.168.1.1" /var/log/openwrt.log  
+& ~
+```
+
+Also comment out the following line:
+```
+$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
+```
+
+Cron line example:
+```
+* * * * *  /home/user/homekit/src/openwrt_logger.py --access-point 1 --file /var/wrtlogfs/openwrt-5.log  >/dev/null
+```
+
+`/var/wrtlogfs` is recommended to be tmpfs, to avoid writes on mmc card, in case
+you use arm sbcs as I do.

src/home/api/__init__.py

@@ -1,11 +1,19 @@
 import importlib
 
-__all__ = ['WebAPIClient', 'RequestParams']
+__all__ = [
+    # web_api_client.py
+    'WebApiClient',
+    'RequestParams',
+
+    # config.py
+    'WebApiConfig'
+]
 
 
 def __getattr__(name):
     if name in __all__:
-        module = importlib.import_module(f'.web_api_client', __name__)
+        file = 'config' if name == 'WebApiConfig' else 'web_api_client'
+        module = importlib.import_module(f'.{file}', __name__)
         return getattr(module, name)
 
     raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

src/home/api/__init__.pyi

@@ -1,4 +1,5 @@
 from .web_api_client import (
     RequestParams as RequestParams,
-    WebAPIClient as WebAPIClient
+    WebApiClient as WebApiClient
 )
+from .config import WebApiConfig as WebApiConfig

src/home/api/config.py

@@ -0,0 +1,15 @@
+from ..config import ConfigUnit
+from typing import Optional, Union
+
+
+class WebApiConfig(ConfigUnit):
+    NAME = 'web_api'
+
+    @classmethod
+    def schema(cls) -> Optional[dict]:
+        return {
+            'listen_addr': cls._addr_schema(required=True),
+            'host': cls._addr_schema(required=True),
+            'token': dict(type='string', required=True),
+            'recordings_dir': dict(type='string', required=True)
+        }

src/home/api/web_api_client.py

@@ -9,13 +9,15 @@ from enum import Enum, auto
 from typing import Optional, Callable, Union, List, Tuple, Dict
 from requests.auth import HTTPBasicAuth
 
+from .config import WebApiConfig
 from .errors import ApiResponseError
 from .types import *
 from ..config import config
 from ..util import stringify
 from ..media import RecordFile, MediaNodeClient
 
-logger = logging.getLogger(__name__)
+_logger = logging.getLogger(__name__)
+_config = WebApiConfig()
 
 
 RequestParams = namedtuple('RequestParams', 'params, files, method')
@@ -26,7 +28,7 @@ class HTTPMethod(Enum):
     POST = auto()
 
 
-class WebAPIClient:
+class WebApiClient:
     token: str
     timeout: Union[float, Tuple[float, float]]
     basic_auth: Optional[HTTPBasicAuth]
@@ -35,22 +37,22 @@ class WebAPIClient:
     async_success_handler: Optional[Callable]
 
     def __init__(self, timeout: Union[float, Tuple[float, float]] = 5):
-        self.token = config['api']['token']
+        self.token = config['token']
         self.timeout = timeout
         self.basic_auth = None
         self.do_async = False
         self.async_error_handler = None
         self.async_success_handler = None
 
-        if 'basic_auth' in config['api']:
-            ba = config['api']['basic_auth']
-            col = ba.index(':')
-
-            user = ba[:col]
-            pw = ba[col+1:]
-
-            logger.debug(f'enabling basic auth: {user}:{pw}')
-            self.basic_auth = HTTPBasicAuth(user, pw)
+        # if 'basic_auth' in config['api']:
+        #     ba = config['api']['basic_auth']
+        #     col = ba.index(':')
+        #
+        #     user = ba[:col]
+        #     pw = ba[col+1:]
+        #
+        #     _logger.debug(f'enabling basic auth: {user}:{pw}')
+        #     self.basic_auth = HTTPBasicAuth(user, pw)
 
     # api methods
     # -----------
@@ -152,7 +154,7 @@ class WebAPIClient:
                       params: dict,
                       method: HTTPMethod = HTTPMethod.GET,
                       files: Optional[Dict[str, str]] = None) -> Optional[any]:
-        domain = config['api']['host']
+        domain = config['host']
         kwargs = {}
 
         if self.basic_auth is not None:
@@ -196,7 +198,7 @@ class WebAPIClient:
                 try:
                     f.close()
                 except Exception as exc:
-                    logger.exception(exc)
+                    _logger.exception(exc)
                     pass
 
     def _make_request_in_thread(self, name, params, method, files):
@@ -204,7 +206,7 @@ class WebAPIClient:
             result = self._make_request(name, params, method, files)
             self._report_async_success(result, name, RequestParams(params=params, method=method, files=files))
         except Exception as e:
-            logger.exception(e)
+            _logger.exception(e)
             self._report_async_error(e, name, RequestParams(params=params, method=method, files=files))
 
     def enable_async(self,

src/home/database/_base.py

@@ -0,0 +1,9 @@
+import os
+
+
+def get_data_root_directory(name: str) -> str:
+    return os.path.join(
+        os.environ['HOME'],
+        '.config',
+        'homekit',
+        'data')

src/home/database/simple_state.py

@@ -2,24 +2,26 @@ import os
 import json
 import atexit
 
+from ._base import get_data_root_directory
+
 
 class SimpleState:
     def __init__(self,
-                 file: str,
-                 default: dict = None,
-                 **kwargs):
+                 name: str,
+                 default: dict = None):
         if default is None:
             default = {}
         elif type(default) is not dict:
             raise TypeError('default must be dictionary')
 
-        if not os.path.exists(file):
+        path = os.path.join(get_data_root_directory(), name)
+        if not os.path.exists(path):
             self._data = default
         else:
-            with open(file, 'r') as f:
+            with open(path, 'r') as f:
                 self._data = json.loads(f.read())
 
-        self._file = file
+        self._file = path
         atexit.register(self.__cleanup)
 
     def __cleanup(self):

src/home/database/sqlite.py

@@ -2,15 +2,13 @@ import sqlite3
 import os.path
 import logging
 
+from ._base import get_data_root_directory
 from ..config import config, is_development_mode
 
 
 def _get_database_path(name: str) -> str:
     return os.path.join(
-        os.environ['HOME'],
-        '.config',
-        'homekit',
-        'data',
+        get_data_root_directory(),
         f'{name}.db')
 
 

src/home/telegram/_botutil.py

@@ -3,7 +3,7 @@ import traceback
 
 from html import escape
 from telegram import User
-from home.api import WebAPIClient as APIClient
+from home.api import WebApiClient as APIClient
 from home.api.types import BotType
 from home.api.errors import ApiResponseError
 

src/home/telegram/bot.py

@@ -21,7 +21,7 @@ from telegram.ext.filters import BaseFilter
 from telegram.error import TimedOut
 
 from home.config import config
-from home.api import WebAPIClient
+from home.api import WebApiClient
 from home.api.types import BotType
 
 from ._botlang import lang, languages
@@ -522,7 +522,7 @@ def _logging_callback_handler(update: Update, context: CallbackContext):
 
 
 def enable_logging(bot_type: BotType):
-    api = WebAPIClient(timeout=3)
+    api = WebApiClient(timeout=3)
     api.enable_async()
 
     global _reporting

src/inverter_bot.py

@@ -28,7 +28,7 @@ from home.inverter.types import (
 )
 from home.database.inverter_time_formats import FormatDate
 from home.api.types import BotType
-from home.api import WebAPIClient
+from home.api import WebApiClient
 from telegram import ReplyKeyboardMarkup, InlineKeyboardMarkup, InlineKeyboardButton
 
 
@@ -718,7 +718,7 @@ class ConsumptionConversation(bot.conversation):
         message = ctx.reply(ctx.lang('consumption_request_sent'),
                   markup=bot.IgnoreMarkup())
 
-        api = WebAPIClient(timeout=60)
+        api = WebApiClient(timeout=60)
         method = 'inverter_get_consumed_energy' if state == self.TOTAL else 'inverter_get_grid_consumed_energy'
 
         try:

src/openwrt_log_analyzer.py

@@ -59,7 +59,7 @@ if __name__ == '__main__':
         state_file = config['simple_state']['file']
         state_file = state_file.replace('.txt', f'-{ap}.txt')
 
-        state = SimpleState(file=state_file,
+        state = SimpleState(name=state_file,
                             default={'last_id': 0})
 
         max_last_id = 0

src/openwrt_logger.py

@@ -2,29 +2,19 @@
 import os
 
 from datetime import datetime
-from typing import Tuple, List
+from typing import Tuple, List, Optional
 from argparse import ArgumentParser
-from home.config import config
+from home.config import config, AppConfigUnit
 from home.database import SimpleState
-from home.api import WebAPIClient
+from home.api import WebApiClient
 
-f"""
-This script is supposed to be run by cron every 5 minutes or so.
-It looks for new lines in log file and sends them to remote server.
 
-OpenWRT must have remote logging enabled (UDP; IP of host this script is launched on; port 514)
-
-/etc/rsyslog.conf contains following (assuming 192.168.1.1 is the router IP):
-
-$ModLoad imudp  
-$UDPServerRun 514  
-:fromhost-ip, isequal, "192.168.1.1" /var/log/openwrt.log  
-& ~
-
-Also comment out the following line: 
-$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
- 
-"""
+class OpenwrtLoggerConfig(AppConfigUnit):
+    @classmethod
+    def schema(cls) -> Optional[dict]:
+        return dict(
+            database_name_template=dict(type='string', required=True)
+        )
 
 
 def parse_line(line: str) -> Tuple[int, str]:
@@ -46,11 +36,10 @@ if __name__ == '__main__':
     parser.add_argument('--access-point', type=int, required=True,
                         help='access point number')
 
-    arg = config.load_app('openwrt_logger', parser=parser)
-
-    state = SimpleState(file=config['simple_state']['file'].replace('{ap}', str(arg.access_point)),
-                        default={'seek': 0, 'size': 0})
+    arg = config.load_app(OpenwrtLoggerConfig, parser=parser)
 
+    state = SimpleState(name=config.app_config['database_name_template'].replace('{ap}', str(arg.access_point)),
+                        default=dict(seek=0, size=0))
     fsize = os.path.getsize(arg.file)
     if fsize < state['size']:
         state['seek'] = 0
@@ -79,5 +68,5 @@ if __name__ == '__main__':
                 except ValueError:
                     lines.append((0, line))
 
-            api = WebAPIClient()
+            api = WebApiClient()
             api.log_openwrt(lines, arg.access_point)

src/sensors_bot.py

@@ -17,7 +17,7 @@ from telegram import ReplyKeyboardMarkup, InlineKeyboardMarkup, InlineKeyboardBu
 from home.config import config
 from home.telegram import bot
 from home.util import chunks, MySimpleSocketClient
-from home.api import WebAPIClient
+from home.api import WebApiClient
 from home.api.types import (
     BotType,
     TemperatureSensorLocation
@@ -111,7 +111,7 @@ def callback_handler(ctx: bot.Context) -> None:
     sensor = TemperatureSensorLocation[match.group(1).upper()]
     hours = int(match.group(2))
 
-    api = WebAPIClient(timeout=20)
+    api = WebApiClient(timeout=20)
     data = api.get_sensors_data(sensor, hours)
 
     title = ctx.lang(sensor.name.lower()) + ' (' + ctx.lang('n_hrs', hours) + ')'

src/sound_bot.py

@@ -9,7 +9,7 @@ from html import escape
 from typing import Optional, List, Dict, Tuple
 
 from home.config import config
-from home.api import WebAPIClient
+from home.api import WebApiClient
 from home.api.types import SoundSensorLocation, BotType
 from home.api.errors import ApiResponseError
 from home.media import SoundNodeClient, SoundRecordClient, SoundRecordFile, CameraNodeClient
@@ -734,7 +734,7 @@ def sound_sensors_last_24h(ctx: bot.Context):
 
     ctx.answer()
 
-    cl = WebAPIClient()
+    cl = WebApiClient()
     data = cl.get_sound_sensor_hits(location=SoundSensorLocation[node.upper()],
                                     after=datetime.now() - timedelta(hours=24))
 
@@ -757,7 +757,7 @@ def sound_sensors_last_anything(ctx: bot.Context):
 
     ctx.answer()
 
-    cl = WebAPIClient()
+    cl = WebApiClient()
     data = cl.get_last_sound_sensor_hits(location=SoundSensorLocation[node.upper()],
                                          last=20)
 

src/sound_sensor_server.py

@@ -7,7 +7,7 @@ from typing import Optional, List, Dict, Tuple
 from functools import partial
 from home.config import config
 from home.util import Addr
-from home.api import WebAPIClient, RequestParams
+from home.api import WebApiClient, RequestParams
 from home.api.types import SoundSensorLocation
 from home.soundsensor import SoundSensorServer, SoundSensorHitHandler
 from home.media import MediaNodeType, SoundRecordClient, CameraRecordClient, RecordClient
@@ -120,7 +120,7 @@ def hits_sender():
         sleep(5)
 
 
-api: Optional[WebAPIClient] = None
+api: Optional[WebApiClient] = None
 hc: Optional[HitCounter] = None
 record_clients: Dict[MediaNodeType, RecordClient] = {}
 
@@ -162,7 +162,7 @@ if __name__ == '__main__':
     config.load_app('sound_sensor_server')
 
     hc = HitCounter()
-    api = WebAPIClient(timeout=(10, 60))
+    api = WebApiClient(timeout=(10, 60))
     api.enable_async(error_handler=api_error_handler)
 
     t = threading.Thread(target=hits_sender)

test/test_api.py

@@ -7,7 +7,7 @@ sys.path.extend([
     )
 ])
 
-from src.home.api import WebAPIClient
+from src.home.api import WebApiClient
 from src.home.api.types import BotType
 from src.home.config import config
 
@@ -15,5 +15,5 @@ from src.home.config import config
 if __name__ == '__main__':
     config.load_app('test_api')
 
-    api = WebAPIClient()
+    api = WebApiClient()
     print(api.log_bot_request(BotType.ADMIN, 1, "test_api.py"))

test/test_record_upload.py

@@ -10,7 +10,7 @@ sys.path.extend([
 
 import time
 
-from src.home.api import WebAPIClient, RequestParams
+from src.home.api import WebApiClient, RequestParams
 from src.home.config import config
 from src.home.media import SoundRecordClient
 from src.home.util import Addr
@@ -74,7 +74,7 @@ if __name__ == '__main__':
                                finished_handler=record_finished,
                                download_on_finish=True)
 
-    api = WebAPIClient()
+    api = WebApiClient()
     api.enable_async(error_handler=api_error_handler,
                      success_handler=api_success_handler)
 

test/test_sound_server_api.py

@@ -10,7 +10,7 @@ import threading
 
 from time import sleep
 from src.home.config import config
-from src.home.api import WebAPIClient
+from src.home.api import WebApiClient
 from src.home.api.types import SoundSensorLocation
 from typing import List, Tuple
 
@@ -59,7 +59,7 @@ if __name__ == '__main__':
     config.load_app('test_api')
 
     hc = HitCounter()
-    api = WebAPIClient()
+    api = WebApiClient()
 
     hc.add('spb1', 1)
     # hc.add('big_house', 123)