Merge branch 'master' of ch1p.io:homekit

doc/ipcam_server.md

@@ -1,3 +1,4 @@
+
 # ipcam_server.py
 
 ## Configuration
@@ -19,6 +20,14 @@ camera:
     recordings_path: "/data3/cam-3"
     motion_path: "/data3/cam-3/motion"
 
+storages:
+  - mountpoint: "/data1"
+    cams: [1]
+  - mountpoint: "/data2"
+    cams: [2]
+  - mountpoint: "/data3"
+    cams: [3]
+
 motion:
   padding: 2
   telegram: true
@@ -30,6 +39,19 @@ telegram:
   token: ""
   chat_id: ""
   parse_mode: HTML
+
+  fragment_url_templates:
+    - ["example", "https://example.ru/cam-{camera}/motion/{file}"]
+
+  original_file_url_templates:
+    - ["example", "https://example.ru/cam-{camera}/{file}"]
+
+fix_interval: 600
+fix_enabled: true
+
+cleanup_min_gb: 200
+cleanup_interval: 86400
+
 ```
 
 ## Usage

localwebsite/classes/MyOpenWrtUtils.php

@@ -119,7 +119,7 @@ class MyOpenWrtUtils {
         $ip = array_shift($words);
         array_pop($words);
         $hostname = trim(implode(' ', $words));
-        if (!$hostname)
+        if (!$hostname || $hostname == '*')
             $hostname = '?';
         return [
             'time' => $time,

localwebsite/config.php

@@ -81,5 +81,7 @@ return [
     'auth_pw_salt' => '',
 
     'grafana_sensors_url' => '',
-    'grafana_inverter_url' => ''
+    'grafana_inverter_url' => '',
+
+    'dhcp_hostname_overrides' => [],
 ];

localwebsite/handlers/ModemHandler.php

@@ -146,7 +146,12 @@ class ModemHandler extends RequestHandler
     }
 
     public function GET_routing_dhcp_page() {
+        $overrides = config::get('dhcp_hostname_overrides');
         $leases = MyOpenWrtUtils::getDHCPLeases();
+        foreach ($leases as &$lease) {
+            if ($lease['hostname'] == '?' && array_key_exists($lease['mac'], $overrides))
+                $lease['hostname'] = $overrides[$lease['mac']];
+        }
         $this->tpl->set([
             'leases' => $leases
         ]);

src/ipcam_server.py

@@ -1,8 +1,10 @@
 #!/usr/bin/env python3
 import logging
 import os
+import re
 import asyncio
 import time
+import shutil
 import home.telegram.aio as telegram
 
 from apscheduler.schedulers.asyncio import AsyncIOScheduler
@@ -16,6 +18,7 @@ from home.camera import util as camutil
 from enum import Enum
 from typing import Optional, Union, List, Tuple
 from datetime import datetime, timedelta
+from functools import cmp_to_key
 
 
 class TimeFilterType(Enum):
@@ -405,8 +408,75 @@ async def fix_job() -> None:
         fix_job_running = False
 
 
+async def cleanup_job() -> None:
+    def fn2dt(name: str) -> datetime:
+        name = os.path.basename(name)
+
+        if name.startswith('record_'):
+            return datetime.strptime(re.match(r'record_(.*?)\.mp4', name).group(1), datetime_format)
+
+        m = re.match(rf'({datetime_format_re})__{datetime_format_re}\.mp4', name)
+        if m:
+            return datetime.strptime(m.group(1), datetime_format)
+
+        raise ValueError(f'unrecognized filename format: {name}')
+
+    def compare(i1: str, i2: str) -> int:
+        dt1 = fn2dt(i1)
+        dt2 = fn2dt(i2)
+
+        if dt1 < dt2:
+            return -1
+        elif dt1 > dt2:
+            return 1
+        else:
+            return 0
+
+    global cleanup_job_running
+    logger.debug('cleanup_job: starting')
+
+    if cleanup_job_running:
+        logger.error('cleanup_job: already running')
+        return
+
+    try:
+        cleanup_job_running = True
+
+        gb = float(1 << 30)
+        for storage in config['storages']:
+            if os.path.exists(storage['mountpoint']):
+                total, used, free = shutil.disk_usage(storage['mountpoint'])
+                free_gb = free // gb
+                if free_gb < config['cleanup_min_gb']:
+                    # print(f"{storage['mountpoint']}: free={free}, free_gb={free_gb}")
+                    cleaned = 0
+                    files = []
+                    for cam in storage['cams']:
+                        for _dir in (config['camera'][cam]['recordings_path'], config['camera'][cam]['motion_path']):
+                            files += list(map(lambda file: os.path.join(_dir, file), os.listdir(_dir)))
+                        files = list(filter(lambda path: os.path.isfile(path) and path.endswith('.mp4'), files))
+                        files.sort(key=cmp_to_key(compare))
+
+                    for file in files:
+                        size = os.stat(file).st_size
+                        try:
+                            os.unlink(file)
+                            cleaned += size
+                        except OSError as e:
+                            logger.exception(e)
+                        if (free + cleaned) // gb >= config['cleanup_min_gb']:
+                            break
+            else:
+                logger.error(f"cleanup_job: {storage['mountpoint']} not found")
+    finally:
+        cleanup_job_running = False
+
+
 fix_job_running = False
+cleanup_job_running = False
+
 datetime_format = '%Y-%m-%d-%H.%M.%S'
+datetime_format_re = r'\d{4}-\d{2}-\d{2}-\d{2}\.\d{2}.\d{2}'
 db: Optional[IPCamServerDatabase] = None
 server: Optional[IPCamWebServer] = None
 logger = logging.getLogger(__name__)
@@ -426,6 +496,7 @@ if __name__ == '__main__':
         if config['fix_enabled']:
             scheduler = AsyncIOScheduler(event_loop=loop)
             scheduler.add_job(fix_job, 'interval', seconds=config['fix_interval'])
+            scheduler.add_job(cleanup_job, 'interval', seconds=config['cleanup_interval'])
             scheduler.start()
     except KeyError:
         pass

src/openwrt_logger.py

@@ -22,6 +22,9 @@ $UDPServerRun 514
 :fromhost-ip, isequal, "192.168.1.1" /var/log/openwrt.log  
 & ~
 
+Also comment out the following line: 
+$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
+ 
 """
 
 

test/test_ipcam_server_cleanup.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+import shutil
+import sys
+import os
+import re
+import logging
+sys.path.extend([
+    os.path.realpath(
+        os.path.join(os.path.dirname(os.path.join(__file__)), '..')
+    )
+])
+
+from functools import cmp_to_key
+from datetime import datetime
+from pprint import pprint
+from src.home.config import config
+
+
+logger = logging.getLogger(__name__)
+datetime_format = '%Y-%m-%d-%H.%M.%S'
+datetime_format_re = r'\d{4}-\d{2}-\d{2}-\d{2}\.\d{2}.\d{2}'
+
+
+def cleanup_job():
+    def fn2dt(name: str) -> datetime:
+        name = os.path.basename(name)
+
+        if name.startswith('record_'):
+            return datetime.strptime(re.match(r'record_(.*?)\.mp4', name).group(1), datetime_format)
+
+        m = re.match(rf'({datetime_format_re})__{datetime_format_re}\.mp4', name)
+        if m:
+            return datetime.strptime(m.group(1), datetime_format)
+
+        raise ValueError(f'unrecognized filename format: {name}')
+
+    def compare(i1: str, i2: str) -> int:
+        dt1 = fn2dt(i1)
+        dt2 = fn2dt(i2)
+
+        if dt1 < dt2:
+            return -1
+        elif dt1 > dt2:
+            return 1
+        else:
+            return 0
+
+    gb = float(1 << 30)
+    for storage in config['storages']:
+        if os.path.exists(storage['mountpoint']):
+            total, used, free = shutil.disk_usage(storage['mountpoint'])
+            free_gb = free // gb
+            if free_gb < config['cleanup_min_gb']:
+                # print(f"{storage['mountpoint']}: free={free}, free_gb={free_gb}")
+                # continue
+                cleaned = 0
+                files = []
+                for cam in storage['cams']:
+                    for _dir in (config['camera'][cam]['recordings_path'], config['camera'][cam]['motion_path']):
+                        files += list(map(lambda file: os.path.join(_dir, file), os.listdir(_dir)))
+                    files = list(filter(lambda path: os.path.isfile(path) and path.endswith('.mp4'), files))
+                    files.sort(key=cmp_to_key(compare))
+                    # files = list(sorted(files, key=compare))
+
+                for file in files:
+                    size = os.stat(file).st_size
+                    try:
+                        # os.unlink(file)
+                        print(f'unlink {file}')
+                        cleaned += size
+                    except OSError as e:
+                        logger.exception(e)
+                    if (free + cleaned) // gb >= config['cleanup_min_gb']:
+                        break
+        else:
+            logger.error(f"cleanup_job: {storage['mountpoint']} not found")
+
+
+if __name__ == '__main__':
+    config.load('ipcam_server')
+    cleanup_job()