api: only open database connection while handling requests

Helps with recovery when database restarts during long-held connections.

Code from https://docs.peewee-orm.com/en/3.15.3/peewee/database.html#flask

src/Dockerfile

@@ -13,7 +13,7 @@ COPY main.py .
 COPY api.py .
 COPY moj_licznik.py .
 COPY log_config.py .
-COPY database_empty.sqlite database.sqlite
+COPY database_empty.sqlite data/database.sqlite
 RUN chmod a+x run.sh
 
 RUN apk add --update --no-cache py3-pip && \

src/README.md

@@ -47,6 +47,11 @@ Wymagane parametry:
 * ENERGA_USERNAME - nazwa użytkownika w aplikacji Energa Mój licznik
 * ENERGA_PASSWORD - hasło użytkownika w aplikacji Energa Mój licznik
 
+Opcjonalne parametry:
+
+* POSTGRESQL_CONNSTRING - namiar na bazę PostgreSQL do przechowywania odczytów;
+format opisany w [dokumentacji PGSQL](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING-URIS)
+(przykładowo `postgresql://uzytkownik:haslo@serwer/mojlicznik`)
 
 ## Konfiguracja sensorów
 Do HA możesz dodać sensory, które zawierają informacje udostępniane przez API

src/api.py

@@ -1,4 +1,4 @@
-from peewee import SqliteDatabase
+from peewee import SqliteDatabase, PostgresqlDatabase
 from flask import Flask, jsonify, request, redirect, url_for, abort
 from waitress import serve
 #from datetime 
@@ -9,13 +9,30 @@ import urllib.parse
 
 logger = logging.getLogger("energaMeter.api")
 
-
+if postgresql_connstring := os.getenv("POSTGRESQL_CONNSTRING"):
+    from psycopg2.extensions import parse_dsn
+    db_name = parse_dsn(postgresql_connstring)['dbname']
+    db = PostgresqlDatabase(db_name, dsn=postgresql_connstring)
+else:
     path = os.path.dirname(os.path.abspath(__file__))
-db_file = 'database.sqlite'
+    db_file = 'data/database.sqlite'
     db = SqliteDatabase(os.path.join(path, db_file))
 
 app = Flask(__name__)
 
+# This hook ensures that a connection is opened to handle any queries
+# generated by the request.
+@app.before_request
+def _db_connect():
+    db.connect()
+
+# This hook ensures that the connection is closed when we've finished
+# processing the request.
+@app.teardown_request
+def _db_close(exc):
+    if not db.is_closed():
+        db.close()
+
 @app.route('/', methods=['GET'])
 def root():
     query = PPETable.select() #.where(PPETable.is_active == True)
@@ -207,7 +224,7 @@ def charts():
     zone = request.args.get('zone', None)
     negative = request.args.get('negative', type=bool, default=False) 
     logger.debug(f"API: GET /charts - {start_date} - {end_date}")
-    query = MainChartTable.select().where((MainChartTable.tm >= int(start_date)) & (MainChartTable.tm <= int(end_date)))
+    query = MainChartTable.select().where((MainChartTable.tm >= int(start_date)/1000) & (MainChartTable.tm <= int(end_date)/1000))
     logger.debug(f"{query}")
     factor = 1
     if negative:
@@ -226,13 +243,13 @@ def charts():
     charts = []
 
     for p in result_ppes:
-        czas = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(p.tm/1000))
+        czas = p.tm.strftime("%Y-%m-%d %H:%M:%S")
         chart = {
             'mp': p.mp,
             'meter_type': p.meter_type,
             'meter_type_url': urllib.parse.quote_plus(p.meter_type),
             'zone': p.zone,
-            'time_tm': p.tm,
+            'time_tm': int(p.tm.timestamp()*1000),
             'time': czas,
             'value': p.value * factor
         }

src/moj_licznik.py

@@ -1,17 +1,24 @@
-from peewee import SqliteDatabase
+from peewee import SqliteDatabase, PostgresqlDatabase
 from datetime import datetime, timedelta, date
 import calendar, requests, re, time, json, os, logging
 import http.cookiejar as cookiejar
 from requests.exceptions import HTTPError
 from bs4 import BeautifulSoup
 from enum import Enum
-from peewee import AutoField, Model, CharField, IntegerField, DateField, BooleanField, CompositeKey, DecimalField, ForeignKeyField, SQL
+from peewee import AutoField, Model, CharField, IntegerField, DateField, BooleanField, CompositeKey, DecimalField, ForeignKeyField, SQL, TextField, TimestampField
 import urllib.parse
 
 logger = logging.getLogger("energaMeter")
 
+if postgresql_connstring := os.getenv("POSTGRESQL_CONNSTRING"):
+    from psycopg2.extensions import parse_dsn
+    db_name = parse_dsn(postgresql_connstring)['dbname']
+    db_host = parse_dsn(postgresql_connstring)['host']
+    db = PostgresqlDatabase(db_name, dsn=postgresql_connstring)
+    logger.info(f"Używam bazy PostgreSQL „{db_name}” na {db_host}")
+else:
     path = os.path.dirname(os.path.abspath(__file__))
-db_file = 'database.sqlite'
+    db_file = 'data/database.sqlite'
     db = SqliteDatabase(os.path.join(path, db_file))
 
 class ChartType(Enum):
@@ -31,7 +38,7 @@ class PPETable(Model):
     class Meta:
         database = db
         table_name = 'PPE'
-        constraints = [SQL('UNIQUE (ppe, tariffCode)')]
+        constraints = [SQL('UNIQUE ("ppe", "tariffCode")')]
 
 class MeterTable(Model):
     id = AutoField() # Meter point
@@ -43,7 +50,7 @@ class MeterTable(Model):
     class Meta:
         database = db
         table_name = 'METER'
-        constraints = [SQL('UNIQUE (ppe_id, meter_type)')]
+        constraints = [SQL('UNIQUE ("ppe_id", "meter_type")')]
 
 class CounterTable(Model):
     id = AutoField()
@@ -71,7 +78,7 @@ class ChartTable(Model):
     year = IntegerField()
     month = IntegerField(null=True)
     day = IntegerField(null=True)
-    value =CharField()
+    value = TextField()
 
     class Meta:
         database = db
@@ -82,7 +89,7 @@ class MainChartTable(Model):
     mp = CharField()
     meter_type = CharField() 
     zone = IntegerField()
-    tm = IntegerField()
+    tm = TimestampField()
     value = DecimalField(max_digits=20, decimal_places=16, null=True)
     tarAvg = DecimalField(max_digits=20, decimal_places=16, null=True)
     est = BooleanField(default=False)
@@ -360,16 +367,17 @@ class MojLicznik:
             try:
                 logger.debug(f"save_main_charts: mp: {mp}, val: {val}, meter_type: {m_type}")
                 z = val["zones"]
+                tm = int(val["tm"]) / 1000 # convert JS timestamp (milliseconds) to unix (seconds)
                 if z[0]:
                     # MainChartTable.get_or_create(tm = val["tm"], zone = 1, value = z[0], tarAvg=val["tarAvg"], est=val["est"], cplt=val["cplt"])
                     try:
-                        existing_record = MainChartTable.get((MainChartTable.meter_type == m_type) & (MainChartTable.mp == mp) & (MainChartTable.tm == val["tm"]) & (MainChartTable.zone == 1))
+                        existing_record = MainChartTable.get((MainChartTable.meter_type == m_type) & (MainChartTable.mp == mp) & (MainChartTable.tm == tm) & (MainChartTable.zone == 1))
                     except MainChartTable.DoesNotExist:
                         # Jeśli rekord nie istnieje, utwórz nowy
                         MainChartTable.create(
                             mp=mp,
                             meter_type=m_type,
-                            tm=val["tm"],
+                            tm=tm,
                             zone=1,
                             value=z[0],
                             tarAvg=val["tarAvg"],
@@ -379,13 +387,13 @@ class MojLicznik:
                 
                 if z[1]:
                     try:
-                        existing_record = MainChartTable.get((MainChartTable.meter_type == m_type) & (MainChartTable.mp == mp) & (MainChartTable.tm == val["tm"]) & (MainChartTable.zone == 2))
+                        existing_record = MainChartTable.get((MainChartTable.meter_type == m_type) & (MainChartTable.mp == mp) & (MainChartTable.tm == tm) & (MainChartTable.zone == 2))
                     except MainChartTable.DoesNotExist:
                         # Jeśli rekord nie istnieje, utwórz nowy
                         MainChartTable.create(
                             mp=mp,
                             meter_type=m_type,
-                            tm=val["tm"],
+                            tm=tm,
                             zone=2,
                             value=z[1],
                             tarAvg=val["tarAvg"],
@@ -395,13 +403,13 @@ class MojLicznik:
                 
                 if z[2]:
                     try:
-                        existing_record = MainChartTable.get((MainChartTable.meter_type == m_type) & (MainChartTable.mp == mp) & (MainChartTable.tm == val["tm"]) & (MainChartTable.zone == 3))
+                        existing_record = MainChartTable.get((MainChartTable.meter_type == m_type) & (MainChartTable.mp == mp) & (MainChartTable.tm == tm) & (MainChartTable.zone == 3))
                     except MainChartTable.DoesNotExist:
                         # Jeśli rekord nie istnieje, utwórz nowy
                         MainChartTable.create(
                             mp=mp,
                             meter_type=m_type,
-                            tm=val["tm"],
+                            tm=tm,
                             zone=3,
                             value=z[2],
                             tarAvg=val["tarAvg"],