Ready for publish

README.md

@@ -1,2 +1,6 @@
 # tasmota-sml-parser
-Tasmota SML Parser
+Webapp to conveniently parse Tasmota SML Dump output and display the actual data contained.
+
+Makes building a Tasmota SML Script much easier.
+
+Online Demo is at https://tasmota-sml-parser.azurewebsites.net/.

app.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+
+from flask import Flask, render_template, request, redirect, url_for
+from flask_bootstrap import Bootstrap
+from flask_nav import Nav
+from flask_nav.elements import Navbar, View, Subgroup, Link, Text, Separator
+from sml_decoder import TasmotaSMLParser
+
+app = Flask(__name__)
+Bootstrap(app)
+app.config["BOOTSTRAP_SERVE_LOCAL"] = True
+app.debug = False
+nav = Nav()
+nav.init_app(app)
+nav.register_element("frontend_top", Navbar(View("Tasmota SML Decoder", ".index")))
+
+
+@app.route("/")
+def index():
+    return render_template("index.html")
+
+
+@app.route("/decode", methods=["POST", "GET"])
+def decode():
+    if request.method == "GET":
+        return redirect("/")
+    elif request.method == "POST":
+        data = request.form["smldump"].splitlines()
+
+        tas = TasmotaSMLParser()
+        msgs = tas.decode_messages(data)
+        messages = []
+        for msg in msgs:
+            details = tas.get_message_details(msg)
+            tasmota_script = tas.build_meter_def(msg)
+            messages.append({"msg": details, "tas": tasmota_script})
+
+        messages = sorted(messages, key=lambda x: x["msg"]["obis"])
+
+        return render_template(
+            "decode.html",
+            smldump=data,
+            parse_errors=tas.parse_errors,
+            messages=messages,
+        )
+
+
+if __name__ == "__main__":
+    app.run()

requirements.txt

@@ -0,0 +1,7 @@
+Flask==2.0.3
+Flask-Bootstrap==3.3.7.1
+flask-nav==0.6
+#smllib==1.1
+# SML Lib with grid freq and current power
+git+https://github.com/spacemanspiff2007/SmlLib.git@3d4a106#egg=smllib
+

sml_decoder.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+
+from binascii import a2b_hex
+from smllib.sml_frame import SmlFrame
+from smllib.const import OBIS_NAMES, UNITS
+
+file = "test-data.txt"
+
+
+class TasmotaSMLParser:
+    def __init__(self):
+        self.obis_decoded = []
+        self.obis_errors = []
+        self.parse_errors = []
+
+    def parse_input(self, input):
+        """Parse a line into a frame"""
+        if " : 77 " in input:
+            # Remove the spaces and return a bytestring
+            frame = a2b_hex("".join((input.split(" : ", 1)[1]).split()))
+        elif input.startswith("77 0"):
+            frame = a2b_hex("".join(input.split()))
+        else:
+            self.parse_errors.append(input)
+            return
+
+        return frame
+
+    def decode_frame(self, frame):
+        f = SmlFrame(frame)
+        try:
+            msgs = f.get_obis()
+            if len(msgs) == 0:
+                return False
+        except:
+            self.obis_errors.append(frame)
+            return None
+        return msgs
+
+    def decode_messages(self, input):
+        messages = []
+        for line in input:
+            frame = self.parse_input(line)
+            if frame is None:
+                continue
+            msgs = self.decode_frame(frame)
+            if msgs in [None, False]:
+                continue
+            for msg in msgs:
+                if msg.obis in self.obis_decoded:
+                    continue
+                else:
+                    self.obis_decoded.append(msg.obis)
+                messages.append(msg)
+        return messages
+
+    def get_message_details(self, msg):
+        name = OBIS_NAMES.get(msg.obis, "Unbekannter Datentyp")
+        unit = UNITS.get(msg.unit, "Unbekannte Einheit")
+        mqtt_topic = (
+            "_".join(OBIS_NAMES.get(msg.obis, "Unbekanntes MQTT Topic").split())
+            .replace("/", "-")
+            .lower()
+        )
+        try:
+            precision = msg.scaler * -1
+        except TypeError:
+            precision = 0
+
+        try:
+            human_readable = f"{msg.value * pow(10, msg.scaler)}{unit} ({name})"
+        except TypeError:
+            if msg.unit in UNITS and msg.name in OBIS_NAMES:
+                human_readable = f"{msg.value}{unit} ({name})"
+            elif msg.unit in UNITS and msg.name not in OBIS_NAMES:
+                human_readable = f"{msg.value}{unit}"
+            else:
+                human_readable = f"{msg.value}"
+
+        data = {
+            "obis": msg.obis,
+            "obis_code": msg.obis.obis_code,
+            "obis_short": msg.obis.obis_short,
+            "name": name,
+            "unit": unit,
+            "topic": mqtt_topic,
+            "precision": precision,
+            "value": msg.value,
+            "status": msg.status,
+            "val_time": msg.val_time,
+            "unit_raw": msg.unit,
+            "scaler": msg.scaler,
+            "value_signature": msg.value_signature,
+            "human_readable": human_readable,
+        }
+
+        return data
+
+    def build_meter_def(self, msg):
+        data = self.get_message_details(msg)
+        return f"1,7707{data['obis'].upper()}@1,{data['name']},{data['unit']},{data['topic']},{data['precision']}"
+
+    def pretty_print(self, msg):
+        print(msg.format_msg())
+
+
+def main():
+    tas = TasmotaSMLParser()
+    msgs = []
+    with open(file, "r") as fp:
+        msgs = tas.decode_messages(fp.read().splitlines())
+
+        for msg in msgs:
+            tas.pretty_print(msg)
+            details = tas.get_message_details(msg)
+            print("Tasmota SML Script meter definition:")
+            print(tas.build_meter_def(msg))
+            print(80 * "#")
+
+
+if __name__ == "__main__":
+    main()

static/app.css

@@ -0,0 +1,17 @@
+.jumbotron {
+    margin-top: 1em;
+}
+
+.parser-error {
+    color: red;
+}
+
+.parser-success {
+    color: green;
+}
+
+footer {
+    border-top: 1px solid black;
+    margin: 5em;
+    margin-top: 2em;
+}

templates/base.html

@@ -0,0 +1,64 @@
+{# ``base.html`` is the template all our other templates derive from. While
+   Flask-Bootstrap ships with its own base, it is good form to create a custom
+   one for our app, as it allows customizing some aspects.
+
+   Deriving from bootstap/base.html gives us a basic page scaffoling.
+
+   You can find additional information about template inheritance at
+
+   http://jinja.pocoo.org/docs/templates/#template-inheritance
+#}
+{%- extends "bootstrap/base.html" %}
+
+{# We also set a default title, usually because we might forget to set one.
+   In our sample app, we will most likely just opt not to change it #}
+{% block title %}Tasmota SML Decoder{% endblock %}
+
+{# While we are at it, we also enable fixes for legacy browsers. First we
+   import the necessary macros: #}
+{% import "bootstrap/fixes.html" as fixes %}
+
+{# Then, inside the head block, we apply these. To not replace the header,
+   ``super()`` is used: #}
+
+{% block head %}
+   {{super()}}
+
+{#- Docs: http://pythonhosted.org/Flask-Bootstrap/macros.html#fixes
+   The sample application already contains the required static files. #}
+{{fixes.ie8()}}
+{%- endblock %}
+
+{# Adding our own CSS files is also done here. Check the documentation at
+   http://pythonhosted.org/Flask-Bootstrap/basic-usage.html#available-blocks
+   for an overview. #}
+{% block styles -%}
+    {{super()}}  {# do not forget to call super or Bootstrap's own stylesheets
+                    will disappear! #}
+
+    <link rel="stylesheet" type="text/css"
+          href="{{url_for('static', filename='app.css')}}">
+
+{% endblock %}
+
+{# Finally, round things out with navigation #}
+{% block navbar %}
+{{nav.frontend_top.render()}}
+{% endblock %}
+
+{#
+   {%- block content %}
+{{ super() }}
+{%- block footer %}
+<footer>&copy; 2022 <a href="https://github.com/ixs/tasmota-sml-parser/">Andreas Thienemann</a></footer>
+{%- endblock footer %}
+{%- endblock content %}
+#}
+
+
+{%- block body %}
+{{ super() }}
+{%- block footer %}
+<footer>&copy; 2022 <a href="https://github.com/ixs/tasmota-sml-parser/">Andreas Thienemann</a></footer>
+{%- endblock footer %}
+{%- endblock body %}

templates/decode.html

@@ -0,0 +1,76 @@
+{# This simple template derives from ``base.html``. See ``base.html`` for
+   more information about template inheritance. #}
+{%- extends "base.html" %}
+
+{# Loads some of the macros included with Flask-Bootstrap. We are using the
+   utils module here to automatically render Flask's flashed messages in a
+   bootstrap friendly manner #}
+{% import "bootstrap/utils.html" as utils %}
+
+
+{# Inside the ``content`` is where you should place most of your own stuff.
+   This will keep scripts at the page end and a navbar you add on later
+   intact. #}
+{% block content %}
+  <div class="container">
+  {%- with messages = get_flashed_messages(with_categories=True) %}
+  {%- if messages %}
+    <div class="row">
+      <div class="col-md-12">
+        {{utils.flashed_messages(messages)}}
+      </div>
+    </div>
+  {%- endif %}
+  {%- endwith %}
+    <div class="jumbotron">
+      <h1>Tasmota SML Dekoder</h1>
+      <p><p>
+    </div>
+    <div>
+{% if messages | length > 0 %}
+      <h2>Dekodierte SML Nachrichten</h2>
+      <table class="table table-striped">
+        <thead class="thead-light">
+          <tr>
+            <th scope="col">OBIS (hex)</th>
+            <th scope="col">OBIS</th>
+            <th scope="col">Name</th>
+            <th scope="col">Wert</th>
+            <th scope="col">Einheit</th>
+            <th scope="col">Parsed</th>
+          </tr>
+        </thead>
+        <tbody>
+{%- for msg in messages %}
+          <tr>
+            <td>0x{{ msg.msg.obis }}</td>
+            <td>{{ msg.msg.obis_short }}</td>
+            <td>{{ msg.msg.name }}</td>
+            <td>{{ msg.msg.value}}</td>
+            <td>{{ msg.msg.unit }}</td>
+            <td>{{ msg.msg.human_readable }}</td>
+          </tr>
+{%- endfor %}
+        </tbody>
+      </table>
+      <h2>Tasmota Meter Definition</h2>
+      <p>Aufgrund der erkannten SML Elemente wäre dies ein Vorschlag für eine Tasmota Meter Definition.</p>
+      <pre>M 1
++1,3,s,0,9600,
+{% for msg in messages -%}
+        {{ msg.tas }}
+{% endfor -%}</pre>
+{% endif %}
+      <hr>
+      <h2>Empfangene Daten</h2>
+      Folgende Daten wurden empfangen.
+      <pre>{% for line in smldump -%}
+{% if line in parse_errors -%}
+        <span class="parser-error">{{ line }}</span>
+{% else -%}
+        <span class="parser-success">{{ line }}</span>
+{% endif -%}
+{% endfor -%}</pre>
+    </div>
+   </div>
+{%- endblock %}

templates/index.html

@@ -0,0 +1,79 @@
+{# This simple template derives from ``base.html``. See ``base.html`` for
+   more information about template inheritance. #}
+{%- extends "base.html" %}
+
+{# Loads some of the macros included with Flask-Bootstrap. We are using the
+   utils module here to automatically render Flask's flashed messages in a
+   bootstrap friendly manner #}
+{% import "bootstrap/utils.html" as utils %}
+
+
+{# Inside the ``content`` is where you should place most of your own stuff.
+   This will keep scripts at the page end and a navbar you add on later
+   intact. #}
+{% block content %}
+  <div class="container">
+  {%- with messages = get_flashed_messages(with_categories=True) %}
+  {%- if messages %}
+    <div class="row">
+      <div class="col-md-12">
+        {{utils.flashed_messages(messages)}}
+      </div>
+    </div>
+  {%- endif %}
+  {%- endwith %}
+    <div class="jumbotron">
+      <h1>Tasmota SML Dekoder</h1>
+      <p><a href="https://tasmota.github.io">Tasmota</a> unterstützt <a href="https://tasmota.github.io/docs/Smart-Meter-Interface/">
+      verschiedene intelligente Stromzähler</a> und kann deren Daten auslesen und z.B. als MQTT Telegramm verschicken.</p>
+      <p>Die Konfiguration des Stromzählers erfolgt hierbei mit der <a href="https://tasmota.github.io/docs/Scripting-Language/">
+      Tasmota Scripting Language</a> um die verschiedenen auf dem Markt gebräuchlichen Stromzähler zu unterstützen.<p>
+      <hr>
+      <p>Diese Webseite hilft, den eigenen Stromzähler richtig zu konfigurieren, indem der "SML dump" von Tasmota dekodiert
+      wird um zu erkennen, was der Stromzähler für Daten sendet. Damit kann dann einfach ein entsprechendes Tasmota Script
+      gebaut werden um die gewünschten Daten zu extrahieren.</p>
+    </div>
+    <div>
+      <h2>Ablauf</h2>
+
+      <ol>
+        <li>Tasmota Befehl <code>sensor53 d1</code> in der Konsole eingeben, damit die empfangenen SML Daten ausgegeben
+        werden anstelle interpretiert zu werden.<br>
+        Beispielhaft enthält die Tasmota Konsole anschließend Zeilen wie diese:<br>
+        <pre>14:10:15.988 : 77 07 01 00 10 07 00 ff 01 01 62 1b 52 00 53 01 c6 01 
+          14:10:16.009 : 77 07 01 00 20 07 00 ff 01 01 62 23 52 ff 63 08 f2 01 
+          14:10:16.029 : 77 07 01 00 34 07 00 ff 01 01 62 23 52 ff 63 08 de 01 
+          14:10:16.050 : 77 07 01 00 48 07 00 ff 01 01 62 23 52 ff 63 08 ee 01 
+          14:10:16.070 : 77 07 01 00 1f 07 00 ff 01 01 62 21 52 fe 62 34 01 
+          14:10:16.089 : 77 07 01 00 33 07 00 ff 01 01 62 21 52 fe 62 23 01 
+          14:10:16.109 : 77 07 01 00 47 07 00 ff 01 01 62 21 52 fe 62 c7 01 
+          14:10:16.128 : 77 07 01 00 51 07 01 ff 01 01 62 08 52 00 62 f1 01 
+          14:10:16.145 : 77 07 01 00 51 07 02 ff 01 01 62 08 52 00 62 </pre></li>
+        <li>Diese Ausgabe der gelesenen SML Daten in die Zwischenablage kopieren.</li>
+        <li>Tasmota Befehl <code>sensor53 d0</code> in der Konsole eingeben um den "Dump" Modus zu beenden.</li>
+        <li>Die in der Zwischenablage gespeicherten Daten in das Feld auf dieser Webseite eingeben und das Formular absenden.</li>
+        <li>Dekodierte Daten anschauen und den Meter Definition Vorschlag gegebenenfalls anpassen.</li>
+      </ol>
+
+      <hr>
+
+      <form name="smldump" action="/decode" method="POST">
+        <div class="form-group">
+          <label for="formGroupSMLDumpInput">SML Dump</label>
+          <textarea class="form-control" id="formGroupSMLDumpInput" name="smldump" rows=10
+            placeholder="14:10:15.988 : 77 07 01 00 10 07 00 ff 01 01 62 1b 52 00 53 01 c6 01 
+14:10:16.009 : 77 07 01 00 20 07 00 ff 01 01 62 23 52 ff 63 08 f2 01 
+14:10:16.029 : 77 07 01 00 34 07 00 ff 01 01 62 23 52 ff 63 08 de 01 
+14:10:16.050 : 77 07 01 00 48 07 00 ff 01 01 62 23 52 ff 63 08 ee 01 
+14:10:16.070 : 77 07 01 00 1f 07 00 ff 01 01 62 21 52 fe 62 34 01 
+14:10:16.089 : 77 07 01 00 33 07 00 ff 01 01 62 21 52 fe 62 23 01 
+14:10:16.109 : 77 07 01 00 47 07 00 ff 01 01 62 21 52 fe 62 c7 01 
+14:10:16.128 : 77 07 01 00 51 07 01 ff 01 01 62 08 52 00 62 f1 01 
+14:10:16.145 : 77 07 01 00 51 07 02 ff 01 01 62 08 52 00 62 "></textarea>
+        </div>
+        <button type="submit" class="btn btn-primary">Dekodieren</button>
+      </form>
+
+    </div>
+   </div>
+{%- endblock %}

test-data.txt

@@ -0,0 +1,59 @@
+>D
+>B
+->sensor53 r
+>M 1
++1,3,s,0,9600,Haus
+1,770701000F0700FF@1,Aktuell,W,Power_curr,0
+1,77070100010800FF@1000,Zählerstand Verb.,kWh,Tariflos,2
+1,77070100020800FF@1000,Zählerstand Einsp.,kWh,Tariflos,2
+#
+
+
+
+
+15:57:05.415 : 77 07 01 00 60 05 00 ff 01 01 01 01 65 00 1c 81 04 01 01 01 63 33 14 00 76 04 00 00 03 62 00 62 00 72 65 00 00 02 01 71 
+15:57:05.435 : 77 01 0b 0a 01 48 4c 59 02 00 01 1c 0f 01 01 f1 04 
+15:57:05.455 : 77 07 01 00 60 32 01 01 01 01 01 01 04 48 4c 59 01 
+15:57:05.483 : 77 07 01 00 60 01 00 ff 01 01 01 01 0b 0a 01 48 4c 59 02 00 01 1c 0f 01 
+15:57:05.516 : 77 07 01 00 01 08 00 ff 65 00 1c 81 04 65 05 a2 99 1e 62 1e 52 ff 65 0b e8 4d cf 01 
+15:57:05.544 : 77 07 01 00 02 08 00 ff 65 00 1c 81 04 65 05 a2 99 1e 62 1e 52 ff 62 00 01 
+15:57:05.565 : 77 07 01 00 10 07 00 ff 01 01 62 1b 52 00 53 01 a0 01 
+15:57:05.586 : 77 07 01 00 20 07 00 ff 01 01 62 23 52 ff 63 09 04 01 
+15:57:05.606 : 77 07 01 00 34 07 00 ff 01 01 62 23 52 ff 63 08 e5 01 
+15:57:05.627 : 77 07 01 00 48 07 00 ff 01 01 62 23 52 ff 63 09 10 01 
+15:57:05.647 : 77 07 01 00 1f 07 00 ff 01 01 62 21 52 fe 62 31 01 
+15:57:05.666 : 77 07 01 00 33 07 00 ff 01 01 62 21 52 fe 62 24 01 
+15:57:05.685 : 77 07 01 00 47 07 00 ff 01 01 62 21 52 fe 62 c6 01 
+15:57:05.706 : 77 07 01 00 51 07 01 ff 01 01 62 08 52 00 62 f0 01 
+15:57:05.725 : 77 07 01 00 51 07 02 ff 01 01 62 08 52 00 62 76 01 
+15:57:05.745 : 77 07 01 00 51 07 04 ff 01 01 62 08 52 00 63 01 2f 01 
+15:57:05.767 : 77 07 01 00 51 07 0f ff 01 01 62 08 52 00 63 01 31 01 
+15:57:05.787 : 77 07 01 00 51 07 1a ff 01 01 62 08 52 00 63 01 37 01 
+15:57:05.808 : 77 07 01 00 0e 07 00 ff 01 01 62 2c 52 ff 63 01 f4 01 
+15:57:05.834 : 77 07 01 00 00 02 00 00 01 01 01 01 09 31 2e 30 32 2e 30 30 37 01 
+15:57:05.854 : 77 07 01 00 60 5a 02 01 01 01 01 01 05 41 30 31 41 01 
+15:57:07.029 : 77 07 01 00 60 05 00 ff 01 01 01 01 65 00 1c 81 04 01 01 01 63 68 a4 00 76 04 00 00 03 62 00 62 00 72 65 00 00 02 01 71 
+15:57:07.049 : 77 01 0b 0a 01 48 4c 59 02 00 01 1c 0f 01 01 f1 04 
+15:57:07.068 : 77 07 01 00 60 32 01 01 01 01 01 01 04 48 4c 59 01 
+15:57:07.111 : 77 07 01 00 60 01 00 ff 01 01 01 01 0b 0a 01 48 4c 59 02 00 01 1c 0f 01 
+15:57:07.127 : 77 07 01 00 01 08 00 ff 65 00 1c 81 04 65 05 a2 99 20 62 1e 52 ff 65 0b e8 4d d1 01 
+15:57:07.157 : 77 07 01 00 02 08 00 ff 65 00 1c 81 04 65 05 a2 99 20 62 1e 52 ff 62 00 01 
+15:57:07.177 : 77 07 01 00 10 07 00 ff 01 01 62 1b 52 00 53 01 9f 01 
+15:57:07.198 : 77 07 01 00 20 07 00 ff 01 01 62 23 52 ff 63 09 04 01 
+15:57:07.219 : 77 07 01 00 34 07 00 ff 01 01 62 23 52 ff 63 08 e4 01 
+15:57:07.239 : 77 07 01 00 48 07 00 ff 01 01 62 23 52 ff 63 09 0e 01 
+15:57:07.259 : 77 07 01 00 1f 07 00 ff 01 01 62 21 52 fe 62 31 01 
+15:57:07.279 : 77 07 01 00 33 07 00 ff 01 01 62 21 52 fe 62 24 01 
+15:57:07.299 : 77 07 01 00 47 07 00 ff 01 01 62 21 52 fe 62 c6 01 
+15:57:07.318 : 77 07 01 00 51 07 01 ff 01 01 62 08 52 00 62 f1 01 
+15:57:07.336 : 77 07 01 00 51 07 02 ff 01 01 62 08 52 00 62 
+15:57:07.338 : 77 01 
+15:57:07.358 : 77 07 01 00 51 07 04 ff 01 01 62 08 52 00 63 01 2f 01 
+15:57:07.380 : 77 07 01 00 51 07 0f ff 01 01 62 08 52 00 63 01 31 01 
+15:57:07.400 : 77 07 01 00 51 07 1a ff 01 01 62 08 52 00 63 01 37 01 
+15:57:07.421 : 77 07 01 00 0e 07 00 ff 01 01 62 2c 52 ff 63 01 f4 01 
+15:57:07.446 : 77 07 01 00 00 02 00 00 01 01 01 01 09 31 2e 30 32 2e 30 30 37 01 
+15:57:07.466 : 77 07 01 00 60 5a 02 01 01 01 01 01 05 41 30 31 41 01 
+
+
+foobar