Implemented message purging and fixed json format on /messages

README.md

@@ -9,15 +9,16 @@ Simple REST-API for sending messages over the [Matrix](https://matrix.org) proto
 ## Features
 
 - Docker support
-- Authentication support
-- Sending encrypted messages
+- Encryption support
 - UTF-8 support
-- Markdown support for messages
+- Markdown support
 - Tag support to embed emojis in the title by using their short codes
-- Title support
 - Web UI to send messages
-- Webhooks
-- Pushing messages as json
+- Webhook support
+- Json support
+- Title support
+- Authentication support
+- Automatic message purging
 
 ## Table of contents
 > Note: Matrix-Notifier probably contains a lot of bugs.

bot/sync.py

@@ -23,17 +23,17 @@ async def save_id(con, cur, id):
 
 
 async def check(messages, client):
-    try:
-        data_dir = config.datadir_bot
-        if not os.path.exists(data_dir):
-            os.mkdir(data_dir)
+    data_dir = config.datadir_bot
+    if not os.path.exists(data_dir):
+        os.mkdir(data_dir)
 
-        con = sqlite3.connect(f"{data_dir}/ids.db")
-        cur = con.cursor()
+    con = sqlite3.connect(f"{data_dir}/ids.db")
+    cur = con.cursor()
 
-        cur.execute(
-            '''CREATE TABLE IF NOT EXISTS ids (id INT PRIMARY KEY)''')
+    cur.execute(
+        '''CREATE TABLE IF NOT EXISTS ids (id INT PRIMARY KEY)''')
 
+    try:
         cur.execute(f'''SELECT * FROM (SELECT * FROM ids ORDER BY id DESC LIMIT 250) sub ORDER BY id ASC''')
         data = cur.fetchall()
         ids = []

docker/example.env

@@ -27,3 +27,8 @@ SERVER_HOSTNAME=server
 
 # Optional Settings:
 # SERVER_PORT=5505
+
+# How long messages should be kept in hours
+# message_preserve_time = 72
+# How often the purge job for clearing all messages should be run in minutes
+# message_purge_interval = 30

docker/release-docker-compose.yml

@@ -3,7 +3,7 @@ version: "3.3"
 services:
   bot:
     container_name: "matrix-notifer-bot"
-    image: nlion/matrix-notifier-bot:v1.0.2
+    image: nlion/matrix-notifier-bot:v1.2.0
     depends_on:
       - server
     env_file:
@@ -14,7 +14,7 @@ services:
 
   server:
     container_name: "matrix-notifer-server"
-    image: nlion/matrix-notifier-server:v1.0.2
+    image: nlion/matrix-notifier-server:v1.2.0
     ports:
     - "5505:5505"
     env_file:

server/Dockerfile

@@ -15,6 +15,9 @@ ENV SERVER_PORT 5505
 ENV authorization false
 ENV auth_secret ""
 
+ENV message_preserve_time 72
+ENV message_purge_interval 30
+
 EXPOSE 5505
 
 CMD [ "python3", "app.py" ]

server/app.py

@@ -3,6 +3,7 @@
 import os
 import json
 import logging
+from apscheduler.schedulers.background import BackgroundScheduler
 
 import config
 import parser
@@ -16,6 +17,7 @@
             static_url_path='/static',
             template_folder='web/templates')
 
+scheduler = BackgroundScheduler()
 
 @app.route("/", methods=['POST'])
 def post_messages():
@@ -34,11 +36,12 @@ def post_messages():
 
     msg = parser.messageparse(parameter=parameter, body=body)
 
-    id = saver.save_to_db(msg)
+    id, message_time = saver.save_to_db(msg)
 
-    msg_json = json.dumps(dict([('Id', id), ('Channels', msg.channels),
-                                ('Title', msg.title), ('Content', msg.content),
-                                ('Tags', msg.tags), ('Markdown', msg.markdown)]))
+    msg_json = json.dumps(dict([('Id', id), ('time', message_time),
+                                ('Channels', msg.channels), ('Title', msg.title),
+                                ('Content', msg.content), ('Tags', msg.tags),
+                                ('Markdown', msg.markdown)]))
 
     return msg_json, 200
 
@@ -59,11 +62,12 @@ def webhook_messages():
 
     msg = parser.webhookmessageparse(parameter=parameter, content=message)
 
-    id = saver.save_to_db(msg)
+    id, message_time = saver.save_to_db(msg)
 
-    msg_json = json.dumps(dict([('Id', id), ('Channels', msg.channels),
-                                ('Title', msg.title), ('Content', msg.content),
-                                ('Tags', msg.tags), ('Markdown', msg.markdown)]))
+    msg_json = json.dumps(dict([('Id', id), ('time', message_time),
+                                ('Channels', msg.channels), ('Title', msg.title),
+                                ('Content', msg.content), ('Tags', msg.tags),
+                                ('Markdown', msg.markdown)]))
 
     return msg_json, 200
 
@@ -86,11 +90,12 @@ def json_messages():
 
     msg = parser.webhookmessageparse(parameter=parameter, content=message)
 
-    id = saver.save_to_db(msg)
+    id, message_time = saver.save_to_db(msg)
 
-    msg_json = json.dumps(dict([('Id', id), ('Channels', msg.channels),
-                                ('Title', msg.title), ('Content', msg.content),
-                                ('Tags', msg.tags), ('Markdown', msg.markdown)]))
+    msg_json = json.dumps(dict([('Id', id), ('time', message_time),
+                                ('Channels', msg.channels), ('Title', msg.title),
+                                ('Content', msg.content), ('Tags', msg.tags),
+                                ('Markdown', msg.markdown)]))
 
     return msg_json, 200
 
@@ -114,17 +119,18 @@ def get_messages():
     cur = con.cursor()
 
     cur.execute(
-        '''CREATE TABLE IF NOT EXISTS messages (id INT PRIMARY KEY, channels text, title text, content text, tags text, markdown text)''')
+        '''CREATE TABLE IF NOT EXISTS messages (id INT PRIMARY KEY, date text, channels text, title text, content text, tags text, markdown text)''')
 
     cur.execute(
         'SELECT * FROM (SELECT * FROM messages ORDER BY id DESC LIMIT :limit) sub ORDER BY id ASC', {"limit": parameter.limit})
 
     data = cur.fetchall()
     message_data_list = []
     for tuple in data:
-        message_data = dict([('Id', tuple[0]), ('Channels', json.loads(tuple[1])),
-                            ('Title', tuple[2]), ('Content', tuple[3]),
-                            ('Tags', json.loads(tuple[4])),('Markdown', tuple[5])])
+        message_data = dict([('Id', tuple[0]), ('time', tuple[1]),
+                             ('Channels', json.loads(tuple[2])), ('Title', tuple[3]),
+                             ('Content', tuple[4]), ('Tags', json.loads(tuple[5])),
+                             ('Markdown', tuple[6])])
         message_data_list.append(message_data)
 
     content_data = json.dumps(message_data_list)
@@ -153,7 +159,7 @@ def get_message_byid(message_id):
     cur = con.cursor()
 
     cur.execute(
-        '''CREATE TABLE IF NOT EXISTS messages (id INT PRIMARY KEY, channels text, title text, content text, tags text, markdown text)''')
+        '''CREATE TABLE IF NOT EXISTS messages (id INT PRIMARY KEY, date text, channels text, title text, content text, tags text, markdown text)''')
 
     ids = parser.id_parse(message_id)
 
@@ -165,9 +171,10 @@ def get_message_byid(message_id):
 
         if not data == []:
             for tuple in data:
-                message_data = dict([('Id', tuple[0]), ('Channels', json.loads(tuple[1])),
-                                     ('Title', tuple[2]), ('Content', tuple[3]),
-                                     ('Tags', json.loads(tuple[4])), ('Markdown', tuple[5])])
+                message_data = dict([('Id', tuple[0]), ('time', tuple[1]),
+                                     ('Channels', json.loads(tuple[2])), ('Title', tuple[3]),
+                                     ('Content', tuple[4]), ('Tags', json.loads(tuple[5])),
+                                     ('Markdown', tuple[6])])
                 message_data_list.append(message_data)
 
     if not message_data_list == []:
@@ -186,4 +193,6 @@ def get_page():
 
 
 if __name__ == "__main__":
+    job = scheduler.add_job(saver.clean_db, 'interval', minutes=config.message_purge_interval)
+    scheduler.start()
     app.run(host='0.0.0.0', port=config.port)

server/config.py

@@ -14,6 +14,10 @@
 # Cant contain any &'s or +'s
 auth_secret = ""
 
+# How long the server should keep messages in hours
+message_preserve_time = 72
+# How often the purge job should be run in minutes
+message_purge_interval = 30
 
 # Do Not Touch
 # If docker is used it will use the environment values
@@ -23,6 +27,8 @@
     datadir_server = "/data"
     authorization = os.environ.get('authorization', False)
     auth_secret = os.environ.get('auth_secret', False)
+    message_preserve_time = os.environ.get('message_preserve_time', False)
+    message_purge_interval = os.environ.get('message_purge_interval', False)
 
 if str(authorization).lower() == "true" or authorization == True:
     authorization = True
@@ -31,3 +37,15 @@
         quit(1)
 else:
     authorization = False
+if not type(message_preserve_time) == type(1):
+    try:
+        message_preserve = int(message_preserve_time)
+    except ValueError:
+        logger.error("Wrong message_preserve format")
+        quit(1)
+if not type(message_purge_interval) == type(1):
+    try:
+        message_purge_interval = int(message_purge_interval)
+    except ValueError:
+        logger.error("Wrong message_purge_interval format")
+        quit(1)

server/requirements.txt

@@ -1 +1,2 @@
-flask==2.2.3
+flask==2.2.3
+apscheduler

server/saver.py

@@ -1,6 +1,7 @@
 import sqlite3
 import os
 import json
+from datetime import datetime
 
 import config
 
@@ -32,14 +33,34 @@ def save_to_db(msg):
     cur = con.cursor()
 
     cur.execute(
-        '''CREATE TABLE IF NOT EXISTS messages (id INT PRIMARY KEY, channels text, title text, content text, tags text, markdown text)''')
+        '''CREATE TABLE IF NOT EXISTS messages (id INT PRIMARY KEY, date text, channels text, title text, content text, tags text, markdown text)''')
     id = get_next_id(cur)
 
     channels = json.dumps(msg.channels)
     tags = json.dumps(msg.tags)
 
+    message_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
     cur.execute(
-        'INSERT INTO messages VALUES (?, ?, ?, ?, ?, ?)', (id, channels, msg.title, msg.content, tags, msg.markdown))
+        'INSERT INTO messages VALUES (?, ?, ?, ?, ?, ?, ?)', (id, message_time, channels, msg.title, msg.content, tags, msg.markdown))
+
+    con.commit()
+    return id, message_time
+
+
+def clean_db():
+    message_preserve_time = config.message_preserve_time
+
+    data_dir = config.datadir_server
+    if not os.path.exists(data_dir):
+        os.mkdir(data_dir)
+
+    con = sqlite3.connect(f"{data_dir}/messages.db")
+    cur = con.cursor()
+
+    cur.execute(
+        '''CREATE TABLE IF NOT EXISTS messages (id INT PRIMARY KEY, date text, channels text, title text, content text, tags text, markdown text)''')
+
+    cur.execute(
+        f'DELETE FROM messages WHERE date < datetime("now", "localtime", "-{message_preserve_time} hours")')
 
     con.commit()
-    return id