Merge branch 'master' into extend-confserver-logging

.dockerignore

@@ -0,0 +1,22 @@
+# Logs
+logs
+
+# Tests
+tests
+
+# Examples
+examples
+
+# Docs
+docs
+
+# Certs
+certs
+
+.coverage
+
+# Markdown
+*.md
+
+# Other
+*.yml

.gitignore

@@ -7,16 +7,21 @@ __pycache__
 # Except .
 !.github/
 !.gitignore
+!.dockerignore
+!.travis.yml
 
 # Ignore items in test (report, cache, etc), except files starting with test
 tests/* 
 !tests/test*
 !tests/test_certs
 !tests/pytest.ini
+!tests/passwd
+!tests/passwd_bad
 
 # Ignore logs/data/certs
 logs/*
 data/*
+!data/web*
 certs/*
 
 # Except README in logs/data/certs

Dockerfile

@@ -2,18 +2,16 @@ FROM python:3.7-alpine as base
 
 FROM base as builder
 
-RUN mkdir /install
+# add build utils (gcc, others)
-WORKDIR /install
+RUN apk add build-base
-
-COPY requirements.txt /requirements.txt
-
-RUN pip install --install-option="--prefix=/install" -r /requirements.txt
 
 FROM base
 
-COPY --from=builder /install /usr/local
 COPY . /bumper
 
 WORKDIR /bumper
 
-ENTRYPOINT ["python3", "-m", "bumper"]
+# install required python packages
+RUN pip3 install -r requirements.txt
+
+ENTRYPOINT ["python3", "-m", "bumper"]

bumper/__init__.py

@@ -321,6 +321,11 @@ def main(argv=None):
             first_run()
             return
 
+        if not (
+            os.path.exists(os.path.join(data_dir, "passwd"))
+        ):
+            with open(os.path.join(data_dir, "passwd"), 'w'): pass
+
         parser = argparse.ArgumentParser()
         parser.add_argument(
             "--listen", type=str, default=None, help="start serving on address"

bumper/confserver.py

@@ -6,6 +6,7 @@ import ssl
 import string
 import random
 import bumper
+import os
 from bumper.models import *
 from datetime import datetime, timedelta
 import asyncio
@@ -143,6 +144,7 @@ class ConfServer:
                     "/api/dim/devmanager.do", self.handle_dim_devmanager
                 ),  # EcoVacs Home
                 web.post("/lookup.do", self.handle_lookup),
+                web.get("/api/pim/file/get/{id}", self.handle_pimFile)
             ]
         )
         # Direct register from app:
@@ -202,10 +204,12 @@ class ConfServer:
             mq_sessions = []
             for sess in mqttserver._sessions:
                 tmpsess = []
-                tmpsess.append({"client_id": mqttserver._sessions[sess][0].client_id})
+                tmpsess.append({
-                tmpsess.append(
+                    "username": mqttserver._sessions[sess][0].username,
-                    {"state": mqttserver._sessions[sess][0].transitions.state}
+                    "client_id": mqttserver._sessions[sess][0].client_id,
-                )
+                    "state": mqttserver._sessions[sess][0].transitions.state,
+                })
+               
                 mq_sessions.append(tmpsess)
             all = {
                 "bots": bots,
@@ -559,7 +563,10 @@ class ConfServer:
                 bumper.user_add_device(tmpuser["userid"], user_devid)
 
             for bot in bots:  # Add all bots to the user
-                bumper.user_add_bot(tmpuser["userid"], bot["did"])
+                if "did" in bot:
+                    bumper.user_add_bot(tmpuser["userid"], bot["did"])
+                else:
+                    confserverlog.error("No DID for bot: {}".format(bot))
 
             if "checkLogin" in request.path:  # If request was to check a token do so
                 checkToken = self.check_token(
@@ -920,74 +927,7 @@ class ConfServer:
         try:
             body = {
                 "code": bumper.RETURN_API_SUCCESS,
-                "data": [
+                "data": EcoVacsHomeProducts,
-                    {
-                        "classid": "dl8fht",
-                        "product": {
-                            "_id": "5acb0fa87c295c0001876ecf",
-                            "name": "DEEBOT 600 Series",
-                            "icon": "5acc32067c295c0001876eea",
-                            "UILogicId": "dl8fht",
-                            "ota": False,
-                            "iconUrl": "https://portal-ww.ecouser.net/api/pim/file/get/5acc32067c295c0001876eea",
-                        },
-                    },
-                    {
-                        "classid": "02uwxm",
-                        "product": {
-                            "_id": "5ae1481e7ccd1a0001e1f69e",
-                            "name": "DEEBOT OZMO Slim10 Series",
-                            "icon": "5b1dddc48bc45700014035a1",
-                            "UILogicId": "02uwxm",
-                            "ota": False,
-                            "iconUrl": "https://portal-ww.ecouser.net/api/pim/file/get/5b1dddc48bc45700014035a1",
-                        },
-                    },
-                    {
-                        "classid": "y79a7u",
-                        "product": {
-                            "_id": "5b04c0227ccd1a0001e1f6a8",
-                            "name": "DEEBOT OZMO 900",
-                            "icon": "5b04c0217ccd1a0001e1f6a7",
-                            "UILogicId": "y79a7u",
-                            "ota": True,
-                            "iconUrl": "https://portal-ww.ecouser.net/api/pim/file/get/5b04c0217ccd1a0001e1f6a7",
-                        },
-                    },
-                    {
-                        "classid": "jr3pqa",
-                        "product": {
-                            "_id": "5b43077b8bc457000140363e",
-                            "name": "DEEBOT 711",
-                            "icon": "5b5ac4cc8d5a56000111e769",
-                            "UILogicId": "jr3pqa",
-                            "ota": True,
-                            "iconUrl": "https://portal-ww.ecouser.net/api/pim/file/get/5b5ac4cc8d5a56000111e769",
-                        },
-                    },
-                    {
-                        "classid": "uv242z",
-                        "product": {
-                            "_id": "5b5149b4ac0b87000148c128",
-                            "name": "DEEBOT 710",
-                            "icon": "5b5ac4e45f21100001882bb9",
-                            "UILogicId": "uv242z",
-                            "ota": True,
-                            "iconUrl": "https://portal-ww.ecouser.net/api/pim/file/get/5b5ac4e45f21100001882bb9",
-                        },
-                    },
-                    {
-                        "classid": "ls1ok3",
-                        "product": {
-                            "_id": "5b6561060506b100015c8868",
-                            "name": "DEEBOT 900 Series",
-                            "icon": "5ba4a2cb6c2f120001c32839",
-                            "UILogicId": "ls1ok3",
-                            "ota": True,
-                            "iconUrl": "https://portal-ww.ecouser.net/api/pim/file/get/5ba4a2cb6c2f120001c32839",
-                        },
-                    },
-                ],
             }
             return web.json_response(body)
 
@@ -1340,6 +1280,16 @@ class ConfServer:
         except Exception as e:
             confserverlog.exception("{}".format(e))
 
+
+    async def handle_pimFile(self, request):
+        try:
+            fileID = request.match_info.get("id", "")
+
+            return web.FileResponse(os.path.join(bumper.data_dir,"web","robotvac_image.jpg"))
+            
+        except Exception as e:
+            confserverlog.exception("{}".format(e))
+
     async def disconnect(self):
         try:
             confserverlog.info("shutting down")

bumper/db.py

@@ -309,7 +309,10 @@ def bot_toEcoVacsHome_JSON(bot):  # EcoVacs Home
 def bot_full_upsert(vacbot):
     bots = db_get().table("bots")
     Bot = Query()
-    bots.upsert(vacbot, Bot.did == vacbot["did"])
+    if "did" in vacbot:
+        bots.upsert(vacbot, Bot.did == vacbot["did"])
+    else:
+        bumperlog.error("No DID in vacbot: {}".format(vacbot))
 
 
 def bot_set_nick(did, nick):

bumper/mqttserver.py

@@ -13,6 +13,7 @@ import bumper
 import json
 from datetime import datetime, timedelta
 import bumper
+from passlib.apps import custom_app_context as pwd_context
 
 helperbotlog = logging.getLogger("helperbot")
 boterrorlog = logging.getLogger("boterror")
@@ -253,11 +254,11 @@ class MQTTServer:
                 },
                 "sys_interval": 10,
                 "auth": {
-                    "allow-anonymous": False,
+                    "allow-anonymous": False, # Set to True to allow anonymous authentication
                     "password-file": os.path.join(
-                        os.path.dirname(os.path.realpath(__file__)), "passwd"
+                        os.path.join(bumper.data_dir, "passwd")
-                    ),
+                    ), # For file auth, set user:hash in passwd file see (https://hbmqtt.readthedocs.io/en/latest/references/hbmqtt.html#configuration-example)
-                    "plugins": ["bumper"],  # No plugins == no auth
+                    "plugins": ["bumper"],  # Bumper plugin provides auth and handling of bots/clients connecting
                 },
                 "topic-check": {"enabled": False},
             }
@@ -271,6 +272,8 @@ class BumperMQTTServer_Plugin:
         self.context = context
         try:
             self.auth_config = self.context.config["auth"]
+            self._users = dict()
+            self._read_password_file()
 
         except KeyError:
             self.context.logger.warning(
@@ -280,6 +283,7 @@ class BumperMQTTServer_Plugin:
             mqttserverlog.exception("{}".format(e))
 
     async def authenticate(self, *args, **kwargs):
+        authenticated = False
         if not self.auth_config:
             # auth config section not found
             self.context.logger.warning(
@@ -287,19 +291,14 @@ class BumperMQTTServer_Plugin:
             )
             return False
 
-        allow_anonymous = self.auth_config.get(
+        
-            "allow-anonymous", True
+        try:
-        )  # allow anonymous by default
+            session = kwargs.get("session", None)
-        if allow_anonymous:
+            username = session.username
-            authenticated = True
+            password = session.password
-            self.context.logger.debug("Authentication success: config allows anonymous")
+            client_id = session.client_id
-        else:
-            try:
-                session = kwargs.get("session", None)
-                username = session.username
-                password = session.password
-                client_id = session.client_id
 
+            if "@" in client_id:
                 didsplit = str(client_id).split("@")
                 if not (  # if ecouser or bumper aren't in details it is a bot
                     "ecouser" in didsplit[1] or "bumper" in didsplit[1]
@@ -312,10 +311,7 @@ class BumperMQTTServer_Plugin:
                         tmpbotdetail[1],
                         "eco-ng",
                     )
-
+                    mqttserverlog.info(f"Bumper Authentication Success - Bot - SN: {username} - DID: {didsplit[0]} - Class: {tmpbotdetail[0]}")                    
-                    mqttserverlog.info(
-                        "bot authenticated SN: {} DID: {}".format(username, didsplit[0])
-                    )
                     authenticated = True
 
                 else:
@@ -325,6 +321,7 @@ class BumperMQTTServer_Plugin:
                     resource = tmpclientdetail[1]
 
                     if userid == "helper1":
+                        mqttserverlog.info(f"Bumper Authentication Success - Helperbot: {client_id}")
                         authenticated = True
                     else:
                         auth = False
@@ -335,20 +332,60 @@ class BumperMQTTServer_Plugin:
 
                         if auth:
                             bumper.client_add(userid, realm, resource)
-                            mqttserverlog.info("client authenticated {}".format(userid))
+                            mqttserverlog.info(f"Bumper Authentication Success - Client - Username: {username} - ClientID: {client_id}")
                             authenticated = True
 
                         else:
                             authenticated = False
 
-            except Exception as e:
+            # Check for File Auth            
-                mqttserverlog.exception(
+            if username and not authenticated: # If there is a username and it isn't already authenticated
-                    "Session: {} - {}".format((kwargs.get("session", None)), e)
+                hash = self._users.get(username, None)
-                )
+                if hash: # If there is a matching entry in passwd, check hash
-                authenticated = False
+                    authenticated = pwd_context.verify(password, hash)
+                    if authenticated:
+                        mqttserverlog.info(f"File Authentication Success - Username: {username} - ClientID: {client_id}")
+                    else:
+                        mqttserverlog.info(f"File Authentication Failed - Username: {username} - ClientID: {client_id}")
+                else:
+                    mqttserverlog.info(f"File Authentication Failed - No Entry for Username: {username} - ClientID: {client_id}")
+
+        except Exception as e:
+            mqttserverlog.exception(
+                "Session: {} - {}".format((kwargs.get("session", None)), e)
+            )
+            authenticated = False
+
+        # Check for allow anonymous
+        allow_anonymous = self.auth_config.get(
+            "allow-anonymous", True
+        )  
+        if allow_anonymous and not authenticated: # If anonymous auth is allowed and it isn't already authenticated
+            authenticated = True
+            self.context.logger.debug(f"Anonymous Authentication Success: config allows anonymous - Username: {username}")
+            mqttserverlog.info(f"Anonymous Authentication Success: config allows anonymous - Username: {username}")
 
         return authenticated
 
+    def _read_password_file(self):
+        password_file = self.auth_config.get('password-file', None)
+        if password_file:
+            try:
+                with open(password_file) as f:
+                    self.context.logger.debug(f"Reading user database from {password_file}")
+                    for l in f:
+                        line = l.strip()
+                        if not line.startswith('#'):    # Allow comments in files
+                            (username, pwd_hash) = line.split(sep=":", maxsplit=3)
+                            if username:
+                                self._users[username] = pwd_hash
+                                self.context.logger.debug(f"user: {username} - hash: {pwd_hash}")
+                self.context.logger.debug(f"{(len(self._users))} user(s) read from file {password_file}")
+            except FileNotFoundError:
+                self.context.logger.warning(f"Password file {password_file} not found")
+        else:
+            self.context.logger.debug("Configuration parameter 'password_file' not found")        
+
     async def on_broker_client_connected(self, client_id):
 
         didsplit = str(client_id).split("@")
@@ -358,7 +395,6 @@ class BumperMQTTServer_Plugin:
             bumper.bot_set_mqtt(bot["did"], True)
             return
 
-        # clientuserid = didsplit[0]
         clientresource = didsplit[1].split("/")[1]
         client = bumper.client_get(clientresource)
         if client:
@@ -374,7 +410,6 @@ class BumperMQTTServer_Plugin:
             bumper.bot_set_mqtt(bot["did"], False)
             return
 
-        # clientuserid = didsplit[0]
         clientresource = didsplit[1].split("/")[1]
         client = bumper.client_get(clientresource)
         if client:

docs/Use_With_App.md

@@ -52,6 +52,30 @@ Bumper *can* be used with the official "Ecovacs" or "Ecovacs Home" app, but with
 
 ### Importing the CA Cert on Android
 
+#### Android 7 and up (requires root)
+
+Android 7 introduced changes to the certificate mechanism. See [blog entry](https://android-developers.googleblog.com/2016/07/changes-to-trusted-certificate.html).
+
+> Apps that target API Level 24 and above no longer trust user or admin-added CAs for secure connections, by default.
+
+To circumvent this problem, the certificate must be added as a system storage.
+
+```bash
+openssl x509 -in ca.crt -noout -text >> ca.crt
+
+# restart adb as root
+adb root
+# remount /system rw
+adb remount
+# add certificate
+adb push ca.crt "/system/etc/security/cacerts/$(openssl x509 -inform PEM -subject_hash_old -in ca.crt | head -1).0"
+adb reboot
+```
+
+After reboot, verify that the certificate was added by checking `Settings > Security > Encryption & credentials > Trusted credentials > System`
+
+#### Android 6 and below
+
 1. Open the e-mail on your Android device
 
 **Quick Method**

tests/passwd

@@ -0,0 +1 @@
+test-client:$6$e9026a738b07b5a1$WaoYMI61aIPhhjfe3FG3uzV1oqyRdLi/TvLbBbvvzFyJ7T6PrileHGkzKkJUMLGQm/dhcq0fUT8mcu2kVcjbX/

tests/passwd_bad

@@ -0,0 +1 @@
+test-client:badhash

tests/test_confserver.py

@@ -133,7 +133,7 @@ async def test_login(aiohttp_client):
     assert "username" in jsonresp["data"]
 
     remove_existing_db()
-    bumper.db = "tests/tmp.db"  # Set db location for testing
+    bumper.db = "tests/tmp.db"  # Set db location for testing 
 
     # Test global_e without user
     resp = await client.get("/v1/private/us/en/dev_1234/global_e/1/0/0/user/login")
@@ -167,6 +167,25 @@ async def test_login(aiohttp_client):
     assert "uid" in jsonresp["data"]
     assert "username" in jsonresp["data"]
 
+    # Add a bot to db that doesn't have a did
+    newbot = {
+            "class": "dev_1234",
+            "company": "com_123",
+            #"did": self.did,
+            "name": "sn_1234",            
+            "resource": "res_1234",            
+    }
+    bumper.bot_full_upsert(newbot)
+
+    resp = await client.get("/v1/private/us/en/dev_1234/ios/1/0/0/user/login")
+    assert resp.status == 200
+    text = await resp.text()
+    jsonresp = json.loads(text)
+    assert jsonresp["code"] == bumper.RETURN_API_SUCCESS
+    assert "accessToken" in jsonresp["data"]
+    assert "uid" in jsonresp["data"]
+    assert "username" in jsonresp["data"]
+
 
 async def test_logout(aiohttp_client):
     remove_existing_db()
@@ -527,6 +546,11 @@ async def test_getProductIotMap(aiohttp_client):
     assert jsonresp["code"] == bumper.RETURN_API_SUCCESS
 
 
+    # Test getPimFile
+    resp = await client.get("/api/pim/file/get/123")
+    assert resp.status == 200
+    
+
 async def test_getUsersAPI(aiohttp_client):
     remove_existing_db()
     bumper.db = "tests/tmp.db"  # Set db location for testing
@@ -936,3 +960,5 @@ async def test_dim_devmanager(aiohttp_client):
     test_resp = json.loads(text)
     assert test_resp["ret"] == "fail"
 
+  
+

tests/test_mqttserver.py

@@ -379,6 +379,26 @@ async def test_mqttserver():
     mqtt_address = ("127.0.0.1", 8883)
 
     mqtt_server = bumper.MQTTServer(mqtt_address)
+
+    mqtt_server.default_config = {
+                "listeners": {
+                    "default": {"type": "tcp"},
+                    "tls1": {
+                        "bind": "{}:{}".format(mqtt_address[0], mqtt_address[1]),
+                        "ssl": "on",
+                        "certfile": bumper.server_cert,
+                        "keyfile": bumper.server_key,
+                    },
+                },
+                "sys_interval": 10,
+                "auth": {
+                    "allow-anonymous": True, # Set to True to allow anonymous authentication
+                    "password-file": "tests/passwd", # For file auth, set user:hash in passwd file see (https://hbmqtt.readthedocs.io/en/latest/references/hbmqtt.html#configuration-example)
+                    "plugins": ["bumper"],  # Bumper plugin provides auth and handling of bots/clients connecting
+                },
+                "topic-check": {"enabled": False},
+            }
+    
     await mqtt_server.broker_coro()
 
     # Test helperbot connect
@@ -420,6 +440,170 @@ async def test_mqttserver():
     )  # Check fake_bot is connected
     await fake_bot.Client.disconnect()
 
+    # Test file auth client connect
+    test_client = bumper.MQTTHelperBot(mqtt_address)
+    test_client.client_id = "test-file-auth"
+    # await test_client.start_helper_bot()
+    test_client.Client = hbmqtt.client.MQTTClient(
+        client_id=test_client.client_id, config={"check_hostname": False, "auto_reconnect": False, "reconnect_retries": 1}
+    )
+
+    # good user/pass
+    await test_client.Client.connect(
+        f"mqtts://test-client:abc123!@{test_client.address[0]}:{test_client.address[1]}/",
+        cafile=bumper.ca_cert, cleansession=True
+    )
+
+    assert (
+        test_client.Client._connected_state._value == True
+    )  # Check client is connected
+    await test_client.Client.disconnect()
+    assert (
+        test_client.Client._connected_state._value == False
+    )  # Check client is disconnected
+    
+    # bad password
+    try:
+        await test_client.Client.connect(
+            f"mqtts://test-client:notvalid!@{test_client.address[0]}:{test_client.address[1]}/",
+            cafile=bumper.ca_cert, cleansession=True
+        )
+
+        assert (
+            test_client.Client._connected_state._value == False
+        )  # Check client is connected
+
+    except Exception as ae:
+        pass
+
+    # no username in file
+    try:
+        await test_client.Client.connect(
+            f"mqtts://test-client-noexist:notvalid!@{test_client.address[0]}:{test_client.address[1]}/",
+            cafile=bumper.ca_cert, cleansession=True
+        )
+
+        assert (
+            test_client.Client._connected_state._value == False
+        )  # Check client is connected
+
+    except Exception as ae:
+        pass    
+        
+   
+
     await asyncio.sleep(0.1)
 
     await mqtt_server.broker.shutdown()
+    await asyncio.sleep(0.1)
+
+async def test_passwordfile_badhash_mqttserver():
+    mqtt_address = ("127.0.0.1", 8883)
+
+    mqtt_server = bumper.MQTTServer(mqtt_address)
+
+    mqtt_server.default_config = {
+                "listeners": {
+                    "default": {"type": "tcp"},
+                    "tls1": {
+                        "bind": "{}:{}".format(mqtt_address[0], mqtt_address[1]),
+                        "ssl": "on",
+                        "certfile": bumper.server_cert,
+                        "keyfile": bumper.server_key,
+                    },
+                },
+                "sys_interval": 10,
+                "auth": {
+                    "allow-anonymous": True, # Set to True to allow anonymous authentication
+                    "password-file": "tests/passwd_bad", # For file auth, set user:hash in passwd file see (https://hbmqtt.readthedocs.io/en/latest/references/hbmqtt.html#configuration-example)
+                    "plugins": ["bumper"],  # Bumper plugin provides auth and handling of bots/clients connecting
+                },
+                "topic-check": {"enabled": False},
+            }
+    
+    await mqtt_server.broker_coro()  
+    await asyncio.sleep(0.1)
+    
+    # bad password
+    try:
+        test_client = bumper.MQTTHelperBot(mqtt_address)
+
+        await test_client.Client.connect(
+            f"mqtts://test-client:notvalid!@{test_client.address[0]}:{test_client.address[1]}/",
+            cafile=bumper.ca_cert, cleansession=True
+        )
+
+        assert (
+            test_client.Client._connected_state._value == False
+        )  # Check client is connected
+   
+    
+    except Exception as ae:
+        pass
+    
+    await mqtt_server.broker.shutdown()
+    await asyncio.sleep(0.1)
+
+async def test_nofileauth_mqttserver():
+    try:
+        
+        mqtt_address = ("127.0.0.1", 8883)
+
+        mqtt_server = bumper.MQTTServer(mqtt_address)
+
+        mqtt_server.default_config = {
+                    "listeners": {
+                        "default": {"type": "tcp"},
+                        "tls1": {
+                            "bind": "{}:{}".format(mqtt_address[0], mqtt_address[1]),
+                            "ssl": "on",
+                            "certfile": bumper.server_cert,
+                            "keyfile": bumper.server_key,
+                        },
+                    },
+                    "sys_interval": 10,
+                    "auth": {
+                        "allow-anonymous": True, # Set to True to allow anonymous authentication
+                        "password-file": "tests/passwd-notfound", # For file auth, set user:hash in passwd file see (https://hbmqtt.readthedocs.io/en/latest/references/hbmqtt.html#configuration-example)
+                        "plugins": ["bumper"],  # Bumper plugin provides auth and handling of bots/clients connecting
+                    },
+                    "topic-check": {"enabled": False},
+                }
+        
+        await mqtt_server.broker_coro()
+
+    except:
+        pass
+    
+
+async def test_passwordfile_opt_missing_mqttserver():
+    try:
+    
+        mqtt_address = ("127.0.0.1", 8883)
+
+        mqtt_server = bumper.MQTTServer(mqtt_address)
+
+        mqtt_server.default_config = {
+                    "listeners": {
+                        "default": {"type": "tcp"},
+                        "tls1": {
+                            "bind": "{}:{}".format(mqtt_address[0], mqtt_address[1]),
+                            "ssl": "on",
+                            "certfile": bumper.server_cert,
+                            "keyfile": bumper.server_key,
+                        },
+                    },
+                    "sys_interval": 10,
+                    "auth": {
+                        "allow-anonymous": True, # Set to True to allow anonymous authentication
+                    
+                        "plugins": ["bumper"],  # Bumper plugin provides auth and handling of bots/clients connecting
+                    },
+                    "topic-check": {"enabled": False},
+                }
+        
+        await mqtt_server.broker_coro()
+        await mqtt_server.broker.shutdown()    
+    
+    except:
+        pass