diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..f4a3930 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,22 @@ +# Logs +logs + +# Tests +tests + +# Examples +examples + +# Docs +docs + +# Certs +certs + +.coverage + +# Markdown +*.md + +# Other +*.yml \ No newline at end of file diff --git a/.gitignore b/.gitignore index b3c8448..9644ac8 100644 --- a/.gitignore +++ b/.gitignore @@ -7,16 +7,21 @@ __pycache__ # Except . !.github/ !.gitignore +!.dockerignore +!.travis.yml # Ignore items in test (report, cache, etc), except files starting with test tests/* !tests/test* !tests/test_certs !tests/pytest.ini +!tests/passwd +!tests/passwd_bad # Ignore logs/data/certs logs/* data/* +!data/web* certs/* # Except README in logs/data/certs diff --git a/Dockerfile b/Dockerfile index 85e9ea5..604ddb2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,18 +2,16 @@ FROM python:3.7-alpine as base FROM base as builder -RUN mkdir /install -WORKDIR /install - -COPY requirements.txt /requirements.txt - -RUN pip install --install-option="--prefix=/install" -r /requirements.txt +# add build utils (gcc, others) +RUN apk add build-base FROM base -COPY --from=builder /install /usr/local COPY . /bumper WORKDIR /bumper -ENTRYPOINT ["python3", "-m", "bumper"] \ No newline at end of file +# install required python packages +RUN pip3 install -r requirements.txt + +ENTRYPOINT ["python3", "-m", "bumper"] diff --git a/bumper/__init__.py b/bumper/__init__.py index 4835994..f791f29 100644 --- a/bumper/__init__.py +++ b/bumper/__init__.py @@ -321,6 +321,11 @@ def main(argv=None): first_run() return + if not ( + os.path.exists(os.path.join(data_dir, "passwd")) + ): + with open(os.path.join(data_dir, "passwd"), 'w'): pass + parser = argparse.ArgumentParser() parser.add_argument( "--listen", type=str, default=None, help="start serving on address" diff --git a/bumper/confserver.py b/bumper/confserver.py index a9237e8..d09c4bb 100644 --- a/bumper/confserver.py +++ b/bumper/confserver.py @@ -6,6 +6,7 @@ import ssl import string import random import bumper +import os from bumper.models import * from datetime import datetime, timedelta import asyncio @@ -143,6 +144,7 @@ class ConfServer: "/api/dim/devmanager.do", self.handle_dim_devmanager ), # EcoVacs Home web.post("/lookup.do", self.handle_lookup), + web.get("/api/pim/file/get/{id}", self.handle_pimFile) ] ) # Direct register from app: @@ -202,10 +204,12 @@ class ConfServer: mq_sessions = [] for sess in mqttserver._sessions: tmpsess = [] - tmpsess.append({"client_id": mqttserver._sessions[sess][0].client_id}) - tmpsess.append( - {"state": mqttserver._sessions[sess][0].transitions.state} - ) + tmpsess.append({ + "username": mqttserver._sessions[sess][0].username, + "client_id": mqttserver._sessions[sess][0].client_id, + "state": mqttserver._sessions[sess][0].transitions.state, + }) + mq_sessions.append(tmpsess) all = { "bots": bots, @@ -559,7 +563,10 @@ class ConfServer: bumper.user_add_device(tmpuser["userid"], user_devid) for bot in bots: # Add all bots to the user - bumper.user_add_bot(tmpuser["userid"], bot["did"]) + if "did" in bot: + bumper.user_add_bot(tmpuser["userid"], bot["did"]) + else: + confserverlog.error("No DID for bot: {}".format(bot)) if "checkLogin" in request.path: # If request was to check a token do so checkToken = self.check_token( @@ -920,74 +927,7 @@ class ConfServer: try: body = { "code": bumper.RETURN_API_SUCCESS, - "data": [ - { - "classid": "dl8fht", - "product": { - "_id": "5acb0fa87c295c0001876ecf", - "name": "DEEBOT 600 Series", - "icon": "5acc32067c295c0001876eea", - "UILogicId": "dl8fht", - "ota": False, - "iconUrl": "https://portal-ww.ecouser.net/api/pim/file/get/5acc32067c295c0001876eea", - }, - }, - { - "classid": "02uwxm", - "product": { - "_id": "5ae1481e7ccd1a0001e1f69e", - "name": "DEEBOT OZMO Slim10 Series", - "icon": "5b1dddc48bc45700014035a1", - "UILogicId": "02uwxm", - "ota": False, - "iconUrl": "https://portal-ww.ecouser.net/api/pim/file/get/5b1dddc48bc45700014035a1", - }, - }, - { - "classid": "y79a7u", - "product": { - "_id": "5b04c0227ccd1a0001e1f6a8", - "name": "DEEBOT OZMO 900", - "icon": "5b04c0217ccd1a0001e1f6a7", - "UILogicId": "y79a7u", - "ota": True, - "iconUrl": "https://portal-ww.ecouser.net/api/pim/file/get/5b04c0217ccd1a0001e1f6a7", - }, - }, - { - "classid": "jr3pqa", - "product": { - "_id": "5b43077b8bc457000140363e", - "name": "DEEBOT 711", - "icon": "5b5ac4cc8d5a56000111e769", - "UILogicId": "jr3pqa", - "ota": True, - "iconUrl": "https://portal-ww.ecouser.net/api/pim/file/get/5b5ac4cc8d5a56000111e769", - }, - }, - { - "classid": "uv242z", - "product": { - "_id": "5b5149b4ac0b87000148c128", - "name": "DEEBOT 710", - "icon": "5b5ac4e45f21100001882bb9", - "UILogicId": "uv242z", - "ota": True, - "iconUrl": "https://portal-ww.ecouser.net/api/pim/file/get/5b5ac4e45f21100001882bb9", - }, - }, - { - "classid": "ls1ok3", - "product": { - "_id": "5b6561060506b100015c8868", - "name": "DEEBOT 900 Series", - "icon": "5ba4a2cb6c2f120001c32839", - "UILogicId": "ls1ok3", - "ota": True, - "iconUrl": "https://portal-ww.ecouser.net/api/pim/file/get/5ba4a2cb6c2f120001c32839", - }, - }, - ], + "data": EcoVacsHomeProducts, } return web.json_response(body) @@ -1340,6 +1280,16 @@ class ConfServer: except Exception as e: confserverlog.exception("{}".format(e)) + + async def handle_pimFile(self, request): + try: + fileID = request.match_info.get("id", "") + + return web.FileResponse(os.path.join(bumper.data_dir,"web","robotvac_image.jpg")) + + except Exception as e: + confserverlog.exception("{}".format(e)) + async def disconnect(self): try: confserverlog.info("shutting down") diff --git a/bumper/db.py b/bumper/db.py index 20181f9..52f6ba1 100644 --- a/bumper/db.py +++ b/bumper/db.py @@ -309,7 +309,10 @@ def bot_toEcoVacsHome_JSON(bot): # EcoVacs Home def bot_full_upsert(vacbot): bots = db_get().table("bots") Bot = Query() - bots.upsert(vacbot, Bot.did == vacbot["did"]) + if "did" in vacbot: + bots.upsert(vacbot, Bot.did == vacbot["did"]) + else: + bumperlog.error("No DID in vacbot: {}".format(vacbot)) def bot_set_nick(did, nick): diff --git a/bumper/mqttserver.py b/bumper/mqttserver.py index c7ff20e..f28a3f5 100644 --- a/bumper/mqttserver.py +++ b/bumper/mqttserver.py @@ -13,6 +13,7 @@ import bumper import json from datetime import datetime, timedelta import bumper +from passlib.apps import custom_app_context as pwd_context helperbotlog = logging.getLogger("helperbot") boterrorlog = logging.getLogger("boterror") @@ -253,11 +254,11 @@ class MQTTServer: }, "sys_interval": 10, "auth": { - "allow-anonymous": False, + "allow-anonymous": False, # Set to True to allow anonymous authentication "password-file": os.path.join( - os.path.dirname(os.path.realpath(__file__)), "passwd" - ), - "plugins": ["bumper"], # No plugins == no auth + os.path.join(bumper.data_dir, "passwd") + ), # For file auth, set user:hash in passwd file see (https://hbmqtt.readthedocs.io/en/latest/references/hbmqtt.html#configuration-example) + "plugins": ["bumper"], # Bumper plugin provides auth and handling of bots/clients connecting }, "topic-check": {"enabled": False}, } @@ -271,6 +272,8 @@ class BumperMQTTServer_Plugin: self.context = context try: self.auth_config = self.context.config["auth"] + self._users = dict() + self._read_password_file() except KeyError: self.context.logger.warning( @@ -280,6 +283,7 @@ class BumperMQTTServer_Plugin: mqttserverlog.exception("{}".format(e)) async def authenticate(self, *args, **kwargs): + authenticated = False if not self.auth_config: # auth config section not found self.context.logger.warning( @@ -287,19 +291,14 @@ class BumperMQTTServer_Plugin: ) return False - allow_anonymous = self.auth_config.get( - "allow-anonymous", True - ) # allow anonymous by default - if allow_anonymous: - authenticated = True - self.context.logger.debug("Authentication success: config allows anonymous") - else: - try: - session = kwargs.get("session", None) - username = session.username - password = session.password - client_id = session.client_id + + try: + session = kwargs.get("session", None) + username = session.username + password = session.password + client_id = session.client_id + if "@" in client_id: didsplit = str(client_id).split("@") if not ( # if ecouser or bumper aren't in details it is a bot "ecouser" in didsplit[1] or "bumper" in didsplit[1] @@ -312,10 +311,7 @@ class BumperMQTTServer_Plugin: tmpbotdetail[1], "eco-ng", ) - - mqttserverlog.info( - "bot authenticated SN: {} DID: {}".format(username, didsplit[0]) - ) + mqttserverlog.info(f"Bumper Authentication Success - Bot - SN: {username} - DID: {didsplit[0]} - Class: {tmpbotdetail[0]}") authenticated = True else: @@ -325,6 +321,7 @@ class BumperMQTTServer_Plugin: resource = tmpclientdetail[1] if userid == "helper1": + mqttserverlog.info(f"Bumper Authentication Success - Helperbot: {client_id}") authenticated = True else: auth = False @@ -335,20 +332,60 @@ class BumperMQTTServer_Plugin: if auth: bumper.client_add(userid, realm, resource) - mqttserverlog.info("client authenticated {}".format(userid)) + mqttserverlog.info(f"Bumper Authentication Success - Client - Username: {username} - ClientID: {client_id}") authenticated = True else: authenticated = False - except Exception as e: - mqttserverlog.exception( - "Session: {} - {}".format((kwargs.get("session", None)), e) - ) - authenticated = False + # Check for File Auth + if username and not authenticated: # If there is a username and it isn't already authenticated + hash = self._users.get(username, None) + if hash: # If there is a matching entry in passwd, check hash + authenticated = pwd_context.verify(password, hash) + if authenticated: + mqttserverlog.info(f"File Authentication Success - Username: {username} - ClientID: {client_id}") + else: + mqttserverlog.info(f"File Authentication Failed - Username: {username} - ClientID: {client_id}") + else: + mqttserverlog.info(f"File Authentication Failed - No Entry for Username: {username} - ClientID: {client_id}") + + except Exception as e: + mqttserverlog.exception( + "Session: {} - {}".format((kwargs.get("session", None)), e) + ) + authenticated = False + + # Check for allow anonymous + allow_anonymous = self.auth_config.get( + "allow-anonymous", True + ) + if allow_anonymous and not authenticated: # If anonymous auth is allowed and it isn't already authenticated + authenticated = True + self.context.logger.debug(f"Anonymous Authentication Success: config allows anonymous - Username: {username}") + mqttserverlog.info(f"Anonymous Authentication Success: config allows anonymous - Username: {username}") return authenticated + def _read_password_file(self): + password_file = self.auth_config.get('password-file', None) + if password_file: + try: + with open(password_file) as f: + self.context.logger.debug(f"Reading user database from {password_file}") + for l in f: + line = l.strip() + if not line.startswith('#'): # Allow comments in files + (username, pwd_hash) = line.split(sep=":", maxsplit=3) + if username: + self._users[username] = pwd_hash + self.context.logger.debug(f"user: {username} - hash: {pwd_hash}") + self.context.logger.debug(f"{(len(self._users))} user(s) read from file {password_file}") + except FileNotFoundError: + self.context.logger.warning(f"Password file {password_file} not found") + else: + self.context.logger.debug("Configuration parameter 'password_file' not found") + async def on_broker_client_connected(self, client_id): didsplit = str(client_id).split("@") @@ -358,7 +395,6 @@ class BumperMQTTServer_Plugin: bumper.bot_set_mqtt(bot["did"], True) return - # clientuserid = didsplit[0] clientresource = didsplit[1].split("/")[1] client = bumper.client_get(clientresource) if client: @@ -374,7 +410,6 @@ class BumperMQTTServer_Plugin: bumper.bot_set_mqtt(bot["did"], False) return - # clientuserid = didsplit[0] clientresource = didsplit[1].split("/")[1] client = bumper.client_get(clientresource) if client: diff --git a/data/web/robotvac_image.jpg b/data/web/robotvac_image.jpg new file mode 100644 index 0000000..124a15e Binary files /dev/null and b/data/web/robotvac_image.jpg differ diff --git a/docs/Use_With_App.md b/docs/Use_With_App.md index a83a287..85a3c94 100644 --- a/docs/Use_With_App.md +++ b/docs/Use_With_App.md @@ -52,6 +52,30 @@ Bumper *can* be used with the official "Ecovacs" or "Ecovacs Home" app, but with ### Importing the CA Cert on Android +#### Android 7 and up (requires root) + +Android 7 introduced changes to the certificate mechanism. See [blog entry](https://android-developers.googleblog.com/2016/07/changes-to-trusted-certificate.html). + +> Apps that target API Level 24 and above no longer trust user or admin-added CAs for secure connections, by default. + +To circumvent this problem, the certificate must be added as a system storage. + +```bash +openssl x509 -in ca.crt -noout -text >> ca.crt + +# restart adb as root +adb root +# remount /system rw +adb remount +# add certificate +adb push ca.crt "/system/etc/security/cacerts/$(openssl x509 -inform PEM -subject_hash_old -in ca.crt | head -1).0" +adb reboot +``` + +After reboot, verify that the certificate was added by checking `Settings > Security > Encryption & credentials > Trusted credentials > System` + +#### Android 6 and below + 1. Open the e-mail on your Android device **Quick Method** diff --git a/tests/passwd b/tests/passwd new file mode 100644 index 0000000..6975944 --- /dev/null +++ b/tests/passwd @@ -0,0 +1 @@ +test-client:$6$e9026a738b07b5a1$WaoYMI61aIPhhjfe3FG3uzV1oqyRdLi/TvLbBbvvzFyJ7T6PrileHGkzKkJUMLGQm/dhcq0fUT8mcu2kVcjbX/ \ No newline at end of file diff --git a/tests/passwd_bad b/tests/passwd_bad new file mode 100644 index 0000000..b739f39 --- /dev/null +++ b/tests/passwd_bad @@ -0,0 +1 @@ +test-client:badhash \ No newline at end of file diff --git a/tests/test_confserver.py b/tests/test_confserver.py index b98e2f8..6e35542 100644 --- a/tests/test_confserver.py +++ b/tests/test_confserver.py @@ -133,7 +133,7 @@ async def test_login(aiohttp_client): assert "username" in jsonresp["data"] remove_existing_db() - bumper.db = "tests/tmp.db" # Set db location for testing + bumper.db = "tests/tmp.db" # Set db location for testing # Test global_e without user resp = await client.get("/v1/private/us/en/dev_1234/global_e/1/0/0/user/login") @@ -167,6 +167,25 @@ async def test_login(aiohttp_client): assert "uid" in jsonresp["data"] assert "username" in jsonresp["data"] + # Add a bot to db that doesn't have a did + newbot = { + "class": "dev_1234", + "company": "com_123", + #"did": self.did, + "name": "sn_1234", + "resource": "res_1234", + } + bumper.bot_full_upsert(newbot) + + resp = await client.get("/v1/private/us/en/dev_1234/ios/1/0/0/user/login") + assert resp.status == 200 + text = await resp.text() + jsonresp = json.loads(text) + assert jsonresp["code"] == bumper.RETURN_API_SUCCESS + assert "accessToken" in jsonresp["data"] + assert "uid" in jsonresp["data"] + assert "username" in jsonresp["data"] + async def test_logout(aiohttp_client): remove_existing_db() @@ -527,6 +546,11 @@ async def test_getProductIotMap(aiohttp_client): assert jsonresp["code"] == bumper.RETURN_API_SUCCESS + # Test getPimFile + resp = await client.get("/api/pim/file/get/123") + assert resp.status == 200 + + async def test_getUsersAPI(aiohttp_client): remove_existing_db() bumper.db = "tests/tmp.db" # Set db location for testing @@ -936,3 +960,5 @@ async def test_dim_devmanager(aiohttp_client): test_resp = json.loads(text) assert test_resp["ret"] == "fail" + + diff --git a/tests/test_mqttserver.py b/tests/test_mqttserver.py index b4f4aa0..6cb9730 100644 --- a/tests/test_mqttserver.py +++ b/tests/test_mqttserver.py @@ -379,6 +379,26 @@ async def test_mqttserver(): mqtt_address = ("127.0.0.1", 8883) mqtt_server = bumper.MQTTServer(mqtt_address) + + mqtt_server.default_config = { + "listeners": { + "default": {"type": "tcp"}, + "tls1": { + "bind": "{}:{}".format(mqtt_address[0], mqtt_address[1]), + "ssl": "on", + "certfile": bumper.server_cert, + "keyfile": bumper.server_key, + }, + }, + "sys_interval": 10, + "auth": { + "allow-anonymous": True, # Set to True to allow anonymous authentication + "password-file": "tests/passwd", # For file auth, set user:hash in passwd file see (https://hbmqtt.readthedocs.io/en/latest/references/hbmqtt.html#configuration-example) + "plugins": ["bumper"], # Bumper plugin provides auth and handling of bots/clients connecting + }, + "topic-check": {"enabled": False}, + } + await mqtt_server.broker_coro() # Test helperbot connect @@ -420,6 +440,170 @@ async def test_mqttserver(): ) # Check fake_bot is connected await fake_bot.Client.disconnect() + # Test file auth client connect + test_client = bumper.MQTTHelperBot(mqtt_address) + test_client.client_id = "test-file-auth" + # await test_client.start_helper_bot() + test_client.Client = hbmqtt.client.MQTTClient( + client_id=test_client.client_id, config={"check_hostname": False, "auto_reconnect": False, "reconnect_retries": 1} + ) + + # good user/pass + await test_client.Client.connect( + f"mqtts://test-client:abc123!@{test_client.address[0]}:{test_client.address[1]}/", + cafile=bumper.ca_cert, cleansession=True + ) + + assert ( + test_client.Client._connected_state._value == True + ) # Check client is connected + await test_client.Client.disconnect() + assert ( + test_client.Client._connected_state._value == False + ) # Check client is disconnected + + # bad password + try: + await test_client.Client.connect( + f"mqtts://test-client:notvalid!@{test_client.address[0]}:{test_client.address[1]}/", + cafile=bumper.ca_cert, cleansession=True + ) + + assert ( + test_client.Client._connected_state._value == False + ) # Check client is connected + + except Exception as ae: + pass + + # no username in file + try: + await test_client.Client.connect( + f"mqtts://test-client-noexist:notvalid!@{test_client.address[0]}:{test_client.address[1]}/", + cafile=bumper.ca_cert, cleansession=True + ) + + assert ( + test_client.Client._connected_state._value == False + ) # Check client is connected + + except Exception as ae: + pass + + + await asyncio.sleep(0.1) await mqtt_server.broker.shutdown() + await asyncio.sleep(0.1) + +async def test_passwordfile_badhash_mqttserver(): + mqtt_address = ("127.0.0.1", 8883) + + mqtt_server = bumper.MQTTServer(mqtt_address) + + mqtt_server.default_config = { + "listeners": { + "default": {"type": "tcp"}, + "tls1": { + "bind": "{}:{}".format(mqtt_address[0], mqtt_address[1]), + "ssl": "on", + "certfile": bumper.server_cert, + "keyfile": bumper.server_key, + }, + }, + "sys_interval": 10, + "auth": { + "allow-anonymous": True, # Set to True to allow anonymous authentication + "password-file": "tests/passwd_bad", # For file auth, set user:hash in passwd file see (https://hbmqtt.readthedocs.io/en/latest/references/hbmqtt.html#configuration-example) + "plugins": ["bumper"], # Bumper plugin provides auth and handling of bots/clients connecting + }, + "topic-check": {"enabled": False}, + } + + await mqtt_server.broker_coro() + await asyncio.sleep(0.1) + + # bad password + try: + test_client = bumper.MQTTHelperBot(mqtt_address) + + await test_client.Client.connect( + f"mqtts://test-client:notvalid!@{test_client.address[0]}:{test_client.address[1]}/", + cafile=bumper.ca_cert, cleansession=True + ) + + assert ( + test_client.Client._connected_state._value == False + ) # Check client is connected + + + except Exception as ae: + pass + + await mqtt_server.broker.shutdown() + await asyncio.sleep(0.1) + +async def test_nofileauth_mqttserver(): + try: + + mqtt_address = ("127.0.0.1", 8883) + + mqtt_server = bumper.MQTTServer(mqtt_address) + + mqtt_server.default_config = { + "listeners": { + "default": {"type": "tcp"}, + "tls1": { + "bind": "{}:{}".format(mqtt_address[0], mqtt_address[1]), + "ssl": "on", + "certfile": bumper.server_cert, + "keyfile": bumper.server_key, + }, + }, + "sys_interval": 10, + "auth": { + "allow-anonymous": True, # Set to True to allow anonymous authentication + "password-file": "tests/passwd-notfound", # For file auth, set user:hash in passwd file see (https://hbmqtt.readthedocs.io/en/latest/references/hbmqtt.html#configuration-example) + "plugins": ["bumper"], # Bumper plugin provides auth and handling of bots/clients connecting + }, + "topic-check": {"enabled": False}, + } + + await mqtt_server.broker_coro() + + except: + pass + + +async def test_passwordfile_opt_missing_mqttserver(): + try: + + mqtt_address = ("127.0.0.1", 8883) + + mqtt_server = bumper.MQTTServer(mqtt_address) + + mqtt_server.default_config = { + "listeners": { + "default": {"type": "tcp"}, + "tls1": { + "bind": "{}:{}".format(mqtt_address[0], mqtt_address[1]), + "ssl": "on", + "certfile": bumper.server_cert, + "keyfile": bumper.server_key, + }, + }, + "sys_interval": 10, + "auth": { + "allow-anonymous": True, # Set to True to allow anonymous authentication + + "plugins": ["bumper"], # Bumper plugin provides auth and handling of bots/clients connecting + }, + "topic-check": {"enabled": False}, + } + + await mqtt_server.broker_coro() + await mqtt_server.broker.shutdown() + + except: + pass