From ea72078ff83d84744f60d427b1b6c83a1ac18b52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Fri, 28 Sep 2018 14:14:51 +0200 Subject: [PATCH 001/108] OV-7. Add the new component and separate logging handlers and formatters --- bin/logging.conf | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/bin/logging.conf b/bin/logging.conf index 8ef248d1..8c9bc224 100644 --- a/bin/logging.conf +++ b/bin/logging.conf @@ -3,12 +3,16 @@ #============================ formatters ====================================== [formatters] -keys=std,console +keys=std,console,benchmark [formatter_std] format=%(asctime)s [%(name)s:%(levelname)s] %(message)s datefmt= +[formatter_benchmark] +format=%(message)s +datefmt= + [formatter_console] format=%(asctime)s %(levelname)s %(message)s datefmt=%H:%M:%S @@ -16,7 +20,7 @@ datefmt=%H:%M:%S #============================ handlers ======================================== [handlers] -keys=std,console +keys=std,console,benchmark [handler_std] class=handlers.RotatingFileHandler @@ -29,10 +33,15 @@ class=StreamHandler args=() formatter=console +[handler_benchmark] +class=handlers.RotatingFileHandler +args=('%(logDir)s/networkEvent.log', 'a', 2000000, 5) +formatter=benchmark + #============================ loggers ========================================= [loggers] -keys=root,eventBusMonitor,openTun,openTunWindows,openTunLinux,eventBusClient,lbrClient,moteConnector,moteProbe,moteProbeUtils,moteState,openLbr,OpenParser,Parser,OpenHdlc,ParserData,ParserPrintf,ParserInfoErrorCritical,ParserStatus,RPL,SourceRoute,JRC,openVisualizerApp,openVisualizerGui,openVisualizerCli,openVisualizerWeb,OVtracer +keys=root,eventBusMonitor,openTun,openTunWindows,openTunLinux,eventBusClient,lbrClient,moteConnector,moteProbe,moteProbeUtils,moteState,openLbr,OpenParser,Parser,OpenHdlc,ParserData,ParserPrintf,ParserInfoErrorCritical,ParserStatus,RPL,SourceRoute,JRC,networkEventLogger,openVisualizerApp,openVisualizerGui,openVisualizerCli,openVisualizerWeb,OVtracer [logger_root] level=ERROR @@ -164,6 +173,12 @@ handlers=std propagate=0 qualname=JRC +[logger_networkEventLogger] +level=INFO +handlers=benchmark +propagate=0 +qualname=networkEventLogger + [logger_OpenHdlc] level=ERROR handlers=std From 005ee5f4ebeb8cb45a775d02caf3554e8f3cc0fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Mon, 1 Oct 2018 17:19:09 +0200 Subject: [PATCH 002/108] OV-7. Log network events as part of moteState class. Communicate mote's address when invoking different StateElem update handlers. --- openvisualizer/moteState/moteState.py | 85 +++++++++++++++++++++------ openvisualizer/openType/typeAsn.py | 14 ++++- 2 files changed, 80 insertions(+), 19 deletions(-) diff --git a/openvisualizer/moteState/moteState.py b/openvisualizer/moteState/moteState.py index c1d675bc..784f5c69 100644 --- a/openvisualizer/moteState/moteState.py +++ b/openvisualizer/moteState/moteState.py @@ -13,6 +13,11 @@ log.setLevel(logging.ERROR) log.addHandler(logging.NullHandler()) +# a special logger that writes to a separate file: each log line is a JSON string corresponding to network events +# with information sufficient to calculate network-wide KPIs +networkEventLogger = logging.getLogger('networkEventLogger') +networkEventLogger.setLevel(logging.INFO) + import copy import time import threading @@ -27,6 +32,8 @@ typeComponent, \ typeRssi +from openvisualizer import openvisualizer_utils as u + class OpenEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, (StateElem,openType.openType)): @@ -121,7 +128,8 @@ def _elemToDict(self,elem): class StateOutputBuffer(StateElem): - def update(self,notif): + def update(self,data): + (my64bID, notif) = data StateElem.update(self) if len(self.data)==0: self.data.append({}) @@ -130,7 +138,8 @@ def update(self,notif): class StateAsn(StateElem): - def update(self,notif): + def update(self,data): + (my64bID, notif) = data StateElem.update(self) if len(self.data)==0: self.data.append({}) @@ -141,19 +150,39 @@ def update(self,notif): notif.asn_4) class StateJoined(StateElem): - def update(self,notif): + def update(self,data): + (my64bID, notif) = data StateElem.update(self) if len(self.data)==0: self.data.append({}) if 'joinedAsn' not in self.data[0]: self.data[0]['joinedAsn'] = typeAsn.typeAsn() - self.data[0]['joinedAsn'].update(notif.joinedAsn_0_1, + + receivedJoinAsn = typeAsn.typeAsn() + receivedJoinAsn.update(notif.joinedAsn_0_1, notif.joinedAsn_2_3, notif.joinedAsn_4) + if self.data[0]['joinedAsn'] != receivedJoinAsn and receivedJoinAsn.asn != [0xff, 0xff, 0xff, 0xff, 0xff]: + + self.data[0]['joinedAsn'].update(notif.joinedAsn_0_1, + notif.joinedAsn_2_3, + notif.joinedAsn_4) + + networkEventLogger.info( + json.dumps( + { + "_type": "secjoin.joined", + "_mote_id": my64bID, + "_asn" : str(self.data[0]['joinedAsn']), + } + ) + ) + class StateMacStats(StateElem): - def update(self,notif): + def update(self,data): + (my64bID, notif) = data StateElem.update(self) if len(self.data)==0: self.data.append({}) @@ -170,7 +199,8 @@ def update(self,notif): class StateScheduleRow(StateElem): - def update(self,notif): + def update(self,data): + (my64bID, notif) = data StateElem.update(self) if len(self.data)==0: self.data.append({}) @@ -196,7 +226,8 @@ def update(self,notif): class StateBackoff(StateElem): - def update(self,notif): + def update(self,data): + (my64bID, notif) = data StateElem.update(self) if len(self.data)==0: self.data.append({}) @@ -225,7 +256,8 @@ def __init__(self): for i in range(20): self.data.append(StateQueueRow()) - def update(self,notif): + def update(self,data): + (my64bID, notif) = data StateElem.update(self) self.data[0].update(notif.creator_0,notif.owner_0) self.data[1].update(notif.creator_1,notif.owner_1) @@ -250,7 +282,8 @@ def update(self,notif): class StateNeighborsRow(StateElem): - def update(self,notif): + def update(self,data): + (my64bID, notif) = data StateElem.update(self) if len(self.data)==0: self.data.append({}) @@ -285,7 +318,8 @@ def update(self,notif): class StateIsSync(StateElem): - def update(self,notif): + def update(self,data): + (my64bID, notif) = data StateElem.update(self) if len(self.data)==0: self.data.append({}) @@ -304,8 +338,16 @@ def get16bAddr(self): return self.data[0]['my16bID'].addr[:] except IndexError: return None - - def update(self,notif): + + def get64bAddr(self): + try: + return self.data[0]['my64bID'].addr[:] + except IndexError: + return None + + def update(self,data): + + (my64bID, notif) = data # update state StateElem.update(self) @@ -376,7 +418,8 @@ def update(self,notif): class StateMyDagRank(StateElem): - def update(self,notif): + def update(self,data): + (my64bID, notif) = data StateElem.update(self) if len(self.data)==0: self.data.append({}) @@ -384,7 +427,8 @@ def update(self,notif): class StatekaPeriod(StateElem): - def update(self,notif): + def update(self,data): + (my64bID, notif) = data StateElem.update(self) if len(self.data)==0: self.data.append({}) @@ -399,11 +443,12 @@ def __init__(self,rowClass,columnOrder=None): self.meta[0]['columnOrder'] = columnOrder self.data = [] - def update(self,notif): + def update(self,data): + (my64bID, notif) = data StateElem.update(self) while len(self.data) Date: Tue, 2 Oct 2018 14:25:49 +0200 Subject: [PATCH 003/108] OV-7. When converting ASN object to string, handle None case. --- openvisualizer/openType/typeAsn.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openvisualizer/openType/typeAsn.py b/openvisualizer/openType/typeAsn.py index 76cc4cb7..1117433b 100644 --- a/openvisualizer/openType/typeAsn.py +++ b/openvisualizer/openType/typeAsn.py @@ -22,6 +22,8 @@ def __init__(self): self.asn = None def __str__(self): + if self.asn is None: + return '?' return '0x{0}'.format(''.join(["%.2x"%b for b in self.asn])) def __eq__(self, other): From 6154c220315c92ed7d8eac95bab0325304aaa192 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Tue, 2 Oct 2018 14:26:09 +0200 Subject: [PATCH 004/108] OV-7. Add a method to return cell type to typeCellType class. --- openvisualizer/openType/typeCellType.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/openvisualizer/openType/typeCellType.py b/openvisualizer/openType/typeCellType.py index e3ac5d01..87b641b1 100644 --- a/openvisualizer/openType/typeCellType.py +++ b/openvisualizer/openType/typeCellType.py @@ -48,6 +48,9 @@ def update(self,type): else: self.desc = 'unknown' self.addr = None + + def getCellType(self): + return self.desc #======================== private ========================================= \ No newline at end of file From 388fec0524ad2c6640253b94aba7e59a1ca6a2b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Tue, 2 Oct 2018 14:28:57 +0200 Subject: [PATCH 005/108] OV-7. Implement StateTable as an abstract class to facilitate logging. Log number of cells in the schedule of each mote, differentiated by type. --- openvisualizer/moteState/moteState.py | 78 ++++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 2 deletions(-) diff --git a/openvisualizer/moteState/moteState.py b/openvisualizer/moteState/moteState.py index 784f5c69..369e596b 100644 --- a/openvisualizer/moteState/moteState.py +++ b/openvisualizer/moteState/moteState.py @@ -434,6 +434,7 @@ def update(self,data): self.data.append({}) self.data[0]['kaPeriod'] = notif.kaPeriod +# abstract class class StateTable(StateElem): def __init__(self,rowClass,columnOrder=None): @@ -450,6 +451,79 @@ def update(self,data): self.data.append(self.meta[0]['rowClass']()) self.data[notif.row].update(data) + if notif.row + 1 == len(self.data): + self.log(my64bID) + + def log(self, id): + raise NotImplementedError + +class StateSchedule(StateTable): + + def __init__(self, *args, **kwargs): + super(StateSchedule, self).__init__(*args, **kwargs) + + def log(self, id): + + try: + numCellsTx = sum(row.getType().getCellType() == "TX" for row in self.data) + numCellsRx = sum(row.getType().getCellType() == "RX" for row in self.data) + numCellsTxRx = sum(row.getType().getCellType() == "TXRX" for row in self.data) + numCells = numCellsTx + numCellsRx + numCellsTxRx + + networkEventLogger.info( + json.dumps( + { + "numCellsTx": str(numCellsTx), + "_type": "tsch.numCellsTx", + "_mote_id": id, + "_timestamp": self.meta[0]['lastUpdated'], + } + ) + ) + + networkEventLogger.info( + json.dumps( + { + "numCellsRx": str(numCellsRx), + "_type": "tsch.numCellsRx", + "_mote_id": id, + "_timestamp": self.meta[0]['lastUpdated'], + } + ) + ) + + networkEventLogger.info( + json.dumps( + { + "numCellsTxRx": str(numCellsTxRx), + "_type": "tsch.numCellsTxRx", + "_mote_id": id, + "_timestamp": self.meta[0]['lastUpdated'], + } + ) + ) + + networkEventLogger.info( + json.dumps( + { + "numCells": str(numCells), + "_type": "tsch.numCells", + "_mote_id": id, + "_timestamp": self.meta[0]['lastUpdated'], + } + ) + ) + except: + pass + + +class StateNeighbors(StateTable): + def __init__(self, *args, **kwargs): + super(StateNeighbors, self).__init__(*args, **kwargs) + + def log(self,id): + pass + class moteState(eventBusClient.eventBusClient): ST_OUPUTBUFFER = 'OutputBuffer' @@ -551,7 +625,7 @@ def __init__(self,moteConnector): self.state[self.ST_ASN] = StateAsn() self.state[self.ST_JOINED] = StateJoined() self.state[self.ST_MACSTATS] = StateMacStats() - self.state[self.ST_SCHEDULE] = StateTable( + self.state[self.ST_SCHEDULE] = StateSchedule( StateScheduleRow, columnOrder = '.'.join( [ @@ -569,7 +643,7 @@ def __init__(self,moteConnector): ) self.state[self.ST_BACKOFF] = StateBackoff() self.state[self.ST_QUEUE] = StateQueue() - self.state[self.ST_NEIGHBORS] = StateTable( + self.state[self.ST_NEIGHBORS] = StateNeighbors( StateNeighborsRow, columnOrder = '.'.join( [ From ed241b777f0dfae027009dfe7c4badd16dabbee9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Tue, 2 Oct 2018 14:30:28 +0200 Subject: [PATCH 006/108] OV-7. Log additional metrics. This commit adds the support for logging of new metrics: - duty cycle - number of transmissions - number of receptions - number of ACKed transmissions --- openvisualizer/moteState/moteState.py | 97 +++++++++++++++++++++++---- 1 file changed, 83 insertions(+), 14 deletions(-) diff --git a/openvisualizer/moteState/moteState.py b/openvisualizer/moteState/moteState.py index 369e596b..b5a24b9d 100644 --- a/openvisualizer/moteState/moteState.py +++ b/openvisualizer/moteState/moteState.py @@ -163,21 +163,25 @@ def update(self,data): notif.joinedAsn_2_3, notif.joinedAsn_4) - if self.data[0]['joinedAsn'] != receivedJoinAsn and receivedJoinAsn.asn != [0xff, 0xff, 0xff, 0xff, 0xff]: - - self.data[0]['joinedAsn'].update(notif.joinedAsn_0_1, - notif.joinedAsn_2_3, - notif.joinedAsn_4) - - networkEventLogger.info( - json.dumps( - { - "_type": "secjoin.joined", - "_mote_id": my64bID, - "_asn" : str(self.data[0]['joinedAsn']), - } + try: + if self.data[0]['joinedAsn'] != receivedJoinAsn and receivedJoinAsn.asn != [0x00, 0x00, 0x00, 0x00, 0x00]: + + self.data[0]['joinedAsn'].update(notif.joinedAsn_0_1, + notif.joinedAsn_2_3, + notif.joinedAsn_4) + + networkEventLogger.info( + json.dumps( + { + "asn" : str(self.data[0]['joinedAsn']), + "_type": "secjoin.joined", + "_mote_id": my64bID, + "_timestamp": self.meta[0]['lastUpdated'], + } + ) ) - ) + except: + pass class StateMacStats(StateElem): @@ -197,6 +201,20 @@ def update(self,data): else: self.data[0]['dutyCycle'] = '?' + try: + networkEventLogger.info( + json.dumps( + { + "dutycycle": str(self.data[0]['dutyCycle']), + "_type": "tsch.dutycycle", + "_mote_id": my64bID, + "_timestamp": self.meta[0]['lastUpdated'], + } + ) + ) + except: + pass + class StateScheduleRow(StateElem): def update(self,data): @@ -224,6 +242,57 @@ def update(self,data): notif.lastUsedAsn_2_3, notif.lastUsedAsn_4) + try: + networkEventLogger.info( + json.dumps( + { + "numTx": str(self.data[0]['numTx']), + "_type": "tsch.numTx", + "_mote_id": my64bID, + "_timestamp": self.meta[0]['lastUpdated'], + } + ) + ) + + networkEventLogger.info( + json.dumps( + { + "numRx": str(self.data[0]['numRx']), + "_type": "tsch.numRx", + "_mote_id": my64bID, + "_timestamp": self.meta[0]['lastUpdated'], + } + ) + ) + + networkEventLogger.info( + json.dumps( + { + "numTxAck": str(self.data[0]['numTxAck']), + "_type": "tsch.numTxAck", + "_mote_id": my64bID, + "_timestamp": self.meta[0]['lastUpdated'], + } + ) + ) + + networkEventLogger.info( + json.dumps( + { + "numTxAck": str(self.data[0]['numTxAck']), + "_type": "tsch.numTxAck", + "_mote_id": my64bID, + "_timestamp": self.meta[0]['lastUpdated'], + } + ) + ) + + except: + pass + + def getType(self): + return self.data[0]['type'] + class StateBackoff(StateElem): def update(self,data): From 82557ec142973703290f6adb402c5273e37a2e32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Sat, 6 Oct 2018 20:28:03 +0200 Subject: [PATCH 007/108] OV-7. Add hostname to the serial port name of a opentestbed mote. --- openvisualizer/moteProbe/moteProbe.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/openvisualizer/moteProbe/moteProbe.py b/openvisualizer/moteProbe/moteProbe.py index da7231cf..10fa1514 100644 --- a/openvisualizer/moteProbe/moteProbe.py +++ b/openvisualizer/moteProbe/moteProbe.py @@ -146,10 +146,19 @@ def _on_mqtt_message(self, client, userdata, message): # get the motes list from payload payload_status = json.loads(message.payload) + + try: + host = payload_status['returnVal']['host_name'] + except KeyError: + host = payload_status['returnVal']['IP_address'] + except: + host = '' for mote in payload_status['returnVal']['motes']: if 'EUI64' in mote: - self.opentestbed_motelist.add(mote['EUI64']) + self.opentestbed_motelist.add( + (host, mote['EUI64']) + ) class moteProbe(threading.Thread): @@ -163,7 +172,7 @@ class moteProbe(threading.Thread): MODE_IOTLAB, MODE_TESTBED, ] - + XOFF = 0x13 XON = 0x11 XONXOFF_ESCAPE = 0x12 @@ -172,25 +181,25 @@ class moteProbe(threading.Thread): # XON is transmitted as [XONXOFF_ESCAPE, XON^XONXOFF_MASK]==[0x12,0x11^0x10]==[0x12,0x01] # XONXOFF_ESCAPE is transmitted as [XONXOFF_ESCAPE, XONXOFF_ESCAPE^XONXOFF_MASK]==[0x12,0x12^0x10]==[0x12,0x02] - def __init__(self,mqtt_broker_address,serialport=None,emulatedMote=None,iotlabmote=None,testbedmote_eui64=None): + def __init__(self,mqtt_broker_address,serialport=None,emulatedMote=None,iotlabmote=None,testbedmote=None): # verify params if serialport: assert not emulatedMote assert not iotlabmote - assert not testbedmote_eui64 + assert not testbedmote self.mode = self.MODE_SERIAL elif emulatedMote: assert not serialport assert not iotlabmote - assert not testbedmote_eui64 + assert not testbedmote self.mode = self.MODE_EMULATED elif iotlabmote: assert not serialport assert not emulatedMote - assert not testbedmote_eui64 + assert not testbedmote self.mode = self.MODE_IOTLAB - elif testbedmote_eui64: + elif testbedmote: assert not serialport assert not emulatedMote assert not iotlabmote @@ -210,8 +219,8 @@ def __init__(self,mqtt_broker_address,serialport=None,emulatedMote=None,iotlabmo self.iotlabmote = iotlabmote self.portname = 'IoT-LAB{0}'.format(iotlabmote) elif self.mode==self.MODE_TESTBED: - self.testbedmote_eui64 = testbedmote_eui64 - self.portname = 'opentestbed_{0}'.format(testbedmote_eui64) + (self.testbed_host, self.testbedmote_eui64) = testbedmote + self.portname = 'opentestbed_{0}_{1}'.format(self.testbed_host, self.testbedmote_eui64) else: raise SystemError() # at this moment, MQTT broker is used even if the mode is not From f30ae75c99a8ee207fb5576793925e3f1b0d3c32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Sat, 6 Oct 2018 20:28:33 +0200 Subject: [PATCH 008/108] OV-7. Log mote_info object encompassing EUI-64 and serial port name. --- openvisualizer/moteState/moteState.py | 69 ++++++++++++++++----------- 1 file changed, 40 insertions(+), 29 deletions(-) diff --git a/openvisualizer/moteState/moteState.py b/openvisualizer/moteState/moteState.py index b5a24b9d..2c5e20aa 100644 --- a/openvisualizer/moteState/moteState.py +++ b/openvisualizer/moteState/moteState.py @@ -129,7 +129,7 @@ def _elemToDict(self,elem): class StateOutputBuffer(StateElem): def update(self,data): - (my64bID, notif) = data + (moteInfo, notif) = data StateElem.update(self) if len(self.data)==0: self.data.append({}) @@ -139,7 +139,7 @@ def update(self,data): class StateAsn(StateElem): def update(self,data): - (my64bID, notif) = data + (moteInfo, notif) = data StateElem.update(self) if len(self.data)==0: self.data.append({}) @@ -151,7 +151,7 @@ def update(self,data): class StateJoined(StateElem): def update(self,data): - (my64bID, notif) = data + (moteInfo, notif) = data StateElem.update(self) if len(self.data)==0: self.data.append({}) @@ -175,7 +175,7 @@ def update(self,data): { "asn" : str(self.data[0]['joinedAsn']), "_type": "secjoin.joined", - "_mote_id": my64bID, + "_mote_info": moteInfo, "_timestamp": self.meta[0]['lastUpdated'], } ) @@ -186,7 +186,7 @@ def update(self,data): class StateMacStats(StateElem): def update(self,data): - (my64bID, notif) = data + (moteInfo, notif) = data StateElem.update(self) if len(self.data)==0: self.data.append({}) @@ -207,7 +207,7 @@ def update(self,data): { "dutycycle": str(self.data[0]['dutyCycle']), "_type": "tsch.dutycycle", - "_mote_id": my64bID, + "_mote_info": moteInfo, "_timestamp": self.meta[0]['lastUpdated'], } ) @@ -218,7 +218,7 @@ def update(self,data): class StateScheduleRow(StateElem): def update(self,data): - (my64bID, notif) = data + (moteInfo, notif) = data StateElem.update(self) if len(self.data)==0: self.data.append({}) @@ -248,7 +248,7 @@ def update(self,data): { "numTx": str(self.data[0]['numTx']), "_type": "tsch.numTx", - "_mote_id": my64bID, + "_mote_info": moteInfo, "_timestamp": self.meta[0]['lastUpdated'], } ) @@ -259,7 +259,7 @@ def update(self,data): { "numRx": str(self.data[0]['numRx']), "_type": "tsch.numRx", - "_mote_id": my64bID, + "_mote_info": moteInfo, "_timestamp": self.meta[0]['lastUpdated'], } ) @@ -270,7 +270,7 @@ def update(self,data): { "numTxAck": str(self.data[0]['numTxAck']), "_type": "tsch.numTxAck", - "_mote_id": my64bID, + "_mote_info": moteInfo, "_timestamp": self.meta[0]['lastUpdated'], } ) @@ -281,7 +281,7 @@ def update(self,data): { "numTxAck": str(self.data[0]['numTxAck']), "_type": "tsch.numTxAck", - "_mote_id": my64bID, + "_mote_info": moteInfo, "_timestamp": self.meta[0]['lastUpdated'], } ) @@ -296,7 +296,7 @@ def getType(self): class StateBackoff(StateElem): def update(self,data): - (my64bID, notif) = data + (moteInfo, notif) = data StateElem.update(self) if len(self.data)==0: self.data.append({}) @@ -326,7 +326,7 @@ def __init__(self): self.data.append(StateQueueRow()) def update(self,data): - (my64bID, notif) = data + (moteInfo, notif) = data StateElem.update(self) self.data[0].update(notif.creator_0,notif.owner_0) self.data[1].update(notif.creator_1,notif.owner_1) @@ -352,7 +352,7 @@ def update(self,data): class StateNeighborsRow(StateElem): def update(self,data): - (my64bID, notif) = data + (moteInfo, notif) = data StateElem.update(self) if len(self.data)==0: self.data.append({}) @@ -388,7 +388,7 @@ def update(self,data): class StateIsSync(StateElem): def update(self,data): - (my64bID, notif) = data + (moteInfo, notif) = data StateElem.update(self) if len(self.data)==0: self.data.append({}) @@ -414,9 +414,20 @@ def get64bAddr(self): except IndexError: return None + def get_serial(self): + return self.moteConnector.serialport + + def get_info(self): + return { + '64bAddr' : u.formatAddr(self.get64bAddr()), + '16bAddr' : u.formatAddr(self.get16bAddr()), + 'isDAGroot' : self.isDAGroot, + 'serial' : self.get_serial(), + } + def update(self,data): - (my64bID, notif) = data + (moteInfo, notif) = data # update state StateElem.update(self) @@ -488,7 +499,7 @@ def update(self,data): class StateMyDagRank(StateElem): def update(self,data): - (my64bID, notif) = data + (moteInfo, notif) = data StateElem.update(self) if len(self.data)==0: self.data.append({}) @@ -497,7 +508,7 @@ def update(self,data): class StatekaPeriod(StateElem): def update(self,data): - (my64bID, notif) = data + (moteInfo, notif) = data StateElem.update(self) if len(self.data)==0: self.data.append({}) @@ -514,14 +525,14 @@ def __init__(self,rowClass,columnOrder=None): self.data = [] def update(self,data): - (my64bID, notif) = data + (moteInfo, notif) = data StateElem.update(self) while len(self.data) Date: Wed, 6 Feb 2019 16:03:57 +0100 Subject: [PATCH 009/108] OV-7. REL-1.2.2 OpenTestbed API implementation and generic MQTT broker argument. --- bin/openVisualizerApp.py | 39 ++++++++++++++------------- openvisualizer/moteProbe/moteProbe.py | 19 ++++++------- 2 files changed, 31 insertions(+), 27 deletions(-) diff --git a/bin/openVisualizerApp.py b/bin/openVisualizerApp.py index 301c9deb..5ba5072f 100644 --- a/bin/openVisualizerApp.py +++ b/bin/openVisualizerApp.py @@ -38,7 +38,7 @@ class OpenVisualizerApp(object): top-level functionality for several UI clients. ''' - def __init__(self,confdir,datadir,logdir,simulatorMode,numMotes,trace,debug,usePageZero,simTopology,iotlabmotes, testbedmotes, pathTopo, mqtt_broker_address, opentun_null): + def __init__(self,confdir,datadir,logdir,simulatorMode,numMotes,trace,debug,usePageZero,simTopology,iotlabmotes,testbed,pathTopo,mqtt_broker_address,opentun_null): # store params self.confdir = confdir @@ -50,7 +50,7 @@ def __init__(self,confdir,datadir,logdir,simulatorMode,numMotes,trace,debug,useP self.debug = debug self.usePageZero = usePageZero self.iotlabmotes = iotlabmotes - self.testbedmotes = testbedmotes + self.testbed = testbed self.pathTopo = pathTopo # local variables @@ -79,9 +79,9 @@ def __init__(self,confdir,datadir,logdir,simulatorMode,numMotes,trace,debug,useP app.close() os.kill(os.getpid(), signal.SIGTERM) - # create a moteProbe for each mote if self.simulatorMode: + self.testEnvironment = 'simulation' # in "simulator" mode, motes are emulated sys.path.append(os.path.join(self.datadir, 'sim_files')) import oos_openwsn @@ -93,19 +93,22 @@ def __init__(self,confdir,datadir,logdir,simulatorMode,numMotes,trace,debug,useP self.simengine.indicateNewMote(moteHandler) self.moteProbes += [moteProbe.moteProbe(mqtt_broker_address, emulatedMote=moteHandler)] elif self.iotlabmotes: + self.testEnvironment = 'iotlab-tcp' # in "IoT-LAB" mode, motes are connected to TCP ports self.moteProbes = [ moteProbe.moteProbe(mqtt_broker_address, iotlabmote=p) for p in self.iotlabmotes.split(',') ] - elif self.testbedmotes: - motesfinder = moteProbe.OpentestbedMoteFinder(mqtt_broker_address) + elif self.testbed: + self.testEnvironment = self.testbed + motesfinder = moteProbe.OpentestbedMoteFinder(testbed=self.testbed, mqttBroker=self.mqtt_broker_address) self.moteProbes = [ moteProbe.moteProbe(mqtt_broker_address, testbedmote_eui64=p) for p in motesfinder.get_opentestbed_motelist() ] else: + self.testEnvironment = 'local' # in "hardware" mode, motes are connected to the serial port self.moteProbes = [ @@ -342,14 +345,13 @@ def main(parser=None): argspace.numMotes = DEFAULT_MOTE_COUNT log.info('Initializing OpenVisualizerApp with options:\n\t{0}'.format( - '\n '.join(['appdir = {0}'.format(argspace.appdir), - 'sim = {0}'.format(argspace.simulatorMode), - 'simCount = {0}'.format(argspace.numMotes), - 'trace = {0}'.format(argspace.trace), - 'debug = {0}'.format(argspace.debug), - 'testbedmotes= {0}'.format(argspace.testbedmotes), - - 'usePageZero = {0}'.format(argspace.usePageZero)], + '\n '.join(['appdir = {0}'.format(argspace.appdir), + 'sim = {0}'.format(argspace.simulatorMode), + 'simCount = {0}'.format(argspace.numMotes), + 'trace = {0}'.format(argspace.trace), + 'debug = {0}'.format(argspace.debug), + 'testbed = {0}'.format(argspace.testbed), + 'usePageZero = {0}'.format(argspace.usePageZero)], ))) log.info('Using external dirs:\n\t{0}'.format( '\n '.join(['conf = {0}'.format(confdir), @@ -369,7 +371,7 @@ def main(parser=None): usePageZero = argspace.usePageZero, simTopology = argspace.simTopology, iotlabmotes = argspace.iotlabmotes, - testbedmotes = argspace.testbedmotes, + testbed = argspace.testbed, pathTopo = argspace.pathTopo, mqtt_broker_address = argspace.mqtt_broker_address, opentun_null = argspace.opentun_null @@ -424,11 +426,12 @@ def _addParserArgs(parser): action = 'store', help = 'comma-separated list of IoT-LAB motes (e.g. "wsn430-9,wsn430-34,wsn430-3")' ) - parser.add_argument('-tb', '--opentestbed', - dest = 'testbedmotes', + parser.add_argument('-tb', '--testbed', + dest = 'testbed', default = False, - action = 'store_true', - help = 'connect motes from opentestbed' + choices = ['opentestbed', 'iotlab', 'wilab'], + action = 'store', + help = 'connect remote motes from a --testbed over OpenTestbed serial-MQTT bridge. uses --mqttBroker' ) parser.add_argument('--mqtt-broker-address', dest = 'mqtt_broker_address', diff --git a/openvisualizer/moteProbe/moteProbe.py b/openvisualizer/moteProbe/moteProbe.py index 10fa1514..1b9b2766 100644 --- a/openvisualizer/moteProbe/moteProbe.py +++ b/openvisualizer/moteProbe/moteProbe.py @@ -103,9 +103,10 @@ class OpentestbedMoteFinder (object): OPENTESTBED_RESP_STATUS_TIMEOUT = 10 - def __init__(self, mqtt_broker_address): - self.opentestbed_motelist = set() + def __init__(self, testbed, mqtt_broker_address): + self.testbed = testbed self.mqtt_broker_address = mqtt_broker_address + self.opentestbed_motelist = set() def get_opentestbed_motelist(self): @@ -130,14 +131,14 @@ def _on_mqtt_connect(self, client, userdata, flags, rc): print "connected to : {0}".format(self.mqtt_broker_address) - client.subscribe('opentestbed/deviceType/box/deviceId/+/resp/status') + client.subscribe('{0}/deviceType/box/deviceId/+/resp/status'.format(self.testbed)) payload_status = { 'token': 123, } # publish the cmd message client.publish( - topic = 'opentestbed/deviceType/box/deviceId/all/cmd/status', + topic = '{0}/deviceType/box/deviceId/all/cmd/status'.format(self.testbed), payload = json.dumps(payload_status), ) @@ -157,7 +158,7 @@ def _on_mqtt_message(self, client, userdata, message): for mote in payload_status['returnVal']['motes']: if 'EUI64' in mote: self.opentestbed_motelist.add( - (host, mote['EUI64']) + (host, mote['EUI64'], self.testbed, self.mqtt_broker_address) ) class moteProbe(threading.Thread): @@ -219,8 +220,8 @@ def __init__(self,mqtt_broker_address,serialport=None,emulatedMote=None,iotlabmo self.iotlabmote = iotlabmote self.portname = 'IoT-LAB{0}'.format(iotlabmote) elif self.mode==self.MODE_TESTBED: - (self.testbed_host, self.testbedmote_eui64) = testbedmote - self.portname = 'opentestbed_{0}_{1}'.format(self.testbed_host, self.testbedmote_eui64) + (self.testbed_host, self.testbedmote_eui64, self.testbed, self.mqtt_broker_address) = testbedmote + self.portname = 'testbed_{0}_{1}_{2}'.format(self.testbed, self.testbed_host, self.testbedmote_eui64) else: raise SystemError() # at this moment, MQTT broker is used even if the mode is not @@ -418,7 +419,7 @@ def _sendData(self,data): payload_buffer['serialbytes'] = [ord(i) for i in hdlcData] # publish the cmd message self.mqttclient.publish( - topic = 'opentestbed/deviceType/mote/deviceId/{0}/cmd/tomoteserialbytes'.format(self.testbedmote_eui64), + topic = '{0}/deviceType/mote/deviceId/{1}/cmd/tomoteserialbytes'.format(self.testbed, self.testbedmote_eui64), payload = json.dumps(payload_buffer), ) else: @@ -429,7 +430,7 @@ def _sendData(self,data): def _on_mqtt_connect(self, client, userdata, flags, rc): - client.subscribe('opentestbed/deviceType/mote/deviceId/{0}/notif/frommoteserialbytes'.format(self.testbedmote_eui64)) + client.subscribe('{0}/deviceType/mote/deviceId/{1}/notif/frommoteserialbytes'.format(self.testbed, self.testbedmote_eui64)) def _on_mqtt_message(self, client, userdata, message): From 7610d6a3ad4a2be3b62a3ce44254f847143609ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Wed, 6 Feb 2019 16:04:50 +0100 Subject: [PATCH 010/108] OV-7. Add separate component to implement OpenBenchmark APIs. --- bin/openVisualizerApp.py | 28 +++++++++++++-- openvisualizer/moteState/moteState.py | 4 --- openvisualizer/openBenchmarkAgent/__init__.py | 0 .../openBenchmarkAgent/openBenchmarkAgent.py | 34 +++++++++++++++++++ 4 files changed, 59 insertions(+), 7 deletions(-) create mode 100644 openvisualizer/openBenchmarkAgent/__init__.py create mode 100644 openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py diff --git a/bin/openVisualizerApp.py b/bin/openVisualizerApp.py index 5ba5072f..a6542b3b 100644 --- a/bin/openVisualizerApp.py +++ b/bin/openVisualizerApp.py @@ -24,6 +24,7 @@ from openvisualizer.moteState import moteState from openvisualizer.RPL import RPL from openvisualizer.JRC import JRC +from openvisualizer.openBenchmarkAgent import openBenchmarkAgent from openvisualizer.openLbr import openLbr from openvisualizer.openTun import openTun from openvisualizer.RPL import topology @@ -38,7 +39,7 @@ class OpenVisualizerApp(object): top-level functionality for several UI clients. ''' - def __init__(self,confdir,datadir,logdir,simulatorMode,numMotes,trace,debug,usePageZero,simTopology,iotlabmotes,testbed,pathTopo,mqtt_broker_address,opentun_null): + def __init__(self,confdir,datadir,logdir,simulatorMode,numMotes,trace,debug,usePageZero,simTopology,iotlabmotes,testbed,benchmark,pathTopo,mqtt_broker_address,opentun_null): # store params self.confdir = confdir @@ -51,6 +52,7 @@ def __init__(self,confdir,datadir,logdir,simulatorMode,numMotes,trace,debug,useP self.usePageZero = usePageZero self.iotlabmotes = iotlabmotes self.testbed = testbed + self.benchmark = benchmark self.pathTopo = pathTopo # local variables @@ -59,6 +61,7 @@ def __init__(self,confdir,datadir,logdir,simulatorMode,numMotes,trace,debug,useP self.rpl = RPL.RPL() self.jrc = JRC.JRC() self.topology = topology.topology() + self.openBenchmarkAgent = None self.DAGrootList = [] # create openTun call last since indicates prefix self.openTun = openTun.create(opentun_null) @@ -101,7 +104,7 @@ def __init__(self,confdir,datadir,logdir,simulatorMode,numMotes,trace,debug,useP ] elif self.testbed: self.testEnvironment = self.testbed - motesfinder = moteProbe.OpentestbedMoteFinder(testbed=self.testbed, mqttBroker=self.mqtt_broker_address) + motesfinder = moteProbe.OpentestbedMoteFinder(testbed=self.testbed, mqtt_broker_address=self.mqtt_broker_address) self.moteProbes = [ moteProbe.moteProbe(mqtt_broker_address, testbedmote_eui64=p) for p in motesfinder.get_opentestbed_motelist() @@ -188,6 +191,16 @@ def __init__(self,confdir,datadir,logdir,simulatorMode,numMotes,trace,debug,useP prefix += "0" moteid = prefix+hexaDAGroot self.DAGrootList.append(moteid) + + # If cloud-based benchmarking service is requested, start the agent + if self.benchmark: + self.openBenchmarkAgent = openBenchmarkAgent.OpenBenchmarkAgent( + mqttBroker=self.mqtt_broker_address, + firmware='openwsn', + testbed=self.testEnvironment, + nodes=[], + scenario=self.benchmark + ) # start tracing threads if self.trace: @@ -351,6 +364,7 @@ def main(parser=None): 'trace = {0}'.format(argspace.trace), 'debug = {0}'.format(argspace.debug), 'testbed = {0}'.format(argspace.testbed), + 'benchmark = {0}'.format(argspace.benchmark), 'usePageZero = {0}'.format(argspace.usePageZero)], ))) log.info('Using external dirs:\n\t{0}'.format( @@ -373,6 +387,7 @@ def main(parser=None): iotlabmotes = argspace.iotlabmotes, testbed = argspace.testbed, pathTopo = argspace.pathTopo, + benchmark = argspace.benchmark, mqtt_broker_address = argspace.mqtt_broker_address, opentun_null = argspace.opentun_null ) @@ -431,7 +446,14 @@ def _addParserArgs(parser): default = False, choices = ['opentestbed', 'iotlab', 'wilab'], action = 'store', - help = 'connect remote motes from a --testbed over OpenTestbed serial-MQTT bridge. uses --mqttBroker' + help = 'connect remote motes from a --testbed over OpenTestbed serial-MQTT bridge.' + ) + parser.add_argument('-b', '--benchmark', + dest = 'benchmark', + default = False, + choices = ['building-automation', 'home-automation', 'industrial-monitoring'], + action = 'store', + help = 'trigger --benchmark scenario using OpenBenchmark cloud service. see benchmark.6tis.ch' ) parser.add_argument('--mqtt-broker-address', dest = 'mqtt_broker_address', diff --git a/openvisualizer/moteState/moteState.py b/openvisualizer/moteState/moteState.py index 2c5e20aa..07d8bc14 100644 --- a/openvisualizer/moteState/moteState.py +++ b/openvisualizer/moteState/moteState.py @@ -13,10 +13,6 @@ log.setLevel(logging.ERROR) log.addHandler(logging.NullHandler()) -# a special logger that writes to a separate file: each log line is a JSON string corresponding to network events -# with information sufficient to calculate network-wide KPIs -networkEventLogger = logging.getLogger('networkEventLogger') -networkEventLogger.setLevel(logging.INFO) import copy import time diff --git a/openvisualizer/openBenchmarkAgent/__init__.py b/openvisualizer/openBenchmarkAgent/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py new file mode 100644 index 00000000..2848189e --- /dev/null +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -0,0 +1,34 @@ +# Copyright (c) 2019, Inria. +# All rights reserved. +# +# Released under the BSD 3-Clause license as published at the link below. +# https://openwsn.atlassian.net/wiki/display/OW/License +''' +Implements the OpenBenchmark Agent component needed to automatically benchmark the implementation in a testbed. +OpenBenchmark APIs are specified at https://benchmark.6tis.ch/docs +''' + +import logging +log = logging.getLogger('openBenchmarkAgent') +log.setLevel(logging.INFO) +log.addHandler(logging.NullHandler()) + +from openvisualizer.eventBus import eventBusClient + +# a special logger that writes to a separate file: each log line is a JSON string corresponding to network events +# with information sufficient to calculate network-wide KPIs +networkEventLogger = logging.getLogger('networkEventLogger') +networkEventLogger.setLevel(logging.INFO) + +OPENBENCHMARK_EXPERIMENT_CONTROL_COMMANDS_API_VERSION = "0.0.1" +OPENBENCHMARK_EXPERIMENT_PERFORMANCE_EVENTS_API_VERSION = "0.0.1" + + +class OpenBenchmarkAgent(eventBusClient.eventBusClient): + def __init__(self, mqttBroker, firmware, testbed, nodes, scenario): + self.mqttBroker = mqttBroker + self.firmware = firmware + self.testbed = testbed + self.nodes = nodes + self.scenario = scenario + pass \ No newline at end of file From 4c884e2c07d4347462dddf23e52adca4ec318e85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Wed, 6 Feb 2019 17:13:37 +0100 Subject: [PATCH 011/108] OV-7. Add openBenchmarkAgent to setup.py and remove trailing whitespaces. --- nativeSetup.py | 36 ++++++++++++++++++------------------ setup.py | 46 +++++++++++++++++++++++----------------------- 2 files changed, 41 insertions(+), 41 deletions(-) diff --git a/nativeSetup.py b/nativeSetup.py index 3f60cb8a..3c7158ed 100644 --- a/nativeSetup.py +++ b/nativeSetup.py @@ -4,20 +4,20 @@ from openvisualizer import appdirs ''' -This implementation of the traditional setup.py uses the application-level -data_files parameter to store data files, rather than the package-level +This implementation of the traditional setup.py uses the application-level +data_files parameter to store data files, rather than the package-level package_data parameter. We store these data files in operating system specific , i.e. "native", locations with the help of the appdirs utility. For example, shared data files on Linux are placed in "/usr/local/share/openvisualizer". We use the site-level data and config directories because we expect the -superuser to run OpenVisualizer, so user-level directories like +superuser to run OpenVisualizer, so user-level directories like "/home//.config" are not available. -For native file storage to work, the installer *must not* modify the location +For native file storage to work, the installer *must not* modify the location of these files at install time. -Use of the legacy distutils package also accommodates existing Linux packaging +Use of the legacy distutils package also accommodates existing Linux packaging tools. ''' @@ -36,26 +36,26 @@ def appdirGlob(globstr, subdir=''): return glob.glob('/'.join([appdir, globstr])) else: return glob.glob('/'.join([appdir, subdir, globstr])) - + setup( name = 'openVisualizer', - packages = ['openvisualizer', - 'openvisualizer.BspEmulator', 'openvisualizer.eventBus', - 'openvisualizer.lbrClient', 'openvisualizer.moteConnector', - 'openvisualizer.moteProbe', 'openvisualizer.moteState', - 'openvisualizer.openLbr', 'openvisualizer.openTun', - 'openvisualizer.openType', 'openvisualizer.openUI', - 'openvisualizer.RPL', 'openvisualizer.SimEngine', - 'openvisualizer.JRC'], + packages = ['openvisualizer', + 'openvisualizer.BspEmulator', 'openvisualizer.eventBus', + 'openvisualizer.lbrClient', 'openvisualizer.moteConnector', + 'openvisualizer.moteProbe', 'openvisualizer.moteState', + 'openvisualizer.openLbr', 'openvisualizer.openTun', + 'openvisualizer.openType', 'openvisualizer.openUI', + 'openvisualizer.RPL', 'openvisualizer.SimEngine', + 'openvisualizer.JRC', 'openvisualizer.openBenchmarkAgent'], package_dir = {'': '.', 'openvisualizer': 'openvisualizer'}, scripts = appdirGlob('openVisualizer*.py'), # Copy simdata files by extension so don't copy .gitignore in that directory. data_files = [(confdir, appdirGlob('*.conf')), - ('/'.join([datadir, webstatic, 'css']), appdirGlob('*', '/'.join([webstatic, 'css']))), - ('/'.join([datadir, webstatic, 'font-awesome', 'css']), appdirGlob('*', '/'.join([webstatic, 'font-awesome', 'css']))), - ('/'.join([datadir, webstatic, 'font-awesome', 'fonts']), appdirGlob('*', '/'.join([webstatic, 'font-awesome', 'fonts']))), - ('/'.join([datadir, webstatic, 'images']), appdirGlob('*', '/'.join([webstatic, 'images']))), + ('/'.join([datadir, webstatic, 'css']), appdirGlob('*', '/'.join([webstatic, 'css']))), + ('/'.join([datadir, webstatic, 'font-awesome', 'css']), appdirGlob('*', '/'.join([webstatic, 'font-awesome', 'css']))), + ('/'.join([datadir, webstatic, 'font-awesome', 'fonts']), appdirGlob('*', '/'.join([webstatic, 'font-awesome', 'fonts']))), + ('/'.join([datadir, webstatic, 'images']), appdirGlob('*', '/'.join([webstatic, 'images']))), ('/'.join([datadir, webstatic, 'js']), appdirGlob('*.js', '/'.join([webstatic, 'js']))), ('/'.join([datadir, webstatic, 'js', 'plugins', 'metisMenu']), appdirGlob('*', '/'.join([webstatic, 'js', 'plugins', 'metisMenu']))), ('/'.join([datadir, webtmpl]), appdirGlob('*', webtmpl)), diff --git a/setup.py b/setup.py index 619ad78c..4cb5323c 100644 --- a/setup.py +++ b/setup.py @@ -7,9 +7,9 @@ from openvisualizer import ovVersion ''' -This implementation of the traditional setup.py uses the root package's -package_data parameter to store data files, rather than the application-level -data_files parameter. This arrangement organizes OpenVisualizer within a +This implementation of the traditional setup.py uses the root package's +package_data parameter to store data files, rather than the application-level +data_files parameter. This arrangement organizes OpenVisualizer within a single tree of directories, and so is more portable. In contrast to the native setup, the installer is free to relocate the tree @@ -25,7 +25,7 @@ simdata = 'data/sim_files' with open('README.md') as f: LONG_DESCRIPTION = f.read() - + # Create list of required modules for 'install_requires' parameter. Cannot create # this list with pip.req.parse_requirements() because it requires the pwd module, # which is Unix only. @@ -42,11 +42,11 @@ def appdirGlob(globstr, subdir=''): return glob.glob('/'.join([appdir, globstr])) else: return glob.glob('/'.join([appdir, subdir, globstr])) - + class build_py(_build_py): ''' Extends setuptools build of openvisualizer package data at installation time. - Selects and copies the architecture-specific simulation module from an OS-based + Selects and copies the architecture-specific simulation module from an OS-based subdirectory up to the parent 'sim_files' directory. Excludes the OS subdirectories from installation. ''' @@ -67,37 +67,37 @@ def build_package_data(self): simPath = os.path.join(build_dir, 'data', 'sim_files') target = os.path.join(simPath, 'oos_openwsn.{0}'.format(fileExt)) self.copy_file(srcfile, target) - + if simPath: shutil.rmtree(os.path.join(simPath, 'linux')) shutil.rmtree(os.path.join(simPath, 'windows')) setup( name = 'openVisualizer', - packages = ['openvisualizer', - 'openvisualizer.BspEmulator', 'openvisualizer.eventBus', - 'openvisualizer.lbrClient', 'openvisualizer.moteConnector', - 'openvisualizer.moteProbe', 'openvisualizer.moteState', - 'openvisualizer.openLbr', 'openvisualizer.openTun', + packages = ['openvisualizer', + 'openvisualizer.BspEmulator', 'openvisualizer.eventBus', + 'openvisualizer.lbrClient', 'openvisualizer.moteConnector', + 'openvisualizer.moteProbe', 'openvisualizer.moteState', + 'openvisualizer.openLbr', 'openvisualizer.openTun', 'openvisualizer.openType', 'openvisualizer.openUI', 'openvisualizer.RPL', 'openvisualizer.SimEngine', 'openvisualizer.remoteConnectorServer', - 'openvisualizer.JRC'], + 'openvisualizer.JRC', 'openvisualizer.openBenchmarkAgent'], scripts = appdirGlob('openVisualizer*.py'), package_dir = {'': '.', 'openvisualizer': 'openvisualizer'}, # Copy simdata files by extension so don't copy .gitignore in that directory. package_data = {'openvisualizer': [ 'data/*.conf', 'data/requirements.txt', - '/'.join([webstatic, 'css', '*']), - '/'.join([webstatic, 'font-awesome', 'css', '*']), - '/'.join([webstatic, 'font-awesome', 'fonts', '*']), - '/'.join([webstatic, 'images', '*']), - '/'.join([webstatic, 'js', '*.js']), - '/'.join([webstatic, 'js', 'plugins', 'metisMenu', '*']), - '/'.join([webtmpl, '*']), - '/'.join([simdata, 'windows', '*.pyd']), - '/'.join([simdata, 'linux', '*.so']), - '/'.join([simdata, '*.h']) + '/'.join([webstatic, 'css', '*']), + '/'.join([webstatic, 'font-awesome', 'css', '*']), + '/'.join([webstatic, 'font-awesome', 'fonts', '*']), + '/'.join([webstatic, 'images', '*']), + '/'.join([webstatic, 'js', '*.js']), + '/'.join([webstatic, 'js', 'plugins', 'metisMenu', '*']), + '/'.join([webtmpl, '*']), + '/'.join([simdata, 'windows', '*.pyd']), + '/'.join([simdata, 'linux', '*.so']), + '/'.join([simdata, '*.h']) ]}, install_requires = deplist, # Must extract zip to edit conf files. From e9b36af1052ee9437b160f8439738cf50687d7b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Wed, 6 Feb 2019 19:26:38 +0100 Subject: [PATCH 012/108] OV-7. Implement interface of openBenchmarkAgent with the rest of openVisualizer. --- SConstruct | 25 +++- bin/SConscript | 14 ++- bin/logging.conf | 8 +- bin/openVisualizerApp.py | 3 +- .../openBenchmarkAgent/openBenchmarkAgent.py | 110 +++++++++++++++++- 5 files changed, 145 insertions(+), 15 deletions(-) diff --git a/SConstruct b/SConstruct index 28749755..cf723030 100644 --- a/SConstruct +++ b/SConstruct @@ -54,6 +54,9 @@ Targets: openwsn-fw directory. --ovdebug Enable debug mode; more detailed logging --usePageZero Use page number 0 in page dispatch of 6lowpan packet (only works within one-hop). + --benchmark Benchmark the network performance using OpenBenchmark cloud service. + --testbed Connect remote motes from a testbed using OpenTestbed software component over MQTT. + --mqttBroker Use specified MQTT broker to interconnect with --testbed motes and --benchmark service. Web UI only --host=
Web server listens on IP address; @@ -157,11 +160,25 @@ AddOption('--trace', action = 'store_true') runnerEnv['TRACEOPT'] = GetOption('traceOpt') -AddOption('--opentestbed', - dest = 'opentestbed', +AddOption('--testbed', + dest = 'testbed', default = False, - action = 'store_true') -runnerEnv['OPENTESTBED'] = GetOption('opentestbed') + choices = ['opentestbed', 'iotlab', 'wilab'], + action = 'store') +runnerEnv['TESTBED'] = GetOption('testbed') + +AddOption('--benchmark', + dest = 'benchmark', + default = False, + choices = ['building-automation', 'home-automation', 'industrial-monitoring'], + action = 'store') +runnerEnv['BENCHMARK'] = GetOption('benchmark') + +AddOption('--mqttBroker', + dest = 'mqttBroker', + default = 'argus.paris.inria.fr', + action = 'store') +runnerEnv['MQTTBROKER'] = GetOption('mqttBroker') AddOption('--mqtt-broker-address', dest = 'mqtt_broker_address', diff --git a/bin/SConscript b/bin/SConscript index 8f39c060..4027b4e4 100644 --- a/bin/SConscript +++ b/bin/SConscript @@ -55,21 +55,29 @@ def uiRunner(target, source, env): if env['TRACEOPT']: argList.append('--trace') + if env['DEBUGOPT']: argList.append('--debug') - if env['OPENTESTBED']: - argList.append('--opentestbed') + + if env['TESTBED']: + argList.append('--testbed={0}'.format(env['TESTBED'])) + if env['MQTT_BROKER_ADDRESS']: argList.append('--mqtt-broker-address=' + '{0}'.format(env['MQTT_BROKER_ADDRESS'])) + if env['OPENTUN_NULL']: argList.append('--opentun-null') - + + if env['BENCHMARK']: + argList.append('--benchmark={0}'.format(env['BENCHMARK'])) + if env['USEPAGEZERO']: argList.append('--usePageZero') if env['HOSTOPT']: argList.append('-H {0}'.format(env['HOSTOPT'])) + if env['PORTOPT']: argList.append('-p {0}'.format(env['PORTOPT'])) diff --git a/bin/logging.conf b/bin/logging.conf index 8c9bc224..393d62d4 100644 --- a/bin/logging.conf +++ b/bin/logging.conf @@ -41,7 +41,7 @@ formatter=benchmark #============================ loggers ========================================= [loggers] -keys=root,eventBusMonitor,openTun,openTunWindows,openTunLinux,eventBusClient,lbrClient,moteConnector,moteProbe,moteProbeUtils,moteState,openLbr,OpenParser,Parser,OpenHdlc,ParserData,ParserPrintf,ParserInfoErrorCritical,ParserStatus,RPL,SourceRoute,JRC,networkEventLogger,openVisualizerApp,openVisualizerGui,openVisualizerCli,openVisualizerWeb,OVtracer +keys=root,eventBusMonitor,openTun,openTunWindows,openTunLinux,eventBusClient,lbrClient,moteConnector,moteProbe,moteProbeUtils,moteState,openLbr,openBenchmarkAgent,OpenParser,Parser,OpenHdlc,ParserData,ParserPrintf,ParserInfoErrorCritical,ParserStatus,RPL,SourceRoute,JRC,networkEventLogger,openVisualizerApp,openVisualizerGui,openVisualizerCli,openVisualizerWeb,OVtracer [logger_root] level=ERROR @@ -113,6 +113,12 @@ handlers=std propagate=0 qualname=moteState +[logger_openBenchmarkAgent] +level=INFO +handlers=std +propagate=0 +qualname=openBenchmarkAgent + [logger_openLbr] level=ERROR handlers=std diff --git a/bin/openVisualizerApp.py b/bin/openVisualizerApp.py index a6542b3b..76430b7f 100644 --- a/bin/openVisualizerApp.py +++ b/bin/openVisualizerApp.py @@ -113,7 +113,6 @@ def __init__(self,confdir,datadir,logdir,simulatorMode,numMotes,trace,debug,useP else: self.testEnvironment = 'local' # in "hardware" mode, motes are connected to the serial port - self.moteProbes = [ moteProbe.moteProbe(mqtt_broker_address, serialport=p) for p in moteProbe.findSerialPorts() ] @@ -198,7 +197,7 @@ def __init__(self,confdir,datadir,logdir,simulatorMode,numMotes,trace,debug,useP mqttBroker=self.mqtt_broker_address, firmware='openwsn', testbed=self.testEnvironment, - nodes=[], + portNames=[mote.getPortName() for mote in self.moteProbes], scenario=self.benchmark ) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 2848189e..67ce1db4 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -9,6 +9,11 @@ ''' import logging +import json +import re +import paho.mqtt.client as mqtt +from time import gmtime, strftime + log = logging.getLogger('openBenchmarkAgent') log.setLevel(logging.INFO) log.addHandler(logging.NullHandler()) @@ -20,15 +25,110 @@ networkEventLogger = logging.getLogger('networkEventLogger') networkEventLogger.setLevel(logging.INFO) -OPENBENCHMARK_EXPERIMENT_CONTROL_COMMANDS_API_VERSION = "0.0.1" -OPENBENCHMARK_EXPERIMENT_PERFORMANCE_EVENTS_API_VERSION = "0.0.1" - +OPENBENCHMARK_API_VERSION = "0.0.1" class OpenBenchmarkAgent(eventBusClient.eventBusClient): - def __init__(self, mqttBroker, firmware, testbed, nodes, scenario): + + def __init__(self, mqttBroker, firmware, testbed, portNames, scenario): + ''' + :param mqttBroker: + Address of the MQTT broker where to connect with OpenBenchmark + :param firmware: + Local identifier of the firmware + :param testbed: + Identifier of the testbed, 'simulation' for OV in simulation mode, or 'local' for OV with locally-connected + motes + :param portNames: + List of port names. If the mote is in testbed, expects 'testbed_{{TESTBED}}_{{HOST}}_{{EUI-64}}, + where {{TESTBED}} is the testbed identifier, {{HOST}} is the identifier of the remote machine in the testbed + where the mote is connected to, and {{EUI-64}} is the EUI-64 address of the mote. + :param scenario: + Identifier of the requested scenario to benchmark the performance. + ''' + + # store params self.mqttBroker = mqttBroker self.firmware = firmware self.testbed = testbed - self.nodes = nodes + self.portNames = portNames self.scenario = scenario + self.experimentId = None + + self.nodes = [] + + # OV is running in simulation mode + if self.testbed is 'simulation': + for port in portNames: + self.nodes += ('simulation', port) + # Motes are attached locally on the physical port + elif self.testbed is 'local': + for port in portNames: + self.nodes += ('local', port) + # General case, motes are in testbed connected over OpenTestbed software + else: + for port in portNames: + m = re.search('testbed_(.+)_(.+)_(.+)', port) + if m: + # (testbed_host, eui64) + assert m.group(1) == self.testbed + self.nodes += (m.group(2), m.group(3)) + + log.info('Initializing OpenBenchmark with options:\n\t{0}'.format( + '\n '.join(['mqttBroker = {0}'.format(self.mqttBroker), + 'firmware = {0}'.format(self.firmware), + 'testbed = {0}'.format(self.testbed), + 'portNames = {0}'.format(self.portNames), + 'scenario = {0}'.format(self.scenario), + 'nodes = {0}'.format(self.nodes)] + ))) + + # mqtt client + self.mqttclient = mqtt.Client('openBenchmarkAgent') + self.mqttclient.on_connect = self._on_mqtt_connect + self.mqttclient.on_message = self._on_mqtt_message + self.mqttclient.connect(self.mqttBroker) + self.mqttclient.loop_start() + + # initialize parent class + eventBusClient.eventBusClient.__init__( + self, + name='openBenchmarkAgent', + registrations=[ + ] + ) + + # ======================== public ========================================== + + # ======================== private ========================================= + + # ==== mqtt callback functions + + def _on_mqtt_connect(self, client, userdata, flags, rc): + + # TODO send startBenchmark command + + payload = { + 'api_version' : OPENBENCHMARK_API_VERSION, + 'token' : 123, + 'date' : strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime()), + 'firmware' : self.firmware, + 'testbed' : self.testbed, + 'nodes' : self.nodes, + 'scenario' : self.scenario + } + + client.publish( + topic = 'openbenchmark/command/startBenchmark'.format(self.testbed), + payload = json.dumps(payload), + ) + + # TODO add timeout and parse response in on_message + # TODO start the network + # FIXME what to do with DAg root + + # TODO subscribe to all topics with a given experimentId + client.subscribe( + 'openbenchmark/experimentId/{0}/#'.format(self.experimentId)) + + def _on_mqtt_message(self, client, userdata, message): pass \ No newline at end of file From e74b2b0aa23a986d98342bc0dcee0728296eddbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Mon, 11 Feb 2019 20:54:32 +0100 Subject: [PATCH 013/108] OV-7. startBenchmark handling. --- bin/openVisualizerApp.py | 2 + .../openBenchmarkAgent/openBenchmarkAgent.py | 161 +++++++++++++----- 2 files changed, 122 insertions(+), 41 deletions(-) diff --git a/bin/openVisualizerApp.py b/bin/openVisualizerApp.py index 76430b7f..909d8690 100644 --- a/bin/openVisualizerApp.py +++ b/bin/openVisualizerApp.py @@ -220,6 +220,8 @@ def close(self): self.jrc.close() for probe in self.moteProbes: probe.close() + if self.openBenchmarkAgent: + self.openBenchmarkAgent.close() def getMoteState(self, moteid): ''' diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 67ce1db4..58b8a361 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -11,9 +11,14 @@ import logging import json import re +import time +import os +import binascii import paho.mqtt.client as mqtt from time import gmtime, strftime +import threading + log = logging.getLogger('openBenchmarkAgent') log.setLevel(logging.INFO) log.addHandler(logging.NullHandler()) @@ -25,10 +30,17 @@ networkEventLogger = logging.getLogger('networkEventLogger') networkEventLogger.setLevel(logging.INFO) -OPENBENCHMARK_API_VERSION = "0.0.1" - class OpenBenchmarkAgent(eventBusClient.eventBusClient): + OPENBENCHMARK_API_VERSION = '0.0.1' + + # MQTT topics + OPENBENCHMARK_STARTBENCHMARK_REQUEST_TOPIC = 'openbenchmark/command/startBenchmark' + OPENBENCHMARK_STARTBENCHMARK_RESPONSE_TOPIC = 'openbenchmark/response/startBenchmark' + + OPENBENCHMARK_RESP_STATUS_TIMEOUT = 10 + OPENBENCHMARK_MAX_RETRIES = 3 + def __init__(self, mqttBroker, firmware, testbed, portNames, scenario): ''' :param mqttBroker: @@ -52,8 +64,15 @@ def __init__(self, mqttBroker, firmware, testbed, portNames, scenario): self.testbed = testbed self.portNames = portNames self.scenario = scenario + + # state self.experimentId = None + # sync primitive for mutual exclusion + self.resourceLockEvent = threading.Event() + + self.mqttClient = None + self.experimentRequestResponse = None self.nodes = [] # OV is running in simulation mode @@ -74,61 +93,121 @@ def __init__(self, mqttBroker, firmware, testbed, portNames, scenario): self.nodes += (m.group(2), m.group(3)) log.info('Initializing OpenBenchmark with options:\n\t{0}'.format( - '\n '.join(['mqttBroker = {0}'.format(self.mqttBroker), + '\n '.join(['mqttBroker = {0}'.format(self.mqttBroker), 'firmware = {0}'.format(self.firmware), - 'testbed = {0}'.format(self.testbed), - 'portNames = {0}'.format(self.portNames), - 'scenario = {0}'.format(self.scenario), - 'nodes = {0}'.format(self.nodes)] + 'testbed = {0}'.format(self.testbed), + 'portNames = {0}'.format(self.portNames), + 'scenario = {0}'.format(self.scenario), + 'nodes = {0}'.format(self.nodes)] ))) # mqtt client - self.mqttclient = mqtt.Client('openBenchmarkAgent') - self.mqttclient.on_connect = self._on_mqtt_connect - self.mqttclient.on_message = self._on_mqtt_message - self.mqttclient.connect(self.mqttBroker) - self.mqttclient.loop_start() - - # initialize parent class - eventBusClient.eventBusClient.__init__( - self, - name='openBenchmarkAgent', - registrations=[ - ] - ) + self.mqttClient = mqtt.Client('openBenchmarkAgent') + self.mqttClient.on_connect = self._on_mqtt_connect + self.mqttClient.on_message = self._on_mqtt_message + self.mqttClient.connect(self.mqttBroker) + self.mqttClient.loop_start() + + # block until client is connected, or give up after 60 seconds + self.resourceLockEvent.wait(60) + + self.experimentId = self._openbenchmark_start_benchmark(self.mqttClient) + + # subscribe to eventBus events only if startBenchmark was successful + if self.experimentId: + + log.info("Experiment #{0} successfuly started".format(self.experimentId)) + + # subscribe to eventBus events + eventBusClient.eventBusClient.__init__( + self, + name='openBenchmarkAgent', + registrations=[ + ] + ) + else: + log.info("Experiment start failed, giving up.") + self.close() # ======================== public ========================================== + def close(self): + self.mqttClient.loop_stop() + # ======================== private ========================================= - # ==== mqtt callback functions + def _openbenchmark_start_benchmark(self, mqttClient): + ''' + :param mqttClient: + paho MQTT client object to use to issue startBenchmark request + :returns: Experiment identifier assigned by OpenBenchmark on success, None on failure. + ''' - def _on_mqtt_connect(self, client, userdata, flags, rc): + # count the number of attempts + attempt = 0 + experimentId = None - # TODO send startBenchmark command + # generate a random token + tokenGenerated = binascii.b2a_hex(os.urandom(8)) payload = { - 'api_version' : OPENBENCHMARK_API_VERSION, - 'token' : 123, - 'date' : strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime()), - 'firmware' : self.firmware, - 'testbed' : self.testbed, - 'nodes' : self.nodes, - 'scenario' : self.scenario + 'api_version': self.OPENBENCHMARK_API_VERSION, + 'token': tokenGenerated, + 'date': strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime()), + 'firmware': self.firmware, + 'testbed': self.testbed, + 'nodes': self.nodes, + 'scenario': self.scenario } - client.publish( - topic = 'openbenchmark/command/startBenchmark'.format(self.testbed), - payload = json.dumps(payload), - ) + mqttClient.subscribe(self.OPENBENCHMARK_STARTBENCHMARK_RESPONSE_TOPIC) + + while attempt < self.OPENBENCHMARK_MAX_RETRIES: + + try: + + mqttClient.publish( + topic=self.OPENBENCHMARK_STARTBENCHMARK_REQUEST_TOPIC, + payload=json.dumps(payload), + ) + + # wait for a while to gather the response + time.sleep(self.OPENBENCHMARK_RESP_STATUS_TIMEOUT) + + # assume response is received + assert self.experimentRequestResponse, "No response from OpenBenchmark" + + # parse it + payload = json.loads(self.experimentRequestResponse) + tokenReceived = payload['token'] + success = payload['success'] - # TODO add timeout and parse response in on_message - # TODO start the network - # FIXME what to do with DAg root + # assume tokens match + assert tokenGenerated is tokenReceived, "Token does not match" + # assume success + assert success is True, "Fail indicated" - # TODO subscribe to all topics with a given experimentId - client.subscribe( - 'openbenchmark/experimentId/{0}/#'.format(self.experimentId)) + experimentId = payload['experimentId'] + + # Retry for all exceptions, including assertions + except Exception as e: + log.info(str(e) + ", retrying...") + attempt += 1 + self.experimentRequestResponse = None + continue + + mqttClient.unsubscribe(self.OPENBENCHMARK_STARTBENCHMARK_RESPONSE_TOPIC) + + return experimentId + + # ==== mqtt callback functions + + def _on_mqtt_connect(self, client, userdata, flags, rc): + # signal to the other thread that we are connected + self.resourceLockEvent.set() def _on_mqtt_message(self, client, userdata, message): - pass \ No newline at end of file + if message.topic is self.OPENBENCHMARK_STARTBENCHMARK_RESPONSE_TOPIC: + self.experimentRequestResponse = message.payload + else: + pass From 702460d5804d858e5a2b3fbed771bfd90c555024 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Tue, 12 Feb 2019 11:38:21 +0100 Subject: [PATCH 014/108] OV-7. More robust MQTT handling. --- .../openBenchmarkAgent/openBenchmarkAgent.py | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 58b8a361..55289524 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -100,22 +100,22 @@ def __init__(self, mqttBroker, firmware, testbed, portNames, scenario): 'scenario = {0}'.format(self.scenario), 'nodes = {0}'.format(self.nodes)] ))) + try: + # mqtt client + self.mqttClient = mqtt.Client('openBenchmarkAgent') + self.mqttClient.on_connect = self._on_mqtt_connect + self.mqttClient.on_message = self._on_mqtt_message + self.mqttClient.connect(self.mqttBroker) + self.mqttClient.loop_start() - # mqtt client - self.mqttClient = mqtt.Client('openBenchmarkAgent') - self.mqttClient.on_connect = self._on_mqtt_connect - self.mqttClient.on_message = self._on_mqtt_message - self.mqttClient.connect(self.mqttBroker) - self.mqttClient.loop_start() + # block until client is connected, or give up after 60 seconds + self.resourceLockEvent.wait(60) - # block until client is connected, or give up after 60 seconds - self.resourceLockEvent.wait(60) + self.experimentId = self._openbenchmark_start_benchmark(self.mqttClient) - self.experimentId = self._openbenchmark_start_benchmark(self.mqttClient) - - # subscribe to eventBus events only if startBenchmark was successful - if self.experimentId: + assert self.experimentId + # assuming startBenchmark was successful, subscribe to event bus events log.info("Experiment #{0} successfuly started".format(self.experimentId)) # subscribe to eventBus events @@ -125,7 +125,9 @@ def __init__(self, mqttBroker, firmware, testbed, portNames, scenario): registrations=[ ] ) - else: + + except Exception as e: + log.info(str(e)) log.info("Experiment start failed, giving up.") self.close() @@ -138,8 +140,7 @@ def close(self): def _openbenchmark_start_benchmark(self, mqttClient): ''' - :param mqttClient: - paho MQTT client object to use to issue startBenchmark request + :param mqttClient: paho MQTT client object to use to issue startBenchmark request :returns: Experiment identifier assigned by OpenBenchmark on success, None on failure. ''' From 2b7baf6a4e4cbcae3d4d67d28f78d7a050025923 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Tue, 12 Feb 2019 16:58:06 +0100 Subject: [PATCH 015/108] OV-7. Add MQTT command handling. --- .../openBenchmarkAgent/openBenchmarkAgent.py | 89 ++++++++++++++++--- 1 file changed, 79 insertions(+), 10 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 55289524..16d5b757 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -18,6 +18,7 @@ from time import gmtime, strftime import threading +import traceback log = logging.getLogger('openBenchmarkAgent') log.setLevel(logging.INFO) @@ -40,6 +41,7 @@ class OpenBenchmarkAgent(eventBusClient.eventBusClient): OPENBENCHMARK_RESP_STATUS_TIMEOUT = 10 OPENBENCHMARK_MAX_RETRIES = 3 + OPENBENCHMARK_PREFIX_CMD_HANDLER_NAME = '_mqtt_handler_' def __init__(self, mqttBroker, firmware, testbed, portNames, scenario): ''' @@ -69,7 +71,8 @@ def __init__(self, mqttBroker, firmware, testbed, portNames, scenario): self.experimentId = None # sync primitive for mutual exclusion - self.resourceLockEvent = threading.Event() + self.mqttConnectedEvent = threading.Event() + self.experimentRequestResponseEvent = threading.Event() self.mqttClient = None self.experimentRequestResponse = None @@ -109,14 +112,14 @@ def __init__(self, mqttBroker, firmware, testbed, portNames, scenario): self.mqttClient.loop_start() # block until client is connected, or give up after 60 seconds - self.resourceLockEvent.wait(60) + self.mqttConnectedEvent.wait(60) self.experimentId = self._openbenchmark_start_benchmark(self.mqttClient) assert self.experimentId - # assuming startBenchmark was successful, subscribe to event bus events - log.info("Experiment #{0} successfuly started".format(self.experimentId)) + # subscribe to all topics on a given experiment ID + self._openbenchmark_subscribe(self.mqttClient, self.experimentId) # subscribe to eventBus events eventBusClient.eventBusClient.__init__( @@ -126,8 +129,10 @@ def __init__(self, mqttBroker, firmware, testbed, portNames, scenario): ] ) + log.info("Experiment #{0} successfuly started".format(self.experimentId)) + except Exception as e: - log.info(str(e)) + log.exception(e) log.info("Experiment start failed, giving up.") self.close() @@ -172,8 +177,8 @@ def _openbenchmark_start_benchmark(self, mqttClient): payload=json.dumps(payload), ) - # wait for a while to gather the response - time.sleep(self.OPENBENCHMARK_RESP_STATUS_TIMEOUT) + # block until response is received, or give up after the timeout + self.experimentRequestResponseEvent.wait(self.OPENBENCHMARK_RESP_STATUS_TIMEOUT) # assume response is received assert self.experimentRequestResponse, "No response from OpenBenchmark" @@ -192,8 +197,9 @@ def _openbenchmark_start_benchmark(self, mqttClient): # Retry for all exceptions, including assertions except Exception as e: - log.info(str(e) + ", retrying...") + log.exception(str(e) + ", retrying...") attempt += 1 + self.experimentRequestResponseEvent.clear() self.experimentRequestResponse = None continue @@ -201,14 +207,77 @@ def _openbenchmark_start_benchmark(self, mqttClient): return experimentId + def _openbenchmark_subscribe(self, mqttClient, experimentId): + mqttClient.subscribe("openbenchmark/experimentId/{0}/command/#".format(experimentId)) + + # command handling adapted from github.com/openwsn-berkeley/opentestbed + def _execute_command_safely(self, topic, payload): + # parse the topic to extract deviceType, deviceId and cmd ([0-9\-]+) + try: + m = re.search("openbenchmark/experimentId/{0}/command/([a-z]+)".format(self.experimentId), topic) + assert m, "Invalid topic, could not parse: '{0}'".format(topic) + + cmd = m.group(1) + + log.debug("Executing command %s", cmd) + returnVal = {} + + payload = payload.decode('utf8') + assert payload, "Could not decode payload" + + tokenReceived = json.loads(payload)['token'] + + # Executes the handler of a command in a try/except environment so exception doesn't crash server. + try: + # find the handler + cmd_handler = getattr(self, '{0}{1}'.format(self.OPENBENCHMARK_PREFIX_CMD_HANDLER_NAME, cmd), None) + + assert cmd_handler, "Unhandled command, ignoring: {0}".format(cmd) + + # call the handler to return dictionary with handler-specific fields in the response + dict = cmd_handler(payload) + + except Exception as err: + log.exception("Exception while executing {0}".format(cmd)) + log.exception(err) + log.exception(traceback.format_exc()) + + returnVal = { + 'success': False, + } + + else: + returnVal['success'] = True + + # update the returnVal dict with handler-specific fields + # handler can set success to False if it wasn't able to fullfill the request + returnVal.update(dict) + + finally: + # echo the token + returnVal['token'] = tokenReceived + + self.mqttClient.publish( + topic='openbenchmark/experimentId/{0}/response/{1}'.format(self.experimentId, cmd), + payload=json.dumps(returnVal), + ) + + except Exception as e: + log.exception(e) + # ==== mqtt callback functions def _on_mqtt_connect(self, client, userdata, flags, rc): # signal to the other thread that we are connected - self.resourceLockEvent.set() + self.mqttConnectedEvent.set() def _on_mqtt_message(self, client, userdata, message): + # check if this is the startBenchmark response if message.topic is self.OPENBENCHMARK_STARTBENCHMARK_RESPONSE_TOPIC: self.experimentRequestResponse = message.payload + self.experimentRequestResponseEvent.set() + # if not, assume this is a command else: - pass + self._execute_command_safely(message.topic, message.payload) + + From a7d9542b0416ffe2fac0766bada3e2584562a26a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Tue, 12 Feb 2019 18:35:31 +0100 Subject: [PATCH 016/108] OV-7. Update command handler API and add mockup handlers. --- .../openBenchmarkAgent/openBenchmarkAgent.py | 47 +++++++++++++++++-- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 16d5b757..fd8949cf 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -235,7 +235,10 @@ def _execute_command_safely(self, topic, payload): assert cmd_handler, "Unhandled command, ignoring: {0}".format(cmd) # call the handler to return dictionary with handler-specific fields in the response - dict = cmd_handler(payload) + # handler return is in format (success, dict), with: + # - success, as a boolean + # - dict, dictionary containing fields to include in the response or None on failure + (success, dict) = cmd_handler(payload) except Exception as err: log.exception("Exception while executing {0}".format(cmd)) @@ -247,11 +250,10 @@ def _execute_command_safely(self, topic, payload): } else: - returnVal['success'] = True - # update the returnVal dict with handler-specific fields - # handler can set success to False if it wasn't able to fullfill the request - returnVal.update(dict) + if success: + returnVal.update(dict) + returnVal['success'] = success finally: # echo the token @@ -280,4 +282,39 @@ def _on_mqtt_message(self, client, userdata, message): else: self._execute_command_safely(message.topic, message.payload) + # ==== mqtt command handlers + + def _mqtt_handler_echo(self, payload): + returnVal = {} + return (True, returnVal) + + def _mqtt_handler_sendPacket(self, payload): + returnVal = {} + + # parse the payload + payloadDecoded = json.loads(payload) + + source = payloadDecoded['source'] + destination = payloadDecoded['destination'] + packetToken = payloadDecoded['packetToken'] + packetPayload = payloadDecoded['packetPayload'] + confirmable = payloadDecoded['confirmable'] + + # TODO lookup corresponding mote probe + # TODO generate an eventbus signal to send a command over serial + + return (True, returnVal) + + def _mqtt_handler_configureTransmitPower(self, payload): + returnVal = {} + + # parse the payload + payloadDecoded = json.loads(payload) + + source = payloadDecoded['source'] + power = payload.Decoded['power'] + + # TODO lookup corresponding mote probe + # TODO generate an eventbus signal to send a command over serial + return (True, returnVal) From c3b2c076860ed04e591d8f28d1d1a9c567cf90b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Wed, 13 Feb 2019 01:08:23 +0100 Subject: [PATCH 017/108] OV-7. Add hooks between openBenchmarkAgent and OpenVisualizer. --- openvisualizer/moteState/moteState.py | 4 + .../openBenchmarkAgent/openBenchmarkAgent.py | 85 ++++++++++++++++--- 2 files changed, 75 insertions(+), 14 deletions(-) diff --git a/openvisualizer/moteState/moteState.py b/openvisualizer/moteState/moteState.py index 07d8bc14..571fcdc5 100644 --- a/openvisualizer/moteState/moteState.py +++ b/openvisualizer/moteState/moteState.py @@ -656,6 +656,8 @@ class moteState(eventBusClient.eventBusClient): COMMAND_SET_UINJECTPERIOD = ['uinjectPeriod', 17, 1] COMMAND_SET_ECHO_REPLY_STATUS = ['echoReply', 18, 1] COMMAND_SET_JOIN_KEY = ['joinKey', 19,16] + COMMAND_SET_TX_POWER = ['txPower', 20, 1] + COMMAND_SEND_PACKET = ['sendPacket', 21, 15] # TODO exact format to be determined COMMAND_ALL = [ COMMAND_SET_EBPERIOD , COMMAND_SET_CHANNEL, @@ -677,6 +679,8 @@ class moteState(eventBusClient.eventBusClient): COMMAND_SET_UINJECTPERIOD, COMMAND_SET_ECHO_REPLY_STATUS, COMMAND_SET_JOIN_KEY, + COMMAND_SET_TX_POWER, + COMMAND_SEND_PACKET, ] TRIGGER_ALL = [ diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index fd8949cf..b46b1dc9 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -25,6 +25,7 @@ log.addHandler(logging.NullHandler()) from openvisualizer.eventBus import eventBusClient +from openvisualizer.moteState import moteState # a special logger that writes to a separate file: each log line is a JSON string corresponding to network events # with information sufficient to calculate network-wide KPIs @@ -76,7 +77,8 @@ def __init__(self, mqttBroker, firmware, testbed, portNames, scenario): self.mqttClient = None self.experimentRequestResponse = None - self.nodes = [] + # dict with keys being eui64, and value corresponding testbed host identifier + self.nodes = {} # OV is running in simulation mode if self.testbed is 'simulation': @@ -93,7 +95,7 @@ def __init__(self, mqttBroker, firmware, testbed, portNames, scenario): if m: # (testbed_host, eui64) assert m.group(1) == self.testbed - self.nodes += (m.group(2), m.group(3)) + self.nodes[m.group(3)] = m.group(2) log.info('Initializing OpenBenchmark with options:\n\t{0}'.format( '\n '.join(['mqttBroker = {0}'.format(self.mqttBroker), @@ -121,11 +123,16 @@ def __init__(self, mqttBroker, firmware, testbed, portNames, scenario): # subscribe to all topics on a given experiment ID self._openbenchmark_subscribe(self.mqttClient, self.experimentId) - # subscribe to eventBus events + # subscribe to eventBus performance-related events eventBusClient.eventBusClient.__init__( self, name='openBenchmarkAgent', registrations=[ + { + 'sender': self.WILDCARD, + 'signal': 'performanceData', + 'callback': self._performance_data_handler, + }, ] ) @@ -267,7 +274,7 @@ def _execute_command_safely(self, topic, payload): except Exception as e: log.exception(e) - # ==== mqtt callback functions + # ==== mqtt and event bus callback functions def _on_mqtt_connect(self, client, userdata, flags, rc): # signal to the other thread that we are connected @@ -282,6 +289,9 @@ def _on_mqtt_message(self, client, userdata, message): else: self._execute_command_safely(message.topic, message.payload) + def _performance_data_handler(self): + pass + # ==== mqtt command handlers def _mqtt_handler_echo(self, payload): @@ -294,14 +304,27 @@ def _mqtt_handler_sendPacket(self, payload): # parse the payload payloadDecoded = json.loads(payload) - source = payloadDecoded['source'] - destination = payloadDecoded['destination'] - packetToken = payloadDecoded['packetToken'] - packetPayload = payloadDecoded['packetPayload'] - confirmable = payloadDecoded['confirmable'] - - # TODO lookup corresponding mote probe - # TODO generate an eventbus signal to send a command over serial + source = payloadDecoded['source'] + destination = payloadDecoded['destination'] + packetsInBurst = payloadDecoded['packetsInBurst'] + packetToken = payloadDecoded['packetToken'] + packetPayload = payloadDecoded['packetPayload'] + confirmable = payloadDecoded['confirmable'] + + # lookup corresponding mote port + destPort = self.nodes[destination] + params = (destination, confirmable, packetsInBurst, packetToken, len(packetPayload)) + action = [moteState.moteState.SET_COMMAND, moteState.moteState.COMMAND_SEND_PACKET, params] + # generate an eventbus signal to send a command over serial + + # dispatch + self.dispatch( + signal = 'cmdToMote', + data = { + 'serialPort': destPort, + 'action': action, + }, + ) return (True, returnVal) @@ -314,7 +337,41 @@ def _mqtt_handler_configureTransmitPower(self, payload): source = payloadDecoded['source'] power = payload.Decoded['power'] - # TODO lookup corresponding mote probe - # TODO generate an eventbus signal to send a command over serial + # lookup corresponding mote port + destPort = self.nodes[source] + action = [moteState.moteState.SET_COMMAND, moteState.moteState.COMMAND_SET_TX_POWER, power] + + # generate an eventbus signal to send a command over serial + + # dispatch + self.dispatch( + signal = 'cmdToMote', + data = { + 'serialPort': destPort, + 'action': action, + }, + ) + + return (True, returnVal) + + def _mqtt_handler_triggerNetworkFormation(self, payload): + returnVal = {} + + # parse the payload + payloadDecoded = json.loads(payload) + + source = payloadDecoded['source'] + + # lookup corresponding mote port + destPort = self.nodes[source] + + # generate an eventbus signal to send a command over serial + self.dispatch( + signal='cmdToMote', + data={ + 'serialPort': destPort, + 'action': moteState.moteState.TRIGGER_DAGROOT, + }, + ) return (True, returnVal) From 4cce2596a2c75aa77cf37919dac456dd4ac4842a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Wed, 13 Feb 2019 01:15:10 +0100 Subject: [PATCH 018/108] OV-7. Hot fixes for simulation and local execution. --- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index b46b1dc9..1618a0ca 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -83,11 +83,11 @@ def __init__(self, mqttBroker, firmware, testbed, portNames, scenario): # OV is running in simulation mode if self.testbed is 'simulation': for port in portNames: - self.nodes += ('simulation', port) + self.nodes[port] = 'simulation' # Motes are attached locally on the physical port elif self.testbed is 'local': for port in portNames: - self.nodes += ('local', port) + self.nodes[port] = 'local' # General case, motes are in testbed connected over OpenTestbed software else: for port in portNames: @@ -179,6 +179,8 @@ def _openbenchmark_start_benchmark(self, mqttClient): try: + self.experimentRequestResponse = None + mqttClient.publish( topic=self.OPENBENCHMARK_STARTBENCHMARK_REQUEST_TOPIC, payload=json.dumps(payload), @@ -207,7 +209,6 @@ def _openbenchmark_start_benchmark(self, mqttClient): log.exception(str(e) + ", retrying...") attempt += 1 self.experimentRequestResponseEvent.clear() - self.experimentRequestResponse = None continue mqttClient.unsubscribe(self.OPENBENCHMARK_STARTBENCHMARK_RESPONSE_TOPIC) @@ -312,7 +313,7 @@ def _mqtt_handler_sendPacket(self, payload): confirmable = payloadDecoded['confirmable'] # lookup corresponding mote port - destPort = self.nodes[destination] + destPort = self.nodes[source] params = (destination, confirmable, packetsInBurst, packetToken, len(packetPayload)) action = [moteState.moteState.SET_COMMAND, moteState.moteState.COMMAND_SEND_PACKET, params] # generate an eventbus signal to send a command over serial From b01ca769612e78968ffab70e8972d8399adad776 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Wed, 13 Feb 2019 11:08:11 +0100 Subject: [PATCH 019/108] OV-7. Improve the logging of connection handling. --- .../openBenchmarkAgent/openBenchmarkAgent.py | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 1618a0ca..45433a06 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -71,12 +71,13 @@ def __init__(self, mqttBroker, firmware, testbed, portNames, scenario): # state self.experimentId = None - # sync primitive for mutual exclusion + # primitive for mutual exclusion self.mqttConnectedEvent = threading.Event() self.experimentRequestResponseEvent = threading.Event() self.mqttClient = None self.experimentRequestResponse = None + # dict with keys being eui64, and value corresponding testbed host identifier self.nodes = {} @@ -118,7 +119,8 @@ def __init__(self, mqttBroker, firmware, testbed, portNames, scenario): self.experimentId = self._openbenchmark_start_benchmark(self.mqttClient) - assert self.experimentId + if not self.experimentId: + raise ValueError("Unable to start an experiment with OpenBenchmark") # subscribe to all topics on a given experiment ID self._openbenchmark_subscribe(self.mqttClient, self.experimentId) @@ -139,8 +141,7 @@ def __init__(self, mqttBroker, firmware, testbed, portNames, scenario): log.info("Experiment #{0} successfuly started".format(self.experimentId)) except Exception as e: - log.exception(e) - log.info("Experiment start failed, giving up.") + log.warning(e) self.close() # ======================== public ========================================== @@ -190,26 +191,33 @@ def _openbenchmark_start_benchmark(self, mqttClient): self.experimentRequestResponseEvent.wait(self.OPENBENCHMARK_RESP_STATUS_TIMEOUT) # assume response is received - assert self.experimentRequestResponse, "No response from OpenBenchmark" + if not self.experimentRequestResponse: + raise ValueError("No response from OpenBenchmark") # parse it payload = json.loads(self.experimentRequestResponse) tokenReceived = payload['token'] success = payload['success'] - # assume tokens match - assert tokenGenerated is tokenReceived, "Token does not match" - # assume success - assert success is True, "Fail indicated" + # check token match + if tokenGenerated is not tokenReceived: + raise ValueError("Token does not match the one sent in the request") + # success? + if success is not True: + raise ValueError("Fail indicated") experimentId = payload['experimentId'] - # Retry for all exceptions, including assertions - except Exception as e: - log.exception(str(e) + ", retrying...") + # Retry for all ValueErrors + except ValueError as valErr: + log.info(str(valErr) + ", retrying...") attempt += 1 self.experimentRequestResponseEvent.clear() continue + # give up + except Exception as e: + log.warning(e) + break mqttClient.unsubscribe(self.OPENBENCHMARK_STARTBENCHMARK_RESPONSE_TOPIC) From 3e71920ac50f61e4e4b336b01c93e32a9f97062b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Wed, 13 Feb 2019 14:16:51 +0100 Subject: [PATCH 020/108] OV-7. Implement sendPacket command in moteConnector. --- openvisualizer/moteConnector/moteConnector.py | 34 ++++++++++++++++++- openvisualizer/moteState/moteState.py | 2 +- openvisualizer/openvisualizer_utils.py | 16 ++++++--- 3 files changed, 46 insertions(+), 6 deletions(-) diff --git a/openvisualizer/moteConnector/moteConnector.py b/openvisualizer/moteConnector/moteConnector.py index 4344ef7e..f4bb990d 100644 --- a/openvisualizer/moteConnector/moteConnector.py +++ b/openvisualizer/moteConnector/moteConnector.py @@ -294,6 +294,39 @@ def _commandToBytes(self,data): except: print "=============================================" print "Wrong joinKey format. Input 16-byte long hex string. e.g. cafebeefcafebeefcafebeefcafebeef" + elif data[0] == 'sendPacket': + try: + if len(data[1]) != commandLen: + raise ValueError + (destination, con, packetsInBurst, packetToken, packetPayloadLen) = data[1] + + # construct command payload as byte-list: + # dest_eui64 (8B) || con (1B) || packetsInBurst (1B) || packetToken (5B) || packetPayloadLen (1B) + payload = [] + payload += u.hex2buf(destination, separator='-') + payload += [int(con)] + payload += [int(packetsInBurst)] + payload += packetToken + payload += [int(packetPayloadLen)] + + if len(payload) != 16: + raise ValueError("Invalid sendPacket payload, expecting 16 bytes") + + dataToSend = [OpenParser.OpenParser.SERFRAME_PC2MOTE_COMMAND, + commandId, + commandLen, + ] + dataToSend += payload + except: + debug = "=============================================\n" + debug += "Wrong sendPacket command format.\n" + debug += "Supported: ( destination, confirmable, packetsInBurst, packetToken, packetPayloadLen )\n" + debug += "destination: dash-separated EUI-64 string, e.g. AA-BB-CC-DD-EE-FF-00-11\n" + debug += "confirmable: boolean\n" + debug += "packetsInBurst: integer\n" + debug += "packetToken: integer array\n" + debug += "packetPayloadLen: integer\n" + log.warning(debug) else: parameter = int(data[1]) if parameter <= 0xffff: @@ -315,7 +348,6 @@ def _commandToBytes(self,data): outcome = True return [outcome,dataToSend] - def _bytesToMesh_handler(self,sender,signal,data): assert type(data)==tuple assert len(data)==2 diff --git a/openvisualizer/moteState/moteState.py b/openvisualizer/moteState/moteState.py index 571fcdc5..2bdb14ac 100644 --- a/openvisualizer/moteState/moteState.py +++ b/openvisualizer/moteState/moteState.py @@ -657,7 +657,7 @@ class moteState(eventBusClient.eventBusClient): COMMAND_SET_ECHO_REPLY_STATUS = ['echoReply', 18, 1] COMMAND_SET_JOIN_KEY = ['joinKey', 19,16] COMMAND_SET_TX_POWER = ['txPower', 20, 1] - COMMAND_SEND_PACKET = ['sendPacket', 21, 15] # TODO exact format to be determined + COMMAND_SEND_PACKET = ['sendPacket', 21, 5] # length is in the number of params in the tuple COMMAND_ALL = [ COMMAND_SET_EBPERIOD , COMMAND_SET_CHANNEL, diff --git a/openvisualizer/openvisualizer_utils.py b/openvisualizer/openvisualizer_utils.py index 998d1d68..d13844f0 100644 --- a/openvisualizer/openvisualizer_utils.py +++ b/openvisualizer/openvisualizer_utils.py @@ -53,24 +53,32 @@ def formatThreadList(): #===== parsing -def hex2buf(s): +def hex2buf(s,separator=None): ''' Convert a string of hex caracters into a byte list. For example: ``'abcdef00' -> [0xab,0xcd,0xef,0x00]`` :param s: [in] The string to convert + :param separator: [in] Optional separator char used in the string, e.g ab-cd-ef-00' :returns: A list of integers, each element in [0x00..0xff]. ''' assert type(s)==str - assert len(s)%2 == 0 + if separator: + assert type(separator)==str + assert len(separator)==1 + newVal = s.replace(separator,'') # remove the separator before proceding + else: + newVal = s + + assert len(newVal)%2 == 0 returnVal = [] - for i in range(len(s)/2): + for i in range(len(newVal)/2): realIdx = i*2 - returnVal.append(int(s[realIdx:realIdx+2],16)) + returnVal.append(int(newVal[realIdx:realIdx+2],16)) return returnVal From e88674b71c0e57954430263985fbff5714f860ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Thu, 21 Feb 2019 11:39:03 +0100 Subject: [PATCH 021/108] OV-7. Expect exactly 16 bytes for sendPacket command payload. --- openvisualizer/moteConnector/moteConnector.py | 5 ++--- openvisualizer/moteState/moteState.py | 2 +- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 1 + 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/openvisualizer/moteConnector/moteConnector.py b/openvisualizer/moteConnector/moteConnector.py index f4bb990d..412a3338 100644 --- a/openvisualizer/moteConnector/moteConnector.py +++ b/openvisualizer/moteConnector/moteConnector.py @@ -296,8 +296,7 @@ def _commandToBytes(self,data): print "Wrong joinKey format. Input 16-byte long hex string. e.g. cafebeefcafebeefcafebeefcafebeef" elif data[0] == 'sendPacket': try: - if len(data[1]) != commandLen: - raise ValueError + (destination, con, packetsInBurst, packetToken, packetPayloadLen) = data[1] # construct command payload as byte-list: @@ -309,7 +308,7 @@ def _commandToBytes(self,data): payload += packetToken payload += [int(packetPayloadLen)] - if len(payload) != 16: + if len(payload) != commandLen: raise ValueError("Invalid sendPacket payload, expecting 16 bytes") dataToSend = [OpenParser.OpenParser.SERFRAME_PC2MOTE_COMMAND, diff --git a/openvisualizer/moteState/moteState.py b/openvisualizer/moteState/moteState.py index 2bdb14ac..1985ff23 100644 --- a/openvisualizer/moteState/moteState.py +++ b/openvisualizer/moteState/moteState.py @@ -657,7 +657,7 @@ class moteState(eventBusClient.eventBusClient): COMMAND_SET_ECHO_REPLY_STATUS = ['echoReply', 18, 1] COMMAND_SET_JOIN_KEY = ['joinKey', 19,16] COMMAND_SET_TX_POWER = ['txPower', 20, 1] - COMMAND_SEND_PACKET = ['sendPacket', 21, 5] # length is in the number of params in the tuple + COMMAND_SEND_PACKET = ['sendPacket', 21, 16] # dest_eui64 (8B) || con (1B) || packetsInBurst (1B) || packetToken (5B) || packetPayloadLen (1B) COMMAND_ALL = [ COMMAND_SET_EBPERIOD , COMMAND_SET_CHANNEL, diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 45433a06..47d78857 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -299,6 +299,7 @@ def _on_mqtt_message(self, client, userdata, message): self._execute_command_safely(message.topic, message.payload) def _performance_data_handler(self): + # TODO pass # ==== mqtt command handlers From dda90b14892dadfb972a475ef7cb86d316c1ca7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Thu, 28 Feb 2019 16:16:46 +0100 Subject: [PATCH 022/108] OV-7. Human-friendly var names for command handling. --- openvisualizer/moteConnector/moteConnector.py | 35 ++++++++++--------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/openvisualizer/moteConnector/moteConnector.py b/openvisualizer/moteConnector/moteConnector.py index 412a3338..71278f13 100644 --- a/openvisualizer/moteConnector/moteConnector.py +++ b/openvisualizer/moteConnector/moteConnector.py @@ -172,8 +172,7 @@ def _cmdToMote_handler(self,sender,signal,data): def _commandToBytes(self,data): - # data[0]: commandID - # data[1]: parameter + outcome = False dataToSend = [] @@ -182,9 +181,13 @@ def _commandToBytes(self,data): # get commandId commandIndex = 0 for cmd in moteState.moteState.COMMAND_ALL: + # data[0]: command name + # data[1]: parameter if data[0] == cmd[0]: + commandName = cmd[0] commandId = cmd[1] commandLen = cmd[2] + parameter = data[1] break else: commandIndex += 1 @@ -198,14 +201,14 @@ def _commandToBytes(self,data): print " }" return [outcome,dataToSend] - if data[0][:2] == '6p': + if '6p' in commandName: try: dataToSend = [OpenParser.OpenParser.SERFRAME_PC2MOTE_COMMAND, commandId, commandLen ] - paramList = data[1].split(',') - if data[0] != '6pClear': + paramList = parameter.split(',') + if commandId != '6pClear': if paramList[0] == 'tx': cellOptions = 1<<0 elif paramList[0] == 'rx': @@ -223,9 +226,9 @@ def _commandToBytes(self,data): dataToSend += [cellOptions] celllist_add = {} celllist_delete = {} - if data[0] == '6pList' and len(paramList)==3: + if commandId == '6pList' and len(paramList)==3: dataToSend += map(int,paramList[ptr:]) - if data[0] == '6pAdd': + if commandId == '6pAdd': # append numCell dataToSend += [int(paramList[ptr])] # append celllist @@ -236,7 +239,7 @@ def _commandToBytes(self,data): assert TRUE dataToSend += map(int,celllist_add['slotoffset']) dataToSend += map(int,celllist_add['channeloffset']) - if data[0] == '6pDelete': + if commandId == '6pDelete': # append numCell dataToSend += [int(paramList[ptr])] # append celllist @@ -247,7 +250,7 @@ def _commandToBytes(self,data): assert TRUE dataToSend += map(int,celllist_delete['slotoffset']) dataToSend += map(int,celllist_delete['channeloffset']) - if data[0] == '6pRelocate': + if commandId == '6pRelocate': dataToSend += [int(paramList[ptr])] # append celllist celllist_delete['slotoffset'] = paramList[ptr+1].split('-') @@ -281,11 +284,11 @@ def _commandToBytes(self,data): print "comma. e.g. set 6pList tx, 5, 3" print "comma. e.g. set 6pClear all" return [outcome,dataToSend] - elif data[0] == 'joinKey': + elif commandName == 'joinKey': try: - if len(data[1]) != commandLen*2: # two hex chars is one byte + if len(parameter) != commandLen*2: # two hex chars is one byte raise ValueError - payload = binascii.unhexlify(data[1]) + payload = binascii.unhexlify(parameter) dataToSend = [OpenParser.OpenParser.SERFRAME_PC2MOTE_COMMAND, commandId, commandLen, @@ -294,7 +297,7 @@ def _commandToBytes(self,data): except: print "=============================================" print "Wrong joinKey format. Input 16-byte long hex string. e.g. cafebeefcafebeefcafebeefcafebeef" - elif data[0] == 'sendPacket': + elif commandName == 'sendPacket': try: (destination, con, packetsInBurst, packetToken, packetPayloadLen) = data[1] @@ -308,8 +311,8 @@ def _commandToBytes(self,data): payload += packetToken payload += [int(packetPayloadLen)] - if len(payload) != commandLen: - raise ValueError("Invalid sendPacket payload, expecting 16 bytes") + if len(parameter) != commandLen: + raise ValueError("Invalid sendPacket payload, expecting {0} bytes".format(commandLen)) dataToSend = [OpenParser.OpenParser.SERFRAME_PC2MOTE_COMMAND, commandId, @@ -327,7 +330,7 @@ def _commandToBytes(self,data): debug += "packetPayloadLen: integer\n" log.warning(debug) else: - parameter = int(data[1]) + parameter = int(parameter) if parameter <= 0xffff: parameter = [(parameter & 0xff),((parameter >> 8) & 0xff)] dataToSend = [OpenParser.OpenParser.SERFRAME_PC2MOTE_COMMAND, From 4e58ed716e55aa3716d8da2bdcd1b81f38cba14f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Thu, 28 Feb 2019 16:18:10 +0100 Subject: [PATCH 023/108] OV-7. Expect formatted byte string as a param for sendPacket. --- openvisualizer/moteConnector/moteConnector.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/openvisualizer/moteConnector/moteConnector.py b/openvisualizer/moteConnector/moteConnector.py index 71278f13..25f0cfe8 100644 --- a/openvisualizer/moteConnector/moteConnector.py +++ b/openvisualizer/moteConnector/moteConnector.py @@ -300,17 +300,6 @@ def _commandToBytes(self,data): elif commandName == 'sendPacket': try: - (destination, con, packetsInBurst, packetToken, packetPayloadLen) = data[1] - - # construct command payload as byte-list: - # dest_eui64 (8B) || con (1B) || packetsInBurst (1B) || packetToken (5B) || packetPayloadLen (1B) - payload = [] - payload += u.hex2buf(destination, separator='-') - payload += [int(con)] - payload += [int(packetsInBurst)] - payload += packetToken - payload += [int(packetPayloadLen)] - if len(parameter) != commandLen: raise ValueError("Invalid sendPacket payload, expecting {0} bytes".format(commandLen)) @@ -318,7 +307,7 @@ def _commandToBytes(self,data): commandId, commandLen, ] - dataToSend += payload + dataToSend += parameter except: debug = "=============================================\n" debug += "Wrong sendPacket command format.\n" From f30f0d7a8a33f4e5746aace756e13c9e57293866 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Thu, 28 Feb 2019 16:19:04 +0100 Subject: [PATCH 024/108] OV-7. Format sendPacket payload in openBenchmarkAgent. --- .../openBenchmarkAgent/openBenchmarkAgent.py | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 47d78857..fa683a8e 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -314,16 +314,28 @@ def _mqtt_handler_sendPacket(self, payload): # parse the payload payloadDecoded = json.loads(payload) - source = payloadDecoded['source'] - destination = payloadDecoded['destination'] - packetsInBurst = payloadDecoded['packetsInBurst'] - packetToken = payloadDecoded['packetToken'] - packetPayload = payloadDecoded['packetPayload'] - confirmable = payloadDecoded['confirmable'] + source = payloadDecoded['source'] + destination = payloadDecoded['destination'] + packetsInBurst = payloadDecoded['packetsInBurst'] + packetToken = payloadDecoded['packetToken'] + packetPayloadLen = payloadDecoded['packetPayloadLen'] + confirmable = payloadDecoded['confirmable'] # lookup corresponding mote port destPort = self.nodes[source] - params = (destination, confirmable, packetsInBurst, packetToken, len(packetPayload)) + + # construct command payload as byte-list: + # dest_eui64 (8B) || con (1B) || packetsInBurst (1B) || packetToken (5B) || packetPayloadLen (1B) + params = [] + params += u.hex2buf(destination, separator='-') + params += [int(confirmable)] + params += [int(packetsInBurst)] + params += packetToken + params += [packetPayloadLen] + + if len(params) != 16: + return False, returnVal + action = [moteState.moteState.SET_COMMAND, moteState.moteState.COMMAND_SEND_PACKET, params] # generate an eventbus signal to send a command over serial @@ -336,7 +348,7 @@ def _mqtt_handler_sendPacket(self, payload): }, ) - return (True, returnVal) + return True, returnVal def _mqtt_handler_configureTransmitPower(self, payload): returnVal = {} From b73a72141c2f4b8ec7e5555d615fb26f55323338 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Fri, 1 Mar 2019 10:44:50 +0100 Subject: [PATCH 025/108] OV-7. Add virtual CoAP server listening on DAG root's IPV6. --- .../openBenchmarkAgent/openBenchmarkAgent.py | 175 +++++++++++++++++- 1 file changed, 174 insertions(+), 1 deletion(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index fa683a8e..203e771d 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -26,6 +26,14 @@ from openvisualizer.eventBus import eventBusClient from openvisualizer.moteState import moteState +import openvisualizer.openvisualizer_utils + +from coap import coap, \ + coapResource, \ + coapDefines as d, \ + coapOption as o, \ + coapUtils as u, \ + coapObjectSecurity as oscoap # a special logger that writes to a separate file: each log line is a JSON string corresponding to network events # with information sufficient to calculate network-wide KPIs @@ -78,6 +86,8 @@ def __init__(self, mqttBroker, firmware, testbed, portNames, scenario): self.mqttClient = None self.experimentRequestResponse = None + self.coapServer = None + # dict with keys being eui64, and value corresponding testbed host identifier self.nodes = {} @@ -122,6 +132,10 @@ def __init__(self, mqttBroker, firmware, testbed, portNames, scenario): if not self.experimentId: raise ValueError("Unable to start an experiment with OpenBenchmark") + # everything is ok, start a coap server + coapResource = openbenchmarkResource() + self.coapServer = coapServer(coapResource) + # subscribe to all topics on a given experiment ID self._openbenchmark_subscribe(self.mqttClient, self.experimentId) @@ -147,7 +161,10 @@ def __init__(self, mqttBroker, firmware, testbed, portNames, scenario): # ======================== public ========================================== def close(self): - self.mqttClient.loop_stop() + if self.mqttClient: + self.mqttClient.loop_stop() + if self.coapServer: + self.coapServer.close() # ======================== private ========================================= @@ -397,3 +414,159 @@ def _mqtt_handler_triggerNetworkFormation(self, payload): ) return (True, returnVal) + + +# ======================== CoAP server ====================================== +class coapServer(eventBusClient.eventBusClient): + + OPENBENCHMARK_COAP_PORT = 5684 + + def __init__(self, coapResource): + # log + log.info("create instance") + + self.coapResource = coapResource + + # run CoAP server in testing mode + # this mode does not open a real socket, rather uses PyDispatcher for sending/receiving messages + # We interface this mode with OpenVisualizer to run JRC co-located with the DAG root + self.coapServer = coap.coap(udpPort=self.OPENBENCHMARK_COAP_PORT, testing=True) + self.coapServer.addResource(coapResource) + + self.coapClient = None + + self.dagRootEui64 = None + + # store params + + # initialize parent class + eventBusClient.eventBusClient.__init__( + self, + name='OpenBenchmark-coapServer', + registrations=[ + { + 'sender': self.WILDCARD, + 'signal': 'registerDagRoot', + 'callback': self._registerDagRoot_notif + }, + { + 'sender': self.WILDCARD, + 'signal': 'unregisterDagRoot', + 'callback': self._unregisterDagRoot_notif + }, + ] + ) + + # ======================== public ========================================== + + def close(self): + # nothing to do + pass + + # ======================== private ========================================= + + # ==== handle EventBus notifications + + def _registerDagRoot_notif(self, sender, signal, data): + # register for the global address of the DAG root + self.register( + sender=self.WILDCARD, + signal=( + tuple(data['prefix'] + data['host']), + self.PROTO_UDP, + self.OPENBENCHMARK_COAP_PORT + ), + callback=self._receiveFromMesh, + ) + + self.dagRootEui64 = data['host'] + + + def _unregisterDagRoot_notif(self, sender, signal, data): + # unregister global address + self.unregister( + sender=self.WILDCARD, + signal=( + tuple(data['prefix'] + data['host']), + self.PROTO_UDP, + self.OPENBENCHMARK_COAP_PORT + ), + callback=self._receiveFromMesh, + ) + + self.dagRootEui64 = None + + def _receiveFromMesh(self, sender, signal, data): + ''' + Receive packet from the mesh destined for the CoAP server. + Forwards the packet to the virtual CoAP server running in test mode (PyDispatcher). + ''' + sender = openvisualizer.openvisualizer_utils.formatIPv6Addr(data[0]) + # FIXME pass source port within the signal and open coap client at this port + self.coapClient = coap.coap(ipAddress=sender, udpPort=d.DEFAULT_UDP_PORT, testing=True, receiveCallback=self._receiveFromCoAP) + self.coapClient.socketUdp.sendUdp(destIp='', destPort=self.OPENBENCHMARK_COAP_PORT, msg=data[1]) # low level forward of the CoAP message + return True + + def _receiveFromCoAP(self, timestamp, sender, data): + ''' + Receive CoAP response and forward it to the mesh network. + Appends UDP and IPv6 headers to the CoAP message and forwards it on the Eventbus towards the mesh. + ''' + self.coapClient.close() + + # UDP + udplen = len(data) + 8 + + udp = u.int2buf(sender[1], 2) # src port + udp += u.int2buf(self.coapClient.udpPort, 2) # dest port + udp += [udplen >> 8, udplen & 0xff] # length + udp += [0x00, 0x00] # checksum + udp += data + + # destination address of the packet is CoAP client's IPv6 address (address of the mote) + dstIpv6Address = u.ipv6AddrString2Bytes(self.coapClient.ipAddress) + assert len(dstIpv6Address)==16 + # source address of the packet is DAG root's IPV6 address + # use the same prefix (link-local or global) as in the destination address + srcIpv6Address = dstIpv6Address[:8] + srcIpv6Address += self.dagRootEui64 + assert len(srcIpv6Address)==16 + + # CRC See https://tools.ietf.org/html/rfc2460. + + udp[6:8] = openvisualizer.openvisualizer_utils.calculatePseudoHeaderCRC( + src=srcIpv6Address, + dst=dstIpv6Address, + length=[0x00, 0x00] + udp[4:6], + nh=[0x00, 0x00, 0x00, 17], # UDP as next header + payload=udp, + ) + + # IPv6 + ip = [6 << 4] # v6 + traffic class (upper nybble) + ip += [0x00, 0x00, 0x00] # traffic class (lower nibble) + flow label + ip += udp[4:6] # payload length + ip += [17] # next header (protocol); UDP=17 + ip += [64] # hop limit (pick a safe value) + ip += srcIpv6Address # source + ip += dstIpv6Address # destination + ip += udp + + # announce network prefix + self.dispatch( + signal = 'v6ToMesh', + data = ip + ) + +# ==================== Implementation of CoAP openbenchmark resource ===================== +class openbenchmarkResource(coapResource.coapResource): + def __init__(self): + + # initialize parent class + coapResource.coapResource.__init__( + self, + path = 'b', + ) + + def POST(self,options=[], payload=[]): + return (d.COAP_RC_2_04_CHANGED, [], []) From b1bf7ebdf3c8d6c32ad2c8be6f64aafda377a062 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Sun, 3 Mar 2019 14:02:22 +0100 Subject: [PATCH 026/108] OV-7. Handle sendPacket on behalf of the DAG root. --- .../openBenchmarkAgent/openBenchmarkAgent.py | 139 ++++++++++++------ 1 file changed, 97 insertions(+), 42 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 203e771d..a9d83e46 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -33,7 +33,8 @@ coapDefines as d, \ coapOption as o, \ coapUtils as u, \ - coapObjectSecurity as oscoap + coapObjectSecurity as oscoap, \ + coapException as e # a special logger that writes to a separate file: each log line is a JSON string corresponding to network events # with information sufficient to calculate network-wide KPIs @@ -331,39 +332,45 @@ def _mqtt_handler_sendPacket(self, payload): # parse the payload payloadDecoded = json.loads(payload) - source = payloadDecoded['source'] - destination = payloadDecoded['destination'] - packetsInBurst = payloadDecoded['packetsInBurst'] - packetToken = payloadDecoded['packetToken'] - packetPayloadLen = payloadDecoded['packetPayloadLen'] - confirmable = payloadDecoded['confirmable'] - - # lookup corresponding mote port - destPort = self.nodes[source] - - # construct command payload as byte-list: - # dest_eui64 (8B) || con (1B) || packetsInBurst (1B) || packetToken (5B) || packetPayloadLen (1B) - params = [] - params += u.hex2buf(destination, separator='-') - params += [int(confirmable)] - params += [int(packetsInBurst)] - params += packetToken - params += [packetPayloadLen] - - if len(params) != 16: - return False, returnVal - - action = [moteState.moteState.SET_COMMAND, moteState.moteState.COMMAND_SEND_PACKET, params] - # generate an eventbus signal to send a command over serial - - # dispatch - self.dispatch( - signal = 'cmdToMote', - data = { - 'serialPort': destPort, - 'action': action, - }, - ) + sourceStr = payloadDecoded['source'] + source = u.hex2buf(sourceStr, separator='-') + destination = u.hex2buf(payloadDecoded['destination'], separator='-') + packetsInBurst = payloadDecoded['packetsInBurst'] + packetToken = payloadDecoded['packetToken'] + packetPayloadLen = payloadDecoded['packetPayloadLen'] + acknowledged = payloadDecoded['confirmable'] + + if self.coapServer.getDagRootEui64() == source: # check if command is for the DAG root whose APP code is implemented here + self.coapServer.triggerSendPacket(destination, acknowledged, packetsInBurst, packetToken, packetPayloadLen) + + else: # command is for one of the motes in the mesh, send it over the serial + + # lookup corresponding mote port + destPort = self.nodes[sourceStr] + + # construct command payload as byte-list: + # dest_eui64 (8B) || con (1B) || packetsInBurst (1B) || packetToken (5B) || packetPayloadLen (1B) + commandPayload = [] + commandPayload += destination + commandPayload += [int(acknowledged)] + commandPayload += [int(packetsInBurst)] + commandPayload += packetToken + commandPayload += [packetPayloadLen] + + if len(commandPayload) != 16: + return False, returnVal + + action = [moteState.moteState.SET_COMMAND, moteState.moteState.COMMAND_SEND_PACKET, commandPayload] + # generate an eventbus signal to send a command over serial + + # dispatch + self.dispatch( + signal = 'cmdToMote', + data = { + 'serialPort': destPort, + 'action': action, + }, + ) return True, returnVal @@ -433,7 +440,7 @@ def __init__(self, coapResource): self.coapServer = coap.coap(udpPort=self.OPENBENCHMARK_COAP_PORT, testing=True) self.coapServer.addResource(coapResource) - self.coapClient = None + self.coapEphemeralClient = None self.dagRootEui64 = None @@ -463,6 +470,51 @@ def close(self): # nothing to do pass + def getDagRootEui64(self): + return self.dagRootEui64 + + def getDagRootIPv6(self): + ipv6buf = self.networkPrefix + self.dagRootEui64 + return openvisualizer.openvisualizer_utils.formatIPv6Addr(ipv6buf) + + def triggerSendPacket(self, destination, acknowledged, packetsInBurst, packetToken, packetPayloadLen): + + destinationIPv6 = openvisualizer.openvisualizer_utils.formatIPv6Addr((self.networkPrefix + destination)) + options = [] + + if not acknowledged: + options += [o.NoResponse([d.DFLT_OPTION_NORESPONSE_SUPRESS_ALL])] + + with self.clientLock: + for packetCounter in range (0, packetsInBurst): + try: + # construct the payload of the POST request + payload = [] + payload += [packetCounter] + payload += [packetToken[1:]] + payload += [0] * packetPayloadLen + + # the call to POST() is blocking unless no response is expected + p = self.coapServer.POST('coap://[{0}:{1}]/b'.format(destinationIPv6, d.DEFAULT_UDP_PORT), + confirmable=False, + options=options, + payload = payload) + + # TODO log if a response is received + except e.coapNoResponseExpected: + pass + + def encodeSendPacketPayload(self, destination, confirmable, packetsInBurst, packetToken, packetPayloadLen): + # construct command payload as byte-list: + # dest_eui64 (8B) || con (1B) || packetsInBurst (1B) || packetToken (5B) || packetPayloadLen (1B) + buf = [] + buf += u.hex2buf(destination, separator='-') + buf += [int(confirmable)] + buf += [int(packetsInBurst)] + buf += packetToken + buf += [packetPayloadLen] + return buf + # ======================== private ========================================= # ==== handle EventBus notifications @@ -479,9 +531,9 @@ def _registerDagRoot_notif(self, sender, signal, data): callback=self._receiveFromMesh, ) + self.networkPrefix = data['prefix'] self.dagRootEui64 = data['host'] - def _unregisterDagRoot_notif(self, sender, signal, data): # unregister global address self.unregister( @@ -493,7 +545,7 @@ def _unregisterDagRoot_notif(self, sender, signal, data): ), callback=self._receiveFromMesh, ) - + self.networkPrefix = None self.dagRootEui64 = None def _receiveFromMesh(self, sender, signal, data): @@ -503,8 +555,8 @@ def _receiveFromMesh(self, sender, signal, data): ''' sender = openvisualizer.openvisualizer_utils.formatIPv6Addr(data[0]) # FIXME pass source port within the signal and open coap client at this port - self.coapClient = coap.coap(ipAddress=sender, udpPort=d.DEFAULT_UDP_PORT, testing=True, receiveCallback=self._receiveFromCoAP) - self.coapClient.socketUdp.sendUdp(destIp='', destPort=self.OPENBENCHMARK_COAP_PORT, msg=data[1]) # low level forward of the CoAP message + self.coapEphemeralClient = coap.coap(ipAddress=sender, udpPort=d.DEFAULT_UDP_PORT, testing=True, receiveCallback=self._receiveFromCoAP) + self.coapEphemeralClient.socketUdp.sendUdp(destIp='', destPort=self.OPENBENCHMARK_COAP_PORT, msg=data[1]) # low level forward of the CoAP message return True def _receiveFromCoAP(self, timestamp, sender, data): @@ -512,19 +564,21 @@ def _receiveFromCoAP(self, timestamp, sender, data): Receive CoAP response and forward it to the mesh network. Appends UDP and IPv6 headers to the CoAP message and forwards it on the Eventbus towards the mesh. ''' - self.coapClient.close() + + # FIXME potential memory leak when there the request contains a No Response option + self.coapEphemeralClient.close() # UDP udplen = len(data) + 8 udp = u.int2buf(sender[1], 2) # src port - udp += u.int2buf(self.coapClient.udpPort, 2) # dest port + udp += u.int2buf(self.coapEphemeralClient.udpPort, 2) # dest port udp += [udplen >> 8, udplen & 0xff] # length udp += [0x00, 0x00] # checksum udp += data # destination address of the packet is CoAP client's IPv6 address (address of the mote) - dstIpv6Address = u.ipv6AddrString2Bytes(self.coapClient.ipAddress) + dstIpv6Address = u.ipv6AddrString2Bytes(self.coapEphemeralClient.ipAddress) assert len(dstIpv6Address)==16 # source address of the packet is DAG root's IPV6 address # use the same prefix (link-local or global) as in the destination address @@ -569,4 +623,5 @@ def __init__(self): ) def POST(self,options=[], payload=[]): + # TODO parse the packet token and log the event return (d.COAP_RC_2_04_CHANGED, [], []) From e4fce801b4ca4dea3939c0998bf6691550b8e762 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Fri, 8 Mar 2019 20:48:28 +0100 Subject: [PATCH 027/108] OV-7. Implement handlers for openbenchmark serial status dumps. --- openvisualizer/moteConnector/ParserStatus.py | 66 ++++++++++ openvisualizer/moteConnector/StackDefines.py | 1 + openvisualizer/moteState/moteState.py | 132 ++++++++++++++++--- 3 files changed, 183 insertions(+), 16 deletions(-) diff --git a/openvisualizer/moteConnector/ParserStatus.py b/openvisualizer/moteConnector/ParserStatus.py index 302fe27b..2399cac3 100644 --- a/openvisualizer/moteConnector/ParserStatus.py +++ b/openvisualizer/moteConnector/ParserStatus.py @@ -254,6 +254,72 @@ def __init__(self): 'joinedAsn_0_1', # H ], ) + self._addFieldsParser ( + 3, + 12, + 'BenchmarkPacketSent', + ' Date: Fri, 8 Mar 2019 20:56:11 +0100 Subject: [PATCH 028/108] OV-19. Add virtual CoAP server files. --- openvisualizer/coapServer/__init__.py | 0 openvisualizer/coapServer/coapServer.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 openvisualizer/coapServer/__init__.py create mode 100644 openvisualizer/coapServer/coapServer.py diff --git a/openvisualizer/coapServer/__init__.py b/openvisualizer/coapServer/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/openvisualizer/coapServer/coapServer.py b/openvisualizer/coapServer/coapServer.py new file mode 100644 index 00000000..e69de29b From 7b5db33dc94247a9c160d0e48d059fb39b765423 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Fri, 8 Mar 2019 21:47:31 +0100 Subject: [PATCH 029/108] OV-19. Make coap server a stand-alone component. Adapt JRC and OpenBenchmark agent. --- bin/logging.conf | 8 +- bin/openVisualizerApp.py | 5 +- openvisualizer/JRC/JRC.py | 215 +++------------- openvisualizer/coapServer/coapServer.py | 193 ++++++++++++++ .../openBenchmarkAgent/openBenchmarkAgent.py | 237 ++++-------------- 5 files changed, 279 insertions(+), 379 deletions(-) diff --git a/bin/logging.conf b/bin/logging.conf index 393d62d4..ce513f22 100644 --- a/bin/logging.conf +++ b/bin/logging.conf @@ -41,7 +41,7 @@ formatter=benchmark #============================ loggers ========================================= [loggers] -keys=root,eventBusMonitor,openTun,openTunWindows,openTunLinux,eventBusClient,lbrClient,moteConnector,moteProbe,moteProbeUtils,moteState,openLbr,openBenchmarkAgent,OpenParser,Parser,OpenHdlc,ParserData,ParserPrintf,ParserInfoErrorCritical,ParserStatus,RPL,SourceRoute,JRC,networkEventLogger,openVisualizerApp,openVisualizerGui,openVisualizerCli,openVisualizerWeb,OVtracer +keys=root,eventBusMonitor,openTun,openTunWindows,openTunLinux,eventBusClient,lbrClient,moteConnector,moteProbe,moteProbeUtils,moteState,openLbr,openBenchmarkAgent,OpenParser,Parser,OpenHdlc,ParserData,ParserPrintf,ParserInfoErrorCritical,ParserStatus,RPL,SourceRoute,JRC,networkEventLogger,openVisualizerApp,openVisualizerGui,openVisualizerCli,openVisualizerWeb,OVtracer,coapServer [logger_root] level=ERROR @@ -221,3 +221,9 @@ handlers=std propagate=0 qualname=OVtracer +[logger_coapServer] +level=INFO +handlers=std +propagate=0 +qualname=coapServer + diff --git a/bin/openVisualizerApp.py b/bin/openVisualizerApp.py index 909d8690..d88ca15f 100644 --- a/bin/openVisualizerApp.py +++ b/bin/openVisualizerApp.py @@ -22,6 +22,7 @@ from openvisualizer.moteProbe import moteProbe from openvisualizer.moteConnector import moteConnector from openvisualizer.moteState import moteState +from openvisualizer.coapServer import coapServer from openvisualizer.RPL import RPL from openvisualizer.JRC import JRC from openvisualizer.openBenchmarkAgent import openBenchmarkAgent @@ -58,8 +59,9 @@ def __init__(self,confdir,datadir,logdir,simulatorMode,numMotes,trace,debug,useP # local variables self.eventBusMonitor = eventBusMonitor.eventBusMonitor() self.openLbr = openLbr.OpenLbr(usePageZero) + self.coapServer = coapServer.coapServer() self.rpl = RPL.RPL() - self.jrc = JRC.JRC() + self.jrc = JRC.JRC(self.coapServer) self.topology = topology.topology() self.openBenchmarkAgent = None self.DAGrootList = [] @@ -195,6 +197,7 @@ def __init__(self,confdir,datadir,logdir,simulatorMode,numMotes,trace,debug,useP if self.benchmark: self.openBenchmarkAgent = openBenchmarkAgent.OpenBenchmarkAgent( mqttBroker=self.mqtt_broker_address, + coapServer=self.coapServer, firmware='openwsn', testbed=self.testEnvironment, portNames=[mote.getPortName() for mote in self.moteProbes], diff --git a/openvisualizer/JRC/JRC.py b/openvisualizer/JRC/JRC.py index 968990b1..1dcd5e62 100644 --- a/openvisualizer/JRC/JRC.py +++ b/openvisualizer/JRC/JRC.py @@ -23,14 +23,43 @@ import os # ======================== Top Level JRC Class ============================= -class JRC(): - def __init__(self): - coapResource = joinResource() - self.coapServer = coapServer(coapResource, contextHandler(coapResource).securityContextLookup) +class JRC(eventBusClient.eventBusClient): + def __init__(self, coapServer): + # store params + self.coapServer = coapServer + + self.coapResource = joinResource() + + self.coapServer.coapServer.addResource(self.coapResource) + self.coapServer.coapServer.addSecurityContextHandler(contextHandler(self.coapResource).securityContextLookup) + + # initialize parent class + eventBusClient.eventBusClient.__init__( + self, + name='JRC', + registrations=[ + { + 'sender': self.WILDCARD, + 'signal': 'getL2SecurityKey', + 'callback': self._getL2SecurityKey_notif, + }, + + ] + ) + + # ======================== public ========================================== def close(self): self.coapServer.close() + # ==== handle EventBus notifications + + def _getL2SecurityKey_notif(self, sender, signal, data): + ''' + Return L2 security key for the network. + ''' + return {'index': [self.coapResource.networkKeyIndex], 'value': self.coapResource.networkKey} + # ======================== Security Context Handler ========================= class contextHandler(): MASTERSECRET = binascii.unhexlify('DEADBEEFCAFEDEADBEEFCAFEDEADBEEF') # value of the OSCORE Master Secret from 6TiSCH TD @@ -66,184 +95,6 @@ def securityContextLookup(self, kid): return context -# ======================== Interface with OpenVisualizer ====================================== -class coapServer(eventBusClient.eventBusClient): - # link-local prefix - LINK_LOCAL_PREFIX = [0xfe, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] - - def __init__(self, coapResource, contextHandler=None): - # log - log.info("create instance") - - self.coapResource = coapResource - - # run CoAP server in testing mode - # this mode does not open a real socket, rather uses PyDispatcher for sending/receiving messages - # We interface this mode with OpenVisualizer to run JRC co-located with the DAG root - self.coapServer = coap.coap(udpPort=d.DEFAULT_UDP_PORT, testing=True) - self.coapServer.addResource(coapResource) - self.coapServer.addSecurityContextHandler(contextHandler) - self.coapServer.maxRetransmit = 1 - - self.coapClient = None - - self.dagRootEui64 = None - - # store params - - # initialize parent class - eventBusClient.eventBusClient.__init__( - self, - name='JRC', - registrations=[ - { - 'sender': self.WILDCARD, - 'signal': 'getL2SecurityKey', - 'callback': self._getL2SecurityKey_notif, - }, - { - 'sender': self.WILDCARD, - 'signal': 'registerDagRoot', - 'callback': self._registerDagRoot_notif - }, - { - 'sender': self.WILDCARD, - 'signal': 'unregisterDagRoot', - 'callback': self._unregisterDagRoot_notif - }, - ] - ) - - # local variables - self.stateLock = threading.Lock() - - # ======================== public ========================================== - - def close(self): - # nothing to do - pass - - # ======================== private ========================================= - - # ==== handle EventBus notifications - - def _getL2SecurityKey_notif(self, sender, signal, data): - ''' - Return L2 security key for the network. - ''' - return {'index' : [self.coapResource.networkKeyIndex], 'value' : self.coapResource.networkKey} - - def _registerDagRoot_notif(self, sender, signal, data): - # register for the global address of the DAG root - self.register( - sender=self.WILDCARD, - signal=( - tuple(data['prefix'] + data['host']), - self.PROTO_UDP, - d.DEFAULT_UDP_PORT - ), - callback=self._receiveFromMesh, - ) - - # register to receive at link-local DAG root's address - self.register( - sender=self.WILDCARD, - signal=( - tuple(self.LINK_LOCAL_PREFIX + data['host']), - self.PROTO_UDP, - d.DEFAULT_UDP_PORT - ), - callback=self._receiveFromMesh, - ) - - self.dagRootEui64 = data['host'] - - def _unregisterDagRoot_notif(self, sender, signal, data): - # unregister global address - self.unregister( - sender=self.WILDCARD, - signal=( - tuple(data['prefix'] + data['host']), - self.PROTO_UDP, - d.DEFAULT_UDP_PORT - ), - callback=self._receiveFromMesh, - ) - # unregister link-local address - self.unregister( - sender=self.WILDCARD, - signal=( - tuple(self.LINK_LOCAL_PREFIX + data['host']), - self.PROTO_UDP, - d.DEFAULT_UDP_PORT - ), - callback=self._receiveFromMesh, - ) - - self.dagRootEui64 = None - - def _receiveFromMesh(self, sender, signal, data): - ''' - Receive packet from the mesh destined for JRC's CoAP server. - Forwards the packet to the virtual CoAP server running in test mode (PyDispatcher). - ''' - sender = openvisualizer.openvisualizer_utils.formatIPv6Addr(data[0]) - # FIXME pass source port within the signal and open coap client at this port - self.coapClient = coap.coap(ipAddress=sender, udpPort=d.DEFAULT_UDP_PORT, testing=True, receiveCallback=self._receiveFromCoAP) - self.coapClient.socketUdp.sendUdp(destIp='', destPort=d.DEFAULT_UDP_PORT, msg=data[1]) # low level forward of the CoAP message - return True - - def _receiveFromCoAP(self, timestamp, sender, data): - ''' - Receive CoAP response and forward it to the mesh network. - Appends UDP and IPv6 headers to the CoAP message and forwards it on the Eventbus towards the mesh. - ''' - self.coapClient.close() - - # UDP - udplen = len(data) + 8 - - udp = u.int2buf(sender[1], 2) # src port - udp += u.int2buf(self.coapClient.udpPort, 2) # dest port - udp += [udplen >> 8, udplen & 0xff] # length - udp += [0x00, 0x00] # checksum - udp += data - - # destination address of the packet is CoAP client's IPv6 address (address of the mote) - dstIpv6Address = u.ipv6AddrString2Bytes(self.coapClient.ipAddress) - assert len(dstIpv6Address)==16 - # source address of the packet is DAG root's IPV6 address - # use the same prefix (link-local or global) as in the destination address - srcIpv6Address = dstIpv6Address[:8] - srcIpv6Address += self.dagRootEui64 - assert len(srcIpv6Address)==16 - - # CRC See https://tools.ietf.org/html/rfc2460. - - udp[6:8] = openvisualizer.openvisualizer_utils.calculatePseudoHeaderCRC( - src=srcIpv6Address, - dst=dstIpv6Address, - length=[0x00, 0x00] + udp[4:6], - nh=[0x00, 0x00, 0x00, 17], # UDP as next header - payload=udp, - ) - - # IPv6 - ip = [6 << 4] # v6 + traffic class (upper nybble) - ip += [0x00, 0x00, 0x00] # traffic class (lower nibble) + flow label - ip += udp[4:6] # payload length - ip += [17] # next header (protocol); UDP=17 - ip += [64] # hop limit (pick a safe value) - ip += srcIpv6Address # source - ip += dstIpv6Address # destination - ip += udp - - # announce network prefix - self.dispatch( - signal = 'v6ToMesh', - data = ip - ) - # ==================== Implementation of CoAP join resource ===================== class joinResource(coapResource.coapResource): def __init__(self): diff --git a/openvisualizer/coapServer/coapServer.py b/openvisualizer/coapServer/coapServer.py index e69de29b..be126689 100644 --- a/openvisualizer/coapServer/coapServer.py +++ b/openvisualizer/coapServer/coapServer.py @@ -0,0 +1,193 @@ +from coap import coap, \ + coapResource, \ + coapDefines as d, \ + coapOption as o, \ + coapUtils as u, \ + coapObjectSecurity as oscoap +import logging.handlers +try: + from openvisualizer.eventBus import eventBusClient + import openvisualizer.openvisualizer_utils +except ImportError: + pass + +log = logging.getLogger('coapServer') +log.setLevel(logging.ERROR) +log.addHandler(logging.NullHandler()) + +import cbor +import binascii +import os +import threading + + +class coapServer(eventBusClient.eventBusClient): + # link-local prefix + LINK_LOCAL_PREFIX = [0xfe, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + + def __init__(self): + # log + log.info("create instance") + + # run CoAP server in testing mode + # this mode does not open a real socket, rather uses PyDispatcher for sending/receiving messages + # We interface this mode with OpenVisualizer to run JRC co-located with the DAG root + self.coapServer = coap.coap(udpPort=d.DEFAULT_UDP_PORT, testing=True) + + self.ephemeralCoapClient = None + + self.dagRootEui64 = None + self.networkPrefix = None + + # store params + + # initialize parent class + eventBusClient.eventBusClient.__init__( + self, + name='coapServer', + registrations=[ + { + 'sender': self.WILDCARD, + 'signal': 'registerDagRoot', + 'callback': self._registerDagRoot_notif + }, + { + 'sender': self.WILDCARD, + 'signal': 'unregisterDagRoot', + 'callback': self._unregisterDagRoot_notif + }, + ] + ) + + # local variables + self.stateLock = threading.Lock() + + # ======================== public ========================================== + + def close(self): + # nothing to do + pass + + def getDagRootEui64(self): + return self.dagRootEui64 + + def getDagRootIPv6(self): + ipv6buf = self.networkPrefix + self.dagRootEui64 + return openvisualizer.openvisualizer_utils.formatIPv6Addr(ipv6buf) + + # ======================== private ========================================= + + # ==== handle EventBus notifications + + def _registerDagRoot_notif(self, sender, signal, data): + # register for the global address of the DAG root + self.register( + sender=self.WILDCARD, + signal=( + tuple(data['prefix'] + data['host']), + self.PROTO_UDP, + d.DEFAULT_UDP_PORT + ), + callback=self._receiveFromMesh, + ) + + # register to receive at link-local DAG root's address + self.register( + sender=self.WILDCARD, + signal=( + tuple(self.LINK_LOCAL_PREFIX + data['host']), + self.PROTO_UDP, + d.DEFAULT_UDP_PORT + ), + callback=self._receiveFromMesh, + ) + + self.dagRootEui64 = data['host'] + self.networkPrefix = data['prefix'] + + def _unregisterDagRoot_notif(self, sender, signal, data): + # unregister global address + self.unregister( + sender=self.WILDCARD, + signal=( + tuple(data['prefix'] + data['host']), + self.PROTO_UDP, + d.DEFAULT_UDP_PORT + ), + callback=self._receiveFromMesh, + ) + # unregister link-local address + self.unregister( + sender=self.WILDCARD, + signal=( + tuple(self.LINK_LOCAL_PREFIX + data['host']), + self.PROTO_UDP, + d.DEFAULT_UDP_PORT + ), + callback=self._receiveFromMesh, + ) + + self.dagRootEui64 = None + self.networkPrefix = None + + def _receiveFromMesh(self, sender, signal, data): + ''' + Receive packet from the mesh destined for JRC's CoAP server. + Forwards the packet to the virtual CoAP server running in test mode (PyDispatcher). + ''' + sender = openvisualizer.openvisualizer_utils.formatIPv6Addr(data[0]) + # FIXME pass source port within the signal and open coap client at this port + self.ephemeralCoapClient = coap.coap(ipAddress=sender, udpPort=d.DEFAULT_UDP_PORT, testing=True, receiveCallback=self._receiveFromCoAP) + self.ephemeralCoapClient.socketUdp.sendUdp(destIp='', destPort=d.DEFAULT_UDP_PORT, msg=data[1]) # low level forward of the CoAP message + return True + + def _receiveFromCoAP(self, timestamp, sender, data): + ''' + Receive CoAP response and forward it to the mesh network. + Appends UDP and IPv6 headers to the CoAP message and forwards it on the Eventbus towards the mesh. + ''' + self.ephemeralCoapClient.close() + + # UDP + udplen = len(data) + 8 + + udp = u.int2buf(sender[1], 2) # src port + udp += u.int2buf(self.ephemeralCoapClient.udpPort, 2) # dest port + udp += [udplen >> 8, udplen & 0xff] # length + udp += [0x00, 0x00] # checksum + udp += data + + # destination address of the packet is CoAP client's IPv6 address (address of the mote) + dstIpv6Address = u.ipv6AddrString2Bytes(self.ephemeralCoapClient.ipAddress) + assert len(dstIpv6Address)==16 + # source address of the packet is DAG root's IPV6 address + # use the same prefix (link-local or global) as in the destination address + srcIpv6Address = dstIpv6Address[:8] + srcIpv6Address += self.dagRootEui64 + assert len(srcIpv6Address)==16 + + # CRC See https://tools.ietf.org/html/rfc2460. + + udp[6:8] = openvisualizer.openvisualizer_utils.calculatePseudoHeaderCRC( + src=srcIpv6Address, + dst=dstIpv6Address, + length=[0x00, 0x00] + udp[4:6], + nh=[0x00, 0x00, 0x00, 17], # UDP as next header + payload=udp, + ) + + # IPv6 + ip = [6 << 4] # v6 + traffic class (upper nybble) + ip += [0x00, 0x00, 0x00] # traffic class (lower nibble) + flow label + ip += udp[4:6] # payload length + ip += [17] # next header (protocol); UDP=17 + ip += [64] # hop limit (pick a safe value) + ip += srcIpv6Address # source + ip += dstIpv6Address # destination + ip += udp + + # announce network prefix + self.dispatch( + signal = 'v6ToMesh', + data = ip + ) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index a9d83e46..37cca6ab 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -53,7 +53,7 @@ class OpenBenchmarkAgent(eventBusClient.eventBusClient): OPENBENCHMARK_MAX_RETRIES = 3 OPENBENCHMARK_PREFIX_CMD_HANDLER_NAME = '_mqtt_handler_' - def __init__(self, mqttBroker, firmware, testbed, portNames, scenario): + def __init__(self, coapServer, mqttBroker, firmware, testbed, portNames, scenario): ''' :param mqttBroker: Address of the MQTT broker where to connect with OpenBenchmark @@ -71,6 +71,7 @@ def __init__(self, mqttBroker, firmware, testbed, portNames, scenario): ''' # store params + self.coapServer = coapServer self.mqttBroker = mqttBroker self.firmware = firmware self.testbed = testbed @@ -87,8 +88,6 @@ def __init__(self, mqttBroker, firmware, testbed, portNames, scenario): self.mqttClient = None self.experimentRequestResponse = None - self.coapServer = None - # dict with keys being eui64, and value corresponding testbed host identifier self.nodes = {} @@ -135,7 +134,7 @@ def __init__(self, mqttBroker, firmware, testbed, portNames, scenario): # everything is ok, start a coap server coapResource = openbenchmarkResource() - self.coapServer = coapServer(coapResource) + self.coapServer.coapServer.addResource(coapResource) # subscribe to all topics on a given experiment ID self._openbenchmark_subscribe(self.mqttClient, self.experimentId) @@ -167,6 +166,44 @@ def close(self): if self.coapServer: self.coapServer.close() + def triggerSendPacket(self, destination, acknowledged, packetsInBurst, packetToken, packetPayloadLen): + + destinationIPv6 = openvisualizer.openvisualizer_utils.formatIPv6Addr((self.networkPrefix + destination)) + options = [] + + if not acknowledged: + options += [o.NoResponse([d.DFLT_OPTION_NORESPONSE_SUPRESS_ALL])] + + with self.clientLock: + for packetCounter in range (0, packetsInBurst): + try: + # construct the payload of the POST request + payload = [] + payload += [packetCounter] + payload += [packetToken[1:]] + payload += [0] * packetPayloadLen + + # the call to POST() is blocking unless no response is expected + p = self.coapServer.POST('coap://[{0}:{1}]/b'.format(destinationIPv6, d.DEFAULT_UDP_PORT), + confirmable=False, + options=options, + payload = payload) + + # TODO log if a response is received + except e.coapNoResponseExpected: + pass + + def encodeSendPacketPayload(self, destination, confirmable, packetsInBurst, packetToken, packetPayloadLen): + # construct command payload as byte-list: + # dest_eui64 (8B) || con (1B) || packetsInBurst (1B) || packetToken (5B) || packetPayloadLen (1B) + buf = [] + buf += u.hex2buf(destination, separator='-') + buf += [int(confirmable)] + buf += [int(packetsInBurst)] + buf += packetToken + buf += [packetPayloadLen] + return buf + # ======================== private ========================================= def _openbenchmark_start_benchmark(self, mqttClient): @@ -341,7 +378,7 @@ def _mqtt_handler_sendPacket(self, payload): acknowledged = payloadDecoded['confirmable'] if self.coapServer.getDagRootEui64() == source: # check if command is for the DAG root whose APP code is implemented here - self.coapServer.triggerSendPacket(destination, acknowledged, packetsInBurst, packetToken, packetPayloadLen) + self.triggerSendPacket(destination, acknowledged, packetsInBurst, packetToken, packetPayloadLen) else: # command is for one of the motes in the mesh, send it over the serial @@ -422,196 +459,6 @@ def _mqtt_handler_triggerNetworkFormation(self, payload): return (True, returnVal) - -# ======================== CoAP server ====================================== -class coapServer(eventBusClient.eventBusClient): - - OPENBENCHMARK_COAP_PORT = 5684 - - def __init__(self, coapResource): - # log - log.info("create instance") - - self.coapResource = coapResource - - # run CoAP server in testing mode - # this mode does not open a real socket, rather uses PyDispatcher for sending/receiving messages - # We interface this mode with OpenVisualizer to run JRC co-located with the DAG root - self.coapServer = coap.coap(udpPort=self.OPENBENCHMARK_COAP_PORT, testing=True) - self.coapServer.addResource(coapResource) - - self.coapEphemeralClient = None - - self.dagRootEui64 = None - - # store params - - # initialize parent class - eventBusClient.eventBusClient.__init__( - self, - name='OpenBenchmark-coapServer', - registrations=[ - { - 'sender': self.WILDCARD, - 'signal': 'registerDagRoot', - 'callback': self._registerDagRoot_notif - }, - { - 'sender': self.WILDCARD, - 'signal': 'unregisterDagRoot', - 'callback': self._unregisterDagRoot_notif - }, - ] - ) - - # ======================== public ========================================== - - def close(self): - # nothing to do - pass - - def getDagRootEui64(self): - return self.dagRootEui64 - - def getDagRootIPv6(self): - ipv6buf = self.networkPrefix + self.dagRootEui64 - return openvisualizer.openvisualizer_utils.formatIPv6Addr(ipv6buf) - - def triggerSendPacket(self, destination, acknowledged, packetsInBurst, packetToken, packetPayloadLen): - - destinationIPv6 = openvisualizer.openvisualizer_utils.formatIPv6Addr((self.networkPrefix + destination)) - options = [] - - if not acknowledged: - options += [o.NoResponse([d.DFLT_OPTION_NORESPONSE_SUPRESS_ALL])] - - with self.clientLock: - for packetCounter in range (0, packetsInBurst): - try: - # construct the payload of the POST request - payload = [] - payload += [packetCounter] - payload += [packetToken[1:]] - payload += [0] * packetPayloadLen - - # the call to POST() is blocking unless no response is expected - p = self.coapServer.POST('coap://[{0}:{1}]/b'.format(destinationIPv6, d.DEFAULT_UDP_PORT), - confirmable=False, - options=options, - payload = payload) - - # TODO log if a response is received - except e.coapNoResponseExpected: - pass - - def encodeSendPacketPayload(self, destination, confirmable, packetsInBurst, packetToken, packetPayloadLen): - # construct command payload as byte-list: - # dest_eui64 (8B) || con (1B) || packetsInBurst (1B) || packetToken (5B) || packetPayloadLen (1B) - buf = [] - buf += u.hex2buf(destination, separator='-') - buf += [int(confirmable)] - buf += [int(packetsInBurst)] - buf += packetToken - buf += [packetPayloadLen] - return buf - - # ======================== private ========================================= - - # ==== handle EventBus notifications - - def _registerDagRoot_notif(self, sender, signal, data): - # register for the global address of the DAG root - self.register( - sender=self.WILDCARD, - signal=( - tuple(data['prefix'] + data['host']), - self.PROTO_UDP, - self.OPENBENCHMARK_COAP_PORT - ), - callback=self._receiveFromMesh, - ) - - self.networkPrefix = data['prefix'] - self.dagRootEui64 = data['host'] - - def _unregisterDagRoot_notif(self, sender, signal, data): - # unregister global address - self.unregister( - sender=self.WILDCARD, - signal=( - tuple(data['prefix'] + data['host']), - self.PROTO_UDP, - self.OPENBENCHMARK_COAP_PORT - ), - callback=self._receiveFromMesh, - ) - self.networkPrefix = None - self.dagRootEui64 = None - - def _receiveFromMesh(self, sender, signal, data): - ''' - Receive packet from the mesh destined for the CoAP server. - Forwards the packet to the virtual CoAP server running in test mode (PyDispatcher). - ''' - sender = openvisualizer.openvisualizer_utils.formatIPv6Addr(data[0]) - # FIXME pass source port within the signal and open coap client at this port - self.coapEphemeralClient = coap.coap(ipAddress=sender, udpPort=d.DEFAULT_UDP_PORT, testing=True, receiveCallback=self._receiveFromCoAP) - self.coapEphemeralClient.socketUdp.sendUdp(destIp='', destPort=self.OPENBENCHMARK_COAP_PORT, msg=data[1]) # low level forward of the CoAP message - return True - - def _receiveFromCoAP(self, timestamp, sender, data): - ''' - Receive CoAP response and forward it to the mesh network. - Appends UDP and IPv6 headers to the CoAP message and forwards it on the Eventbus towards the mesh. - ''' - - # FIXME potential memory leak when there the request contains a No Response option - self.coapEphemeralClient.close() - - # UDP - udplen = len(data) + 8 - - udp = u.int2buf(sender[1], 2) # src port - udp += u.int2buf(self.coapEphemeralClient.udpPort, 2) # dest port - udp += [udplen >> 8, udplen & 0xff] # length - udp += [0x00, 0x00] # checksum - udp += data - - # destination address of the packet is CoAP client's IPv6 address (address of the mote) - dstIpv6Address = u.ipv6AddrString2Bytes(self.coapEphemeralClient.ipAddress) - assert len(dstIpv6Address)==16 - # source address of the packet is DAG root's IPV6 address - # use the same prefix (link-local or global) as in the destination address - srcIpv6Address = dstIpv6Address[:8] - srcIpv6Address += self.dagRootEui64 - assert len(srcIpv6Address)==16 - - # CRC See https://tools.ietf.org/html/rfc2460. - - udp[6:8] = openvisualizer.openvisualizer_utils.calculatePseudoHeaderCRC( - src=srcIpv6Address, - dst=dstIpv6Address, - length=[0x00, 0x00] + udp[4:6], - nh=[0x00, 0x00, 0x00, 17], # UDP as next header - payload=udp, - ) - - # IPv6 - ip = [6 << 4] # v6 + traffic class (upper nybble) - ip += [0x00, 0x00, 0x00] # traffic class (lower nibble) + flow label - ip += udp[4:6] # payload length - ip += [17] # next header (protocol); UDP=17 - ip += [64] # hop limit (pick a safe value) - ip += srcIpv6Address # source - ip += dstIpv6Address # destination - ip += udp - - # announce network prefix - self.dispatch( - signal = 'v6ToMesh', - data = ip - ) - # ==================== Implementation of CoAP openbenchmark resource ===================== class openbenchmarkResource(coapResource.coapResource): def __init__(self): From d0795f570cc8d09be2009b568a916f22b666f2cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Mon, 18 Mar 2019 21:48:26 +0100 Subject: [PATCH 030/108] OV-7. Remove remnant calls to networkEventLogger. --- openvisualizer/moteState/moteState.py | 132 ++------------------------ 1 file changed, 6 insertions(+), 126 deletions(-) diff --git a/openvisualizer/moteState/moteState.py b/openvisualizer/moteState/moteState.py index 09cbc38c..6b02af62 100644 --- a/openvisualizer/moteState/moteState.py +++ b/openvisualizer/moteState/moteState.py @@ -159,25 +159,11 @@ def update(self,data): notif.joinedAsn_2_3, notif.joinedAsn_4) - try: - if self.data[0]['joinedAsn'] != receivedJoinAsn and receivedJoinAsn.asn != [0x00, 0x00, 0x00, 0x00, 0x00]: - - self.data[0]['joinedAsn'].update(notif.joinedAsn_0_1, - notif.joinedAsn_2_3, - notif.joinedAsn_4) - - networkEventLogger.info( - json.dumps( - { - "asn" : str(self.data[0]['joinedAsn']), - "_type": "secjoin.joined", - "_mote_info": moteInfo, - "_timestamp": self.meta[0]['lastUpdated'], - } - ) - ) - except: - pass + if self.data[0]['joinedAsn'] != receivedJoinAsn and receivedJoinAsn.asn != [0x00, 0x00, 0x00, 0x00, 0x00]: + + self.data[0]['joinedAsn'].update(notif.joinedAsn_0_1, + notif.joinedAsn_2_3, + notif.joinedAsn_4) class StateMacStats(StateElem): @@ -197,20 +183,6 @@ def update(self,data): else: self.data[0]['dutyCycle'] = '?' - try: - networkEventLogger.info( - json.dumps( - { - "dutycycle": str(self.data[0]['dutyCycle']), - "_type": "tsch.dutycycle", - "_mote_info": moteInfo, - "_timestamp": self.meta[0]['lastUpdated'], - } - ) - ) - except: - pass - class StateScheduleRow(StateElem): def update(self,data): @@ -238,54 +210,6 @@ def update(self,data): notif.lastUsedAsn_2_3, notif.lastUsedAsn_4) - try: - networkEventLogger.info( - json.dumps( - { - "numTx": str(self.data[0]['numTx']), - "_type": "tsch.numTx", - "_mote_info": moteInfo, - "_timestamp": self.meta[0]['lastUpdated'], - } - ) - ) - - networkEventLogger.info( - json.dumps( - { - "numRx": str(self.data[0]['numRx']), - "_type": "tsch.numRx", - "_mote_info": moteInfo, - "_timestamp": self.meta[0]['lastUpdated'], - } - ) - ) - - networkEventLogger.info( - json.dumps( - { - "numTxAck": str(self.data[0]['numTxAck']), - "_type": "tsch.numTxAck", - "_mote_info": moteInfo, - "_timestamp": self.meta[0]['lastUpdated'], - } - ) - ) - - networkEventLogger.info( - json.dumps( - { - "numTxAck": str(self.data[0]['numTxAck']), - "_type": "tsch.numTxAck", - "_mote_info": moteInfo, - "_timestamp": self.meta[0]['lastUpdated'], - } - ) - ) - - except: - pass - def getType(self): return self.data[0]['type'] @@ -539,60 +463,16 @@ def __init__(self, *args, **kwargs): super(StateSchedule, self).__init__(*args, **kwargs) def log(self, moteInfo): - try: numCellsTx = sum(row.getType().getCellType() == "TX" for row in self.data) numCellsRx = sum(row.getType().getCellType() == "RX" for row in self.data) numCellsTxRx = sum(row.getType().getCellType() == "TXRX" for row in self.data) numCells = numCellsTx + numCellsRx + numCellsTxRx - networkEventLogger.info( - json.dumps( - { - "numCellsTx": str(numCellsTx), - "_type": "tsch.numCellsTx", - "_mote_info": moteInfo, - "_timestamp": self.meta[0]['lastUpdated'], - } - ) - ) - - networkEventLogger.info( - json.dumps( - { - "numCellsRx": str(numCellsRx), - "_type": "tsch.numCellsRx", - "_mote_info": moteInfo, - "_timestamp": self.meta[0]['lastUpdated'], - } - ) - ) - - networkEventLogger.info( - json.dumps( - { - "numCellsTxRx": str(numCellsTxRx), - "_type": "tsch.numCellsTxRx", - "_mote_info": moteInfo, - "_timestamp": self.meta[0]['lastUpdated'], - } - ) - ) - - networkEventLogger.info( - json.dumps( - { - "numCells": str(numCells), - "_type": "tsch.numCells", - "_mote_info": moteInfo, - "_timestamp": self.meta[0]['lastUpdated'], - } - ) - ) + # TODO publish except: pass - class StateNeighbors(StateTable): def __init__(self, *args, **kwargs): super(StateNeighbors, self).__init__(*args, **kwargs) From 46881af04b5510a04aaa32ad2eb25d1bfabd6d75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Tue, 19 Mar 2019 16:53:28 +0100 Subject: [PATCH 031/108] OV-7. Add a special parser for benchmarking-related events. --- bin/logging.conf | 8 ++- openvisualizer/moteConnector/OpenParser.py | 8 +++ .../moteConnector/ParserBenchmark.py | 60 +++++++++++++++++++ 3 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 openvisualizer/moteConnector/ParserBenchmark.py diff --git a/bin/logging.conf b/bin/logging.conf index ce513f22..5953b9fb 100644 --- a/bin/logging.conf +++ b/bin/logging.conf @@ -41,7 +41,7 @@ formatter=benchmark #============================ loggers ========================================= [loggers] -keys=root,eventBusMonitor,openTun,openTunWindows,openTunLinux,eventBusClient,lbrClient,moteConnector,moteProbe,moteProbeUtils,moteState,openLbr,openBenchmarkAgent,OpenParser,Parser,OpenHdlc,ParserData,ParserPrintf,ParserInfoErrorCritical,ParserStatus,RPL,SourceRoute,JRC,networkEventLogger,openVisualizerApp,openVisualizerGui,openVisualizerCli,openVisualizerWeb,OVtracer,coapServer +keys=root,eventBusMonitor,openTun,openTunWindows,openTunLinux,eventBusClient,lbrClient,moteConnector,moteProbe,moteProbeUtils,moteState,openLbr,openBenchmarkAgent,OpenParser,Parser,OpenHdlc,ParserData,ParserPrintf,ParserInfoErrorCritical,ParserStatus,ParserBenchmark,RPL,SourceRoute,JRC,networkEventLogger,openVisualizerApp,openVisualizerGui,openVisualizerCli,openVisualizerWeb,OVtracer,coapServer [logger_root] level=ERROR @@ -161,6 +161,12 @@ handlers=std propagate=0 qualname=ParserStatus +[logger_ParserBenchmark] +level=DEBUG +handlers=std +propagate=0 +qualname=ParserBenchmark + [logger_RPL] level=INFO handlers=std diff --git a/openvisualizer/moteConnector/OpenParser.py b/openvisualizer/moteConnector/OpenParser.py index cf85976f..9886e122 100644 --- a/openvisualizer/moteConnector/OpenParser.py +++ b/openvisualizer/moteConnector/OpenParser.py @@ -15,6 +15,7 @@ import ParserData import ParserPacket import ParserPrintf +import ParserBenchmark class OpenParser(Parser.Parser): @@ -27,6 +28,7 @@ class OpenParser(Parser.Parser): SERFRAME_MOTE2PC_CRITICAL = ParserIEC.ParserInfoErrorCritical.SEVERITY_CRITICAL SERFRAME_MOTE2PC_SNIFFED_PACKET = ord('P') SERFRAME_MOTE2PC_PRINTF = ord('F') + SERFRAME_MOTE2PC_BENCHMARK = ord('B') SERFRAME_PC2MOTE_SETDAGROOT = ord('R') SERFRAME_PC2MOTE_DATA = ord('D') @@ -53,6 +55,7 @@ def __init__(self, mqtt_broker_address): self.parserData = ParserData.ParserData(mqtt_broker_address) self.parserPacket = ParserPacket.ParserPacket() self.parserPrintf = ParserPrintf.ParserPrintf() + self.parserBenchmark = ParserBenchmark.ParserBenchmark() # register subparsers self._addSubParser( @@ -90,6 +93,11 @@ def __init__(self, mqtt_broker_address): val = self.SERFRAME_MOTE2PC_PRINTF, parser = self.parserPrintf.parseInput, ) + self._addSubParser( + index = 0, + val = self.SERFRAME_MOTE2PC_BENCHMARK, + parser = self.parserBenchmark.parseInput, + ) #======================== public ========================================== diff --git a/openvisualizer/moteConnector/ParserBenchmark.py b/openvisualizer/moteConnector/ParserBenchmark.py new file mode 100644 index 00000000..62d5c2cc --- /dev/null +++ b/openvisualizer/moteConnector/ParserBenchmark.py @@ -0,0 +1,60 @@ +# Copyright (c) 2010-2013, Regents of the University of California. +# All rights reserved. +# +# Released under the BSD 3-Clause license as published at the link below. +# https://openwsn.atlassian.net/wiki/display/OW/License +import logging + +log = logging.getLogger('ParserBenchmark') +log.setLevel(logging.ERROR) +log.addHandler(logging.NullHandler()) + +import struct + +from pydispatch import dispatcher + +from ParserException import ParserException +import Parser + +from openvisualizer.openType import typeAsn + +class ParserBenchmark(Parser.Parser): + HEADER_LENGTH = 2 + + def __init__(self): + + # log + log.info("create instance") + + # initialize parent class + Parser.Parser.__init__(self, self.HEADER_LENGTH) + + self._asn = ['asn_4', # B + 'asn_2_3', # H + 'asn_0_1', # H + ] + + # ======================== public ========================================== + + def parseInput(self, input): + # log + log.debug("received data {0}".format(input)) + + # ensure input not short longer than header + self._checkLength(input) + + source = input[:2] + event = input[2] + + asnParsed = struct.unpack(' Date: Tue, 19 Mar 2019 16:56:31 +0100 Subject: [PATCH 032/108] OV-7. Remove remnant status handlers and parsers. --- bin/openVisualizerWeb.py | 1 - bin/web_files/templates/moteview.tmpl | 5 - openvisualizer/moteConnector/ParserStatus.py | 77 ------------ openvisualizer/moteState/moteState.py | 126 ------------------- 4 files changed, 209 deletions(-) diff --git a/bin/openVisualizerWeb.py b/bin/openVisualizerWeb.py index 8873d554..b3f6129b 100755 --- a/bin/openVisualizerWeb.py +++ b/bin/openVisualizerWeb.py @@ -294,7 +294,6 @@ def _getMoteData(self, moteid): ms.ST_SCHEDULE : ms.getStateElem(ms.ST_SCHEDULE).toJson('data'), ms.ST_QUEUE : ms.getStateElem(ms.ST_QUEUE).toJson('data'), ms.ST_NEIGHBORS : ms.getStateElem(ms.ST_NEIGHBORS).toJson('data'), - ms.ST_JOINED : ms.getStateElem(ms.ST_JOINED).toJson('data'), } else: if log.isEnabledFor(logging.DEBUG): diff --git a/bin/web_files/templates/moteview.tmpl b/bin/web_files/templates/moteview.tmpl index 3a01554a..680d6aec 100644 --- a/bin/web_files/templates/moteview.tmpl +++ b/bin/web_files/templates/moteview.tmpl @@ -158,11 +158,6 @@ - - Join ASN - - -
Output Buffer
diff --git a/openvisualizer/moteConnector/ParserStatus.py b/openvisualizer/moteConnector/ParserStatus.py index 2399cac3..0d4f9598 100644 --- a/openvisualizer/moteConnector/ParserStatus.py +++ b/openvisualizer/moteConnector/ParserStatus.py @@ -243,83 +243,6 @@ def __init__(self): 'kaPeriod', # H ], ) - self._addFieldsParser ( - 3, - 11, - 'Joined', - ' Date: Tue, 19 Mar 2019 16:56:57 +0100 Subject: [PATCH 033/108] OV-7. Have OpenBenchmarkAgent subscribe to performanceData events. --- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 37cca6ab..760ae702 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -146,7 +146,7 @@ def __init__(self, coapServer, mqttBroker, firmware, testbed, portNames, scenari registrations=[ { 'sender': self.WILDCARD, - 'signal': 'performanceData', + 'signal': 'fromMote.performanceData', 'callback': self._performance_data_handler, }, ] From 2f39fbf0d7ff588bf49d27de5c9d24665baa1300 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Tue, 19 Mar 2019 17:26:36 +0100 Subject: [PATCH 034/108] OV-7. Remove remnants in web_files. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mališa Vučinić --- bin/web_files/templates/moteview.tmpl | 2 -- 1 file changed, 2 deletions(-) diff --git a/bin/web_files/templates/moteview.tmpl b/bin/web_files/templates/moteview.tmpl index 680d6aec..bd790399 100644 --- a/bin/web_files/templates/moteview.tmpl +++ b/bin/web_files/templates/moteview.tmpl @@ -286,7 +286,6 @@ queueJson = $.parseJSON(json.Queue); nbrsJson = $.parseJSON(json.Neighbors); kaPeriodJson = $.parseJSON(json.kaPeriod)[0]; - joinedJson = $.parseJSON(json.Joined)[0]; } // Exclude tailing description from server @@ -298,7 +297,6 @@ $("#sync_fld").text( hasJson && syncJson.isSync > 0 ? 'Synchronized!' : 'Not synchronized'); $("#pan_fld").text( hasJson ? idJson.myPANID : ''); $("#asn_fld").text( hasJson ? asnJson.asn : ''); - $("#join_fld").text( hasJson ? joinedJson.joinedAsn : ''); $("#dagrank_fld").text( hasJson ? dagrankJson.myDAGrank : ''); $("#outread_fld").text( hasJson ? outbufJson.index_read : ''); $("#outwrite_fld").text( hasJson ? outbufJson.index_write : ''); From b552cdf0a815d636ca00157bb36ed28b03223395 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Tue, 19 Mar 2019 17:32:39 +0100 Subject: [PATCH 035/108] OV-7. Rename class. --- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 760ae702..05a71697 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -133,7 +133,7 @@ def __init__(self, coapServer, mqttBroker, firmware, testbed, portNames, scenari raise ValueError("Unable to start an experiment with OpenBenchmark") # everything is ok, start a coap server - coapResource = openbenchmarkResource() + coapResource = OpenbenchmarkResource() self.coapServer.coapServer.addResource(coapResource) # subscribe to all topics on a given experiment ID @@ -460,7 +460,7 @@ def _mqtt_handler_triggerNetworkFormation(self, payload): return (True, returnVal) # ==================== Implementation of CoAP openbenchmark resource ===================== -class openbenchmarkResource(coapResource.coapResource): +class OpenbenchmarkResource(coapResource.coapResource): def __init__(self): # initialize parent class From 423f09401f366cfbe83ee76c8fdd0d5c0f405de1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Tue, 19 Mar 2019 19:21:22 +0100 Subject: [PATCH 036/108] OV-7. Expect full EUI-64 when parsing benchmark serial frames. --- openvisualizer/moteConnector/ParserBenchmark.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/openvisualizer/moteConnector/ParserBenchmark.py b/openvisualizer/moteConnector/ParserBenchmark.py index 62d5c2cc..8c035317 100644 --- a/openvisualizer/moteConnector/ParserBenchmark.py +++ b/openvisualizer/moteConnector/ParserBenchmark.py @@ -43,15 +43,15 @@ def parseInput(self, input): # ensure input not short longer than header self._checkLength(input) - source = input[:2] - event = input[2] + source = input[:8] + event = input[8] - asnParsed = struct.unpack(' Date: Tue, 19 Mar 2019 19:22:29 +0100 Subject: [PATCH 037/108] OV-7. Implement PerformanceEvent class to handle events from firmware. --- .../openBenchmarkAgent/openBenchmarkAgent.py | 151 +++++++++++++++++- 1 file changed, 146 insertions(+), 5 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 05a71697..a4941dd5 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -87,6 +87,7 @@ def __init__(self, coapServer, mqttBroker, firmware, testbed, portNames, scenari self.mqttClient = None self.experimentRequestResponse = None + self.performanceEvent = None # dict with keys being eui64, and value corresponding testbed host identifier self.nodes = {} @@ -139,6 +140,9 @@ def __init__(self, coapServer, mqttBroker, firmware, testbed, portNames, scenari # subscribe to all topics on a given experiment ID self._openbenchmark_subscribe(self.mqttClient, self.experimentId) + # instantiate performance event handlers from firmware + self.performanceEvent = PerformanceEvent(self.experimentId, self.mqttClient) + # subscribe to eventBus performance-related events eventBusClient.eventBusClient.__init__( self, @@ -147,7 +151,7 @@ def __init__(self, coapServer, mqttBroker, firmware, testbed, portNames, scenari { 'sender': self.WILDCARD, 'signal': 'fromMote.performanceData', - 'callback': self._performance_data_handler, + 'callback': self.performanceEvent.handler, }, ] ) @@ -353,10 +357,6 @@ def _on_mqtt_message(self, client, userdata, message): else: self._execute_command_safely(message.topic, message.payload) - def _performance_data_handler(self): - # TODO - pass - # ==== mqtt command handlers def _mqtt_handler_echo(self, payload): @@ -460,6 +460,7 @@ def _mqtt_handler_triggerNetworkFormation(self, payload): return (True, returnVal) # ==================== Implementation of CoAP openbenchmark resource ===================== + class OpenbenchmarkResource(coapResource.coapResource): def __init__(self): @@ -472,3 +473,143 @@ def __init__(self): def POST(self,options=[], payload=[]): # TODO parse the packet token and log the event return (d.COAP_RC_2_04_CHANGED, [], []) + +class PerformanceEvent(object): + + # event #name # id + EV_PACKET_SENT = ['packetSent', 0 ] + EV_PACKET_RECEIVED = ['packetReceived', 1 ] + EV_SECURE_JOIN_COMPLETED = ['secureJoinCompleted', 2 ] + EV_BANDWIDTH_ASSIGNED = ['bandwidthAssigned', 3 ] + EV_SYNCHRONIZATION_COMPLETED = ['synchronizationCompleted', 256 ] + EV_NETWORK_FORMATION_COMPLETED = ['networkFormationCompleted', 257 ] + EV_RADIO_DUTY_CYCLE_MEASUREMENT = ['radioDutyCycleMeasurement', 258 ] + EV_CLOCK_DRIFT_MEASUREMENT = ['clockDriftMeasurement', 259 ] + + EV_ALL = [ + EV_PACKET_SENT, + EV_PACKET_RECEIVED, + EV_SECURE_JOIN_COMPLETED, + EV_BANDWIDTH_ASSIGNED, + EV_SYNCHRONIZATION_COMPLETED, + EV_NETWORK_FORMATION_COMPLETED, + EV_RADIO_DUTY_CYCLE_MEASUREMENT, + EV_CLOCK_DRIFT_MEASUREMENT, + ] + + PERFORMANCE_EVENT_HANDLER_NAME = "_handler_event_" + + def __init__(self, experimentId, mqttClient): + + #assert experimentId + #assert mqttClient + + self.experimentId = experimentId + self.mqttClient = mqttClient + + # ======================== public ========================================= + + def handler(self, sender, signal, data): + (source, event, timestamp, buf) = data + + returnVal = {} + + + log.debug( + "sender: {0}\n signal: {1}\n source: {2}\n event: {3}\n timestamp: {4}\n data: {5}".format(sender, signal, + source, event, + timestamp, data)) + try: + + # find the event + event_name = None + for ev in self.EV_ALL: + if ev[1] == event: + event_name = ev[0] + break + + assert event_name, "Unhandled event, ignoring: {0}".format(event) + + # find the handler + event_handler = getattr(self, '{0}{1}'.format(self.PERFORMANCE_EVENT_HANDLER_NAME, event_name, None)) + + assert event_handler, "Event recognized but cannot find handler, event: {0}".format(event) + + # call the handler to return dictionary with handler-specific fields in the response + # handler return is in format (success, dict), with: + # - success, as a boolean + # - dict, dictionary containing fields to include in the response or None on failure + (success, dict) = event_handler(buf) + + if success: + # common fields + returnVal['event'] = event_name + returnVal['timestamp'] = str(timestamp) + returnVal['source'] = openvisualizer.openvisualizer_utils.formatAddr(source) + + # handler-specific fields + returnVal.update(dict) + + log.debug(returnVal) + + #self.mqttClient.publish( + # topic='openbenchmark/experimentId/{0}/nodeId/{0}/performanceData'.format(self.experimentId, source), + # payload=json.dumps(returnVal), + #) + + except Exception as err: + log.exception("Exception while executing {0}".format(event)) + log.exception(err) + log.exception(traceback.format_exc()) + + + # ======================== private ========================================= + + # packetSent + def _handler_event_packetSent(self, buf): + returnVal = {} + + packetToken = buf[:5] + destBuf = buf[5:13] + dest = openvisualizer.openvisualizer_utils.formatAddr(destBuf) + hopLimit = buf[13] + + returnVal['packetToken'] = packetToken + returnVal['destination'] = dest + returnVal['hopLimit'] = hopLimit + + return (True, returnVal) + + # packetReceived, same syntax as packetSent + def _handler_event_packetReceived(self, buf): + return self._handler_event_packetSent(buf) + + # synchronizationCompleted + def _handler_event_synchronizationCompleted(self, buf): + return (True, {}) + + # secureJoinCompleted + def _handler_event_secureJoinCompleted(self, buf): + return (True, {}) + + # bandwidthAssigned + def _handler_event_bandwidthAssigned(self, buf): + return (True, {}) + + # networkFormationCompleted + def _handler_event_networkFormationCompleted(self, buf): + # TODO + return (True, {}) + + # radioDutyCycleMeasurement + def _handler_event_radioDutyCycleMeasurement(self, buf): + returnVal = {} + # TODO + return (True, returnVal) + + # clockDriftMeasurement + def _handler_event_clockDriftMeasurement(self, buf): + returnVal = {} + # TODO + return (True, returnVal) + From d010142a0bd04bafc7e92770cf2cc851eedd7362 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Tue, 19 Mar 2019 21:08:36 +0100 Subject: [PATCH 038/108] OV-7. Align event IDs with firmware. --- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index a4941dd5..6fbd163a 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -479,9 +479,9 @@ class PerformanceEvent(object): # event #name # id EV_PACKET_SENT = ['packetSent', 0 ] EV_PACKET_RECEIVED = ['packetReceived', 1 ] - EV_SECURE_JOIN_COMPLETED = ['secureJoinCompleted', 2 ] - EV_BANDWIDTH_ASSIGNED = ['bandwidthAssigned', 3 ] - EV_SYNCHRONIZATION_COMPLETED = ['synchronizationCompleted', 256 ] + EV_SYNCHRONIZATION_COMPLETED = ['synchronizationCompleted', 2 ] + EV_SECURE_JOIN_COMPLETED = ['secureJoinCompleted', 3 ] + EV_BANDWIDTH_ASSIGNED = ['bandwidthAssigned', 4 ] EV_NETWORK_FORMATION_COMPLETED = ['networkFormationCompleted', 257 ] EV_RADIO_DUTY_CYCLE_MEASUREMENT = ['radioDutyCycleMeasurement', 258 ] EV_CLOCK_DRIFT_MEASUREMENT = ['clockDriftMeasurement', 259 ] From b78287d5a82ad97a6dde5fedc45000dac1df285a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Tue, 19 Mar 2019 21:09:27 +0100 Subject: [PATCH 039/108] OV-7. Rename event handler to remove ambiguity with updates. --- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 6fbd163a..9d0fc86d 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -151,7 +151,7 @@ def __init__(self, coapServer, mqttBroker, firmware, testbed, portNames, scenari { 'sender': self.WILDCARD, 'signal': 'fromMote.performanceData', - 'callback': self.performanceEvent.handler, + 'callback': self.performanceEvent.handle_event, }, ] ) @@ -509,7 +509,7 @@ def __init__(self, experimentId, mqttClient): # ======================== public ========================================= - def handler(self, sender, signal, data): + def handle_event(self, sender, signal, data): (source, event, timestamp, buf) = data returnVal = {} From 835a4b06629ac21c01080a418258c07375c796ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Tue, 19 Mar 2019 21:10:50 +0100 Subject: [PATCH 040/108] OV-7. Add performance event poller that will poll for periodic measurements. --- openvisualizer/moteState/moteState.py | 34 ++++++++ .../openBenchmarkAgent/openBenchmarkAgent.py | 82 +++++++++++++++++++ 2 files changed, 116 insertions(+) diff --git a/openvisualizer/moteState/moteState.py b/openvisualizer/moteState/moteState.py index 6baa9e24..ee8ce8e3 100644 --- a/openvisualizer/moteState/moteState.py +++ b/openvisualizer/moteState/moteState.py @@ -145,6 +145,11 @@ def update(self,data): notif.asn_2_3, notif.asn_4) + def getAsn(self): + if len(self.data) != 0: + return self.data[0]['asn'] + return None + class StateMacStats(StateElem): def update(self,data): @@ -163,6 +168,11 @@ def update(self,data): else: self.data[0]['dutyCycle'] = '?' + def getDutyCycle(self): + if len(self.data) != 0: + return self.data[0]['dutyCycle'] + return '?' + class StateScheduleRow(StateElem): def update(self,data): @@ -647,6 +657,11 @@ def __init__(self,moteConnector): 'signal' : 'fromMote.status', 'callback' : self._receivedStatus_notif, }, + { + 'sender': self.WILDCARD, + 'signal': 'getDutyCycleMeasurement', + 'callback': self._getDutyCycle, + }, ] ) @@ -715,3 +730,22 @@ def _receivedStatus_notif(self,data): def _isnamedtupleinstance(self,var,tupleInstance): return var._fields==tupleInstance._fields + + def _getDutyCycle(self, sender, signal, data): + try: + # data = { + # 'source' : self.moteConnector.name, + # 'timestamp' : str(self.state[self.ST_ASN].getAsn()), + # 'dutyCycle' : self.state[self.ST_MACSTATS].getDutyCycle(), + # } + + data = { + 'source' : self.moteConnector.name, + 'timestamp' : 0, + 'dutyCycle' : 0, + } + + # dispatch + self.dispatch('dutyCycleMeasurement', data) + except: + pass diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 9d0fc86d..08bbd0f1 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -49,6 +49,7 @@ class OpenBenchmarkAgent(eventBusClient.eventBusClient): OPENBENCHMARK_STARTBENCHMARK_REQUEST_TOPIC = 'openbenchmark/command/startBenchmark' OPENBENCHMARK_STARTBENCHMARK_RESPONSE_TOPIC = 'openbenchmark/response/startBenchmark' + OPENBENCHMARK_MEASUREMENT_PERIOD = 3 OPENBENCHMARK_RESP_STATUS_TIMEOUT = 10 OPENBENCHMARK_MAX_RETRIES = 3 OPENBENCHMARK_PREFIX_CMD_HANDLER_NAME = '_mqtt_handler_' @@ -143,6 +144,9 @@ def __init__(self, coapServer, mqttBroker, firmware, testbed, portNames, scenari # instantiate performance event handlers from firmware self.performanceEvent = PerformanceEvent(self.experimentId, self.mqttClient) + # instantiate performance poller thread to gather duty cycle measurements periodically + self.performanceUpdatePoller = PerformanceUpdatePoller(self.OPENBENCHMARK_MEASUREMENT_PERIOD) + # subscribe to eventBus performance-related events eventBusClient.eventBusClient.__init__( self, @@ -169,6 +173,8 @@ def close(self): self.mqttClient.loop_stop() if self.coapServer: self.coapServer.close() + if self.performanceUpdatePoller: + self.performanceUpdatePoller.close() def triggerSendPacket(self, destination, acknowledged, packetsInBurst, packetToken, packetPayloadLen): @@ -613,3 +619,79 @@ def _handler_event_clockDriftMeasurement(self, buf): # TODO return (True, returnVal) +class PerformanceUpdatePoller(eventBusClient.eventBusClient, threading.Thread): + + def __init__(self, period): + # log + log.info("creating a thread for periodic polling of performance metrics") + + # local variables + self.period = period + self.dutyCycleMeasurementList = set() + self.dataLock = threading.Lock() + # flag to permit exit from read loop + self.goOn = True + + # initialize the parent class + threading.Thread.__init__(self) + + # give this thread a name + self.name = 'performanceUpdatePollerThread' + + # subscribe to eventBus performance-related events + eventBusClient.eventBusClient.__init__( + self, + name='performanceUpdatePoller', + registrations=[ + { + 'sender': self.WILDCARD, + 'signal': 'dutyCycleMeasurement', + 'callback': self.handle_dutyCycleMeasurement, + }, + ] + ) + # start myself + self.start() + + def run(self): + try: + # log + log.info("start running") + + while self.goOn: + + log.debug("poller is polling") + + # poll moteState for latest measurements + self.dispatch('getDutyCycleMeasurement', []) + + # wait for a while to gather the response from motes + time.sleep(1) + + with self.dataLock: + log.debug("collected responses:") + log.debug(self.dutyCycleMeasurementList) + + self.dutyCycleMeasurementList = set() + + time.sleep(self.period) + + except Exception as err: + errMsg = u.formatCrashMessage(self.name, err) + print errMsg + log.critical(errMsg) + self.close() + finally: + pass + + def close(self): + self.goOn = False + + def handle_dutyCycleMeasurement(self, sender, signal, data): + log.debug( + "handle_update sender: {0}\n signal: {1}\n data: {2}".format(sender, signal, data)) + with self.dataLock: + self.dutyCycleMeasurementList.add( + ( data['source'], data['timestamp'], data['dutyCycle'] ) + ) + From 7ecfd0d1a1b7369bf996890962bb9d97ad8dd4eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Wed, 20 Mar 2019 12:20:29 +0100 Subject: [PATCH 041/108] OV-7. Parse source address and convert it to string immediately. --- openvisualizer/moteConnector/ParserBenchmark.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/openvisualizer/moteConnector/ParserBenchmark.py b/openvisualizer/moteConnector/ParserBenchmark.py index 8c035317..b8996fcd 100644 --- a/openvisualizer/moteConnector/ParserBenchmark.py +++ b/openvisualizer/moteConnector/ParserBenchmark.py @@ -17,6 +17,7 @@ import Parser from openvisualizer.openType import typeAsn +from openvisualizer import openvisualizer_utils as u class ParserBenchmark(Parser.Parser): HEADER_LENGTH = 2 @@ -44,6 +45,8 @@ def parseInput(self, input): self._checkLength(input) source = input[:8] + source = u.formatAddr(source) + event = input[8] asnParsed = struct.unpack(' Date: Wed, 20 Mar 2019 12:20:56 +0100 Subject: [PATCH 042/108] OV-7. In case idmanager state is not available, return empty list. --- openvisualizer/moteState/moteState.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openvisualizer/moteState/moteState.py b/openvisualizer/moteState/moteState.py index ee8ce8e3..faf6edce 100644 --- a/openvisualizer/moteState/moteState.py +++ b/openvisualizer/moteState/moteState.py @@ -316,13 +316,13 @@ def get16bAddr(self): try: return self.data[0]['my16bID'].addr[:] except IndexError: - return None + return [] def get64bAddr(self): try: return self.data[0]['my64bID'].addr[:] except IndexError: - return None + return [] def get_serial(self): return self.moteConnector.serialport From 85773266ead5dabd1108199082f5b969264e255e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Wed, 20 Mar 2019 12:22:01 +0100 Subject: [PATCH 043/108] OV-7. Return None if duty cycle measurement is not available in mote state. --- openvisualizer/moteState/moteState.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openvisualizer/moteState/moteState.py b/openvisualizer/moteState/moteState.py index faf6edce..ca8e420d 100644 --- a/openvisualizer/moteState/moteState.py +++ b/openvisualizer/moteState/moteState.py @@ -169,9 +169,9 @@ def update(self,data): self.data[0]['dutyCycle'] = '?' def getDutyCycle(self): - if len(self.data) != 0: + if len(self.data)!=0: return self.data[0]['dutyCycle'] - return '?' + return None class StateScheduleRow(StateElem): From af8c1335227b35083913ed0eafb6390164dfbb40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Wed, 20 Mar 2019 12:25:27 +0100 Subject: [PATCH 044/108] OV-7. Redo duty cycle measurement signal handler in moteState. --- openvisualizer/moteState/moteState.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/openvisualizer/moteState/moteState.py b/openvisualizer/moteState/moteState.py index ca8e420d..3274706a 100644 --- a/openvisualizer/moteState/moteState.py +++ b/openvisualizer/moteState/moteState.py @@ -732,20 +732,20 @@ def _isnamedtupleinstance(self,var,tupleInstance): return var._fields==tupleInstance._fields def _getDutyCycle(self, sender, signal, data): - try: - # data = { - # 'source' : self.moteConnector.name, - # 'timestamp' : str(self.state[self.ST_ASN].getAsn()), - # 'dutyCycle' : self.state[self.ST_MACSTATS].getDutyCycle(), - # } + # source + source = self.state[self.ST_IDMANAGER].get_info() + # get last duty cycle measurement + dutyCycle = self.state[self.ST_MACSTATS].getDutyCycle() + # asn of the dutyCycle measurement is an approximation as the exact timestamp is not available + timestamp = str(self.state[self.ST_ASN].getAsn()) + + if source and dutyCycle and timestamp: data = { - 'source' : self.moteConnector.name, - 'timestamp' : 0, - 'dutyCycle' : 0, + 'source' : source['64bAddr'], + 'timestamp' : timestamp, + 'dutyCycle' : dutyCycle, } # dispatch self.dispatch('dutyCycleMeasurement', data) - except: - pass From 461ad7e666398c0e62d69e8c103ecffac8e93e17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Wed, 20 Mar 2019 12:26:41 +0100 Subject: [PATCH 045/108] OV-7. Pass mqttClient to Performance Update Poller constructor. --- .../openBenchmarkAgent/openBenchmarkAgent.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 08bbd0f1..cbd9adf9 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -145,7 +145,8 @@ def __init__(self, coapServer, mqttBroker, firmware, testbed, portNames, scenari self.performanceEvent = PerformanceEvent(self.experimentId, self.mqttClient) # instantiate performance poller thread to gather duty cycle measurements periodically - self.performanceUpdatePoller = PerformanceUpdatePoller(self.OPENBENCHMARK_MEASUREMENT_PERIOD) + self.performanceUpdatePoller = PerformanceUpdatePoller(self.experimentId, self.mqttClient, + self.OPENBENCHMARK_MEASUREMENT_PERIOD) # subscribe to eventBus performance-related events eventBusClient.eventBusClient.__init__( @@ -621,15 +622,20 @@ def _handler_event_clockDriftMeasurement(self, buf): class PerformanceUpdatePoller(eventBusClient.eventBusClient, threading.Thread): - def __init__(self, period): + def __init__(self, experimentId, mqttClient, period): # log log.info("creating a thread for periodic polling of performance metrics") - # local variables + # params + self.experimentId = experimentId + self.mqttClient = mqttClient self.period = period - self.dutyCycleMeasurementList = set() + + # local vars + self.dutyCycleMeasurements = set() self.dataLock = threading.Lock() - # flag to permit exit from read loop + + # flag to permit exit from infinite loop self.goOn = True # initialize the parent class From a827aa39c78e2d32290d173526731de9f3d1e8dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Wed, 20 Mar 2019 12:29:02 +0100 Subject: [PATCH 046/108] OV-7. Publish periodic measurement on MQTT once they are available. --- .../openBenchmarkAgent/openBenchmarkAgent.py | 46 +++++++++++++------ 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index cbd9adf9..83bb776d 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -552,17 +552,18 @@ def handle_event(self, sender, signal, data): # common fields returnVal['event'] = event_name returnVal['timestamp'] = str(timestamp) - returnVal['source'] = openvisualizer.openvisualizer_utils.formatAddr(source) + returnVal['source'] = source # handler-specific fields returnVal.update(dict) + topic = 'openbenchmark/experimentId/{0}/nodeId/{1}/performanceData'.format(self.experimentId, source) - log.debug(returnVal) + log.debug("Publishing on topic: {0} Payload: {1}".format(topic, returnVal)) - #self.mqttClient.publish( - # topic='openbenchmark/experimentId/{0}/nodeId/{0}/performanceData'.format(self.experimentId, source), - # payload=json.dumps(returnVal), - #) + self.mqttClient.publish( + topic=topic, + payload=json.dumps(returnVal), + ) except Exception as err: log.exception("Exception while executing {0}".format(event)) @@ -674,11 +675,32 @@ def run(self): # wait for a while to gather the response from motes time.sleep(1) - with self.dataLock: - log.debug("collected responses:") - log.debug(self.dutyCycleMeasurementList) + log.debug("Performance Event Poller, received measurements:") + log.debug(self.dutyCycleMeasurements) + + for measurement in self.dutyCycleMeasurements: + + # dispatch each as an individual message + (source, timestamp, dutyCycle) = measurement - self.dutyCycleMeasurementList = set() + topic = 'openbenchmark/experimentId/{0}/nodeId/{1}/performanceData'.format(self.experimentId, source) + payload = { + 'event' : 'radioDutyCycleMeasurement', + 'timestamp' : timestamp, + 'source' : source, + 'dutyCycle' : dutyCycle, + } + + log.debug("Publishing on topic: {0} Payload: {1}".format(topic, payload)) + + self.mqttClient.publish( + topic='openbenchmark/experimentId/{0}/nodeId/{0}/performanceData'.format(self.experimentId, source), + payload=json.dumps(payload), + ) + + with self.dataLock: + # reset for the next measurement + self.dutyCycleMeasurements = set() time.sleep(self.period) @@ -694,10 +716,8 @@ def close(self): self.goOn = False def handle_dutyCycleMeasurement(self, sender, signal, data): - log.debug( - "handle_update sender: {0}\n signal: {1}\n data: {2}".format(sender, signal, data)) with self.dataLock: - self.dutyCycleMeasurementList.add( + self.dutyCycleMeasurements.add( ( data['source'], data['timestamp'], data['dutyCycle'] ) ) From 39af9e33863e579b797f8335fd49e2122bafc5d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Wed, 20 Mar 2019 12:44:13 +0100 Subject: [PATCH 047/108] OV-7. Instantiate PerformanceEventPoller within PerformanceEvent class. --- .../openBenchmarkAgent/openBenchmarkAgent.py | 85 ++++++++++--------- 1 file changed, 45 insertions(+), 40 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 83bb776d..9525281b 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -49,7 +49,6 @@ class OpenBenchmarkAgent(eventBusClient.eventBusClient): OPENBENCHMARK_STARTBENCHMARK_REQUEST_TOPIC = 'openbenchmark/command/startBenchmark' OPENBENCHMARK_STARTBENCHMARK_RESPONSE_TOPIC = 'openbenchmark/response/startBenchmark' - OPENBENCHMARK_MEASUREMENT_PERIOD = 3 OPENBENCHMARK_RESP_STATUS_TIMEOUT = 10 OPENBENCHMARK_MAX_RETRIES = 3 OPENBENCHMARK_PREFIX_CMD_HANDLER_NAME = '_mqtt_handler_' @@ -144,10 +143,6 @@ def __init__(self, coapServer, mqttBroker, firmware, testbed, portNames, scenari # instantiate performance event handlers from firmware self.performanceEvent = PerformanceEvent(self.experimentId, self.mqttClient) - # instantiate performance poller thread to gather duty cycle measurements periodically - self.performanceUpdatePoller = PerformanceUpdatePoller(self.experimentId, self.mqttClient, - self.OPENBENCHMARK_MEASUREMENT_PERIOD) - # subscribe to eventBus performance-related events eventBusClient.eventBusClient.__init__( self, @@ -174,8 +169,8 @@ def close(self): self.mqttClient.loop_stop() if self.coapServer: self.coapServer.close() - if self.performanceUpdatePoller: - self.performanceUpdatePoller.close() + if self.performanceEvent: + self.performanceEvent.close() def triggerSendPacket(self, destination, acknowledged, packetsInBurst, packetToken, packetPayloadLen): @@ -466,42 +461,28 @@ def _mqtt_handler_triggerNetworkFormation(self, payload): return (True, returnVal) -# ==================== Implementation of CoAP openbenchmark resource ===================== - -class OpenbenchmarkResource(coapResource.coapResource): - def __init__(self): - - # initialize parent class - coapResource.coapResource.__init__( - self, - path = 'b', - ) - - def POST(self,options=[], payload=[]): - # TODO parse the packet token and log the event - return (d.COAP_RC_2_04_CHANGED, [], []) +# ================= Implementation of Experiments Performance Events API ===================== class PerformanceEvent(object): - # event #name # id + # period for measurement polling + PERFORMANCE_EVENT_MEASUREMENT_PERIOD = 10 + + # asynchronous event #name # id EV_PACKET_SENT = ['packetSent', 0 ] EV_PACKET_RECEIVED = ['packetReceived', 1 ] EV_SYNCHRONIZATION_COMPLETED = ['synchronizationCompleted', 2 ] EV_SECURE_JOIN_COMPLETED = ['secureJoinCompleted', 3 ] EV_BANDWIDTH_ASSIGNED = ['bandwidthAssigned', 4 ] EV_NETWORK_FORMATION_COMPLETED = ['networkFormationCompleted', 257 ] - EV_RADIO_DUTY_CYCLE_MEASUREMENT = ['radioDutyCycleMeasurement', 258 ] - EV_CLOCK_DRIFT_MEASUREMENT = ['clockDriftMeasurement', 259 ] - EV_ALL = [ + EV_ASYNC_ALL = [ EV_PACKET_SENT, EV_PACKET_RECEIVED, EV_SECURE_JOIN_COMPLETED, EV_BANDWIDTH_ASSIGNED, EV_SYNCHRONIZATION_COMPLETED, EV_NETWORK_FORMATION_COMPLETED, - EV_RADIO_DUTY_CYCLE_MEASUREMENT, - EV_CLOCK_DRIFT_MEASUREMENT, ] PERFORMANCE_EVENT_HANDLER_NAME = "_handler_event_" @@ -511,9 +492,14 @@ def __init__(self, experimentId, mqttClient): #assert experimentId #assert mqttClient + # params self.experimentId = experimentId self.mqttClient = mqttClient + # start poller thread for periodic measurements + self.performanceUpdatePoller = PerformanceUpdatePoller(self.experimentId, self.mqttClient, + self.PERFORMANCE_EVENT_MEASUREMENT_PERIOD) + # ======================== public ========================================= def handle_event(self, sender, signal, data): @@ -530,7 +516,7 @@ def handle_event(self, sender, signal, data): # find the event event_name = None - for ev in self.EV_ALL: + for ev in self.EV_ASYNC_ALL: if ev[1] == event: event_name = ev[0] break @@ -570,6 +556,9 @@ def handle_event(self, sender, signal, data): log.exception(err) log.exception(traceback.format_exc()) + def close(self): + if self.performanceUpdatePoller: + self.performanceUpdatePoller.close() # ======================== private ========================================= @@ -609,20 +598,19 @@ def _handler_event_networkFormationCompleted(self, buf): # TODO return (True, {}) - # radioDutyCycleMeasurement - def _handler_event_radioDutyCycleMeasurement(self, buf): - returnVal = {} - # TODO - return (True, returnVal) - - # clockDriftMeasurement - def _handler_event_clockDriftMeasurement(self, buf): - returnVal = {} - # TODO - return (True, returnVal) +# ================= helper class that polls for periodic measurements ===================== class PerformanceUpdatePoller(eventBusClient.eventBusClient, threading.Thread): + # periodic event names + EV_RADIO_DUTY_CYCLE_MEASUREMENT = ['radioDutyCycleMeasurement', 258 ] + EV_CLOCK_DRIFT_MEASUREMENT = ['clockDriftMeasurement', 259 ] + + EV_SYNC_ALL = [ + EV_RADIO_DUTY_CYCLE_MEASUREMENT, + EV_CLOCK_DRIFT_MEASUREMENT, + ] + def __init__(self, experimentId, mqttClient, period): # log log.info("creating a thread for periodic polling of performance metrics") @@ -655,11 +643,14 @@ def __init__(self, experimentId, mqttClient, period): 'signal': 'dutyCycleMeasurement', 'callback': self.handle_dutyCycleMeasurement, }, + # TODO clock drift measurements ] ) # start myself self.start() + # ======================== public ========================================= + def run(self): try: # log @@ -685,7 +676,7 @@ def run(self): topic = 'openbenchmark/experimentId/{0}/nodeId/{1}/performanceData'.format(self.experimentId, source) payload = { - 'event' : 'radioDutyCycleMeasurement', + 'event' : self.EV_RADIO_DUTY_CYCLE_MEASUREMENT[0], 'timestamp' : timestamp, 'source' : source, 'dutyCycle' : dutyCycle, @@ -721,3 +712,17 @@ def handle_dutyCycleMeasurement(self, sender, signal, data): ( data['source'], data['timestamp'], data['dutyCycle'] ) ) +# ==================== Implementation of CoAP openbenchmark resource ===================== + +class OpenbenchmarkResource(coapResource.coapResource): + + def __init__(self): + # initialize parent class + coapResource.coapResource.__init__( + self, + path='b', + ) + + def POST(self, options=[], payload=[]): + # TODO parse the packet token and log the event + return (d.COAP_RC_2_04_CHANGED, [], []) From da6285631f6101d073d64a9e18ceb7285a0a5792 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Wed, 20 Mar 2019 14:26:11 +0100 Subject: [PATCH 048/108] OV-7. Make event publishing a method of PerformanceEvent. --- .../openBenchmarkAgent/openBenchmarkAgent.py | 35 +++++++++++-------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 9525281b..99e948ee 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -535,21 +535,7 @@ def handle_event(self, sender, signal, data): (success, dict) = event_handler(buf) if success: - # common fields - returnVal['event'] = event_name - returnVal['timestamp'] = str(timestamp) - returnVal['source'] = source - - # handler-specific fields - returnVal.update(dict) - topic = 'openbenchmark/experimentId/{0}/nodeId/{1}/performanceData'.format(self.experimentId, source) - - log.debug("Publishing on topic: {0} Payload: {1}".format(topic, returnVal)) - - self.mqttClient.publish( - topic=topic, - payload=json.dumps(returnVal), - ) + self.publish_event(event=event_name, timestamp=str(timestamp), source=source, eventSpecificFields=dict) except Exception as err: log.exception("Exception while executing {0}".format(event)) @@ -560,6 +546,25 @@ def close(self): if self.performanceUpdatePoller: self.performanceUpdatePoller.close() + def publish_event(self, event, timestamp, source, eventSpecificFields): + payload = {} + + payload['event'] = event + payload['timestamp'] = timestamp + payload['source'] = source + + # update the payload with event specific fields + payload.update(eventSpecificFields) + + topic = 'openbenchmark/experimentId/{0}/nodeId/{1}/performanceData'.format(self.experimentId, source) + + log.debug("Publishing on topic: {0} Payload: {1}".format(topic, payload)) + + self.mqttClient.publish( + topic=topic, + payload=json.dumps(returnVal), + ) + # ======================== private ========================================= # packetSent From 359193cfdb333601ea5472eae643bff58973f4c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Wed, 20 Mar 2019 14:45:41 +0100 Subject: [PATCH 049/108] OV-7. Parametrize hop limit sent by coap server within the OV. --- openvisualizer/coapServer/coapServer.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/openvisualizer/coapServer/coapServer.py b/openvisualizer/coapServer/coapServer.py index be126689..688d22e6 100644 --- a/openvisualizer/coapServer/coapServer.py +++ b/openvisualizer/coapServer/coapServer.py @@ -20,11 +20,13 @@ import os import threading - class coapServer(eventBusClient.eventBusClient): # link-local prefix LINK_LOCAL_PREFIX = [0xfe, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + # default IPv6 hop limit + COAP_SERVER_DEFAULT_IPv6_HOP_LIMIT = 65 + def __init__(self): # log log.info("create instance") @@ -181,7 +183,7 @@ def _receiveFromCoAP(self, timestamp, sender, data): ip += [0x00, 0x00, 0x00] # traffic class (lower nibble) + flow label ip += udp[4:6] # payload length ip += [17] # next header (protocol); UDP=17 - ip += [64] # hop limit (pick a safe value) + ip += [self.COAP_SERVER_DEFAULT_IPv6_HOP_LIMIT] # hop limit (pick a safe value) ip += srcIpv6Address # source ip += dstIpv6Address # destination ip += udp From 7e5881429770f86d2cc746de9dd9e7dc8cdba318 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Wed, 20 Mar 2019 17:23:17 +0100 Subject: [PATCH 050/108] OV-7. packetToken as the last thing in the app payload. --- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 99e948ee..d200e715 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -185,9 +185,9 @@ def triggerSendPacket(self, destination, acknowledged, packetsInBurst, packetTok try: # construct the payload of the POST request payload = [] + payload += [0] * packetPayloadLen payload += [packetCounter] payload += [packetToken[1:]] - payload += [0] * packetPayloadLen # the call to POST() is blocking unless no response is expected p = self.coapServer.POST('coap://[{0}:{1}]/b'.format(destinationIPv6, d.DEFAULT_UDP_PORT), From c3a4743df84e5940e31d69dc28f906f0f1515454 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Wed, 20 Mar 2019 17:26:01 +0100 Subject: [PATCH 051/108] OV-7. Get the packetSent timestamp for dag root sending packets. --- .../openBenchmarkAgent/openBenchmarkAgent.py | 65 ++++++++++++++----- 1 file changed, 48 insertions(+), 17 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index d200e715..42df226a 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -78,16 +78,16 @@ def __init__(self, coapServer, mqttBroker, firmware, testbed, portNames, scenari self.portNames = portNames self.scenario = scenario - # state - self.experimentId = None - # primitive for mutual exclusion self.mqttConnectedEvent = threading.Event() self.experimentRequestResponseEvent = threading.Event() + # local vars + self.experimentId = None self.mqttClient = None self.experimentRequestResponse = None self.performanceEvent = None + self.dagRootEui64 = None # dict with keys being eui64, and value corresponding testbed host identifier self.nodes = {} @@ -443,9 +443,11 @@ def _mqtt_handler_triggerNetworkFormation(self, payload): returnVal = {} # parse the payload - payloadDecoded = json.loads(payload) + payloadDecoded = json.loads(payload) + source = payloadDecoded['source'] - source = payloadDecoded['source'] + # remember DAG root's EUI64 + self.dagRootEui64 = source # lookup corresponding mote port destPort = self.nodes[source] @@ -466,15 +468,16 @@ def _mqtt_handler_triggerNetworkFormation(self, payload): class PerformanceEvent(object): # period for measurement polling - PERFORMANCE_EVENT_MEASUREMENT_PERIOD = 10 + PERFORMANCE_EVENT_MEASUREMENT_PERIOD = 3 - # asynchronous event #name # id - EV_PACKET_SENT = ['packetSent', 0 ] - EV_PACKET_RECEIVED = ['packetReceived', 1 ] - EV_SYNCHRONIZATION_COMPLETED = ['synchronizationCompleted', 2 ] - EV_SECURE_JOIN_COMPLETED = ['secureJoinCompleted', 3 ] - EV_BANDWIDTH_ASSIGNED = ['bandwidthAssigned', 4 ] - EV_NETWORK_FORMATION_COMPLETED = ['networkFormationCompleted', 257 ] + # asynchronous event #name #id #openbenchmark ID + EV_PACKET_SENT = ['packetSent', 0, 'packetSent' ] + EV_PACKET_RECEIVED = ['packetReceived', 1, 'packetReceived' ] + EV_SYNCHRONIZATION_COMPLETED = ['synchronizationCompleted', 2, 'synchronizationCompleted' ] + EV_SECURE_JOIN_COMPLETED = ['secureJoinCompleted', 3, 'secureJoinCompleted' ] + EV_BANDWIDTH_ASSIGNED = ['bandwidthAssigned', 4, 'bandwidthAssigned' ] + EV_PACKET_SENT_DAGROOT = ['packetSentDagRoot', 5, 'packetSent' ] # special event to precisely get the ASN when dag root sent a packet + EV_NETWORK_FORMATION_COMPLETED = ['networkFormationCompleted', 257, 'networkFormationCompleted' ] EV_ASYNC_ALL = [ EV_PACKET_SENT, @@ -482,6 +485,7 @@ class PerformanceEvent(object): EV_SECURE_JOIN_COMPLETED, EV_BANDWIDTH_ASSIGNED, EV_SYNCHRONIZATION_COMPLETED, + EV_PACKET_SENT_DAGROOT, EV_NETWORK_FORMATION_COMPLETED, ] @@ -495,6 +499,7 @@ def __init__(self, experimentId, mqttClient): # params self.experimentId = experimentId self.mqttClient = mqttClient + self.outstandingPacketsFromDagRoot = [] # start poller thread for periodic measurements self.performanceUpdatePoller = PerformanceUpdatePoller(self.experimentId, self.mqttClient, @@ -515,16 +520,17 @@ def handle_event(self, sender, signal, data): try: # find the event - event_name = None + handler_name = None for ev in self.EV_ASYNC_ALL: if ev[1] == event: - event_name = ev[0] + handler_name = ev[0] + event_name = ev[2] break - assert event_name, "Unhandled event, ignoring: {0}".format(event) + assert handler_name, "Unhandled event, ignoring: {0}".format(event) # find the handler - event_handler = getattr(self, '{0}{1}'.format(self.PERFORMANCE_EVENT_HANDLER_NAME, event_name, None)) + event_handler = getattr(self, '{0}{1}'.format(self.PERFORMANCE_EVENT_HANDLER_NAME, handler_name, None)) assert event_handler, "Event recognized but cannot find handler, event: {0}".format(event) @@ -565,6 +571,11 @@ def publish_event(self, event, timestamp, source, eventSpecificFields): payload=json.dumps(returnVal), ) + def add_outstanding_packet(self, packet): + if len(self.outstandingPacketsFromDagRoot) > 10: + self.outstandingPacketsFromDagRoot.pop(0) + self.outstandingPacketsFromDagRoot.append(packet) + # ======================== private ========================================= # packetSent @@ -582,6 +593,26 @@ def _handler_event_packetSent(self, buf): return (True, returnVal) + def _handler_event_packetSentDagRoot(self, buf): + receivedToken = buf + newBuf = [] + + for packet in self.outstandingPacketsFromDagRoot: + (token, destination, hopLimit) = packet + if receivedToken == token: + # we have a hit, remove it from outstanding packets + log.debug("packetSentDagRoot event: hit for packet {0}".format(packet)) + self.outstandingPacketsFromDagRoot.remove(packet) + + # construct the missing fields from the saved values + newBuf += token + newBuf += destination + newBuf += [hopLimit] + return self._handler_event_packetSent(newBuf) + # not found, not all packets sent by dag root are originated by openbenchmark: ignore it + log.debug("packetSentDagRoot event: miss for token {0}".format(receivedToken)) + return (False, {}) + # packetReceived, same syntax as packetSent def _handler_event_packetReceived(self, buf): return self._handler_event_packetSent(buf) From a8c1a062e18c8e03c2a3906decf097b37326e451 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Wed, 20 Mar 2019 20:00:58 +0100 Subject: [PATCH 052/108] OV-7. Small tuning on resource instantiation. --- .../openBenchmarkAgent/openBenchmarkAgent.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 42df226a..5df1dd55 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -133,16 +133,16 @@ def __init__(self, coapServer, mqttBroker, firmware, testbed, portNames, scenari if not self.experimentId: raise ValueError("Unable to start an experiment with OpenBenchmark") - # everything is ok, start a coap server - coapResource = OpenbenchmarkResource() - self.coapServer.coapServer.addResource(coapResource) - # subscribe to all topics on a given experiment ID self._openbenchmark_subscribe(self.mqttClient, self.experimentId) # instantiate performance event handlers from firmware self.performanceEvent = PerformanceEvent(self.experimentId, self.mqttClient) + # everything is ok, start a coap server + coapResource = OpenbenchmarkResource(self.performanceEvent) + self.coapServer.coapServer.addResource(coapResource) + # subscribe to eventBus performance-related events eventBusClient.eventBusClient.__init__( self, @@ -752,7 +752,11 @@ def handle_dutyCycleMeasurement(self, sender, signal, data): class OpenbenchmarkResource(coapResource.coapResource): - def __init__(self): + def __init__(self, performanceEvent): + + # params + self.performanceEvent = performanceEvent + # initialize parent class coapResource.coapResource.__init__( self, From d09414b7069b771cffe384616e2dcfb0416d09bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Wed, 20 Mar 2019 20:02:06 +0100 Subject: [PATCH 053/108] OV-7. Adapt coap resource handler to accept metadata. --- .../openBenchmarkAgent/openBenchmarkAgent.py | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 5df1dd55..5b8ae349 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -140,7 +140,7 @@ def __init__(self, coapServer, mqttBroker, firmware, testbed, portNames, scenari self.performanceEvent = PerformanceEvent(self.experimentId, self.mqttClient) # everything is ok, start a coap server - coapResource = OpenbenchmarkResource(self.performanceEvent) + self.openbenchmarkResource = OpenbenchmarkResource(self.performanceEvent, self.dagRootEui64) self.coapServer.coapServer.addResource(coapResource) # subscribe to eventBus performance-related events @@ -210,6 +210,10 @@ def encodeSendPacketPayload(self, destination, confirmable, packetsInBurst, pack buf += [packetPayloadLen] return buf + def updateDagRootEui64(self, address): + self.dagRootEui64 = address + self.openbenchmarkResource.dagRootEui64 = self.dagRootEui64 + # ======================== private ========================================= def _openbenchmark_start_benchmark(self, mqttClient): @@ -447,7 +451,7 @@ def _mqtt_handler_triggerNetworkFormation(self, payload): source = payloadDecoded['source'] # remember DAG root's EUI64 - self.dagRootEui64 = source + self.updateDagRootEui64(source) # lookup corresponding mote port destPort = self.nodes[source] @@ -752,10 +756,11 @@ def handle_dutyCycleMeasurement(self, sender, signal, data): class OpenbenchmarkResource(coapResource.coapResource): - def __init__(self, performanceEvent): + def __init__(self, performanceEvent, dagRootEui64): # params self.performanceEvent = performanceEvent + self.dagRootEui64 = dagRootEui64 # initialize parent class coapResource.coapResource.__init__( @@ -763,6 +768,22 @@ def __init__(self, performanceEvent): path='b', ) - def POST(self, options=[], payload=[]): - # TODO parse the packet token and log the event + def POST(self, options=[], payload=[], metaData={}): + # token is in the last 5 bytes of payload + token = payload[-5:] + timestamp = metaData['generic_1'] + source = self.dagRootEui64 + + destination = u.ipv6AddrString2Bytes(metaData['srcIP']) + destinationEui64String = openvisualizer.openvisualizer_utils.formatAddr(destination[8:]) + + log.debug("OpenbenchmarkResource: POST handler received metadata: {0}".format(metaData)) + + dict = { + 'packetToken' : token, + 'destination' : destinationEui64String, + 'hopLimit' : metaData['generic_0'], + } + + self.performanceEvent.publish_event(PerformanceEvent.EV_PACKET_RECEIVED[0], timestamp, source, dict) return (d.COAP_RC_2_04_CHANGED, [], []) From e2d54dfde83c1ee3eedffdd8d30ad565f5df07c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Wed, 20 Mar 2019 20:05:20 +0100 Subject: [PATCH 054/108] OV-7. Adapt the JRC resource handler to accept metadata. --- openvisualizer/JRC/JRC.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openvisualizer/JRC/JRC.py b/openvisualizer/JRC/JRC.py index 1dcd5e62..7998183a 100644 --- a/openvisualizer/JRC/JRC.py +++ b/openvisualizer/JRC/JRC.py @@ -111,7 +111,7 @@ def __init__(self): self.addSecurityBinding((None, [d.METHOD_POST])) # security context should be returned by the callback - def POST(self,options=[], payload=[]): + def POST(self,options=[], payload=[], metaData={}): link_layer_keyset = [self.networkKeyIndex, u.buf2str(self.networkKey)] From eff34a9bc917a207dc5c50801155e0dd8c7a1565 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Wed, 20 Mar 2019 20:02:43 +0100 Subject: [PATCH 055/108] OV-7. Add forgoten call to add_outstanding_packet. --- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 5b8ae349..10ea27f5 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -190,6 +190,10 @@ def triggerSendPacket(self, destination, acknowledged, packetsInBurst, packetTok payload += [packetToken[1:]] # the call to POST() is blocking unless no response is expected + token = [packetCounter] + packetToken[1:] + self.performanceEvent.add_outstanding_packet((token, destination, self.coapServer.COAP_SERVER_DEFAULT_IPv6_HOP_LIMIT)) + + log.debug("Publishing on topic: {0} Payload: {1}".format(topic, returnVal)) p = self.coapServer.POST('coap://[{0}:{1}]/b'.format(destinationIPv6, d.DEFAULT_UDP_PORT), confirmable=False, options=options, From bf81fae04a83b00a1866396083cb146d6d141332 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Wed, 20 Mar 2019 20:03:20 +0100 Subject: [PATCH 056/108] OV-20. ParserData now passes the timestamp to upper layers. --- openvisualizer/moteConnector/ParserData.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/openvisualizer/moteConnector/ParserData.py b/openvisualizer/moteConnector/ParserData.py index cbbf2943..f1b02793 100644 --- a/openvisualizer/moteConnector/ParserData.py +++ b/openvisualizer/moteConnector/ParserData.py @@ -12,6 +12,8 @@ from pydispatch import dispatcher +from openvisualizer.openType import typeAsn + from ParserException import ParserException import Parser @@ -95,7 +97,9 @@ def parseInput(self,input): #asn comes in the next 5bytes. asnbytes=input[2:7] - (self._asn) = struct.unpack(' Date: Wed, 20 Mar 2019 20:06:10 +0100 Subject: [PATCH 057/108] OV-20. Process timestamp in coap server and pass it as metadata to the coap lib. --- openvisualizer/coapServer/coapServer.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/openvisualizer/coapServer/coapServer.py b/openvisualizer/coapServer/coapServer.py index 688d22e6..551c2528 100644 --- a/openvisualizer/coapServer/coapServer.py +++ b/openvisualizer/coapServer/coapServer.py @@ -138,9 +138,13 @@ def _receiveFromMesh(self, sender, signal, data): Forwards the packet to the virtual CoAP server running in test mode (PyDispatcher). ''' sender = openvisualizer.openvisualizer_utils.formatIPv6Addr(data[0]) + + hopLimit = data[2] # IPv6 metadata + timestamp = str(data[3]) # timestamp of the received packet + # FIXME pass source port within the signal and open coap client at this port self.ephemeralCoapClient = coap.coap(ipAddress=sender, udpPort=d.DEFAULT_UDP_PORT, testing=True, receiveCallback=self._receiveFromCoAP) - self.ephemeralCoapClient.socketUdp.sendUdp(destIp='', destPort=d.DEFAULT_UDP_PORT, msg=data[1]) # low level forward of the CoAP message + self.ephemeralCoapClient.socketUdp.sendUdp(destIp='', destPort=d.DEFAULT_UDP_PORT, msg=data[1],metaData=(hopLimit, timestamp)) # low level forward of the CoAP message return True def _receiveFromCoAP(self, timestamp, sender, data): From efd7af6b5767184a27e373cc5f48ce5e825b2019 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Wed, 20 Mar 2019 20:04:59 +0100 Subject: [PATCH 058/108] OV-20. Adapt openLbr and eventBusMonitor to accept the packet timestamp. --- openvisualizer/eventBus/eventBusMonitor.py | 2 +- openvisualizer/openLbr/openLbr.py | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/openvisualizer/eventBus/eventBusMonitor.py b/openvisualizer/eventBus/eventBusMonitor.py index 7e2b8b35..25375ac5 100644 --- a/openvisualizer/eventBus/eventBusMonitor.py +++ b/openvisualizer/eventBus/eventBusMonitor.py @@ -132,7 +132,7 @@ def _eventBusNotification(self,signal,sender,data): if signal=='fromMote.data': # Forwards a copy of the data received from a mode # to the Internet interface for debugging. - (previousHop,lowpan) = data + (previousHop,lowpan,timestamp) = data zep = self._wrapMacAndZep( previousHop = previousHop, diff --git a/openvisualizer/openLbr/openLbr.py b/openvisualizer/openLbr/openLbr.py index a17439a4..64c7e44a 100644 --- a/openvisualizer/openLbr/openLbr.py +++ b/openvisualizer/openLbr/openLbr.py @@ -283,7 +283,11 @@ def _meshToV6_notif(self,sender,signal,data): try: ipv6dic={} #build lowpan dictionary from the data - ipv6dic = self.lowpan_to_ipv6(data) + + source = data[0] + buf = data[1] + timestamp = data[2] + ipv6dic = self.lowpan_to_ipv6(source, buf) success = True dispatchSignal = None @@ -302,7 +306,7 @@ def _meshToV6_notif(self,sender,signal,data): #ipv6 header (inner) ipv6dic_inner = {} # parsing the iphc inner header and get the next_header - ipv6dic_inner = self.lowpan_to_ipv6([ipv6dic['pre_hop'],ipv6dic['payload']]) + ipv6dic_inner = self.lowpan_to_ipv6(ipv6dic['pre_hop'],ipv6dic['payload']) ipv6dic['next_header'] = ipv6dic_inner['next_header'] ipv6dic['payload'] = ipv6dic_inner['payload'] ipv6dic['payload_length'] = ipv6dic_inner['payload_length'] @@ -416,7 +420,7 @@ def _meshToV6_notif(self,sender,signal,data): #as source address is being retrieved from the IPHC header, the signal includes it in case #receiver such as RPL DAO processing needs to know the source. - success = self._dispatchProtocol(dispatchSignal,(ipv6dic['src_addr'],ipv6dic['app_payload'])) + success = self._dispatchProtocol(dispatchSignal,(ipv6dic['src_addr'],ipv6dic['app_payload'],ipv6dic['hop_limit'],timestamp)) if success: return @@ -754,11 +758,9 @@ def reassemble_lowpan(self,lowpan): #===== 6LoWPAN -> IPv6 - def lowpan_to_ipv6(self,data): + def lowpan_to_ipv6(self, mac_prev_hop, pkt_lowpan): pkt_ipv6 = {} - mac_prev_hop=data[0] - pkt_lowpan=data[1] if pkt_lowpan[0]==self.PAGE_ONE_DISPATCH: ptr = 1 From 8615be3eae46a24ee3aefb11a733a011618354dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Wed, 20 Mar 2019 20:06:28 +0100 Subject: [PATCH 059/108] OV-7. Bug fix in exception handling. --- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 10ea27f5..d11aa821 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -740,7 +740,7 @@ def run(self): time.sleep(self.period) except Exception as err: - errMsg = u.formatCrashMessage(self.name, err) + errMsg = openvisualizer.openvisualizer_utils.formatCrashMessage(self.name, err) print errMsg log.critical(errMsg) self.close() From 18b897bc1cccea08ac157ea88d208b64004ce6e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Wed, 20 Mar 2019 20:06:55 +0100 Subject: [PATCH 060/108] OV-7. Remove unuseful debugs. --- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index d11aa821..b126deeb 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -702,15 +702,12 @@ def run(self): while self.goOn: - log.debug("poller is polling") - # poll moteState for latest measurements self.dispatch('getDutyCycleMeasurement', []) # wait for a while to gather the response from motes time.sleep(1) - log.debug("Performance Event Poller, received measurements:") log.debug(self.dutyCycleMeasurements) for measurement in self.dutyCycleMeasurements: From 5bd135fe2721db58bee40c7676fce11a4d856d0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Sat, 23 Mar 2019 20:32:18 +0100 Subject: [PATCH 061/108] OV-7. Add demo scenario as a choice to --benchmark argument. --- SConstruct | 2 +- bin/openVisualizerApp.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/SConstruct b/SConstruct index cf723030..42b50aa0 100644 --- a/SConstruct +++ b/SConstruct @@ -170,7 +170,7 @@ runnerEnv['TESTBED'] = GetOption('testbed') AddOption('--benchmark', dest = 'benchmark', default = False, - choices = ['building-automation', 'home-automation', 'industrial-monitoring'], + choices = ['building-automation', 'home-automation', 'industrial-monitoring', 'demo-scenario'], action = 'store') runnerEnv['BENCHMARK'] = GetOption('benchmark') diff --git a/bin/openVisualizerApp.py b/bin/openVisualizerApp.py index d88ca15f..330bba47 100644 --- a/bin/openVisualizerApp.py +++ b/bin/openVisualizerApp.py @@ -455,7 +455,7 @@ def _addParserArgs(parser): parser.add_argument('-b', '--benchmark', dest = 'benchmark', default = False, - choices = ['building-automation', 'home-automation', 'industrial-monitoring'], + choices = ['building-automation', 'home-automation', 'industrial-monitoring', 'demo-scenario'], action = 'store', help = 'trigger --benchmark scenario using OpenBenchmark cloud service. see benchmark.6tis.ch' ) From c63c9ed3d3177b71db016251fbf430c8238e1ea9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Sat, 23 Mar 2019 20:48:01 +0100 Subject: [PATCH 062/108] OV-7. Fix operator misuse. --- .../openBenchmarkAgent/openBenchmarkAgent.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index b126deeb..a5bb8388 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -93,11 +93,11 @@ def __init__(self, coapServer, mqttBroker, firmware, testbed, portNames, scenari self.nodes = {} # OV is running in simulation mode - if self.testbed is 'simulation': + if self.testbed == 'simulation': for port in portNames: self.nodes[port] = 'simulation' # Motes are attached locally on the physical port - elif self.testbed is 'local': + elif self.testbed == 'local': for port in portNames: self.nodes[port] = 'local' # General case, motes are in testbed connected over OpenTestbed software @@ -269,10 +269,10 @@ def _openbenchmark_start_benchmark(self, mqttClient): success = payload['success'] # check token match - if tokenGenerated is not tokenReceived: + if tokenGenerated != tokenReceived: raise ValueError("Token does not match the one sent in the request") # success? - if success is not True: + if success != True: raise ValueError("Fail indicated") experimentId = payload['experimentId'] @@ -360,7 +360,7 @@ def _on_mqtt_connect(self, client, userdata, flags, rc): def _on_mqtt_message(self, client, userdata, message): # check if this is the startBenchmark response - if message.topic is self.OPENBENCHMARK_STARTBENCHMARK_RESPONSE_TOPIC: + if message.topic == self.OPENBENCHMARK_STARTBENCHMARK_RESPONSE_TOPIC: self.experimentRequestResponse = message.payload self.experimentRequestResponseEvent.set() # if not, assume this is a command From d56a3573d299d482e5bf76939b8a0d32fe3aeb76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Sat, 23 Mar 2019 20:57:40 +0100 Subject: [PATCH 063/108] OV-7. Add debug in MQTT on_message handler to facilitate debugging. --- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index a5bb8388..d6d86662 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -359,6 +359,7 @@ def _on_mqtt_connect(self, client, userdata, flags, rc): self.mqttConnectedEvent.set() def _on_mqtt_message(self, client, userdata, message): + log.debug("MQTT message received: topic={0} payload={1}".format(message.topic,message.payload)) # check if this is the startBenchmark response if message.topic == self.OPENBENCHMARK_STARTBENCHMARK_RESPONSE_TOPIC: self.experimentRequestResponse = message.payload From 769bb9b1109569aebf148c73cb06414f5123b7a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Sat, 23 Mar 2019 21:01:13 +0100 Subject: [PATCH 064/108] OV-7. Threading lock may not need to be cleared after a timeout. --- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index d6d86662..11d52cd1 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -281,7 +281,6 @@ def _openbenchmark_start_benchmark(self, mqttClient): except ValueError as valErr: log.info(str(valErr) + ", retrying...") attempt += 1 - self.experimentRequestResponseEvent.clear() continue # give up except Exception as e: From 4bd00b2cf7680c0e9e4075c8060176f634c801e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Sat, 23 Mar 2019 21:13:25 +0100 Subject: [PATCH 065/108] OV-7. Return success when success it is. --- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 11d52cd1..b07b2b45 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -276,6 +276,7 @@ def _openbenchmark_start_benchmark(self, mqttClient): raise ValueError("Fail indicated") experimentId = payload['experimentId'] + break # Retry for all ValueErrors except ValueError as valErr: From d8e864f47e5fe31082ffefef2f2bab0826c65aeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Sat, 23 Mar 2019 21:19:14 +0100 Subject: [PATCH 066/108] OV-7. Additional debug info. --- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index b07b2b45..777c4c90 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -293,7 +293,10 @@ def _openbenchmark_start_benchmark(self, mqttClient): return experimentId def _openbenchmark_subscribe(self, mqttClient, experimentId): - mqttClient.subscribe("openbenchmark/experimentId/{0}/command/#".format(experimentId)) + topic = "openbenchmark/experimentId/{0}/command/#".format(experimentId) + mqttClient.subscribe(topic) + + log.debug("Subscribed to MQTT topic: {0}".format(topic)) # command handling adapted from github.com/openwsn-berkeley/opentestbed def _execute_command_safely(self, topic, payload): From 90195da18086e2fc3fef029acdaf74d167033032 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Sun, 24 Mar 2019 10:40:01 +0100 Subject: [PATCH 067/108] OV-7. Add a debug in close method and use exception logging. --- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 777c4c90..906dfb72 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -159,12 +159,13 @@ def __init__(self, coapServer, mqttBroker, firmware, testbed, portNames, scenari log.info("Experiment #{0} successfuly started".format(self.experimentId)) except Exception as e: - log.warning(e) + log.exception(e) self.close() # ======================== public ========================================== def close(self): + log.debug("close called") if self.mqttClient: self.mqttClient.loop_stop() if self.coapServer: @@ -285,7 +286,7 @@ def _openbenchmark_start_benchmark(self, mqttClient): continue # give up except Exception as e: - log.warning(e) + log.exception(e) break mqttClient.unsubscribe(self.OPENBENCHMARK_STARTBENCHMARK_RESPONSE_TOPIC) From 906199595195fe1b7332d460fea3cc407ff37e78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Sun, 24 Mar 2019 10:56:23 +0100 Subject: [PATCH 068/108] OV-7. Fix CoAP resource registration. --- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 906dfb72..50837ea8 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -141,7 +141,7 @@ def __init__(self, coapServer, mqttBroker, firmware, testbed, portNames, scenari # everything is ok, start a coap server self.openbenchmarkResource = OpenbenchmarkResource(self.performanceEvent, self.dagRootEui64) - self.coapServer.coapServer.addResource(coapResource) + self.coapServer.coapServer.addResource(self.openbenchmarkResource) # subscribe to eventBus performance-related events eventBusClient.eventBusClient.__init__( @@ -165,11 +165,9 @@ def __init__(self, coapServer, mqttBroker, firmware, testbed, portNames, scenari # ======================== public ========================================== def close(self): - log.debug("close called") + log.debug("shutting down") if self.mqttClient: self.mqttClient.loop_stop() - if self.coapServer: - self.coapServer.close() if self.performanceEvent: self.performanceEvent.close() From 073042b28d72def376234d7cdbede0ff0c83edb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Sun, 24 Mar 2019 11:01:40 +0100 Subject: [PATCH 069/108] OV-7. Allow for upper case in command topic names. --- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 50837ea8..f4a5abd0 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -301,7 +301,7 @@ def _openbenchmark_subscribe(self, mqttClient, experimentId): def _execute_command_safely(self, topic, payload): # parse the topic to extract deviceType, deviceId and cmd ([0-9\-]+) try: - m = re.search("openbenchmark/experimentId/{0}/command/([a-z]+)".format(self.experimentId), topic) + m = re.search("openbenchmark/experimentId/{0}/command/([a-zA-Z]+)".format(self.experimentId), topic) assert m, "Invalid topic, could not parse: '{0}'".format(topic) cmd = m.group(1) From 216e2ca740e06a0c715e19568b6d391222753dfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Sun, 24 Mar 2019 11:05:49 +0100 Subject: [PATCH 070/108] OV-7. Syntax fix in configure transmit power handler. --- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index f4a5abd0..812c0425 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -431,7 +431,7 @@ def _mqtt_handler_configureTransmitPower(self, payload): payloadDecoded = json.loads(payload) source = payloadDecoded['source'] - power = payload.Decoded['power'] + power = payloadDecoded['power'] # lookup corresponding mote port destPort = self.nodes[source] From c5fdef7b782d3159e5e550e72a5f17b15669e2ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Sun, 24 Mar 2019 11:28:36 +0100 Subject: [PATCH 071/108] OV-7. Fix wrong util module reference when converting EUI64. --- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 812c0425..3029da3c 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -206,7 +206,7 @@ def encodeSendPacketPayload(self, destination, confirmable, packetsInBurst, pack # construct command payload as byte-list: # dest_eui64 (8B) || con (1B) || packetsInBurst (1B) || packetToken (5B) || packetPayloadLen (1B) buf = [] - buf += u.hex2buf(destination, separator='-') + buf += openvisualizer.openvisualizer_utils.hex2buf(destination, separator='-') buf += [int(confirmable)] buf += [int(packetsInBurst)] buf += packetToken @@ -383,8 +383,8 @@ def _mqtt_handler_sendPacket(self, payload): payloadDecoded = json.loads(payload) sourceStr = payloadDecoded['source'] - source = u.hex2buf(sourceStr, separator='-') - destination = u.hex2buf(payloadDecoded['destination'], separator='-') + source = openvisualizer.openvisualizer_utils.hex2buf(sourceStr, separator='-') + destination = openvisualizer.openvisualizer_utils.hex2buf(payloadDecoded['destination'], separator='-') packetsInBurst = payloadDecoded['packetsInBurst'] packetToken = payloadDecoded['packetToken'] packetPayloadLen = payloadDecoded['packetPayloadLen'] From 5004e86f5a039c898670cb89a29ad78a932e0280 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Sun, 24 Mar 2019 11:37:36 +0100 Subject: [PATCH 072/108] OV-7. Record the portname of each node and use it to send commands. --- .../openBenchmarkAgent/openBenchmarkAgent.py | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 3029da3c..76408b87 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -89,17 +89,19 @@ def __init__(self, coapServer, mqttBroker, firmware, testbed, portNames, scenari self.performanceEvent = None self.dagRootEui64 = None - # dict with keys being eui64, and value corresponding testbed host identifier + # dict with keys being eui64, and value a tuple (localPort, testbedHost) self.nodes = {} # OV is running in simulation mode if self.testbed == 'simulation': for port in portNames: - self.nodes[port] = 'simulation' + # FIXME get eui64 in simulation + self.nodes[port] = (port, 'simulation') # Motes are attached locally on the physical port elif self.testbed == 'local': for port in portNames: - self.nodes[port] = 'local' + # FIXME get eui64 when executing locally + self.nodes[port] = (port, 'local') # General case, motes are in testbed connected over OpenTestbed software else: for port in portNames: @@ -107,7 +109,7 @@ def __init__(self, coapServer, mqttBroker, firmware, testbed, portNames, scenari if m: # (testbed_host, eui64) assert m.group(1) == self.testbed - self.nodes[m.group(3)] = m.group(2) + self.nodes[m.group(3)] = (port, m.group(2)) log.info('Initializing OpenBenchmark with options:\n\t{0}'.format( '\n '.join(['mqttBroker = {0}'.format(self.mqttBroker), @@ -232,13 +234,18 @@ def _openbenchmark_start_benchmark(self, mqttClient): # generate a random token tokenGenerated = binascii.b2a_hex(os.urandom(8)) + # format nodes to the format expected by OpenBenchmark + nodes = {} + for key, (portName, testbedId) in self.nodes.iteritems(): + nodes[key] = testbedId + payload = { 'api_version': self.OPENBENCHMARK_API_VERSION, 'token': tokenGenerated, 'date': strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime()), 'firmware': self.firmware, 'testbed': self.testbed, - 'nodes': self.nodes, + 'nodes': nodes, 'scenario': self.scenario } @@ -396,7 +403,7 @@ def _mqtt_handler_sendPacket(self, payload): else: # command is for one of the motes in the mesh, send it over the serial # lookup corresponding mote port - destPort = self.nodes[sourceStr] + (destPort, testbedHost) = self.nodes[sourceStr] # construct command payload as byte-list: # dest_eui64 (8B) || con (1B) || packetsInBurst (1B) || packetToken (5B) || packetPayloadLen (1B) @@ -434,7 +441,7 @@ def _mqtt_handler_configureTransmitPower(self, payload): power = payloadDecoded['power'] # lookup corresponding mote port - destPort = self.nodes[source] + (destPort, testbedHost) = self.nodes[source] action = [moteState.moteState.SET_COMMAND, moteState.moteState.COMMAND_SET_TX_POWER, power] # generate an eventbus signal to send a command over serial @@ -461,7 +468,7 @@ def _mqtt_handler_triggerNetworkFormation(self, payload): self.updateDagRootEui64(source) # lookup corresponding mote port - destPort = self.nodes[source] + (destPort, testbedHost) = self.nodes[source] # generate an eventbus signal to send a command over serial self.dispatch( From abecf8046ba5c7c0742d9d7c4877bcd571429df0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Sun, 24 Mar 2019 12:05:23 +0100 Subject: [PATCH 073/108] OV-7. Command signal is the command string, not list. --- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 76408b87..a1fbc346 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -417,7 +417,7 @@ def _mqtt_handler_sendPacket(self, payload): if len(commandPayload) != 16: return False, returnVal - action = [moteState.moteState.SET_COMMAND, moteState.moteState.COMMAND_SEND_PACKET, commandPayload] + action = [moteState.moteState.SET_COMMAND, moteState.moteState.COMMAND_SEND_PACKET[0], commandPayload] # generate an eventbus signal to send a command over serial # dispatch @@ -442,7 +442,7 @@ def _mqtt_handler_configureTransmitPower(self, payload): # lookup corresponding mote port (destPort, testbedHost) = self.nodes[source] - action = [moteState.moteState.SET_COMMAND, moteState.moteState.COMMAND_SET_TX_POWER, power] + action = [moteState.moteState.SET_COMMAND, moteState.moteState.COMMAND_SET_TX_POWER[0], power] # generate an eventbus signal to send a command over serial From b9162b0801db8b485d952b8dbb4860cf16a44958 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Sun, 24 Mar 2019 14:12:53 +0100 Subject: [PATCH 074/108] OV-7. Fix remnant return var. --- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index a1fbc346..6260284f 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -586,7 +586,7 @@ def publish_event(self, event, timestamp, source, eventSpecificFields): self.mqttClient.publish( topic=topic, - payload=json.dumps(returnVal), + payload=json.dumps(payload), ) def add_outstanding_packet(self, packet): From 177cd6eed5f1b080cddefe743e0ff96c13c3cb7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Sun, 24 Mar 2019 14:20:07 +0100 Subject: [PATCH 075/108] OV-7. Pass string to hex2buf converter, not unicode. --- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 6260284f..ac96e22b 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -208,7 +208,7 @@ def encodeSendPacketPayload(self, destination, confirmable, packetsInBurst, pack # construct command payload as byte-list: # dest_eui64 (8B) || con (1B) || packetsInBurst (1B) || packetToken (5B) || packetPayloadLen (1B) buf = [] - buf += openvisualizer.openvisualizer_utils.hex2buf(destination, separator='-') + buf += openvisualizer.openvisualizer_utils.hex2buf(str(destination), separator='-') buf += [int(confirmable)] buf += [int(packetsInBurst)] buf += packetToken @@ -389,9 +389,9 @@ def _mqtt_handler_sendPacket(self, payload): # parse the payload payloadDecoded = json.loads(payload) - sourceStr = payloadDecoded['source'] + sourceStr = str(payloadDecoded['source']) source = openvisualizer.openvisualizer_utils.hex2buf(sourceStr, separator='-') - destination = openvisualizer.openvisualizer_utils.hex2buf(payloadDecoded['destination'], separator='-') + destination = openvisualizer.openvisualizer_utils.hex2buf(str(payloadDecoded['destination']), separator='-') packetsInBurst = payloadDecoded['packetsInBurst'] packetToken = payloadDecoded['packetToken'] packetPayloadLen = payloadDecoded['packetPayloadLen'] From 0e51006d4f6d8404bd8a00419265e38b437c225a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Sun, 24 Mar 2019 14:37:43 +0100 Subject: [PATCH 076/108] OV-7. Get dag root and network prefix address from the event bus. --- .../openBenchmarkAgent/openBenchmarkAgent.py | 44 ++++++++++++++----- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index ac96e22b..c6ecdfbd 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -87,7 +87,8 @@ def __init__(self, coapServer, mqttBroker, firmware, testbed, portNames, scenari self.mqttClient = None self.experimentRequestResponse = None self.performanceEvent = None - self.dagRootEui64 = None + self.dagRootEui64Buf = None + self.networkPrefixBuf = None # dict with keys being eui64, and value a tuple (localPort, testbedHost) self.nodes = {} @@ -142,7 +143,7 @@ def __init__(self, coapServer, mqttBroker, firmware, testbed, portNames, scenari self.performanceEvent = PerformanceEvent(self.experimentId, self.mqttClient) # everything is ok, start a coap server - self.openbenchmarkResource = OpenbenchmarkResource(self.performanceEvent, self.dagRootEui64) + self.openbenchmarkResource = OpenbenchmarkResource(self.performanceEvent, self.dagRootEui64Buf) self.coapServer.coapServer.addResource(self.openbenchmarkResource) # subscribe to eventBus performance-related events @@ -155,6 +156,16 @@ def __init__(self, coapServer, mqttBroker, firmware, testbed, portNames, scenari 'signal': 'fromMote.performanceData', 'callback': self.performanceEvent.handle_event, }, + { + 'sender': self.WILDCARD, + 'signal': 'registerDagRoot', + 'callback': self._registerDagRoot_notif + }, + { + 'sender': self.WILDCARD, + 'signal': 'unregisterDagRoot', + 'callback': self._unregisterDagRoot_notif + }, ] ) @@ -175,7 +186,9 @@ def close(self): def triggerSendPacket(self, destination, acknowledged, packetsInBurst, packetToken, packetPayloadLen): - destinationIPv6 = openvisualizer.openvisualizer_utils.formatIPv6Addr((self.networkPrefix + destination)) + assert self.networkPrefixBuf + + destinationIPv6 = openvisualizer.openvisualizer_utils.formatIPv6Addr((self.networkPrefixBuf + destination)) options = [] if not acknowledged: @@ -215,10 +228,6 @@ def encodeSendPacketPayload(self, destination, confirmable, packetsInBurst, pack buf += [packetPayloadLen] return buf - def updateDagRootEui64(self, address): - self.dagRootEui64 = address - self.openbenchmarkResource.dagRootEui64 = self.dagRootEui64 - # ======================== private ========================================= def _openbenchmark_start_benchmark(self, mqttClient): @@ -377,6 +386,17 @@ def _on_mqtt_message(self, client, userdata, message): else: self._execute_command_safely(message.topic, message.payload) + # save dag root address and network prefix + def _registerDagRoot_notif(self, sender, signal, data): + self.dagRootEui64Buf = data['host'] + self.openbenchmarkResource.dagRootEui64Buf = self.dagRootEui64Buf + self.networkPrefixBuf = data['prefix'] + + def _unregisterDagRoot_notif(self, sender, signal, data): + self.dagRootEui64Buf = None + self.openbenchmarkResource.dagRootEui64Buf = None + self.networkPrefixBuf = None + # ==== mqtt command handlers def _mqtt_handler_echo(self, payload): @@ -464,9 +484,6 @@ def _mqtt_handler_triggerNetworkFormation(self, payload): payloadDecoded = json.loads(payload) source = payloadDecoded['source'] - # remember DAG root's EUI64 - self.updateDagRootEui64(source) - # lookup corresponding mote port (destPort, testbedHost) = self.nodes[source] @@ -771,7 +788,7 @@ def __init__(self, performanceEvent, dagRootEui64): # params self.performanceEvent = performanceEvent - self.dagRootEui64 = dagRootEui64 + self.dagRootEui64Buf = dagRootEui64 # initialize parent class coapResource.coapResource.__init__( @@ -780,10 +797,13 @@ def __init__(self, performanceEvent, dagRootEui64): ) def POST(self, options=[], payload=[], metaData={}): + + assert self.dagRootEui64Buf + # token is in the last 5 bytes of payload token = payload[-5:] timestamp = metaData['generic_1'] - source = self.dagRootEui64 + source = openvisualizer.openvisualizer_utils.formatAddr(self.dagRootEui64Buf) destination = u.ipv6AddrString2Bytes(metaData['srcIP']) destinationEui64String = openvisualizer.openvisualizer_utils.formatAddr(destination[8:]) From 20fa8dc40d085e2b375fae2f18d141750d3747c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Sun, 24 Mar 2019 14:53:44 +0100 Subject: [PATCH 077/108] OV-7. Fixes in triggerSendPacket(). --- .../openBenchmarkAgent/openBenchmarkAgent.py | 43 +++++++++---------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index c6ecdfbd..48ae21ea 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -194,28 +194,27 @@ def triggerSendPacket(self, destination, acknowledged, packetsInBurst, packetTok if not acknowledged: options += [o.NoResponse([d.DFLT_OPTION_NORESPONSE_SUPRESS_ALL])] - with self.clientLock: - for packetCounter in range (0, packetsInBurst): - try: - # construct the payload of the POST request - payload = [] - payload += [0] * packetPayloadLen - payload += [packetCounter] - payload += [packetToken[1:]] - - # the call to POST() is blocking unless no response is expected - token = [packetCounter] + packetToken[1:] - self.performanceEvent.add_outstanding_packet((token, destination, self.coapServer.COAP_SERVER_DEFAULT_IPv6_HOP_LIMIT)) - - log.debug("Publishing on topic: {0} Payload: {1}".format(topic, returnVal)) - p = self.coapServer.POST('coap://[{0}:{1}]/b'.format(destinationIPv6, d.DEFAULT_UDP_PORT), - confirmable=False, - options=options, - payload = payload) - - # TODO log if a response is received - except e.coapNoResponseExpected: - pass + for packetCounter in range (0, packetsInBurst): + try: + # construct the payload of the POST request + payload = [] + payload += [0] * packetPayloadLen + payload += [packetCounter] + payload += [packetToken[1:]] + + # FIXME the call to POST() is blocking unless no response is expected + token = [packetCounter] + packetToken[1:] + self.performanceEvent.add_outstanding_packet((token, destination, self.coapServer.COAP_SERVER_DEFAULT_IPv6_HOP_LIMIT)) + + log.debug("Publishing on topic: {0} Payload: {1}".format(topic, returnVal)) + p = self.coapServer.coapServer.POST('coap://[{0}:{1}]/b'.format(destinationIPv6, d.DEFAULT_UDP_PORT), + confirmable=False, + options=options, + payload = payload) + + # TODO log if a response is received + except e.coapNoResponseExpected: + pass def encodeSendPacketPayload(self, destination, confirmable, packetsInBurst, packetToken, packetPayloadLen): # construct command payload as byte-list: From 025c4081c27f54e25bcde4963539ecdf68744a95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Sun, 24 Mar 2019 14:57:54 +0100 Subject: [PATCH 078/108] OV-7. Remove remnant debug throwing an exception. --- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 48ae21ea..a8707eaa 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -206,7 +206,6 @@ def triggerSendPacket(self, destination, acknowledged, packetsInBurst, packetTok token = [packetCounter] + packetToken[1:] self.performanceEvent.add_outstanding_packet((token, destination, self.coapServer.COAP_SERVER_DEFAULT_IPv6_HOP_LIMIT)) - log.debug("Publishing on topic: {0} Payload: {1}".format(topic, returnVal)) p = self.coapServer.coapServer.POST('coap://[{0}:{1}]/b'.format(destinationIPv6, d.DEFAULT_UDP_PORT), confirmable=False, options=options, From d3b5b56706c82d6ecf2afbd39bf6acf1ea5a786b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Sun, 24 Mar 2019 15:05:30 +0100 Subject: [PATCH 079/108] OV-7. Add a debug when sending POST requests. --- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index a8707eaa..4f12b6ac 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -206,6 +206,8 @@ def triggerSendPacket(self, destination, acknowledged, packetsInBurst, packetTok token = [packetCounter] + packetToken[1:] self.performanceEvent.add_outstanding_packet((token, destination, self.coapServer.COAP_SERVER_DEFAULT_IPv6_HOP_LIMIT)) + log.debug("Sending a POST request to 'coap://[{0}:{1}]/b".format(destinationIPv6, d.DEFAULT_UDP_PORT)) + p = self.coapServer.coapServer.POST('coap://[{0}:{1}]/b'.format(destinationIPv6, d.DEFAULT_UDP_PORT), confirmable=False, options=options, From b0600c37b5754747fcdcf7267b8ef21dfdeaa4a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Sun, 24 Mar 2019 15:11:10 +0100 Subject: [PATCH 080/108] OV-7. Fix syntax in CoAP URI. --- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 4f12b6ac..4d6f3f3c 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -206,9 +206,9 @@ def triggerSendPacket(self, destination, acknowledged, packetsInBurst, packetTok token = [packetCounter] + packetToken[1:] self.performanceEvent.add_outstanding_packet((token, destination, self.coapServer.COAP_SERVER_DEFAULT_IPv6_HOP_LIMIT)) - log.debug("Sending a POST request to 'coap://[{0}:{1}]/b".format(destinationIPv6, d.DEFAULT_UDP_PORT)) + log.debug("Sending a POST request to 'coap://[{0}]:{1}/b".format(destinationIPv6, d.DEFAULT_UDP_PORT)) - p = self.coapServer.coapServer.POST('coap://[{0}:{1}]/b'.format(destinationIPv6, d.DEFAULT_UDP_PORT), + p = self.coapServer.coapServer.POST('coap://[{0}]:{1}/b'.format(destinationIPv6, d.DEFAULT_UDP_PORT), confirmable=False, options=options, payload = payload) From 8a4efa822771ac66d68c6a7e5268e25ef89156e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Sun, 24 Mar 2019 15:39:31 +0100 Subject: [PATCH 081/108] OV-7. Remove remnant log. --- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 4d6f3f3c..6194130c 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -735,8 +735,6 @@ def run(self): # wait for a while to gather the response from motes time.sleep(1) - log.debug(self.dutyCycleMeasurements) - for measurement in self.dutyCycleMeasurements: # dispatch each as an individual message From 0a6366da7e480c9f1d27fd7dee33c723f8e375fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Mon, 25 Mar 2019 17:48:05 +0100 Subject: [PATCH 082/108] OV-7. Invoke CoAP client in an ephemeral thread. --- .../openBenchmarkAgent/openBenchmarkAgent.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 6194130c..ba58e80d 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -202,7 +202,6 @@ def triggerSendPacket(self, destination, acknowledged, packetsInBurst, packetTok payload += [packetCounter] payload += [packetToken[1:]] - # FIXME the call to POST() is blocking unless no response is expected token = [packetCounter] + packetToken[1:] self.performanceEvent.add_outstanding_packet((token, destination, self.coapServer.COAP_SERVER_DEFAULT_IPv6_HOP_LIMIT)) @@ -216,6 +215,8 @@ def triggerSendPacket(self, destination, acknowledged, packetsInBurst, packetTok # TODO log if a response is received except e.coapNoResponseExpected: pass + except e.coapTimeout: + pass def encodeSendPacketPayload(self, destination, confirmable, packetsInBurst, packetToken, packetPayloadLen): # construct command payload as byte-list: @@ -418,7 +419,13 @@ def _mqtt_handler_sendPacket(self, payload): acknowledged = payloadDecoded['confirmable'] if self.coapServer.getDagRootEui64() == source: # check if command is for the DAG root whose APP code is implemented here - self.triggerSendPacket(destination, acknowledged, packetsInBurst, packetToken, packetPayloadLen) + coapClientThread = threading.Thread(target = self.triggerSendPacket, args = (destination, + acknowledged, + packetsInBurst, + packetToken, + packetPayloadLen) + ) + coapClientThread.start() else: # command is for one of the motes in the mesh, send it over the serial From 7e84893eced9c5baf7311947fb13e704780c99d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Mon, 25 Mar 2019 19:32:21 +0100 Subject: [PATCH 083/108] OV-7. Add debugs in CoAP packet handling. --- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index ba58e80d..10f0fd21 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -214,8 +214,10 @@ def triggerSendPacket(self, destination, acknowledged, packetsInBurst, packetTok # TODO log if a response is received except e.coapNoResponseExpected: + log.debug("No CoAP response expected.") pass except e.coapTimeout: + log.debug("CoAP response timed out.") pass def encodeSendPacketPayload(self, destination, confirmable, packetsInBurst, packetToken, packetPayloadLen): From 335addee2854826d6cadba1a799384d58ce336de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Thu, 28 Mar 2019 23:54:28 +0100 Subject: [PATCH 084/108] OV-7. Send ASN as int, not string. --- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 10f0fd21..b6e8e86b 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -600,7 +600,7 @@ def publish_event(self, event, timestamp, source, eventSpecificFields): payload = {} payload['event'] = event - payload['timestamp'] = timestamp + payload['timestamp'] = int(timestamp,16) payload['source'] = source # update the payload with event specific fields From 627d4baac5d156431deffb3a0c276209e038e0df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Sat, 30 Mar 2019 15:20:31 +0100 Subject: [PATCH 085/108] OV-7. Add a script to test sendPacket command handling. --- .../openBenchmarkAgent/generateSendPacket.py | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 openvisualizer/openBenchmarkAgent/generateSendPacket.py diff --git a/openvisualizer/openBenchmarkAgent/generateSendPacket.py b/openvisualizer/openBenchmarkAgent/generateSendPacket.py new file mode 100644 index 00000000..2c312133 --- /dev/null +++ b/openvisualizer/openBenchmarkAgent/generateSendPacket.py @@ -0,0 +1,77 @@ +import paho.mqtt.client as mqtt +from argparse import ArgumentParser + +class PacketGenerator(): + OPENBENCHMARK_SENDPACKET_TOPIC = 'openbenchmark/experimentId/000/command/sendPacket' + + def __init__(self): + self.parser = ArgumentParser() + self._addParserArgs() + + self.argspace = self.parser.parse_args() + + self.confirmable = self.argspace.confirmable + self.mqttBroker = self.argspace.mqttBroker + self.payloadLen = self.argspace.mqttBroker + self.destEui64 = self.argspace.destEui64 + self.srcEui64 = self.argspace.srcEui64 + + # mqtt client + self.mqttClient = mqtt.Client('generateSendPacket') + self.mqttClient.connect(self.mqttBroker) + + command = { + 'token' : '123', + 'source' : self.srcEui64, + 'destination' : self.destEui64, + 'packetsInBurst' : 1, + 'packetToken' : [0, 1, 2, 3, 4], + 'packetPayloadLen' : 10, + 'confirmable' : self.confirmable + } + + self.mqttClient.publish( + topic=self.OPENBENCHMARK_SENDPACKET_TOPIC, + payload=json.dumps(command), + ) + + def _addParserArgs(self): + self.parser.add_argument('-c', '--confirmable', + dest='confirmable', + default=False, + action='store', + help='Whether a packet should be ack-ed at the app layer' + ) + self.parser.add_argument('-p', '--payloadLen', + dest='payloadLen', + default=10, + action='store', + help='Length of the payload in the packet to be sent.' + ) + self.parser.add_argument('-d', '--destEui64', + dest='destEui64', + default='', + action='store', + help='Destination EUI-64.' + ) + self.parser.add_argument('-s', '--srcEui64', + dest='srcEui64', + default='', + action='store', + help='Source EUI-64.' + ) + self.parser.add_argument('-b', '--mqttBroker', + dest='mqttBroker', + default='argus.paris.inria.fr', + action='store_true', + help='MQTT broker to use' + ) + + def _on_mqtt_connect(self): + pass + + def _on_mqtt_message(self): + pass + +if __name__ == '__main__': + PacketGenerator() From b2d96d1e3440178b865cfe3e532019c2c81900d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Sat, 30 Mar 2019 23:35:31 +0100 Subject: [PATCH 086/108] OV-19. Refactoring of the coapServer and adapting it to the coap lib. --- bin/openVisualizerApp.py | 12 +- openvisualizer/JRC/JRC.py | 6 +- openvisualizer/coapServer/coapServer.py | 211 ++++++++++-------- .../openBenchmarkAgent/openBenchmarkAgent.py | 9 +- 4 files changed, 133 insertions(+), 105 deletions(-) diff --git a/bin/openVisualizerApp.py b/bin/openVisualizerApp.py index 330bba47..5454c6c3 100644 --- a/bin/openVisualizerApp.py +++ b/bin/openVisualizerApp.py @@ -17,6 +17,9 @@ log = logging.getLogger('openVisualizerApp') +from coap import coap, \ + coapDefines + from openvisualizer.eventBus import eventBusMonitor from openvisualizer.eventLogger import eventLogger from openvisualizer.moteProbe import moteProbe @@ -59,7 +62,13 @@ def __init__(self,confdir,datadir,logdir,simulatorMode,numMotes,trace,debug,useP # local variables self.eventBusMonitor = eventBusMonitor.eventBusMonitor() self.openLbr = openLbr.OpenLbr(usePageZero) - self.coapServer = coapServer.coapServer() + + # run CoAP server in testing mode + # this mode does not open a real socket, rather uses PyDispatcher for sending/receiving messages + # We interface this mode with OpenVisualizer to run JRC co-located with the DAG root + self.coapServer = coap.coap(udpPort=coapDefines.DEFAULT_UDP_PORT, + testing=True, + socketUdp=coapServer.coapDispatcher) self.rpl = RPL.RPL() self.jrc = JRC.JRC(self.coapServer) self.topology = topology.topology() @@ -225,6 +234,7 @@ def close(self): probe.close() if self.openBenchmarkAgent: self.openBenchmarkAgent.close() + self.coapServer.close() def getMoteState(self, moteid): ''' diff --git a/openvisualizer/JRC/JRC.py b/openvisualizer/JRC/JRC.py index 7998183a..670a4da4 100644 --- a/openvisualizer/JRC/JRC.py +++ b/openvisualizer/JRC/JRC.py @@ -30,8 +30,8 @@ def __init__(self, coapServer): self.coapResource = joinResource() - self.coapServer.coapServer.addResource(self.coapResource) - self.coapServer.coapServer.addSecurityContextHandler(contextHandler(self.coapResource).securityContextLookup) + self.coapServer.addResource(self.coapResource) + self.coapServer.addSecurityContextHandler(contextHandler(self.coapResource).securityContextLookup) # initialize parent class eventBusClient.eventBusClient.__init__( @@ -50,7 +50,7 @@ def __init__(self, coapServer): # ======================== public ========================================== def close(self): - self.coapServer.close() + pass # ==== handle EventBus notifications diff --git a/openvisualizer/coapServer/coapServer.py b/openvisualizer/coapServer/coapServer.py index 551c2528..235700f9 100644 --- a/openvisualizer/coapServer/coapServer.py +++ b/openvisualizer/coapServer/coapServer.py @@ -3,7 +3,8 @@ coapDefines as d, \ coapOption as o, \ coapUtils as u, \ - coapObjectSecurity as oscoap + coapObjectSecurity as oscoap, \ + socketUdp import logging.handlers try: from openvisualizer.eventBus import eventBusClient @@ -18,35 +19,34 @@ import cbor import binascii import os +import time import threading -class coapServer(eventBusClient.eventBusClient): +# default IPv6 hop limit +COAP_SERVER_DEFAULT_IPv6_HOP_LIMIT = 65 + +class coapDispatcher(socketUdp.socketUdp, eventBusClient.eventBusClient): + # link-local prefix LINK_LOCAL_PREFIX = [0xfe, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] - # default IPv6 hop limit - COAP_SERVER_DEFAULT_IPv6_HOP_LIMIT = 65 - - def __init__(self): + def __init__(self, ipAddress, udpPort, callback): # log - log.info("create instance") - - # run CoAP server in testing mode - # this mode does not open a real socket, rather uses PyDispatcher for sending/receiving messages - # We interface this mode with OpenVisualizer to run JRC co-located with the DAG root - self.coapServer = coap.coap(udpPort=d.DEFAULT_UDP_PORT, testing=True) - - self.ephemeralCoapClient = None + log.debug('creating instance') + # params + self.udpPort = udpPort self.dagRootEui64 = None self.networkPrefix = None + self.callback = callback - # store params + # initialize the parent socketUdp class + socketUdp.socketUdp.__init__(self, ipAddress, self.udpPort, self.callback) - # initialize parent class + # initialize the parent class eventBusClient class eventBusClient.eventBusClient.__init__( self, - name='coapServer', + name='coapDispatcher', registrations=[ { 'sender': self.WILDCARD, @@ -61,24 +61,107 @@ def __init__(self): ] ) - # local variables - self.stateLock = threading.Lock() + # change name + self.name = 'coapDispatcher@DagRootIpv6:{0}'.format(self.udpPort) + self.gotMsgSem = threading.Semaphore() + + # start myself + self.start() # ======================== public ========================================== + # TODO rework the class for it to be completely stateless and not depend on self.dagRootEui64 + def sendUdp(self, destIp, destPort, msg): + ''' + Receive CoAP response and forward it to the mesh network. + Appends UDP and IPv6 headers to the CoAP message and forwards it on the Eventbus towards the mesh. + ''' - def close(self): - # nothing to do - pass + assert self.dagRootEui64 + assert self.networkPrefix + + log.debug("sendUdp to {0}:{1} message {2}".format(destIp,destPort,msg)) - def getDagRootEui64(self): - return self.dagRootEui64 + # UDP + udplen = len(msg) + 8 - def getDagRootIPv6(self): - ipv6buf = self.networkPrefix + self.dagRootEui64 - return openvisualizer.openvisualizer_utils.formatIPv6Addr(ipv6buf) + # FIXME need to signal the source port from the packet + udp = u.int2buf(self.udpPort, 2) # src port + udp += u.int2buf(destPort, 2) # dest port + udp += [udplen >> 8, udplen & 0xff] # length + udp += [0x00, 0x00] # checksum + udp += msg + + # destination address of the packet is CoAP client's IPv6 address (address of the mote) + dstIpv6Address = u.ipv6AddrString2Bytes(destIp) + assert len(dstIpv6Address) == 16 + # source address of the packet is DAG root's IPV6 address + # use the same prefix (link-local or global) as in the destination address + srcIpv6Address = dstIpv6Address[:8] + srcIpv6Address += self.dagRootEui64 + assert len(srcIpv6Address) == 16 + + # CRC See https://tools.ietf.org/html/rfc2460. + + udp[6:8] = openvisualizer.openvisualizer_utils.calculatePseudoHeaderCRC( + src=srcIpv6Address, + dst=dstIpv6Address, + length=[0x00, 0x00] + udp[4:6], + nh=[0x00, 0x00, 0x00, 17], # UDP as next header + payload=udp, + ) + + # IPv6 + ip = [6 << 4] # v6 + traffic class (upper nybble) + ip += [0x00, 0x00, 0x00] # traffic class (lower nibble) + flow label + ip += udp[4:6] # payload length + ip += [17] # next header (protocol); UDP=17 + ip += [COAP_SERVER_DEFAULT_IPv6_HOP_LIMIT] # hop limit (pick a safe value) + ip += srcIpv6Address # source + ip += dstIpv6Address # destination + ip += udp + + self.dispatch( + signal='v6ToMesh', + data=ip + ) + + # update stats + self._incrementTx() + + def close(self): + # stop + self.goOn = False + self.gotMsgSem.release() # ======================== private ========================================= + def _messageNotification(self, sender, signal, data): + # log + log.debug("messageNotification: got {1} from {0}".format(sender, data)) + + srcIpv6 = openvisualizer.openvisualizer_utils.formatIPv6Addr(data[0]) + rawbytes = data[1] + hopLimit = data[2] # IPv6 metadata + timestamp = str(data[3]) # timestamp of the received packet + + sender = (srcIpv6, self.udpPort, (hopLimit, timestamp)) + + # call the callback + self.callback(timestamp, sender, rawbytes) + + # update stats + self._incrementRx() + + # release the lock + self.gotMsgSem.release() + + # return success in order to acknowledge the reception + return True + + def run(self): + while self.goOn: + self.gotMsgSem.acquire() + # ==== handle EventBus notifications def _registerDagRoot_notif(self, sender, signal, data): @@ -90,7 +173,7 @@ def _registerDagRoot_notif(self, sender, signal, data): self.PROTO_UDP, d.DEFAULT_UDP_PORT ), - callback=self._receiveFromMesh, + callback=self._messageNotification, ) # register to receive at link-local DAG root's address @@ -101,7 +184,7 @@ def _registerDagRoot_notif(self, sender, signal, data): self.PROTO_UDP, d.DEFAULT_UDP_PORT ), - callback=self._receiveFromMesh, + callback=self._messageNotification, ) self.dagRootEui64 = data['host'] @@ -116,7 +199,7 @@ def _unregisterDagRoot_notif(self, sender, signal, data): self.PROTO_UDP, d.DEFAULT_UDP_PORT ), - callback=self._receiveFromMesh, + callback=self._messageNotification, ) # unregister link-local address self.unregister( @@ -126,74 +209,8 @@ def _unregisterDagRoot_notif(self, sender, signal, data): self.PROTO_UDP, d.DEFAULT_UDP_PORT ), - callback=self._receiveFromMesh, + callback=self._messageNotification, ) self.dagRootEui64 = None self.networkPrefix = None - - def _receiveFromMesh(self, sender, signal, data): - ''' - Receive packet from the mesh destined for JRC's CoAP server. - Forwards the packet to the virtual CoAP server running in test mode (PyDispatcher). - ''' - sender = openvisualizer.openvisualizer_utils.formatIPv6Addr(data[0]) - - hopLimit = data[2] # IPv6 metadata - timestamp = str(data[3]) # timestamp of the received packet - - # FIXME pass source port within the signal and open coap client at this port - self.ephemeralCoapClient = coap.coap(ipAddress=sender, udpPort=d.DEFAULT_UDP_PORT, testing=True, receiveCallback=self._receiveFromCoAP) - self.ephemeralCoapClient.socketUdp.sendUdp(destIp='', destPort=d.DEFAULT_UDP_PORT, msg=data[1],metaData=(hopLimit, timestamp)) # low level forward of the CoAP message - return True - - def _receiveFromCoAP(self, timestamp, sender, data): - ''' - Receive CoAP response and forward it to the mesh network. - Appends UDP and IPv6 headers to the CoAP message and forwards it on the Eventbus towards the mesh. - ''' - self.ephemeralCoapClient.close() - - # UDP - udplen = len(data) + 8 - - udp = u.int2buf(sender[1], 2) # src port - udp += u.int2buf(self.ephemeralCoapClient.udpPort, 2) # dest port - udp += [udplen >> 8, udplen & 0xff] # length - udp += [0x00, 0x00] # checksum - udp += data - - # destination address of the packet is CoAP client's IPv6 address (address of the mote) - dstIpv6Address = u.ipv6AddrString2Bytes(self.ephemeralCoapClient.ipAddress) - assert len(dstIpv6Address)==16 - # source address of the packet is DAG root's IPV6 address - # use the same prefix (link-local or global) as in the destination address - srcIpv6Address = dstIpv6Address[:8] - srcIpv6Address += self.dagRootEui64 - assert len(srcIpv6Address)==16 - - # CRC See https://tools.ietf.org/html/rfc2460. - - udp[6:8] = openvisualizer.openvisualizer_utils.calculatePseudoHeaderCRC( - src=srcIpv6Address, - dst=dstIpv6Address, - length=[0x00, 0x00] + udp[4:6], - nh=[0x00, 0x00, 0x00, 17], # UDP as next header - payload=udp, - ) - - # IPv6 - ip = [6 << 4] # v6 + traffic class (upper nybble) - ip += [0x00, 0x00, 0x00] # traffic class (lower nibble) + flow label - ip += udp[4:6] # payload length - ip += [17] # next header (protocol); UDP=17 - ip += [self.COAP_SERVER_DEFAULT_IPv6_HOP_LIMIT] # hop limit (pick a safe value) - ip += srcIpv6Address # source - ip += dstIpv6Address # destination - ip += udp - - # announce network prefix - self.dispatch( - signal = 'v6ToMesh', - data = ip - ) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index b6e8e86b..a121c558 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -26,6 +26,7 @@ from openvisualizer.eventBus import eventBusClient from openvisualizer.moteState import moteState +from openvisualizer.coapServer import coapServer import openvisualizer.openvisualizer_utils from coap import coap, \ @@ -144,7 +145,7 @@ def __init__(self, coapServer, mqttBroker, firmware, testbed, portNames, scenari # everything is ok, start a coap server self.openbenchmarkResource = OpenbenchmarkResource(self.performanceEvent, self.dagRootEui64Buf) - self.coapServer.coapServer.addResource(self.openbenchmarkResource) + self.coapServer.addResource(self.openbenchmarkResource) # subscribe to eventBus performance-related events eventBusClient.eventBusClient.__init__( @@ -203,11 +204,11 @@ def triggerSendPacket(self, destination, acknowledged, packetsInBurst, packetTok payload += [packetToken[1:]] token = [packetCounter] + packetToken[1:] - self.performanceEvent.add_outstanding_packet((token, destination, self.coapServer.COAP_SERVER_DEFAULT_IPv6_HOP_LIMIT)) + self.performanceEvent.add_outstanding_packet((token, destination, coapServer.COAP_SERVER_DEFAULT_IPv6_HOP_LIMIT)) log.debug("Sending a POST request to 'coap://[{0}]:{1}/b".format(destinationIPv6, d.DEFAULT_UDP_PORT)) - p = self.coapServer.coapServer.POST('coap://[{0}]:{1}/b'.format(destinationIPv6, d.DEFAULT_UDP_PORT), + p = self.coapServer.POST('coap://[{0}]/b'.format(destinationIPv6), confirmable=False, options=options, payload = payload) @@ -420,7 +421,7 @@ def _mqtt_handler_sendPacket(self, payload): packetPayloadLen = payloadDecoded['packetPayloadLen'] acknowledged = payloadDecoded['confirmable'] - if self.coapServer.getDagRootEui64() == source: # check if command is for the DAG root whose APP code is implemented here + if self.dagRootEui64Buf == source: # check if command is for the DAG root whose APP code is implemented here coapClientThread = threading.Thread(target = self.triggerSendPacket, args = (destination, acknowledged, packetsInBurst, From d32b5d6f5d8b81682aca005f55b861e7061ee4f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Sat, 30 Mar 2019 23:36:16 +0100 Subject: [PATCH 087/108] OV-7. Bug fixes. --- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index a121c558..7f085890 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -201,7 +201,7 @@ def triggerSendPacket(self, destination, acknowledged, packetsInBurst, packetTok payload = [] payload += [0] * packetPayloadLen payload += [packetCounter] - payload += [packetToken[1:]] + payload += packetToken[1:] token = [packetCounter] + packetToken[1:] self.performanceEvent.add_outstanding_packet((token, destination, coapServer.COAP_SERVER_DEFAULT_IPv6_HOP_LIMIT)) @@ -753,7 +753,7 @@ def run(self): topic = 'openbenchmark/experimentId/{0}/nodeId/{1}/performanceData'.format(self.experimentId, source) payload = { 'event' : self.EV_RADIO_DUTY_CYCLE_MEASUREMENT[0], - 'timestamp' : timestamp, + 'timestamp' : int(timestamp,16), 'source' : source, 'dutyCycle' : dutyCycle, } From 327568472a323b5689136796f892e81fe06fbece Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Sat, 30 Mar 2019 23:36:43 +0100 Subject: [PATCH 088/108] OV-7. Default params and random token for generateSendPacket.py --- openvisualizer/openBenchmarkAgent/generateSendPacket.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/generateSendPacket.py b/openvisualizer/openBenchmarkAgent/generateSendPacket.py index 2c312133..1766c7d3 100644 --- a/openvisualizer/openBenchmarkAgent/generateSendPacket.py +++ b/openvisualizer/openBenchmarkAgent/generateSendPacket.py @@ -1,5 +1,7 @@ import paho.mqtt.client as mqtt from argparse import ArgumentParser +import json +import random class PacketGenerator(): OPENBENCHMARK_SENDPACKET_TOPIC = 'openbenchmark/experimentId/000/command/sendPacket' @@ -25,7 +27,7 @@ def __init__(self): 'source' : self.srcEui64, 'destination' : self.destEui64, 'packetsInBurst' : 1, - 'packetToken' : [0, 1, 2, 3, 4], + 'packetToken' : [0, random.randint(0,255), random.randint(0,255), random.randint(0,255), random.randint(0,255)], 'packetPayloadLen' : 10, 'confirmable' : self.confirmable } @@ -50,13 +52,13 @@ def _addParserArgs(self): ) self.parser.add_argument('-d', '--destEui64', dest='destEui64', - default='', + default='14-15-92-cc-00-00-00-03', action='store', help='Destination EUI-64.' ) self.parser.add_argument('-s', '--srcEui64', dest='srcEui64', - default='', + default='14-15-92-cc-00-00-00-01', action='store', help='Source EUI-64.' ) From 2beee79cc99b40eb2f43554e09b0c7909b50a0b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Sun, 31 Mar 2019 19:32:01 +0200 Subject: [PATCH 089/108] OV-7. Generate responses as needed and events for ACK packets. --- .../openBenchmarkAgent/openBenchmarkAgent.py | 40 +++++++++++++++++-- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 7f085890..fa7de694 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -208,17 +208,32 @@ def triggerSendPacket(self, destination, acknowledged, packetsInBurst, packetTok log.debug("Sending a POST request to 'coap://[{0}]:{1}/b".format(destinationIPv6, d.DEFAULT_UDP_PORT)) - p = self.coapServer.POST('coap://[{0}]/b'.format(destinationIPv6), + token, metaData = self.coapServer.POST('coap://[{0}]:{1}/b'.format(destinationIPv6, d.DEFAULT_UDP_PORT), confirmable=False, options=options, payload = payload) - # TODO log if a response is received + # response received, publish the event + hopLimit = metaData['generic_0'] + timestamp = metaData['generic_1'] + source = openvisualizer.openvisualizer_utils.formatAddr(self.dagRootEui64Buf) + dict = { + 'packetToken': token, + 'destination': destination, + 'hopLimit': hopLimit, + } + + self.performanceEvent.publish_event(event=PerformanceEvent.EV_PACKET_RECEIVED[2], + timestamp=timestamp, + source=source, + eventSpecificFields=dict) + except e.coapNoResponseExpected: log.debug("No CoAP response expected.") pass except e.coapTimeout: log.debug("CoAP response timed out.") + # should we add an event to log packet drop here pass def encodeSendPacketPayload(self, destination, confirmable, packetsInBurst, packetToken, packetPayloadLen): @@ -808,6 +823,9 @@ def POST(self, options=[], payload=[], metaData={}): assert self.dagRootEui64Buf + respPayload = [] + respOptions = [] + # token is in the last 5 bytes of payload token = payload[-5:] timestamp = metaData['generic_1'] @@ -824,5 +842,19 @@ def POST(self, options=[], payload=[], metaData={}): 'hopLimit' : metaData['generic_0'], } - self.performanceEvent.publish_event(PerformanceEvent.EV_PACKET_RECEIVED[0], timestamp, source, dict) - return (d.COAP_RC_2_04_CHANGED, [], []) + self.performanceEvent.publish_event(PerformanceEvent.EV_PACKET_RECEIVED[2], timestamp, source, dict) + + noResponse = False + for option in options: + if isinstance(option, o.NoResponse): + noResponse = True + + # prepare the response + if not noResponse: + respPayload = token + respPayload[4] = (respPayload[4] + 1) % 255 + self.performanceEvent.add_outstanding_packet( + (respPayload, destination, coapServer.COAP_SERVER_DEFAULT_IPv6_HOP_LIMIT) + ) + + return d.COAP_RC_2_04_CHANGED, respOptions, respPayload From e852ed02a1dc3f00ffc084553d8953949982f83f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Sun, 31 Mar 2019 21:00:32 +0200 Subject: [PATCH 090/108] OV-7. Fix a bug in timestamp generation in ParserData. --- openvisualizer/moteConnector/ParserData.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openvisualizer/moteConnector/ParserData.py b/openvisualizer/moteConnector/ParserData.py index f1b02793..1b769539 100644 --- a/openvisualizer/moteConnector/ParserData.py +++ b/openvisualizer/moteConnector/ParserData.py @@ -97,7 +97,7 @@ def parseInput(self,input): #asn comes in the next 5bytes. asnbytes=input[2:7] - self._asn = struct.unpack(' Date: Sun, 31 Mar 2019 21:01:15 +0200 Subject: [PATCH 091/108] OV-7. Fix generateSendPacket to support a boolean argument. --- openvisualizer/openBenchmarkAgent/generateSendPacket.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/generateSendPacket.py b/openvisualizer/openBenchmarkAgent/generateSendPacket.py index 1766c7d3..d3bedc97 100644 --- a/openvisualizer/openBenchmarkAgent/generateSendPacket.py +++ b/openvisualizer/openBenchmarkAgent/generateSendPacket.py @@ -12,7 +12,7 @@ def __init__(self): self.argspace = self.parser.parse_args() - self.confirmable = self.argspace.confirmable + self.confirmable = True if self.argspace.confirmable == 'True' else False self.mqttBroker = self.argspace.mqttBroker self.payloadLen = self.argspace.mqttBroker self.destEui64 = self.argspace.destEui64 @@ -40,8 +40,9 @@ def __init__(self): def _addParserArgs(self): self.parser.add_argument('-c', '--confirmable', dest='confirmable', - default=False, + default='False', action='store', + choices=['False', 'True'], help='Whether a packet should be ack-ed at the app layer' ) self.parser.add_argument('-p', '--payloadLen', From dde740e5012fd7b94928f8950ca936062d74f8cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Sun, 31 Mar 2019 21:18:58 +0200 Subject: [PATCH 092/108] OV-7. Rework openBenchmarkAgent constructor and the motes param. --- bin/openVisualizerApp.py | 7 ++- .../openBenchmarkAgent/openBenchmarkAgent.py | 54 ++++++++----------- 2 files changed, 27 insertions(+), 34 deletions(-) diff --git a/bin/openVisualizerApp.py b/bin/openVisualizerApp.py index 5454c6c3..92ae1e2f 100644 --- a/bin/openVisualizerApp.py +++ b/bin/openVisualizerApp.py @@ -12,6 +12,7 @@ import os import logging import json +import time from openvisualizer.OVtracer import OVtracer @@ -204,12 +205,16 @@ def __init__(self,confdir,datadir,logdir,simulatorMode,numMotes,trace,debug,useP # If cloud-based benchmarking service is requested, start the agent if self.benchmark: + # give some time to OV to discover nodes' EUI-64 addresses + time.sleep(5) self.openBenchmarkAgent = openBenchmarkAgent.OpenBenchmarkAgent( mqttBroker=self.mqtt_broker_address, coapServer=self.coapServer, firmware='openwsn', testbed=self.testEnvironment, - portNames=[mote.getPortName() for mote in self.moteProbes], + motes={ + ms.getStateElem(ms.ST_IDMANAGER).get_info()['64bAddr'] : { 'serialPort' : ms.getStateElem(ms.ST_IDMANAGER).get_info()['serial'] } for ms in self.moteStates + }, scenario=self.benchmark ) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index fa7de694..81dee146 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -54,7 +54,7 @@ class OpenBenchmarkAgent(eventBusClient.eventBusClient): OPENBENCHMARK_MAX_RETRIES = 3 OPENBENCHMARK_PREFIX_CMD_HANDLER_NAME = '_mqtt_handler_' - def __init__(self, coapServer, mqttBroker, firmware, testbed, portNames, scenario): + def __init__(self, coapServer, mqttBroker, firmware, testbed, motes, scenario): ''' :param mqttBroker: Address of the MQTT broker where to connect with OpenBenchmark @@ -63,10 +63,11 @@ def __init__(self, coapServer, mqttBroker, firmware, testbed, portNames, scenari :param testbed: Identifier of the testbed, 'simulation' for OV in simulation mode, or 'local' for OV with locally-connected motes - :param portNames: - List of port names. If the mote is in testbed, expects 'testbed_{{TESTBED}}_{{HOST}}_{{EUI-64}}, - where {{TESTBED}} is the testbed identifier, {{HOST}} is the identifier of the remote machine in the testbed - where the mote is connected to, and {{EUI-64}} is the EUI-64 address of the mote. + :param motes: + Dictionary of motes with keys being EUI-64 and value a {'serialPort' : serialPortName} dict. If the mote is + in testbed, serialPortName is expected in 'testbed_{{TESTBED}}_{{HOST}}_{{EUI-64}} format, where {{TESTBED}} + is the testbed identifier, {{HOST}} is the identifier of the remote machine in the testbed where the mote is + connected to, and {{EUI-64}} is the EUI-64 address of the mote. :param scenario: Identifier of the requested scenario to benchmark the performance. ''' @@ -76,7 +77,7 @@ def __init__(self, coapServer, mqttBroker, firmware, testbed, portNames, scenari self.mqttBroker = mqttBroker self.firmware = firmware self.testbed = testbed - self.portNames = portNames + self.motes = motes self.scenario = scenario # primitive for mutual exclusion @@ -91,35 +92,22 @@ def __init__(self, coapServer, mqttBroker, firmware, testbed, portNames, scenari self.dagRootEui64Buf = None self.networkPrefixBuf = None - # dict with keys being eui64, and value a tuple (localPort, testbedHost) - self.nodes = {} - - # OV is running in simulation mode - if self.testbed == 'simulation': - for port in portNames: - # FIXME get eui64 in simulation - self.nodes[port] = (port, 'simulation') - # Motes are attached locally on the physical port - elif self.testbed == 'local': - for port in portNames: - # FIXME get eui64 when executing locally - self.nodes[port] = (port, 'local') - # General case, motes are in testbed connected over OpenTestbed software - else: - for port in portNames: - m = re.search('testbed_(.+)_(.+)_(.+)', port) + # Update the self.motes dict with testbed host id to be compatible with OpenBenchmark + for k, v in self.motes.iteritems(): + if self.testbed == 'simulation' or self.testbed == 'local': + self.motes[k]['host'] = self.testbed + else: + m = re.search('testbed_(.+)_(.+)_(.+)', v['serialPort']) if m: - # (testbed_host, eui64) assert m.group(1) == self.testbed - self.nodes[m.group(3)] = (port, m.group(2)) + self.motes[k]['host'] = m.group(2) log.info('Initializing OpenBenchmark with options:\n\t{0}'.format( '\n '.join(['mqttBroker = {0}'.format(self.mqttBroker), 'firmware = {0}'.format(self.firmware), 'testbed = {0}'.format(self.testbed), - 'portNames = {0}'.format(self.portNames), - 'scenario = {0}'.format(self.scenario), - 'nodes = {0}'.format(self.nodes)] + 'motes = {0}'.format(self.motes), + 'scenario = {0}'.format(self.scenario)] ))) try: # mqtt client @@ -264,8 +252,8 @@ def _openbenchmark_start_benchmark(self, mqttClient): # format nodes to the format expected by OpenBenchmark nodes = {} - for key, (portName, testbedId) in self.nodes.iteritems(): - nodes[key] = testbedId + for k,v in self.motes.iteritems(): + nodes[k] = v['host'] payload = { 'api_version': self.OPENBENCHMARK_API_VERSION, @@ -448,7 +436,7 @@ def _mqtt_handler_sendPacket(self, payload): else: # command is for one of the motes in the mesh, send it over the serial # lookup corresponding mote port - (destPort, testbedHost) = self.nodes[sourceStr] + destPort = self.motes[sourceStr]['serialPort'] # construct command payload as byte-list: # dest_eui64 (8B) || con (1B) || packetsInBurst (1B) || packetToken (5B) || packetPayloadLen (1B) @@ -486,7 +474,7 @@ def _mqtt_handler_configureTransmitPower(self, payload): power = payloadDecoded['power'] # lookup corresponding mote port - (destPort, testbedHost) = self.nodes[source] + destPort = self.motes[source]['serialPort'] action = [moteState.moteState.SET_COMMAND, moteState.moteState.COMMAND_SET_TX_POWER[0], power] # generate an eventbus signal to send a command over serial @@ -510,7 +498,7 @@ def _mqtt_handler_triggerNetworkFormation(self, payload): source = payloadDecoded['source'] # lookup corresponding mote port - (destPort, testbedHost) = self.nodes[source] + destPort = self.motes[source]['serialPort'] # generate an eventbus signal to send a command over serial self.dispatch( From fa021503b176b969cf9ba77e93a46648f1a21193 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Sun, 31 Mar 2019 21:19:24 +0200 Subject: [PATCH 093/108] OV-7. When logging hopLimit of packets send by DAG root, add 1. --- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 81dee146..35b70abb 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -655,7 +655,7 @@ def _handler_event_packetSentDagRoot(self, buf): # construct the missing fields from the saved values newBuf += token newBuf += destination - newBuf += [hopLimit] + newBuf += [hopLimit + 1] # +1 to account for the fact that root does not decrement hop limit return self._handler_event_packetSent(newBuf) # not found, not all packets sent by dag root are originated by openbenchmark: ignore it log.debug("packetSentDagRoot event: miss for token {0}".format(receivedToken)) From a673a97c6cef4a687d8f0cab86df9bd5bf1c2e22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Sun, 31 Mar 2019 21:19:53 +0200 Subject: [PATCH 094/108] OV-7. Bug fix when generating ACK packet event. --- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 35b70abb..4982c0d0 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -819,8 +819,8 @@ def POST(self, options=[], payload=[], metaData={}): timestamp = metaData['generic_1'] source = openvisualizer.openvisualizer_utils.formatAddr(self.dagRootEui64Buf) - destination = u.ipv6AddrString2Bytes(metaData['srcIP']) - destinationEui64String = openvisualizer.openvisualizer_utils.formatAddr(destination[8:]) + destinationBuf = u.ipv6AddrString2Bytes(metaData['srcIP'])[8:] + destinationEui64String = openvisualizer.openvisualizer_utils.formatAddr(destinationBuf) log.debug("OpenbenchmarkResource: POST handler received metadata: {0}".format(metaData)) @@ -842,7 +842,7 @@ def POST(self, options=[], payload=[], metaData={}): respPayload = token respPayload[4] = (respPayload[4] + 1) % 255 self.performanceEvent.add_outstanding_packet( - (respPayload, destination, coapServer.COAP_SERVER_DEFAULT_IPv6_HOP_LIMIT) + (respPayload, destinationBuf, coapServer.COAP_SERVER_DEFAULT_IPv6_HOP_LIMIT) ) return d.COAP_RC_2_04_CHANGED, respOptions, respPayload From 074708a884f38b4277ffc76d92ba5ef273e5c60b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Fri, 21 Jun 2019 14:54:04 +0200 Subject: [PATCH 095/108] OV-7. Fixes after a rebase. --- SConstruct | 6 ------ bin/openVisualizerApp.py | 6 ++++-- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/SConstruct b/SConstruct index 42b50aa0..12343a85 100644 --- a/SConstruct +++ b/SConstruct @@ -174,12 +174,6 @@ AddOption('--benchmark', action = 'store') runnerEnv['BENCHMARK'] = GetOption('benchmark') -AddOption('--mqttBroker', - dest = 'mqttBroker', - default = 'argus.paris.inria.fr', - action = 'store') -runnerEnv['MQTTBROKER'] = GetOption('mqttBroker') - AddOption('--mqtt-broker-address', dest = 'mqtt_broker_address', type = 'string') diff --git a/bin/openVisualizerApp.py b/bin/openVisualizerApp.py index 92ae1e2f..f301b96a 100644 --- a/bin/openVisualizerApp.py +++ b/bin/openVisualizerApp.py @@ -59,6 +59,7 @@ def __init__(self,confdir,datadir,logdir,simulatorMode,numMotes,trace,debug,useP self.testbed = testbed self.benchmark = benchmark self.pathTopo = pathTopo + self.mqtt_broker_address = mqtt_broker_address # local variables self.eventBusMonitor = eventBusMonitor.eventBusMonitor() @@ -118,7 +119,7 @@ def __init__(self,confdir,datadir,logdir,simulatorMode,numMotes,trace,debug,useP self.testEnvironment = self.testbed motesfinder = moteProbe.OpentestbedMoteFinder(testbed=self.testbed, mqtt_broker_address=self.mqtt_broker_address) self.moteProbes = [ - moteProbe.moteProbe(mqtt_broker_address, testbedmote_eui64=p) + moteProbe.moteProbe(mqtt_broker_address, testbedmote=p) for p in motesfinder.get_opentestbed_motelist() ] @@ -143,7 +144,7 @@ def __init__(self,confdir,datadir,logdir,simulatorMode,numMotes,trace,debug,useP eventLogger.eventLogger(ms) for ms in self.moteStates ] - if self.testbedmotes: + if self.testbed: # at least, when we use OpenTestbed, we don't need # Rover. Don't instantiate remoteConnectorServer which # consumes a lot of CPU. @@ -384,6 +385,7 @@ def main(parser=None): 'debug = {0}'.format(argspace.debug), 'testbed = {0}'.format(argspace.testbed), 'benchmark = {0}'.format(argspace.benchmark), + 'mqttBroker = {0}'.format(argspace.mqtt_broker_address), 'usePageZero = {0}'.format(argspace.usePageZero)], ))) log.info('Using external dirs:\n\t{0}'.format( From 04bcaf83485d8394d7079bf6191fc812ac5a9be7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Mon, 1 Jul 2019 16:10:59 +0200 Subject: [PATCH 096/108] OV-7. When in simulation mode, pass serial port name as testbed host. --- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 4982c0d0..f5a9fcde 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -94,8 +94,8 @@ def __init__(self, coapServer, mqttBroker, firmware, testbed, motes, scenario): # Update the self.motes dict with testbed host id to be compatible with OpenBenchmark for k, v in self.motes.iteritems(): - if self.testbed == 'simulation' or self.testbed == 'local': - self.motes[k]['host'] = self.testbed + if self.testbed == 'opensim' or self.testbed == 'local': + self.motes[k]['host'] = self.motes[k]['serialPort'] else: m = re.search('testbed_(.+)_(.+)_(.+)', v['serialPort']) if m: From a8404c7b1880a2a4d6bb1cbd7703a30c5a0cafc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Mon, 1 Jul 2019 16:11:30 +0200 Subject: [PATCH 097/108] OV-7. Rename test environment name when in simulation mode to opensim. --- bin/openVisualizerApp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/openVisualizerApp.py b/bin/openVisualizerApp.py index f301b96a..399ba178 100644 --- a/bin/openVisualizerApp.py +++ b/bin/openVisualizerApp.py @@ -97,7 +97,7 @@ def __init__(self,confdir,datadir,logdir,simulatorMode,numMotes,trace,debug,useP # create a moteProbe for each mote if self.simulatorMode: - self.testEnvironment = 'simulation' + self.testEnvironment = 'opensim' # in "simulator" mode, motes are emulated sys.path.append(os.path.join(self.datadir, 'sim_files')) import oos_openwsn From 83c8fd4274cae86472fd37ade4006b227bf76d96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Thu, 4 Jul 2019 15:17:24 +0200 Subject: [PATCH 098/108] OV-7. Increase measurement period to 30s. --- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index f5a9fcde..9c2a346e 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -516,7 +516,7 @@ def _mqtt_handler_triggerNetworkFormation(self, payload): class PerformanceEvent(object): # period for measurement polling - PERFORMANCE_EVENT_MEASUREMENT_PERIOD = 3 + PERFORMANCE_EVENT_MEASUREMENT_PERIOD = 30 # asynchronous event #name #id #openbenchmark ID EV_PACKET_SENT = ['packetSent', 0, 'packetSent' ] From 7443e47f2980e546ea9650fbd6b2fa5e4fd639ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Mon, 15 Jul 2019 16:50:50 +0200 Subject: [PATCH 099/108] OV-7. Fix a bug when retransmitting failed request. --- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 9c2a346e..3e88dfca 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -286,9 +286,9 @@ def _openbenchmark_start_benchmark(self, mqttClient): raise ValueError("No response from OpenBenchmark") # parse it - payload = json.loads(self.experimentRequestResponse) - tokenReceived = payload['token'] - success = payload['success'] + payloadResponse = json.loads(self.experimentRequestResponse) + tokenReceived = payloadResponse['token'] + success = payloadResponse['success'] # check token match if tokenGenerated != tokenReceived: @@ -297,7 +297,7 @@ def _openbenchmark_start_benchmark(self, mqttClient): if success != True: raise ValueError("Fail indicated") - experimentId = payload['experimentId'] + experimentId = payloadResponse['experimentId'] break # Retry for all ValueErrors From d3f69ee363d787f752555736ed2c645c1e8e3baf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Tue, 16 Jul 2019 16:46:17 +0200 Subject: [PATCH 100/108] OV-7. Add commit number in SUT name. --- bin/openVisualizerApp.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/openVisualizerApp.py b/bin/openVisualizerApp.py index 399ba178..374b93eb 100644 --- a/bin/openVisualizerApp.py +++ b/bin/openVisualizerApp.py @@ -13,6 +13,7 @@ import logging import json import time +import subprocess from openvisualizer.OVtracer import OVtracer @@ -211,7 +212,7 @@ def __init__(self,confdir,datadir,logdir,simulatorMode,numMotes,trace,debug,useP self.openBenchmarkAgent = openBenchmarkAgent.OpenBenchmarkAgent( mqttBroker=self.mqtt_broker_address, coapServer=self.coapServer, - firmware='openwsn', + firmware='openwsn-{0}'.format(subprocess.check_output(["git", "describe", "--tags"]).strip()), testbed=self.testEnvironment, motes={ ms.getStateElem(ms.ST_IDMANAGER).get_info()['64bAddr'] : { 'serialPort' : ms.getStateElem(ms.ST_IDMANAGER).get_info()['serial'] } for ms in self.moteStates From 7149d8658bfad0eed5d68dfc75651db1d7579f9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Wed, 17 Jul 2019 17:52:31 +0200 Subject: [PATCH 101/108] OV-7. Add desynchronized event handling. --- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 3e88dfca..5378dfda 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -523,8 +523,9 @@ class PerformanceEvent(object): EV_PACKET_RECEIVED = ['packetReceived', 1, 'packetReceived' ] EV_SYNCHRONIZATION_COMPLETED = ['synchronizationCompleted', 2, 'synchronizationCompleted' ] EV_SECURE_JOIN_COMPLETED = ['secureJoinCompleted', 3, 'secureJoinCompleted' ] - EV_BANDWIDTH_ASSIGNED = ['bandwidthAssigned', 4, 'bandwidthAssigned' ] + EV_BANDWIDTH_ASSIGNED = ['bandwidthAssigned', 4, 'bandwidthAssigned' ] EV_PACKET_SENT_DAGROOT = ['packetSentDagRoot', 5, 'packetSent' ] # special event to precisely get the ASN when dag root sent a packet + EV_DESYNCHRONIZED = ['desynchronized', 6, 'desynchronized' ] EV_NETWORK_FORMATION_COMPLETED = ['networkFormationCompleted', 257, 'networkFormationCompleted' ] EV_ASYNC_ALL = [ @@ -534,6 +535,7 @@ class PerformanceEvent(object): EV_BANDWIDTH_ASSIGNED, EV_SYNCHRONIZATION_COMPLETED, EV_PACKET_SENT_DAGROOT, + EV_DESYNCHRONIZED, EV_NETWORK_FORMATION_COMPLETED, ] @@ -669,6 +671,10 @@ def _handler_event_packetReceived(self, buf): def _handler_event_synchronizationCompleted(self, buf): return (True, {}) + # desynchronized + def _handler_event_desynchronized(self, buf): + return (True, {}) + # secureJoinCompleted def _handler_event_secureJoinCompleted(self, buf): return (True, {}) From 52dff994fae925f4bf2efded69642a0634b9e9ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Fri, 19 Jul 2019 13:54:54 +0200 Subject: [PATCH 102/108] OV-7. More robust EUI-64 discovery process. --- bin/openVisualizerApp.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/bin/openVisualizerApp.py b/bin/openVisualizerApp.py index 374b93eb..321b8aa3 100644 --- a/bin/openVisualizerApp.py +++ b/bin/openVisualizerApp.py @@ -207,16 +207,26 @@ def __init__(self,confdir,datadir,logdir,simulatorMode,numMotes,trace,debug,useP # If cloud-based benchmarking service is requested, start the agent if self.benchmark: + # give some time to OV to discover nodes' EUI-64 addresses - time.sleep(5) + motes = {} + for ms in self.moteStates: + attempt = 0 + while ms.getStateElem(ms.ST_IDMANAGER).get_info()['64bAddr'] == '': + if attempt >= 10: + motes['invalid_eui64_' + ms.getStateElem(ms.ST_IDMANAGER).get_info()['serial']] = { + 'serialPort': ms.getStateElem(ms.ST_IDMANAGER).get_info()['serial']} + break + attempt += 1 + time.sleep(1) + motes[ ms.getStateElem(ms.ST_IDMANAGER).get_info()['64bAddr'] ] = { 'serialPort' : ms.getStateElem(ms.ST_IDMANAGER).get_info()['serial'] } + self.openBenchmarkAgent = openBenchmarkAgent.OpenBenchmarkAgent( mqttBroker=self.mqtt_broker_address, coapServer=self.coapServer, firmware='openwsn-{0}'.format(subprocess.check_output(["git", "describe", "--tags"]).strip()), testbed=self.testEnvironment, - motes={ - ms.getStateElem(ms.ST_IDMANAGER).get_info()['64bAddr'] : { 'serialPort' : ms.getStateElem(ms.ST_IDMANAGER).get_info()['serial'] } for ms in self.moteStates - }, + motes=motes, scenario=self.benchmark ) From f0d99af81fef53b335e27b99e4644a3956b9c3b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Wed, 4 Sep 2019 15:17:07 +0200 Subject: [PATCH 103/108] DAG root should not decrement hop limit. --- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 5378dfda..96b1e1d0 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -657,7 +657,7 @@ def _handler_event_packetSentDagRoot(self, buf): # construct the missing fields from the saved values newBuf += token newBuf += destination - newBuf += [hopLimit + 1] # +1 to account for the fact that root does not decrement hop limit + newBuf += [hopLimit] return self._handler_event_packetSent(newBuf) # not found, not all packets sent by dag root are originated by openbenchmark: ignore it log.debug("packetSentDagRoot event: miss for token {0}".format(receivedToken)) From 9ca1a345bb0ebb6b5e8b65981073befff2978092 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Thu, 12 Sep 2019 16:49:35 +0200 Subject: [PATCH 104/108] OV-7. Check if timestamp is None. --- openvisualizer/moteState/moteState.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/openvisualizer/moteState/moteState.py b/openvisualizer/moteState/moteState.py index 3274706a..2cc13b32 100644 --- a/openvisualizer/moteState/moteState.py +++ b/openvisualizer/moteState/moteState.py @@ -738,7 +738,11 @@ def _getDutyCycle(self, sender, signal, data): # get last duty cycle measurement dutyCycle = self.state[self.ST_MACSTATS].getDutyCycle() # asn of the dutyCycle measurement is an approximation as the exact timestamp is not available - timestamp = str(self.state[self.ST_ASN].getAsn()) + timestamp = self.state[self.ST_ASN].getAsn() + if timestamp: + timestamp = str(timestamp) + else: + timestamp = '0' if source and dutyCycle and timestamp: data = { From 16ea998667947911ea6493551bc21d6d5c351ee5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Mon, 16 Sep 2019 10:31:16 +0200 Subject: [PATCH 105/108] OV-7. Remove variable length payload packets from OV. --- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 96b1e1d0..44e31898 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -187,7 +187,6 @@ def triggerSendPacket(self, destination, acknowledged, packetsInBurst, packetTok try: # construct the payload of the POST request payload = [] - payload += [0] * packetPayloadLen payload += [packetCounter] payload += packetToken[1:] @@ -820,8 +819,8 @@ def POST(self, options=[], payload=[], metaData={}): respPayload = [] respOptions = [] - # token is in the last 5 bytes of payload - token = payload[-5:] + # token is the payload + token = payload timestamp = metaData['generic_1'] source = openvisualizer.openvisualizer_utils.formatAddr(self.dagRootEui64Buf) From ba69ba8cb986a5c897ff50444fae862e6b10e3db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Tue, 17 Sep 2019 18:40:18 +0200 Subject: [PATCH 106/108] OV-7. Remove mqtt client naming to allow multiple instances. --- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 44e31898..279c9ef2 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -111,7 +111,7 @@ def __init__(self, coapServer, mqttBroker, firmware, testbed, motes, scenario): ))) try: # mqtt client - self.mqttClient = mqtt.Client('openBenchmarkAgent') + self.mqttClient = mqtt.Client() self.mqttClient.on_connect = self._on_mqtt_connect self.mqttClient.on_message = self._on_mqtt_message self.mqttClient.connect(self.mqttBroker) From 1c1fc5ae0ca17426f743f9c38195419eb8df4cee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Tue, 24 Sep 2019 16:24:41 +0200 Subject: [PATCH 107/108] Add short sleep time in between packets in a burst. --- openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py index 279c9ef2..aa05b41c 100644 --- a/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py +++ b/openvisualizer/openBenchmarkAgent/openBenchmarkAgent.py @@ -217,6 +217,7 @@ def triggerSendPacket(self, destination, acknowledged, packetsInBurst, packetTok except e.coapNoResponseExpected: log.debug("No CoAP response expected.") + time.sleep(0.2) pass except e.coapTimeout: log.debug("CoAP response timed out.") From 9e8a38a3f4ed35b0f1c0041adf450a7b4771cb5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mali=C5=A1a=20Vu=C4=8Dini=C4=87?= Date: Tue, 24 Sep 2019 21:53:20 +0200 Subject: [PATCH 108/108] Do not crash when parsing HDLC if something is wrong. --- openvisualizer/moteProbe/OpenHdlc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openvisualizer/moteProbe/OpenHdlc.py b/openvisualizer/moteProbe/OpenHdlc.py index f581aebd..b325830b 100644 --- a/openvisualizer/moteProbe/OpenHdlc.py +++ b/openvisualizer/moteProbe/OpenHdlc.py @@ -95,8 +95,8 @@ def dehdlcify(self,inBuf): :returns: the extracted frame, or -1 if wrong checksum ''' - assert inBuf[ 0]==self.HDLC_FLAG - assert inBuf[-1]==self.HDLC_FLAG + if inBuf[0]!=self.HDLC_FLAG or inBuf[-1]!=self.HDLC_FLAG: + raise HdlcException('unexpected frame') # make copy of input outBuf = inBuf[:]