From 3a37f6b96717063a93ff2c06a7d551156c454424 Mon Sep 17 00:00:00 2001 From: rtrimana Date: Tue, 9 Oct 2018 16:39:42 -0700 Subject: [PATCH] Adding a script to compare 2 timestamps files to validate the detection process. --- .../main/java/edu/uci/iotproject/Main.java | 45 +++++++++++------ .../detection/SignatureDetector.java | 10 ++-- python_ml/validate-detection.py | 50 +++++++++++++++++++ 3 files changed, 86 insertions(+), 19 deletions(-) create mode 100644 python_ml/validate-detection.py diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java index 00b0b9b..f22bf2b 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java @@ -49,10 +49,10 @@ public class Main { final String offPairsPath = "/scratch/July-2018/off.txt"; // 1) D-Link July 26 experiment - final String inputPcapFile = path + "/2018-07/dlink/dlink.wlan1.local.pcap"; - final String outputPcapFile = path + "/2018-07/dlink/dlink-processed.pcap"; - final String triggerTimesFile = path + "/2018-07/dlink/dlink-july-26-2018.timestamps"; - final String deviceIp = "192.168.1.199"; // .246 == phone; .199 == dlink plug? +// final String inputPcapFile = path + "/2018-07/dlink/dlink.wlan1.local.pcap"; +// final String outputPcapFile = path + "/2018-07/dlink/dlink-processed.pcap"; +// final String triggerTimesFile = path + "/2018-07/dlink/dlink-july-26-2018.timestamps"; +// final String deviceIp = "192.168.1.199"; // .246 == phone; .199 == dlink plug? // 2) TP-Link July 25 experiment // final String inputPcapFile = path + "/2018-07/tplink/tplink.wlan1.local.pcap"; @@ -99,11 +99,11 @@ public class Main { // final String deviceIp = "192.168.1.246"; // .246 == phone; .142 == SmartThings Hub (note: use eth0 capture for this!) // September 12, 2018 - includes both wlan1 and eth1 interfaces - //final String inputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset3.wlan1.local.pcap"; -// final String inputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset3.eth1.local.pcap"; +// final String inputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset3.wlan1.local.pcap"; +// //final String inputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset3.eth1.local.pcap"; // final String outputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset3-processed.pcap"; // final String triggerTimesFile = path + "/2018-08/kwikset-doorlock/kwikset-doorlock-sept-12-2018.timestamps"; -// final String deviceIp = "192.168.1.142"; // .246 == phone; .142 == SmartThings Hub (note: use eth0 capture for this!) +// final String deviceIp = "192.168.1.246"; // .246 == phone; .142 == SmartThings Hub (note: use eth0 capture for this!) // 8) Hue Bulb August 7 experiment // final String inputPcapFile = path + "/2018-08/hue-bulb/hue-bulb.wlan1.local.pcap"; @@ -136,10 +136,10 @@ public class Main { // final String deviceIp = "192.168.1.246"; // .246 == phone; .229 == sprinkler // // 13) DLink siren August 14 experiment -// final String inputPcapFile = path + "/2018-08/dlink-siren/dlink-siren.wlan1.local.pcap"; -// final String outputPcapFile = path + "/2018-08/dlink-siren/dlink-siren-processed.pcap"; -// final String triggerTimesFile = path + "/2018-08/dlink-siren/dlink-siren-aug-14-2018.timestamps"; -// final String deviceIp = "192.168.1.246"; // .246 == phone; .183 == siren + final String inputPcapFile = path + "/2018-08/dlink-siren/dlink-siren.wlan1.local.pcap"; + final String outputPcapFile = path + "/2018-08/dlink-siren/dlink-siren-processed.pcap"; + final String triggerTimesFile = path + "/2018-08/dlink-siren/dlink-siren-aug-14-2018.timestamps"; + final String deviceIp = "192.168.1.183"; // .246 == phone; .183 == siren // 14) Nest thermostat August 15 experiment // final String inputPcapFile = path + "/2018-08/nest/nest.wlan1.local.pcap"; @@ -163,6 +163,17 @@ public class Main { // final String outputPcapFile = path + "/2018-08/noise/noise-processed.pcap"; // final String triggerTimesFile = path + "/2018-08/noise/noise-sept-17-2018.timestamps"; // final String deviceIp = "192.168.1.142"; // .142 == SmartThings Hub; .199 == dlink plug; .183 == siren + // September 26 - D-Link noise +// final String inputPcapFile = path + "/2018-08/noise/noise.dlink.wlan1.pcap"; +// final String outputPcapFile = path + "/2018-08/noise/noise-processed.pcap"; +// final String triggerTimesFile = path + "/2018-08/noise/dlink-noise-sept-26-2018.timestamps"; +// final String deviceIp = "192.168.1.183"; // .199 == dlink plug; .183 == siren + // September 27 - Kwikset noise +// final String inputPcapFile = path + "/2018-08/noise/noise.kwikset.eth1.pcap"; +// final String outputPcapFile = path + "/2018-08/noise/noise-processed.pcap"; +// final String triggerTimesFile = path + "/2018-08/noise/kwikset-doorlock-noise-sept-27-2018.timestamps"; +// final String deviceIp = "192.168.1.142"; // .142 == SmartThings Hub; + TriggerTimesFileReader ttfr = new TriggerTimesFileReader(); List triggerTimes = ttfr.readTriggerTimes(triggerTimesFile, false); @@ -298,10 +309,12 @@ public class Main { PcapPacketUtils.mergeSignatures(ppListOfListListOn, sortedAllConversation); PcapPacketUtils.sortSignatures(ppListOfListListOn); count = 0; - for (List> ll : ppListOfListListOn) { + /*for (List> ll : ppListOfListListOn) { PrintUtils.serializeClustersIntoFile("./onSignature" + ++count + ".sig", ll); ppListOfListReadOn.add(PrintUtils.deserializeClustersFromFile("./onSignature" + count + ".sig")); - } + }*/ + PrintUtils.serializeSignatureIntoFile("./onSignature.sig", ppListOfListListOn); + ppListOfListReadOn = PrintUtils.deserializeSignatureFromFile("./onSignature.sig"); System.out.println("========================================"); System.out.println(" Clustering results for OFF "); @@ -322,10 +335,12 @@ public class Main { PcapPacketUtils.mergeSignatures(ppListOfListListOff, sortedAllConversation); PcapPacketUtils.sortSignatures(ppListOfListListOff); count = 0; - for (List> ll : ppListOfListListOff) { + /*for (List> ll : ppListOfListListOff) { PrintUtils.serializeClustersIntoFile("./offSignature" + ++count + ".sig", ll); ppListOfListReadOff.add(PrintUtils.deserializeClustersFromFile("./offSignature" + count + ".sig")); - } + }*/ + PrintUtils.serializeSignatureIntoFile("./offSignature.sig", ppListOfListListOff); + ppListOfListReadOff = PrintUtils.deserializeSignatureFromFile("./offSignature.sig"); System.out.println("========================================"); // ============================================================================================================ diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/SignatureDetector.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/SignatureDetector.java index 6d1f54d..7c9bfdd 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/SignatureDetector.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/SignatureDetector.java @@ -27,8 +27,8 @@ public class SignatureDetector implements PacketListener, ClusterMatcher.Cluster // Test client public static void main(String[] args) throws PcapNativeException, NotOpenException { - // String path = "/scratch/July-2018"; // Rahmadi - String path = "/Users/varmarken/temp/UCI IoT Project/experiments"; // Janus + String path = "/scratch/July-2018"; // Rahmadi + //String path = "/Users/varmarken/temp/UCI IoT Project/experiments"; // Janus // D-Link Plug experiment final String inputPcapFile = path + "/evaluation/dlink/dlink-plug.data.wlan1.pcap"; @@ -91,8 +91,10 @@ public class SignatureDetector implements PacketListener, ClusterMatcher.Cluster default: throw new AssertionError("unhandled event type"); } - String output = String.format("[ !!! %s SIGNATURE DETECTED at %s !!! ]", - eventDescription, dateTimeFormatter.format(ua.getTimestamp())); + //String output = String.format("[ !!! %s SIGNATURE DETECTED at %s !!! ]", + // eventDescription, dateTimeFormatter.format(ua.getTimestamp())); + String output = String.format("%s", + dateTimeFormatter.format(ua.getTimestamp())); System.out.println(output); }; diff --git a/python_ml/validate-detection.py b/python_ml/validate-detection.py new file mode 100644 index 0000000..b22e0a3 --- /dev/null +++ b/python_ml/validate-detection.py @@ -0,0 +1,50 @@ +from datetime import datetime + + +path = "/scratch/July-2018/evaluation/" +device = "dlink" +fileExperiment = "dlink-plug-8hr-data-oct-8-2018.timestamps" +fileDetection = "dlink-plug.detection.timestamps" +TIME_WINDOW = 15 # detection/signature window of 15 seconds +#NEG_TIME_WINDOW = -15 # detection/signature window of 15 seconds + +# Open training timestamps file and store into a list +with open(path + device + "/" + fileExperiment, "r") as experiment: + tsExperimentList = [] + for line in experiment: + # Format "%m/%d/%Y %I:%M:%S %p" + tsE = datetime.strptime(line, "%m/%d/%Y %I:%M:%S %p\n") + tsExperimentList.append(tsE) + +# Open detection timestamps file and store into a list +with open(path + device + "/" + fileDetection, "r") as detection: + tsDetectionList = [] + for line in detection: + # Format "%b %d, %Y %I:%M:%S %p" + tsD = datetime.strptime(line, "%b %d, %Y %I:%M:%S %p\n") + tsDetectionList.append(tsD) + +if (len(tsExperimentList) > len(tsDetectionList)): + maxTimestamps = len(tsExperimentList) +else: + maxTimestamps = len(tsDetectionList) + +i = 0 +j = 0 +while i < maxTimestamps: + tsE = tsExperimentList[i] + tsD = tsDetectionList[j] + # Detection is always a bit later than training trigger + delta = tsD - tsE + # The following happens when we could detect less triggers than the experiment + if (delta.seconds > TIME_WINDOW): + print("Missing trigger at line: " + str(i) + ", t_experiment: " + str(tsE) + " and t_detection: " + str(tsD)) + i = i + 1 + # The following should not happen (we have more detected triggers than the experiment) + #elif (delta.seconds < NEG_TIME_WINDOW): + # print("Mismatch at t_experiment: " + str(tsE) + " and t_detection: " + str(tsD)) + # j = j + 1 + i = i + 1 + j = j + 1 + +print("Done parsing: " + str(i) + " lines") -- 2.34.1