From: rtrimana Date: Tue, 26 Mar 2019 18:26:10 +0000 (-0700) Subject: Adding flexibility to detection result analyzer: to not consider event type, e.g... X-Git-Url: http://demsky.eecs.uci.edu/git/?a=commitdiff_plain;h=4a4f956b06431ba99ff4c0676d6c4bde5d7610e7;p=pingpong.git Adding flexibility to detection result analyzer: to not consider event type, e.g., in the case of Blossom sprinkler. --- diff --git a/Code/Projects/PacketLevelSignatureExtractor/execute_layer2_smarthome_all_detection_results_analysis.sh b/Code/Projects/PacketLevelSignatureExtractor/execute_layer2_smarthome_all_detection_results_analysis.sh index 7d0689d..fd6228e 100755 --- a/Code/Projects/PacketLevelSignatureExtractor/execute_layer2_smarthome_all_detection_results_analysis.sh +++ b/Code/Projects/PacketLevelSignatureExtractor/execute_layer2_smarthome_all_detection_results_analysis.sh @@ -16,9 +16,10 @@ TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/arlo-camera/timestamps/arlo-camera-smartho RESULTS_FILE="$RESULTS_BASE_DIR/arlo-camera/arlo-camera.wlan1.detection.pcap___phone-side.detectionresults" # Put the analysis results in the same folder as the detection results. ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +EXACT_MATCH="true" -PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'" ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" # ====================================================================================================================== @@ -30,13 +31,15 @@ TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/blossom-sprinkler/timestamps/blossom-sprin # DEVICE SIDE RESULTS_FILE="$RESULTS_BASE_DIR/blossom-sprinkler/blossom-sprinkler.wlan1.detection.pcap___device-side.detectionresults" ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" -PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +EXACT_MATCH="false" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'" ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" # PHONE SIDE RESULTS_FILE="$RESULTS_BASE_DIR/blossom-sprinkler/blossom-sprinkler.wlan1.detection.pcap___phone-side.detectionresults" ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" -PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +EXACT_MATCH="false" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'" ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" # ====================================================================================================================== @@ -48,13 +51,15 @@ TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/dlink-plug/timestamps/dlink-plug-smarthome # DEVICE SIDE RESULTS_FILE="$RESULTS_BASE_DIR/dlink-plug/dlink-plug.wlan1.detection.pcap___device-side.detectionresults" ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" -PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +EXACT_MATCH="true" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'" ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" # PHONE SIDE RESULTS_FILE="$RESULTS_BASE_DIR/dlink-plug/dlink-plug.wlan1.detection.pcap___phone-side.detectionresults" ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" -PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +EXACT_MATCH="true" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'" ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" # ====================================================================================================================== @@ -66,7 +71,8 @@ TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/dlink-siren/timestamps/dlink-siren-smartho #PHONE SIDE RESULTS_FILE="$RESULTS_BASE_DIR/dlink-siren/dlink-siren.wlan1.detection.pcap___phone-side.detectionresults" ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" -PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +EXACT_MATCH="true" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'" ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" # ====================================================================================================================== @@ -80,8 +86,9 @@ TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/hue-bulb/timestamps/hue-bulb-smarthome-nov # PHONE SIDE RESULTS_FILE="$RESULTS_BASE_DIR/hue-bulb/hue-bulb.wlan1.detection.pcap___phone-side.detectionresults" ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" -PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" -./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" +EXACT_MATCH="true" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'" +#./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" # ====================================================================================================================== @@ -94,7 +101,8 @@ TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/kwikset-doorlock/timestamps/kwikset-doorlo # PHONE SIDE RESULTS_FILE="$RESULTS_BASE_DIR/kwikset-doorlock/kwikset-doorlock.wlan1.detection.pcap___phone-side.detectionresults" ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" -PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +EXACT_MATCH="true" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'" ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" # ====================================================================================================================== @@ -108,7 +116,8 @@ TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/nest-thermostat/timestamps/nest-thermostat # PHONE SIDE RESULTS_FILE="$RESULTS_BASE_DIR/nest-thermostat/nest-thermostat.wlan1.detection.pcap___phone-side.detectionresults" ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" -PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +EXACT_MATCH="true" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'" ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" # ====================================================================================================================== @@ -122,7 +131,8 @@ TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/st-plug/timestamps/st-plug-smarthome-nov-1 # PHONE SIDE RESULTS_FILE="$RESULTS_BASE_DIR/st-plug/st-plug.wlan1.detection.pcap___phone-side.detectionresults" ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" -PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +EXACT_MATCH="true" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'" ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" # ====================================================================================================================== @@ -136,7 +146,8 @@ TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/tplink-bulb/timestamps/tplink-bulb-smartho # PHONE SIDE RESULTS_FILE="$RESULTS_BASE_DIR/tplink-bulb/tplink-bulb.wlan1.detection.pcap___phone-side.detectionresults" ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" -PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +EXACT_MATCH="true" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'" ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" # ====================================================================================================================== @@ -148,13 +159,15 @@ TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/tplink-plug/timestamps/tplink-plug-smartho # DEVICE SIDE RESULTS_FILE="$RESULTS_BASE_DIR/tplink-plug/tplink-plug.wlan1.detection.pcap___device-side.detectionresults" ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" -PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +EXACT_MATCH="true" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'" ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" # DEVICE SIDE OUTBOUND RESULTS_FILE="$RESULTS_BASE_DIR/tplink-plug/tplink-plug.wlan1.detection.pcap___device-side-outbound.detectionresults" ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" -PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +EXACT_MATCH="true" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'" ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" # ====================================================================================================================== @@ -168,7 +181,8 @@ TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/wemo-insight-plug/timestamps/wemo-insight- # PHONE SIDE RESULTS_FILE="$RESULTS_BASE_DIR/wemo-insight-plug/wemo-insight-plug.wlan1.detection.pcap___phone-side.detectionresults" ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" -PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +EXACT_MATCH="true" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'" ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" # ====================================================================================================================== @@ -182,6 +196,7 @@ TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/wemo-plug/timestamps/wemo-plug-smarthome-n # PHONE SIDE RESULTS_FILE="$RESULTS_BASE_DIR/wemo-plug/wemo-plug.wlan1.detection.pcap___phone-side.detectionresults" ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" -PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +EXACT_MATCH="true" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'" ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" # ====================================================================================================================== \ No newline at end of file diff --git a/Code/Projects/PacketLevelSignatureExtractor/execute_layer3_smarthome_all_detection.sh b/Code/Projects/PacketLevelSignatureExtractor/execute_layer3_smarthome_all_detection.sh index 74a3810..30f9cc8 100755 --- a/Code/Projects/PacketLevelSignatureExtractor/execute_layer3_smarthome_all_detection.sh +++ b/Code/Projects/PacketLevelSignatureExtractor/execute_layer3_smarthome_all_detection.sh @@ -57,7 +57,7 @@ ON_ANALYSIS="$SIGNATURES_BASE_DIR/blossom-sprinkler/analysis/blossom-sprinkler-o OFF_ANALYSIS="$SIGNATURES_BASE_DIR/blossom-sprinkler/analysis/blossom-sprinkler-offClusters-phone-side.cls" ON_SIGNATURE="$SIGNATURES_BASE_DIR/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-phone-side.sig" OFF_SIGNATURE="$SIGNATURES_BASE_DIR/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-phone-side.sig" -RESULTS_FILE="$OUTPUT_DIR/blossom-sprinkler/blossom-sprinkler.wlan1.detection.pcap___phone-side.detectionresults" +RESULTS_FILE="$OUTPUT_DIR/blossom-sprinkler/blossom-sprinkler.eth0.detection.pcap___phone-side.detectionresults" SIGNATURE_DURATION="3670" EPSILON="10.0" @@ -185,7 +185,7 @@ ON_ANALYSIS="$SIGNATURES_BASE_DIR/st-plug/analysis/st-plug-onClusters-phone-side OFF_ANALYSIS="$SIGNATURES_BASE_DIR/st-plug/analysis/st-plug-offClusters-phone-side.cls" ON_SIGNATURE="$SIGNATURES_BASE_DIR/st-plug/signatures/st-plug-onSignature-phone-side.sig" OFF_SIGNATURE="$SIGNATURES_BASE_DIR/st-plug/signatures/st-plug-offSignature-phone-side.sig" -RESULTS_FILE="$OUTPUT_DIR/st-plug/st-plug.wlan1.detection.pcap___phone-side.detectionresults" +RESULTS_FILE="$OUTPUT_DIR/st-plug/st-plug.eth0.detection.pcap___phone-side.detectionresults" SIGNATURE_DURATION="2445" EPSILON="10.0" @@ -216,7 +216,7 @@ PROGRAM_ARGS="'$PCAP_FILE' '$ON_ANALYSIS' '$OFF_ANALYSIS' '$ON_SIGNATURE' '$OFF_ # ==================================================== TP-LINK PLUG ==================================================== -PCAP_FILE="$PCAPS_BASE_DIR/tplink-plug/eth0/tplink-plug.eth0.detection.pcap" +PCAP_FILE="$PCAPS_BASE_DIR/tplink-plug/wlan1/tplink-plug.wlan1.detection.pcap" # DEVICE SIDE (both the 112, 115 and 556, 1293 sequences) ON_ANALYSIS="$SIGNATURES_BASE_DIR/tplink-plug/analysis/tplink-plug-onClusters.cls" @@ -230,12 +230,14 @@ EPSILON="10.0" PROGRAM_ARGS="'$PCAP_FILE' '$ON_ANALYSIS' '$OFF_ANALYSIS' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION' '$EPSILON'" ./gradlew run -DmainClass=edu.uci.iotproject.detection.layer3.Layer3SignatureDetector --args="$PROGRAM_ARGS" +PCAP_FILE="$PCAPS_BASE_DIR/tplink-plug/eth0/tplink-plug.eth0.detection.pcap" + # DEVICE SIDE OUTBOUND (contains only those packets that go through the WAN port, i.e., only the 556, 1293 sequence) ON_ANALYSIS="$SIGNATURES_BASE_DIR/tplink-plug/analysis/tplink-plug-onClusters.cls" OFF_ANALYSIS="$SIGNATURES_BASE_DIR/tplink-plug/analysis/tplink-plug-offClusters.cls" ON_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-plug/signatures/tplink-plug-onSignature-device-side-outbound.sig" OFF_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-plug/signatures/tplink-plug-offSignature-device-side-outbound.sig" -RESULTS_FILE="$OUTPUT_DIR/tplink-plug/tplink-plug.wlan1.wan-detection.pcap___device-side-outbound.detectionresults" +RESULTS_FILE="$OUTPUT_DIR/tplink-plug/tplink-plug.eth0.detection.pcap___device-side-outbound.detectionresults" SIGNATURE_DURATION="224" EPSILON="10.0" diff --git a/Code/Projects/PacketLevelSignatureExtractor/execute_layer3_smarthome_all_detection_results_analysis.sh b/Code/Projects/PacketLevelSignatureExtractor/execute_layer3_smarthome_all_detection_results_analysis.sh new file mode 100755 index 0000000..7eb0d3e --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/execute_layer3_smarthome_all_detection_results_analysis.sh @@ -0,0 +1,206 @@ +#!/bin/bash + +# Base directory where the smarthome evaluation traces and timestamp files are stored, +# (i.e., /some/arbitrary/local/path/experimental_result/smarthome) +TIMESTAMPS_BASE_DIR=$1 +readonly TIMESTAMPS_BASE_DIR + +# Base directory for the detection results files for the smarthome experiment +RESULTS_BASE_DIR=$2 +readonly RESULTS_BASE_DIR + + + +# ==================================================== ARLO CAMERA ===================================================== +TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/arlo-camera/timestamps/arlo-camera-smarthome-nov-15-2018.timestamps" +RESULTS_FILE="$RESULTS_BASE_DIR/arlo-camera/arlo-camera.eth0.detection.pcap___phone-side.detectionresults" +# Put the analysis results in the same folder as the detection results. +ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +EXACT_MATCH="true" + + +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'" +./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ================================================= BLOSSOM SPRINKLER ================================================== +TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/blossom-sprinkler/timestamps/blossom-sprinkler-smarthome-jan-14-2019.timestamps" + +# DEVICE SIDE +RESULTS_FILE="$RESULTS_BASE_DIR/blossom-sprinkler/blossom-sprinkler.eth0.detection.pcap___device-side.detectionresults" +ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +EXACT_MATCH="false" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'" +./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" + +# PHONE SIDE +RESULTS_FILE="$RESULTS_BASE_DIR/blossom-sprinkler/blossom-sprinkler.eth0.detection.pcap___phone-side.detectionresults" +ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +EXACT_MATCH="false" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'" +./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ==================================================== D-LINK PLUG ===================================================== +TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/dlink-plug/timestamps/dlink-plug-smarthome-nov-8-2018.timestamps" + +# DEVICE SIDE +# This one is going to generate 95 FPs because every event is counted twice (same signatures for ON and OFF). +RESULTS_FILE="$RESULTS_BASE_DIR/dlink-plug/dlink-plug.eth0.detection.pcap___device-side.detectionresults" +ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +EXACT_MATCH="true" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'" +./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" + +# PHONE SIDE +RESULTS_FILE="$RESULTS_BASE_DIR/dlink-plug/dlink-plug.eth0.detection.pcap___phone-side.detectionresults" +ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +EXACT_MATCH="true" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'" +./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ==================================================== D-LINK SIREN ==================================================== +TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/dlink-siren/timestamps/dlink-siren-smarthome-nov-10-2018.timestamps" + +#PHONE SIDE +RESULTS_FILE="$RESULTS_BASE_DIR/dlink-siren/dlink-siren.eth0.detection.pcap___phone-side.detectionresults" +ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +EXACT_MATCH="true" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'" +./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ===================================================== HUE BULB ======================================================= +TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/hue-bulb/timestamps/hue-bulb-smarthome-nov-20-2018.timestamps" + +# Has no device side signature. + +# PHONE SIDE +RESULTS_FILE="$RESULTS_BASE_DIR/hue-bulb/hue-bulb.eth0.detection.pcap___phone-side.detectionresults" +ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +EXACT_MATCH="true" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'" +# Note: Hue bulb's signature was gone after September/October 2018, perhaps due to some firmware update. +#./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ================================================= KWIKSET DOORLOCK =================================================== +TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/kwikset-doorlock/timestamps/kwikset-doorlock-smarthome-nov-10-2018.timestamps" + +# Has no device side signature. + +# PHONE SIDE +RESULTS_FILE="$RESULTS_BASE_DIR/kwikset-doorlock/kwikset-doorlock.eth0.detection.pcap___phone-side.detectionresults" +ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +EXACT_MATCH="true" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'" +./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ================================================= NEST THERMOSTAT ==================================================== +TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/nest-thermostat/timestamps/nest-thermostat-smarthome-nov-16-2018.timestamps" + +# Has no device side signature. + +# PHONE SIDE +RESULTS_FILE="$RESULTS_BASE_DIR/nest-thermostat/nest-thermostat.eth0.detection.pcap___phone-side.detectionresults" +ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +EXACT_MATCH="true" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'" +./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ====================================================== ST PLUG ======================================================= +TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/st-plug/timestamps/st-plug-smarthome-nov-13-2018.timestamps" + +# Has no device side signature. + +# PHONE SIDE +RESULTS_FILE="$RESULTS_BASE_DIR/st-plug/st-plug.eth0.detection.pcap___phone-side.detectionresults" +ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +EXACT_MATCH="true" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'" +./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ==================================================== TP-LINK BULB ==================================================== +TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/tplink-bulb/timestamps/tplink-bulb-smarthome-nov-19-2018.timestamps" + +# Has no device side signature. + +# PHONE SIDE +RESULTS_FILE="$RESULTS_BASE_DIR/tplink-bulb/tplink-bulb.wlan1.wan-detection.pcap___phone-side.detectionresults" +ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +EXACT_MATCH="true" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'" +./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ==================================================== TP-LINK PLUG ==================================================== +TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/tplink-plug/timestamps/tplink-plug-smarthome-nov-9-2018.timestamps" + +# DEVICE SIDE +RESULTS_FILE="$RESULTS_BASE_DIR/tplink-plug/tplink-plug.wlan1.wan-detection.pcap___device-side.detectionresults" +ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +EXACT_MATCH="true" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'" +./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" + +# DEVICE SIDE OUTBOUND +RESULTS_FILE="$RESULTS_BASE_DIR/tplink-plug/tplink-plug.eth0.detection.pcap___device-side-outbound.detectionresults" +ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +EXACT_MATCH="true" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'" +./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ================================================== WEMO INSIGHT PLUG ================================================= +TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/wemo-insight-plug/timestamps/wemo-insight-plug-smarthome-nov-22-2018.timestamps" + +# Has no device side signature. + +# PHONE SIDE +# This one is going to generate >100 FPs because every event is counted twice (same signatures for ON and OFF). +RESULTS_FILE="$RESULTS_BASE_DIR/wemo-insight-plug/wemo-insight-plug.wlan1.wan-detection.pcap___phone-side.detectionresults" +ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +EXACT_MATCH="true" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'" +./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ===================================================== WEMO PLUG ====================================================== +TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/wemo-plug/timestamps/wemo-plug-smarthome-nov-21-2018.timestamps" + +# Has no device side signature. + +# PHONE SIDE +# This one is going to generate >100 FPs because every event is counted twice (same signatures for ON and OFF). +RESULTS_FILE="$RESULTS_BASE_DIR/wemo-plug/wemo-plug.wlan1.wan-detection.pcap___phone-side.detectionresults" +ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +EXACT_MATCH="true" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'" +./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" +# ====================================================================================================================== \ No newline at end of file diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SignatureDetector.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SignatureDetector.java index 83b5b91..2be6de3 100644 --- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SignatureDetector.java +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SignatureDetector.java @@ -87,12 +87,6 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb final int signatureDuration = Integer.parseInt(args[6]); final double eps = Double.parseDouble(args[7]); -// final String pcapFile = args[0]; -// final String onSignatureFile = args[1]; -// final String offSignatureFile = args[2]; -// final String resultsFile = args[3]; -// final int signatureDuration = Integer.parseInt(args[4]); - // Parse optional parameters. List> onSignatureMacFilters = null, offSignatureMacFilters = null; final int optParamsStartIdx = 7; @@ -176,9 +170,9 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb // Parse the file reader.readFromHandle(); - String resultOn = "Number of detected events of type " + UserAction.Type.TOGGLE_ON + ": " + + String resultOn = "# Number of detected events of type " + UserAction.Type.TOGGLE_ON + ": " + detectedEvents.stream().filter(ua -> ua.getType() == UserAction.Type.TOGGLE_ON).count(); - String resultOff = "Number of detected events of type " + UserAction.Type.TOGGLE_OFF + ": " + + String resultOff = "# Number of detected events of type " + UserAction.Type.TOGGLE_OFF + ": " + detectedEvents.stream().filter(ua -> ua.getType() == UserAction.Type.TOGGLE_OFF).count(); PrintWriterUtils.println(resultOn, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); PrintWriterUtils.println(resultOff, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3SignatureDetector.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3SignatureDetector.java index 4afa216..0fc43be 100644 --- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3SignatureDetector.java +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3SignatureDetector.java @@ -177,9 +177,9 @@ public class Layer3SignatureDetector implements PacketListener, ClusterMatcherOb // Output the detected events detectedEvents.forEach(outputter); - String resultOn = "Number of detected events of type " + UserAction.Type.TOGGLE_ON + ": " + + String resultOn = "# Number of detected events of type " + UserAction.Type.TOGGLE_ON + ": " + detectedEvents.stream().filter(ua -> ua.getType() == UserAction.Type.TOGGLE_ON).count(); - String resultOff = "Number of detected events of type " + UserAction.Type.TOGGLE_OFF + ": " + + String resultOff = "# Number of detected events of type " + UserAction.Type.TOGGLE_OFF + ": " + detectedEvents.stream().filter(ua -> ua.getType() == UserAction.Type.TOGGLE_OFF).count(); PrintWriterUtils.println(resultOn, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); PrintWriterUtils.println(resultOff, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/evaluation/DetectionResultsAnalyzer.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/evaluation/DetectionResultsAnalyzer.java index 34063d5..8eaa78b 100644 --- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/evaluation/DetectionResultsAnalyzer.java +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/evaluation/DetectionResultsAnalyzer.java @@ -20,22 +20,28 @@ import java.util.Optional; public class DetectionResultsAnalyzer { private static boolean DUPLICATE_OUTPUT_TO_STD_OUT = true; + private static boolean DETECTED_EVENT_EXACT_MATCH; public static void main(String[] args) throws IOException { - if (args.length < 3) { + if (args.length < 4) { String errMsg = String.format("Usage: %s triggerTimesFile detectionOutputFile [stdOut]" + "\n - triggerTimesFile: the file that contains the timestamps for the user actions" + "\n - detectionOutputFile: the file that contains the detected events" + "\n - analysisResultsFile: where to write the results of the detection analysis" + - "\n - stdOut: optional true/false literal indicating if output should also be printed to std out; default is true", + "\n - matchEventType: true/false literal indicating if a detected event should" + + " have a matching type" + + "\n - stdOut: optional true/false literal indicating if output should also be printed to" + + " std out; default is true", DetectionResultsAnalyzer.class.getSimpleName()); + System.out.println(errMsg); return; } String triggerTimesFile = args[0]; File detectionOutputFile = new File(args[1]); String analysisResultsFile = args[2]; - if (args.length > 3) { - DUPLICATE_OUTPUT_TO_STD_OUT = Boolean.parseBoolean(args[3]); + DETECTED_EVENT_EXACT_MATCH = Boolean.parseBoolean(args[3]); + if (args.length > 4) { + DUPLICATE_OUTPUT_TO_STD_OUT = Boolean.parseBoolean(args[4]); } // -------------------------------------- Parse the input files -------------------------------------- @@ -68,20 +74,39 @@ public class DetectionResultsAnalyzer { // To contain all detected events that could be mapped to a trigger List truePositives = new ArrayList<>(); - for (UserAction detectedEvent : detectedEvents) { - Optional matchingTrigger = triggers.stream() - .filter(t -> t.getType() == detectedEvent.getType() && - t.getTimestamp().isBefore(detectedEvent.getTimestamp()) && - t.getTimestamp().plusMillis(TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS). - isAfter(detectedEvent.getTimestamp()) - ).findFirst(); - matchingTrigger.ifPresent(mt -> { - // We've consumed the trigger (matched it with a detected event), so remove it so we don't match with - // another detected event. - triggers.remove(mt); - // The current detected event was a true positive as we could match it with a trigger. - truePositives.add(detectedEvent); - }); + if (DETECTED_EVENT_EXACT_MATCH) { + for (UserAction detectedEvent : detectedEvents) { + Optional matchingTrigger = triggers.stream() + .filter(t -> t.getType() == detectedEvent.getType() && + t.getTimestamp().isBefore(detectedEvent.getTimestamp()) && + t.getTimestamp().plusMillis(TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS). + isAfter(detectedEvent.getTimestamp()) + ).findFirst(); + matchingTrigger.ifPresent(mt -> { + // We've consumed the trigger (matched it with a detected event), so remove it so we don't match with + // another detected event. + triggers.remove(mt); + // The current detected event was a true positive as we could match it with a trigger. + truePositives.add(detectedEvent); + }); + } + // TODO: Experimental + } else { // DETECTED_EVENT_EXACT_MATCH == false + for (UserAction detectedEvent : detectedEvents) { + Optional matchingTrigger = triggers.stream() + .filter(t -> + t.getTimestamp().isBefore(detectedEvent.getTimestamp()) && + t.getTimestamp().plusMillis(TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS). + isAfter(detectedEvent.getTimestamp()) + ).findFirst(); + matchingTrigger.ifPresent(mt -> { + // We've consumed the trigger (matched it with a detected event), so remove it so we don't match with + // another detected event. + triggers.remove(mt); + // The current detected event was a true positive as we could match it with a trigger. + truePositives.add(detectedEvent); + }); + } } // Now the false positives are those elements in detectedEvents that are not in truePositives List falsePositives = new ArrayList<>(); @@ -90,7 +115,7 @@ public class DetectionResultsAnalyzer { // Output the results... PrintWriter outputter = new PrintWriter(new FileWriter(analysisResultsFile)); - PrintWriterUtils.println("---------- False negatives (events that where not detected) ----------", outputter, DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println("---------- False negatives (events that were not detected) ----------", outputter, DUPLICATE_OUTPUT_TO_STD_OUT); for (UserAction missing : triggers) { PrintWriterUtils.println(missing, outputter, DUPLICATE_OUTPUT_TO_STD_OUT); }