From: rtrimana Date: Tue, 5 Feb 2019 01:41:22 +0000 (-0800) Subject: Adding PacketLevelSignatureExtractor. X-Git-Url: http://demsky.eecs.uci.edu/git/?a=commitdiff_plain;h=17a06200889140f90c1e735d1307085c87c8fc41;p=pingpong.git Adding PacketLevelSignatureExtractor. --- diff --git a/Code/Projects/PacketLevelSignatureExtractor/.gitignore b/Code/Projects/PacketLevelSignatureExtractor/.gitignore new file mode 100644 index 0000000..b0ff774 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/.gitignore @@ -0,0 +1,65 @@ +# Borrowed from https://github.com/github/gitignore/blob/master/Global/JetBrains.gitignore + +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff: +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/dictionaries + +# Sensitive or high-churn files: +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.xml +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml + +# Gradle: (combination of the JetBrains gitiginre and Gradle gitignore at ) +.idea/**/gradle.xml +.idea/**/libraries +.gradle +/build/ +# Ignore Gradle GUI config +gradle-app.setting +# Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) +!gradle-wrapper.jar +# Cache of project +.gradletasknamecache +# # Work around https://youtrack.jetbrains.com/issue/IDEA-116898 +# gradle/wrapper/gradle-wrapper.properties + +# CMake +cmake-build-debug/ + +# Mongo Explorer plugin: +.idea/**/mongoSettings.xml + +## File-based project format: +*.iws + +## Plugin-specific files: + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# ignore misc as it changes a lot depending on local settings -- however may need to be included later on. +.idea/misc.xml + diff --git a/Code/Projects/PacketLevelSignatureExtractor/.idea/.name b/Code/Projects/PacketLevelSignatureExtractor/.idea/.name new file mode 100644 index 0000000..297868a --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/.idea/.name @@ -0,0 +1 @@ +PacketLevelSignatureExtractor \ No newline at end of file diff --git a/Code/Projects/PacketLevelSignatureExtractor/.idea/compiler.xml b/Code/Projects/PacketLevelSignatureExtractor/.idea/compiler.xml new file mode 100644 index 0000000..4ac14e9 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/.idea/compiler.xml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/Code/Projects/PacketLevelSignatureExtractor/.idea/copyright/profiles_settings.xml b/Code/Projects/PacketLevelSignatureExtractor/.idea/copyright/profiles_settings.xml new file mode 100644 index 0000000..e7bedf3 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/.idea/copyright/profiles_settings.xml @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/Code/Projects/PacketLevelSignatureExtractor/.idea/encodings.xml b/Code/Projects/PacketLevelSignatureExtractor/.idea/encodings.xml new file mode 100644 index 0000000..15a15b2 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/.idea/encodings.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/Code/Projects/PacketLevelSignatureExtractor/.idea/modules.xml b/Code/Projects/PacketLevelSignatureExtractor/.idea/modules.xml new file mode 100644 index 0000000..0b5d8f6 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/.idea/modules.xml @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file diff --git a/Code/Projects/PacketLevelSignatureExtractor/.idea/modules/PacketLevelSignatureExtractor.iml b/Code/Projects/PacketLevelSignatureExtractor/.idea/modules/PacketLevelSignatureExtractor.iml new file mode 100644 index 0000000..21e02c3 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/.idea/modules/PacketLevelSignatureExtractor.iml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/Code/Projects/PacketLevelSignatureExtractor/.idea/modules/PacketLevelSignatureExtractor_main.iml b/Code/Projects/PacketLevelSignatureExtractor/.idea/modules/PacketLevelSignatureExtractor_main.iml new file mode 100644 index 0000000..e1b3b2d --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/.idea/modules/PacketLevelSignatureExtractor_main.iml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Code/Projects/PacketLevelSignatureExtractor/.idea/modules/PacketLevelSignatureExtractor_test.iml b/Code/Projects/PacketLevelSignatureExtractor/.idea/modules/PacketLevelSignatureExtractor_test.iml new file mode 100644 index 0000000..c8b68de --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/.idea/modules/PacketLevelSignatureExtractor_test.iml @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Code/Projects/PacketLevelSignatureExtractor/.idea/vcs.xml b/Code/Projects/PacketLevelSignatureExtractor/.idea/vcs.xml new file mode 100644 index 0000000..c2365ab --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/Code/Projects/PacketLevelSignatureExtractor/build.gradle b/Code/Projects/PacketLevelSignatureExtractor/build.gradle new file mode 100644 index 0000000..2027ed3 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/build.gradle @@ -0,0 +1,40 @@ +group 'edu.uci.iotproject' +version '1.0-SNAPSHOT' + +apply plugin: 'java' +apply plugin: 'application' + +// Increase max memory +applicationDefaultJvmArgs = ["-Xmx300g"] + +sourceCompatibility = 1.8 + +//mainClassName = "edu.uci.iotproject.Main" +//mainClassName = "edu.uci.iotproject.detection.SignatureDetector" +//mainClassName = "edu.uci.iotproject.detection.layer2.Layer2SignatureDetector" +//mainClassName = "edu.uci.iotproject.evaluation.DetectionResultsAnalyzer" +mainClassName = System.getProperty("mainClass") + + +repositories { + mavenCentral() +} + +dependencies { + testCompile group: 'junit', name: 'junit', version: '4.11' + + // pcap4j + // Updated to v2 alpha as the stable release does not include packet timestamps + // v2 should add support for TCP session reassembly as well, although it does not appear to be part of the lib yet. + compile 'org.pcap4j:pcap4j-core:2.0.0-alpha' + compile 'org.pcap4j:pcap4j-packetfactory-static:2.0.0-alpha' + + // pcap4j logging dependency + compile 'org.slf4j:slf4j-jdk14:1.8.0-beta2' + + // Apache Commons Math for clustering + compile 'org.apache.commons:commons-math3:3.6.1' + + // JGraphT: Java Graph library + compile 'org.jgrapht:jgrapht-core:1.2.0' +} \ No newline at end of file diff --git a/Code/Projects/PacketLevelSignatureExtractor/execute_layer2_smarthome_all_detection.sh b/Code/Projects/PacketLevelSignatureExtractor/execute_layer2_smarthome_all_detection.sh new file mode 100755 index 0000000..1cf9087 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/execute_layer2_smarthome_all_detection.sh @@ -0,0 +1,241 @@ +#!/bin/bash + +#set -x # echo invoked commands to std out + +# Base dir should point to the experimental_result folder which contains the subfolders: +# - 'smarthome' which contains the traces collected while other devices are idle +# - 'standalone' which contains signatures and the traces used to generate the signatures. +BASE_DIR=$1 +readonly BASE_DIR + +OUTPUT_DIR=$2 +readonly OUTPUT_DIR + +PCAPS_BASE_DIR="$BASE_DIR/smarthome" +readonly PCAPS_BASE_DIR + +SIGNATURES_BASE_DIR="$BASE_DIR/standalone" +readonly SIGNATURES_BASE_DIR + +# ==================================================== ARLO CAMERA ===================================================== +PCAP_FILE="$PCAPS_BASE_DIR/arlo-camera/wlan1/arlo-camera.wlan1.detection.pcap" + +# Has no device side signature. + +# PHONE SIDE (TODO: may possibly be the .incomplete signatures) +ON_SIGNATURE="$SIGNATURES_BASE_DIR/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig" +OFF_SIGNATURE="$SIGNATURES_BASE_DIR/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig" +RESULTS_FILE="$OUTPUT_DIR/arlo-camera/arlo-camera.wlan1.detection.pcap___phone-side.detectionresults" +SIGNATURE_DURATION="213" + +PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'" +./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ================================================= BLOSSOM SPRINKLER ================================================== +PCAP_FILE="$PCAPS_BASE_DIR/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.detection.pcap" + +# DEVICE SIDE +ON_SIGNATURE="$SIGNATURES_BASE_DIR/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-device-side.sig" +OFF_SIGNATURE="$SIGNATURES_BASE_DIR/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-device-side.sig" +RESULTS_FILE="$OUTPUT_DIR/blossom-sprinkler/blossom-sprinkler.wlan1.detection.pcap___device-side.detectionresults" +SIGNATURE_DURATION="9274" + +PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'" +./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" + +# PHONE SIDE +ON_SIGNATURE="$SIGNATURES_BASE_DIR/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-phone-side.sig" +OFF_SIGNATURE="$SIGNATURES_BASE_DIR/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-phone-side.sig" +RESULTS_FILE="$OUTPUT_DIR/blossom-sprinkler/blossom-sprinkler.wlan1.detection.pcap___phone-side.detectionresults" +SIGNATURE_DURATION="3670" + +PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'" +./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ==================================================== D-LINK PLUG ===================================================== +PCAP_FILE="$PCAPS_BASE_DIR/dlink-plug/wlan1/dlink-plug.wlan1.detection.pcap" + +# DEVICE SIDE +ON_SIGNATURE="$SIGNATURES_BASE_DIR/dlink-plug/signatures/dlink-plug-onSignature-device-side.sig" +OFF_SIGNATURE="$SIGNATURES_BASE_DIR/dlink-plug/signatures/dlink-plug-offSignature-device-side.sig" +RESULTS_FILE="$OUTPUT_DIR/dlink-plug/dlink-plug.wlan1.detection.pcap___device-side.detectionresults" +SIGNATURE_DURATION="8866" + +PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'" +./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" + +# PHONE SIDE +ON_SIGNATURE="$SIGNATURES_BASE_DIR/dlink-plug/signatures/dlink-plug-onSignature-phone-side.sig" +OFF_SIGNATURE="$SIGNATURES_BASE_DIR/dlink-plug/signatures/dlink-plug-offSignature-phone-side.sig" +RESULTS_FILE="$OUTPUT_DIR/dlink-plug/dlink-plug.wlan1.detection.pcap___phone-side.detectionresults" +SIGNATURE_DURATION="193" + +PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'" +./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ==================================================== D-LINK SIREN ==================================================== +PCAP_FILE="$PCAPS_BASE_DIR/dlink-siren/wlan1/dlink-siren.wlan1.detection.pcap" + +# PHONE SIDE +ON_SIGNATURE="$SIGNATURES_BASE_DIR/dlink-siren/signatures/dlink-siren-onSignature-phone-side.sig" +OFF_SIGNATURE="$SIGNATURES_BASE_DIR/dlink-siren/signatures/dlink-siren-offSignature-phone-side.sig" +RESULTS_FILE="$OUTPUT_DIR/dlink-siren/dlink-siren.wlan1.detection.pcap___phone-side.detectionresults" +SIGNATURE_DURATION="71" + +PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'" +./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ===================================================== HUE BULB ======================================================= +PCAP_FILE="$PCAPS_BASE_DIR/hue-bulb/wlan1/hue-bulb.wlan1.detection.pcap" + +# Has no device side signature. + +# PHONE SIDE +ON_SIGNATURE="$SIGNATURES_BASE_DIR/hue-bulb/signatures/hue-bulb-onSignature-phone-side.sig" +OFF_SIGNATURE="$SIGNATURES_BASE_DIR/hue-bulb/signatures/hue-bulb-offSignature-phone-side.sig" +RESULTS_FILE="$OUTPUT_DIR/hue-bulb/hue-bulb.wlan1.detection.pcap___phone-side.detectionresults" +SIGNATURE_DURATION="27" + +PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'" +./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ================================================= KWIKSET DOORLOCK =================================================== +PCAP_FILE="$PCAPS_BASE_DIR/kwikset-doorlock/wlan1/kwikset-doorlock.wlan1.detection.pcap" + +# Has no device side signature. + +# PHONE SIDE +ON_SIGNATURE="$SIGNATURES_BASE_DIR/kwikset-doorlock/signatures/kwikset-doorlock-onSignature-phone-side.sig" +OFF_SIGNATURE="$SIGNATURES_BASE_DIR/kwikset-doorlock/signatures/kwikset-doorlock-offSignature-phone-side.sig" +RESULTS_FILE="$OUTPUT_DIR/kwikset-doorlock/kwikset-doorlock.wlan1.detection.pcap___phone-side.detectionresults" +SIGNATURE_DURATION="3161" + +PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'" +./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ================================================= NEST THERMOSTAT ==================================================== +PCAP_FILE="$PCAPS_BASE_DIR/nest-thermostat/wlan1/nest-thermostat.wlan1.detection.pcap" + +# Has no device side signature. + +# PHONE SIDE +ON_SIGNATURE="$SIGNATURES_BASE_DIR/nest-thermostat/signatures/nest-thermostat-onSignature-phone-side.sig" +OFF_SIGNATURE="$SIGNATURES_BASE_DIR/nest-thermostat/signatures/nest-thermostat-offSignature-phone-side.sig" +RESULTS_FILE="$OUTPUT_DIR/nest-thermostat/nest-thermostat.wlan1.detection.pcap___phone-side.detectionresults" +SIGNATURE_DURATION="1179" + +PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'" +./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ====================================================== ST PLUG ======================================================= +PCAP_FILE="$PCAPS_BASE_DIR/st-plug/wlan1/st-plug.wlan1.detection.pcap" + +# Has no device side signature. + +# PHONE SIDE +ON_SIGNATURE="$SIGNATURES_BASE_DIR/st-plug/signatures/st-plug-onSignature-phone-side.sig" +OFF_SIGNATURE="$SIGNATURES_BASE_DIR/st-plug/signatures/st-plug-offSignature-phone-side.sig" +RESULTS_FILE="$OUTPUT_DIR/st-plug/st-plug.wlan1.detection.pcap___phone-side.detectionresults" +SIGNATURE_DURATION="2445" + +PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'" +./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ==================================================== TP-LINK BULB ==================================================== +PCAP_FILE="$PCAPS_BASE_DIR/tplink-bulb/wlan1/tplink-bulb.wlan1.detection.pcap" + +# Has no device side signature. + +# PHONE SIDE +ON_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-bulb/signatures/tplink-bulb-onSignature-phone-side.sig" +OFF_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-bulb/signatures/tplink-bulb-offSignature-phone-side.sig" +RESULTS_FILE="$OUTPUT_DIR/tplink-bulb/tplink-bulb.wlan1.detection.pcap___phone-side.detectionresults" +SIGNATURE_DURATION="162" + +PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION' -onmacfilters 50:c7:bf:.* -offmacfilters 50:c7:bf:.*" +./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ==================================================== TP-LINK PLUG ==================================================== +PCAP_FILE="$PCAPS_BASE_DIR/tplink-plug/wlan1/tplink-plug.wlan1.detection.pcap" + +# DEVICE SIDE (both the 112, 115 and 556, 1293 sequences) +ON_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-plug/signatures/tplink-plug-onSignature-device-side.sig" +OFF_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-plug/signatures/tplink-plug-offSignature-device-side.sig" +RESULTS_FILE="$OUTPUT_DIR/tplink-plug/tplink-plug.wlan1.detection.pcap___device-side.detectionresults" +SIGNATURE_DURATION="3660" + +PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION' -onmacfilters 50:c7:bf:.*;50:c7:bf:.* -offmacfilters 50:c7:bf:.*;50:c7:bf:.*" +./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" + +# DEVICE SIDE OUTBOUND (contains only those packets that go through the WAN port, i.e., only the 556, 1293 sequence) +ON_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-plug/signatures/tplink-plug-onSignature-device-side-outbound.sig" +OFF_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-plug/signatures/tplink-plug-offSignature-device-side-outbound.sig" +RESULTS_FILE="$OUTPUT_DIR/tplink-plug/tplink-plug.wlan1.detection.pcap___device-side-outbound.detectionresults" +SIGNATURE_DURATION="224" + +PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION' -onmacfilters 50:c7:bf:.* -offmacfilters 50:c7:bf:.*" +./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" + +# Phone side does not make sense as it is merely a subset of the device side and does not differentiate ONs from OFFs. +# ====================================================================================================================== + + + +# ================================================== WEMO INSIGHT PLUG ================================================= +PCAP_FILE="$PCAPS_BASE_DIR/wemo-insight-plug/wlan1/wemo-insight-plug.wlan1.detection.pcap" + +# Has no device side signature. + +# PHONE SIDE +ON_SIGNATURE="$SIGNATURES_BASE_DIR/wemo-insight-plug/signatures/wemo-insight-plug-onSignature-phone-side.sig" +OFF_SIGNATURE="$SIGNATURES_BASE_DIR/wemo-insight-plug/signatures/wemo-insight-plug-offSignature-phone-side.sig" +RESULTS_FILE="$OUTPUT_DIR/wemo-insight-plug/wemo-insight-plug.wlan1.detection.pcap___phone-side.detectionresults" +SIGNATURE_DURATION="106" + +PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'" +./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ===================================================== WEMO PLUG ====================================================== +PCAP_FILE="$PCAPS_BASE_DIR/wemo-plug/wlan1/wemo-plug.wlan1.detection.pcap" + +# Has no device side signature. + +# PHONE SIDE +ON_SIGNATURE="$SIGNATURES_BASE_DIR/wemo-plug/signatures/wemo-plug-onSignature-phone-side.sig" +OFF_SIGNATURE="$SIGNATURES_BASE_DIR/wemo-plug/signatures/wemo-plug-offSignature-phone-side.sig" +RESULTS_FILE="$OUTPUT_DIR/wemo-plug/wemo-plug.wlan1.detection.pcap___phone-side.detectionresults" +SIGNATURE_DURATION="147" + +PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'" +./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" +# ====================================================================================================================== \ No newline at end of file diff --git a/Code/Projects/PacketLevelSignatureExtractor/execute_layer2_smarthome_all_detection_results_analysis.sh b/Code/Projects/PacketLevelSignatureExtractor/execute_layer2_smarthome_all_detection_results_analysis.sh new file mode 100755 index 0000000..7d0689d --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/execute_layer2_smarthome_all_detection_results_analysis.sh @@ -0,0 +1,187 @@ +#!/bin/bash + +# Base directory where the smarthome evaluation traces and timestamp files are stored, +# (i.e., /some/arbitrary/local/path/experimental_result/smarthome) +TIMESTAMPS_BASE_DIR=$1 +readonly TIMESTAMPS_BASE_DIR + +# Base directory for the detection results files for the smarthome experiment +RESULTS_BASE_DIR=$2 +readonly RESULTS_BASE_DIR + + + +# ==================================================== ARLO CAMERA ===================================================== +TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/arlo-camera/timestamps/arlo-camera-smarthome-nov-15-2018.timestamps" +RESULTS_FILE="$RESULTS_BASE_DIR/arlo-camera/arlo-camera.wlan1.detection.pcap___phone-side.detectionresults" +# Put the analysis results in the same folder as the detection results. +ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" + + +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ================================================= BLOSSOM SPRINKLER ================================================== +TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/blossom-sprinkler/timestamps/blossom-sprinkler-smarthome-jan-14-2019.timestamps" + +# DEVICE SIDE +RESULTS_FILE="$RESULTS_BASE_DIR/blossom-sprinkler/blossom-sprinkler.wlan1.detection.pcap___device-side.detectionresults" +ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" + +# PHONE SIDE +RESULTS_FILE="$RESULTS_BASE_DIR/blossom-sprinkler/blossom-sprinkler.wlan1.detection.pcap___phone-side.detectionresults" +ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ==================================================== D-LINK PLUG ===================================================== +TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/dlink-plug/timestamps/dlink-plug-smarthome-nov-8-2018.timestamps" + +# DEVICE SIDE +RESULTS_FILE="$RESULTS_BASE_DIR/dlink-plug/dlink-plug.wlan1.detection.pcap___device-side.detectionresults" +ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" + +# PHONE SIDE +RESULTS_FILE="$RESULTS_BASE_DIR/dlink-plug/dlink-plug.wlan1.detection.pcap___phone-side.detectionresults" +ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ==================================================== D-LINK SIREN ==================================================== +TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/dlink-siren/timestamps/dlink-siren-smarthome-nov-10-2018.timestamps" + +#PHONE SIDE +RESULTS_FILE="$RESULTS_BASE_DIR/dlink-siren/dlink-siren.wlan1.detection.pcap___phone-side.detectionresults" +ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ===================================================== HUE BULB ======================================================= +TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/hue-bulb/timestamps/hue-bulb-smarthome-nov-20-2018.timestamps" + +# Has no device side signature. + +# PHONE SIDE +RESULTS_FILE="$RESULTS_BASE_DIR/hue-bulb/hue-bulb.wlan1.detection.pcap___phone-side.detectionresults" +ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ================================================= KWIKSET DOORLOCK =================================================== +TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/kwikset-doorlock/timestamps/kwikset-doorlock-smarthome-nov-10-2018.timestamps" + +# Has no device side signature. + +# PHONE SIDE +RESULTS_FILE="$RESULTS_BASE_DIR/kwikset-doorlock/kwikset-doorlock.wlan1.detection.pcap___phone-side.detectionresults" +ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ================================================= NEST THERMOSTAT ==================================================== +TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/nest-thermostat/timestamps/nest-thermostat-smarthome-nov-16-2018.timestamps" + +# Has no device side signature. + +# PHONE SIDE +RESULTS_FILE="$RESULTS_BASE_DIR/nest-thermostat/nest-thermostat.wlan1.detection.pcap___phone-side.detectionresults" +ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ====================================================== ST PLUG ======================================================= +TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/st-plug/timestamps/st-plug-smarthome-nov-13-2018.timestamps" + +# Has no device side signature. + +# PHONE SIDE +RESULTS_FILE="$RESULTS_BASE_DIR/st-plug/st-plug.wlan1.detection.pcap___phone-side.detectionresults" +ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ==================================================== TP-LINK BULB ==================================================== +TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/tplink-bulb/timestamps/tplink-bulb-smarthome-nov-19-2018.timestamps" + +# Has no device side signature. + +# PHONE SIDE +RESULTS_FILE="$RESULTS_BASE_DIR/tplink-bulb/tplink-bulb.wlan1.detection.pcap___phone-side.detectionresults" +ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ==================================================== TP-LINK PLUG ==================================================== +TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/tplink-plug/timestamps/tplink-plug-smarthome-nov-9-2018.timestamps" + +# DEVICE SIDE +RESULTS_FILE="$RESULTS_BASE_DIR/tplink-plug/tplink-plug.wlan1.detection.pcap___device-side.detectionresults" +ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" + +# DEVICE SIDE OUTBOUND +RESULTS_FILE="$RESULTS_BASE_DIR/tplink-plug/tplink-plug.wlan1.detection.pcap___device-side-outbound.detectionresults" +ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ================================================== WEMO INSIGHT PLUG ================================================= +TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/wemo-insight-plug/timestamps/wemo-insight-plug-smarthome-nov-22-2018.timestamps" + +# Has no device side signature. + +# PHONE SIDE +RESULTS_FILE="$RESULTS_BASE_DIR/wemo-insight-plug/wemo-insight-plug.wlan1.detection.pcap___phone-side.detectionresults" +ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ===================================================== WEMO PLUG ====================================================== +TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/wemo-plug/timestamps/wemo-plug-smarthome-nov-21-2018.timestamps" + +# Has no device side signature. + +# PHONE SIDE +RESULTS_FILE="$RESULTS_BASE_DIR/wemo-plug/wemo-plug.wlan1.detection.pcap___phone-side.detectionresults" +ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" +# ====================================================================================================================== \ No newline at end of file diff --git a/Code/Projects/PacketLevelSignatureExtractor/execute_layer2_smarthome_nest.sh b/Code/Projects/PacketLevelSignatureExtractor/execute_layer2_smarthome_nest.sh new file mode 100755 index 0000000..e32d6b9 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/execute_layer2_smarthome_nest.sh @@ -0,0 +1,241 @@ +#!/bin/bash + +#set -x # echo invoked commands to std out + +# Base dir should point to the experimental_result folder which contains the subfolders: +# - 'smarthome' which contains the traces collected while other devices are idle +# - 'standalone' which contains signatures and the traces used to generate the signatures. +BASE_DIR=$1 +readonly BASE_DIR + +OUTPUT_DIR=$2 +readonly OUTPUT_DIR + +PCAPS_BASE_DIR="$BASE_DIR/smarthome" +readonly PCAPS_BASE_DIR + +SIGNATURES_BASE_DIR="$BASE_DIR/standalone" +readonly SIGNATURES_BASE_DIR + +# # ==================================================== ARLO CAMERA ===================================================== +# PCAP_FILE="$PCAPS_BASE_DIR/arlo-camera/wlan1/arlo-camera.wlan1.detection.pcap" + +# # Has no device side signature. + +# # PHONE SIDE (TODO: may possibly be the .incomplete signatures) +# ON_SIGNATURE="$SIGNATURES_BASE_DIR/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig" +# OFF_SIGNATURE="$SIGNATURES_BASE_DIR/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig" +# RESULTS_FILE="$OUTPUT_DIR/arlo-camera/arlo-camera.wlan1.detection.pcap___phone-side.detectionresults" +# SIGNATURE_DURATION="213" + +# PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'" +# ./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" +# # ====================================================================================================================== + + + +# # ================================================= BLOSSOM SPRINKLER ================================================== +# PCAP_FILE="$PCAPS_BASE_DIR/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.detection.pcap" + +# # DEVICE SIDE +# ON_SIGNATURE="$SIGNATURES_BASE_DIR/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-device-side.sig" +# OFF_SIGNATURE="$SIGNATURES_BASE_DIR/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-device-side.sig" +# RESULTS_FILE="$OUTPUT_DIR/blossom-sprinkler/blossom-sprinkler.wlan1.detection.pcap___device-side.detectionresults" +# SIGNATURE_DURATION="9274" + +# PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'" +# ./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" + +# # PHONE SIDE +# ON_SIGNATURE="$SIGNATURES_BASE_DIR/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-phone-side.sig" +# OFF_SIGNATURE="$SIGNATURES_BASE_DIR/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-phone-side.sig" +# RESULTS_FILE="$OUTPUT_DIR/blossom-sprinkler/blossom-sprinkler.wlan1.detection.pcap___phone-side.detectionresults" +# SIGNATURE_DURATION="3670" + +# PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'" +# ./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" +# # ====================================================================================================================== + + + +# # ==================================================== D-LINK PLUG ===================================================== +# PCAP_FILE="$PCAPS_BASE_DIR/dlink-plug/wlan1/dlink-plug.wlan1.detection.pcap" + +# # DEVICE SIDE +# ON_SIGNATURE="$SIGNATURES_BASE_DIR/dlink-plug/signatures/dlink-plug-onSignature-device-side.sig" +# OFF_SIGNATURE="$SIGNATURES_BASE_DIR/dlink-plug/signatures/dlink-plug-offSignature-device-side.sig" +# RESULTS_FILE="$OUTPUT_DIR/dlink-plug/dlink-plug.wlan1.detection.pcap___device-side.detectionresults" +# SIGNATURE_DURATION="8866" + +# PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'" +# ./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" + +# # PHONE SIDE +# ON_SIGNATURE="$SIGNATURES_BASE_DIR/dlink-plug/signatures/dlink-plug-onSignature-phone-side.sig" +# OFF_SIGNATURE="$SIGNATURES_BASE_DIR/dlink-plug/signatures/dlink-plug-offSignature-phone-side.sig" +# RESULTS_FILE="$OUTPUT_DIR/dlink-plug/dlink-plug.wlan1.detection.pcap___phone-side.detectionresults" +# SIGNATURE_DURATION="193" + +# PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'" +# ./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" +# # ====================================================================================================================== + + + +# # ==================================================== D-LINK SIREN ==================================================== +# PCAP_FILE="$PCAPS_BASE_DIR/dlink-siren/wlan1/dlink-siren.wlan1.detection.pcap" + +# # PHONE SIDE +# ON_SIGNATURE="$SIGNATURES_BASE_DIR/dlink-siren/signatures/dlink-siren-onSignature-phone-side.sig" +# OFF_SIGNATURE="$SIGNATURES_BASE_DIR/dlink-siren/signatures/dlink-siren-offSignature-phone-side.sig" +# RESULTS_FILE="$OUTPUT_DIR/dlink-siren/dlink-siren.wlan1.detection.pcap___phone-side.detectionresults" +# SIGNATURE_DURATION="71" + +# PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'" +# ./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" +# # ====================================================================================================================== + + + +# # ===================================================== HUE BULB ======================================================= +# PCAP_FILE="$PCAPS_BASE_DIR/hue-bulb/wlan1/hue-bulb.wlan1.detection.pcap" + +# # Has no device side signature. + +# # PHONE SIDE +# ON_SIGNATURE="$SIGNATURES_BASE_DIR/hue-bulb/signatures/hue-bulb-onSignature-phone-side.sig" +# OFF_SIGNATURE="$SIGNATURES_BASE_DIR/hue-bulb/signatures/hue-bulb-offSignature-phone-side.sig" +# RESULTS_FILE="$OUTPUT_DIR/hue-bulb/hue-bulb.wlan1.detection.pcap___phone-side.detectionresults" +# SIGNATURE_DURATION="27" + +# PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'" +# ./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" +# # ====================================================================================================================== + + + +# # ================================================= KWIKSET DOORLOCK =================================================== +# PCAP_FILE="$PCAPS_BASE_DIR/kwikset-doorlock/wlan1/kwikset-doorlock.wlan1.detection.pcap" + +# # Has no device side signature. + +# # PHONE SIDE +# ON_SIGNATURE="$SIGNATURES_BASE_DIR/kwikset-doorlock/signatures/kwikset-doorlock-onSignature-phone-side.sig" +# OFF_SIGNATURE="$SIGNATURES_BASE_DIR/kwikset-doorlock/signatures/kwikset-doorlock-offSignature-phone-side.sig" +# RESULTS_FILE="$OUTPUT_DIR/kwikset-doorlock/kwikset-doorlock.wlan1.detection.pcap___phone-side.detectionresults" +# SIGNATURE_DURATION="3161" + +# PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'" +# ./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" +# # ====================================================================================================================== + + + +# ================================================= NEST THERMOSTAT ==================================================== +PCAP_FILE="$PCAPS_BASE_DIR/nest-thermostat/wlan1/nest-thermostat.wlan1.detection.pcap" + +# Has no device side signature. + +# PHONE SIDE +ON_SIGNATURE="$SIGNATURES_BASE_DIR/nest-thermostat/signatures/nest-thermostat-onSignature-phone-side.sig" +OFF_SIGNATURE="$SIGNATURES_BASE_DIR/nest-thermostat/signatures/nest-thermostat-offSignature-phone-side.sig" +RESULTS_FILE="$OUTPUT_DIR/nest-thermostat/nest-thermostat.wlan1.detection.pcap___phone-side.detectionresults" +SIGNATURE_DURATION="1179" + +PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'" +./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# # ====================================================== ST PLUG ======================================================= +# PCAP_FILE="$PCAPS_BASE_DIR/st-plug/wlan1/st-plug.wlan1.detection.pcap" + +# # Has no device side signature. + +# # PHONE SIDE +# ON_SIGNATURE="$SIGNATURES_BASE_DIR/st-plug/signatures/st-plug-onSignature-phone-side.sig" +# OFF_SIGNATURE="$SIGNATURES_BASE_DIR/st-plug/signatures/st-plug-offSignature-phone-side.sig" +# RESULTS_FILE="$OUTPUT_DIR/st-plug/st-plug.wlan1.detection.pcap___phone-side.detectionresults" +# SIGNATURE_DURATION="2445" + +# PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'" +# ./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" +# # ====================================================================================================================== + + + +# # ==================================================== TP-LINK BULB ==================================================== +# PCAP_FILE="$PCAPS_BASE_DIR/tplink-bulb/wlan1/tplink-bulb.wlan1.detection.pcap" + +# # Has no device side signature. + +# # PHONE SIDE +# ON_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-bulb/signatures/tplink-bulb-onSignature-phone-side.sig" +# OFF_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-bulb/signatures/tplink-bulb-offSignature-phone-side.sig" +# RESULTS_FILE="$OUTPUT_DIR/tplink-bulb/tplink-bulb.wlan1.detection.pcap___phone-side.detectionresults" +# SIGNATURE_DURATION="162" + +# PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION' -onmacfilters 50:c7:bf:.* -offmacfilters 50:c7:bf:.*" +# ./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" +# # ====================================================================================================================== + + + +# # ==================================================== TP-LINK PLUG ==================================================== +# PCAP_FILE="$PCAPS_BASE_DIR/tplink-plug/wlan1/tplink-plug.wlan1.detection.pcap" + +# # DEVICE SIDE (both the 112, 115 and 556, 1293 sequences) +# ON_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-plug/signatures/tplink-plug-onSignature-device-side.sig" +# OFF_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-plug/signatures/tplink-plug-offSignature-device-side.sig" +# RESULTS_FILE="$OUTPUT_DIR/tplink-plug/tplink-plug.wlan1.detection.pcap___device-side.detectionresults" +# SIGNATURE_DURATION="3660" + +# PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION' -onmacfilters 50:c7:bf:.*;50:c7:bf:.* -offmacfilters 50:c7:bf:.*;50:c7:bf:.*" +# ./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" + +# # DEVICE SIDE OUTBOUND (contains only those packets that go through the WAN port, i.e., only the 556, 1293 sequence) +# ON_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-plug/signatures/tplink-plug-onSignature-device-side-outbound.sig" +# OFF_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-plug/signatures/tplink-plug-offSignature-device-side-outbound.sig" +# RESULTS_FILE="$OUTPUT_DIR/tplink-plug/tplink-plug.wlan1.detection.pcap___device-side-outbound.detectionresults" +# SIGNATURE_DURATION="224" + +# PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION' -onmacfilters 50:c7:bf:.* -offmacfilters 50:c7:bf:.*" +# ./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" + +# # Phone side does not make sense as it is merely a subset of the device side and does not differentiate ONs from OFFs. +# # ====================================================================================================================== + + + +# # ================================================== WEMO INSIGHT PLUG ================================================= +# PCAP_FILE="$PCAPS_BASE_DIR/wemo-insight-plug/wlan1/wemo-insight-plug.wlan1.detection.pcap" + +# # Has no device side signature. + +# # PHONE SIDE +# ON_SIGNATURE="$SIGNATURES_BASE_DIR/wemo-insight-plug/signatures/wemo-insight-plug-onSignature-phone-side.sig" +# OFF_SIGNATURE="$SIGNATURES_BASE_DIR/wemo-insight-plug/signatures/wemo-insight-plug-offSignature-phone-side.sig" +# RESULTS_FILE="$OUTPUT_DIR/wemo-insight-plug/wemo-insight-plug.wlan1.detection.pcap___phone-side.detectionresults" +# SIGNATURE_DURATION="106" + +# PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'" +# ./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" +# # ====================================================================================================================== + + + +# # ===================================================== WEMO PLUG ====================================================== +# PCAP_FILE="$PCAPS_BASE_DIR/wemo-plug/wlan1/wemo-plug.wlan1.detection.pcap" + +# # Has no device side signature. + +# # PHONE SIDE +# ON_SIGNATURE="$SIGNATURES_BASE_DIR/wemo-plug/signatures/wemo-plug-onSignature-phone-side.sig" +# OFF_SIGNATURE="$SIGNATURES_BASE_DIR/wemo-plug/signatures/wemo-plug-offSignature-phone-side.sig" +# RESULTS_FILE="$OUTPUT_DIR/wemo-plug/wemo-plug.wlan1.detection.pcap___phone-side.detectionresults" +# SIGNATURE_DURATION="147" + +# PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'" +# ./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" +# # ====================================================================================================================== \ No newline at end of file diff --git a/Code/Projects/PacketLevelSignatureExtractor/execute_layer2_smarthome_nest_results_analysis.sh b/Code/Projects/PacketLevelSignatureExtractor/execute_layer2_smarthome_nest_results_analysis.sh new file mode 100644 index 0000000..61e029c --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/execute_layer2_smarthome_nest_results_analysis.sh @@ -0,0 +1,187 @@ +#!/bin/bash + +# Base directory where the smarthome evaluation traces and timestamp files are stored, +# (i.e., /some/arbitrary/local/path/experimental_result/smarthome) +TIMESTAMPS_BASE_DIR=$1 +readonly TIMESTAMPS_BASE_DIR + +# Base directory for the detection results files for the smarthome experiment +RESULTS_BASE_DIR=$2 +readonly RESULTS_BASE_DIR + + + +# # ==================================================== ARLO CAMERA ===================================================== +# TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/arlo-camera/timestamps/arlo-camera-smarthome-nov-15-2018.timestamps" +# RESULTS_FILE="$RESULTS_BASE_DIR/arlo-camera/arlo-camera.wlan1.detection.pcap___phone-side.detectionresults" +# # Put the analysis results in the same folder as the detection results. +# ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" + + +# PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +# ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" +# # ====================================================================================================================== + + + +# # ================================================= BLOSSOM SPRINKLER ================================================== +# TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/blossom-sprinkler/timestamps/blossom-sprinkler-smarthome-jan-14-2019.timestamps" + +# # DEVICE SIDE +# RESULTS_FILE="$RESULTS_BASE_DIR/blossom-sprinkler/blossom-sprinkler.wlan1.detection.pcap___device-side.detectionresults" +# ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +# PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +# ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" + +# # PHONE SIDE +# RESULTS_FILE="$RESULTS_BASE_DIR/blossom-sprinkler/blossom-sprinkler.wlan1.detection.pcap___phone-side.detectionresults" +# ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +# PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +# ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" +# # ====================================================================================================================== + + + +# # ==================================================== D-LINK PLUG ===================================================== +# TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/dlink-plug/timestamps/dlink-plug-smarthome-nov-8-2018.timestamps" + +# # DEVICE SIDE +# RESULTS_FILE="$RESULTS_BASE_DIR/dlink-plug/dlink-plug.wlan1.detection.pcap___device-side.detectionresults" +# ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +# PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +# ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" + +# # PHONE SIDE +# RESULTS_FILE="$RESULTS_BASE_DIR/dlink-plug/dlink-plug.wlan1.detection.pcap___phone-side.detectionresults" +# ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +# PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +# ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" +# # ====================================================================================================================== + + + +# # ==================================================== D-LINK SIREN ==================================================== +# TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/dlink-siren/timestamps/dlink-siren-smarthome-nov-10-2018.timestamps" + +# #PHONE SIDE +# RESULTS_FILE="$RESULTS_BASE_DIR/dlink-siren/dlink-siren.wlan1.detection.pcap___phone-side.detectionresults" +# ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +# PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +# ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" +# # ====================================================================================================================== + + + +# # ===================================================== HUE BULB ======================================================= +# TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/hue-bulb/timestamps/hue-bulb-smarthome-nov-20-2018.timestamps" + +# # Has no device side signature. + +# # PHONE SIDE +# RESULTS_FILE="$RESULTS_BASE_DIR/hue-bulb/hue-bulb.wlan1.detection.pcap___phone-side.detectionresults" +# ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +# PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +# ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" +# # ====================================================================================================================== + + + +# # ================================================= KWIKSET DOORLOCK =================================================== +# TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/kwikset-doorlock/timestamps/kwikset-doorlock-smarthome-nov-10-2018.timestamps" + +# # Has no device side signature. + +# # PHONE SIDE +# RESULTS_FILE="$RESULTS_BASE_DIR/kwikset-doorlock/kwikset-doorlock.wlan1.detection.pcap___phone-side.detectionresults" +# ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +# PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +# ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" +# # ====================================================================================================================== + + + +# ================================================= NEST THERMOSTAT ==================================================== +TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/nest-thermostat/timestamps/nest-thermostat-smarthome-nov-16-2018.timestamps" + +# Has no device side signature. + +# PHONE SIDE +RESULTS_FILE="$RESULTS_BASE_DIR/nest-thermostat/nest-thermostat.wlan1.detection.pcap___phone-side.detectionresults" +ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# # ====================================================== ST PLUG ======================================================= +# TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/st-plug/timestamps/st-plug-smarthome-nov-13-2018.timestamps" + +# # Has no device side signature. + +# # PHONE SIDE +# RESULTS_FILE="$RESULTS_BASE_DIR/st-plug/st-plug.wlan1.detection.pcap___phone-side.detectionresults" +# ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +# PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +# ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" +# # ====================================================================================================================== + + + +# # ==================================================== TP-LINK BULB ==================================================== +# TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/tplink-bulb/timestamps/tplink-bulb-smarthome-nov-19-2018.timestamps" + +# # Has no device side signature. + +# # PHONE SIDE +# RESULTS_FILE="$RESULTS_BASE_DIR/tplink-bulb/tplink-bulb.wlan1.detection.pcap___phone-side.detectionresults" +# ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +# PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +# ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" +# # ====================================================================================================================== + + + +# # ==================================================== TP-LINK PLUG ==================================================== +# TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/tplink-plug/timestamps/tplink-plug-smarthome-nov-9-2018.timestamps" + +# # DEVICE SIDE +# RESULTS_FILE="$RESULTS_BASE_DIR/tplink-plug/tplink-plug.wlan1.detection.pcap___device-side.detectionresults" +# ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +# PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +# ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" + +# # DEVICE SIDE OUTBOUND +# RESULTS_FILE="$RESULTS_BASE_DIR/tplink-plug/tplink-plug.wlan1.detection.pcap___device-side-outbound.detectionresults" +# ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +# PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +# ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" +# # ====================================================================================================================== + + + +# # ================================================== WEMO INSIGHT PLUG ================================================= +# TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/wemo-insight-plug/timestamps/wemo-insight-plug-smarthome-nov-22-2018.timestamps" + +# # Has no device side signature. + +# # PHONE SIDE +# RESULTS_FILE="$RESULTS_BASE_DIR/wemo-insight-plug/wemo-insight-plug.wlan1.detection.pcap___phone-side.detectionresults" +# ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +# PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +# ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" +# # ====================================================================================================================== + + + +# # ===================================================== WEMO PLUG ====================================================== +# TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/wemo-plug/timestamps/wemo-plug-smarthome-nov-21-2018.timestamps" + +# # Has no device side signature. + +# # PHONE SIDE +# RESULTS_FILE="$RESULTS_BASE_DIR/wemo-plug/wemo-plug.wlan1.detection.pcap___phone-side.detectionresults" +# ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis" +# PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'" +# ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS" +# # ====================================================================================================================== \ No newline at end of file diff --git a/Code/Projects/PacketLevelSignatureExtractor/execute_layer2_unb_all_detection.sh b/Code/Projects/PacketLevelSignatureExtractor/execute_layer2_unb_all_detection.sh new file mode 100755 index 0000000..0a44835 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/execute_layer2_unb_all_detection.sh @@ -0,0 +1,215 @@ +#!/bin/bash + +#set -x # echo invoked commands to std out + +# Arg1 should point to the UNB trace (PCAP w/o any expected events). +PCAP_FILE=$1 + +readonly PCAP_FILE + +# Arg2 should point to the base directory for signature files (i.e., /some/local/path/experimental_result/standalone) +SIGNATURES_BASE_DIR=$2 +readonly SIGNATURES_BASE_DIR + +# Arg3 should point to folder where the detection results for the UNB trace are to be output. +OUTPUT_DIR=$3 +readonly OUTPUT_DIR + +# ==================================================== ARLO CAMERA ===================================================== +# Has no device side signature. + +# PHONE SIDE (TODO: may possibly be the .incomplete signatures) +ON_SIGNATURE="$SIGNATURES_BASE_DIR/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig" +OFF_SIGNATURE="$SIGNATURES_BASE_DIR/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig" +RESULTS_FILE="$OUTPUT_DIR/arlo-camera/arlo-camera.wlan1.detection.pcap___phone-side.detectionresults" +SIGNATURE_DURATION="213" + +PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'" +./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ================================================= BLOSSOM SPRINKLER ================================================== +# DEVICE SIDE +ON_SIGNATURE="$SIGNATURES_BASE_DIR/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-device-side.sig" +OFF_SIGNATURE="$SIGNATURES_BASE_DIR/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-device-side.sig" +RESULTS_FILE="$OUTPUT_DIR/blossom-sprinkler/blossom-sprinkler.wlan1.detection.pcap___device-side.detectionresults" +SIGNATURE_DURATION="9274" + +PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'" +./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" + +# PHONE SIDE +ON_SIGNATURE="$SIGNATURES_BASE_DIR/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-phone-side.sig" +OFF_SIGNATURE="$SIGNATURES_BASE_DIR/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-phone-side.sig" +RESULTS_FILE="$OUTPUT_DIR/blossom-sprinkler/blossom-sprinkler.wlan1.detection.pcap___phone-side.detectionresults" +SIGNATURE_DURATION="3670" + +PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'" +./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ==================================================== D-LINK PLUG ===================================================== +# DEVICE SIDE +ON_SIGNATURE="$SIGNATURES_BASE_DIR/dlink-plug/signatures/dlink-plug-onSignature-device-side.sig" +OFF_SIGNATURE="$SIGNATURES_BASE_DIR/dlink-plug/signatures/dlink-plug-offSignature-device-side.sig" +RESULTS_FILE="$OUTPUT_DIR/dlink-plug/dlink-plug.wlan1.detection.pcap___device-side.detectionresults" +SIGNATURE_DURATION="8866" + +PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'" +./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" + +# PHONE SIDE +ON_SIGNATURE="$SIGNATURES_BASE_DIR/dlink-plug/signatures/dlink-plug-onSignature-phone-side.sig" +OFF_SIGNATURE="$SIGNATURES_BASE_DIR/dlink-plug/signatures/dlink-plug-offSignature-phone-side.sig" +RESULTS_FILE="$OUTPUT_DIR/dlink-plug/dlink-plug.wlan1.detection.pcap___phone-side.detectionresults" +SIGNATURE_DURATION="193" + +PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'" +./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ==================================================== D-LINK SIREN ==================================================== +# PHONE SIDE +ON_SIGNATURE="$SIGNATURES_BASE_DIR/dlink-siren/signatures/dlink-siren-onSignature-phone-side.sig" +OFF_SIGNATURE="$SIGNATURES_BASE_DIR/dlink-siren/signatures/dlink-siren-offSignature-phone-side.sig" +RESULTS_FILE="$OUTPUT_DIR/dlink-siren/dlink-siren.wlan1.detection.pcap___phone-side.detectionresults" +SIGNATURE_DURATION="71" + +PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'" +./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ===================================================== HUE BULB ======================================================= +# Has no device side signature. + +# PHONE SIDE +ON_SIGNATURE="$SIGNATURES_BASE_DIR/hue-bulb/signatures/hue-bulb-onSignature-phone-side.sig" +OFF_SIGNATURE="$SIGNATURES_BASE_DIR/hue-bulb/signatures/hue-bulb-offSignature-phone-side.sig" +RESULTS_FILE="$OUTPUT_DIR/hue-bulb/hue-bulb.wlan1.detection.pcap___phone-side.detectionresults" +SIGNATURE_DURATION="27" + +PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'" +./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ================================================= KWIKSET DOORLOCK =================================================== +# Has no device side signature. + +# PHONE SIDE +ON_SIGNATURE="$SIGNATURES_BASE_DIR/kwikset-doorlock/signatures/kwikset-doorlock-onSignature-phone-side.sig" +OFF_SIGNATURE="$SIGNATURES_BASE_DIR/kwikset-doorlock/signatures/kwikset-doorlock-offSignature-phone-side.sig" +RESULTS_FILE="$OUTPUT_DIR/kwikset-doorlock/kwikset-doorlock.wlan1.detection.pcap___phone-side.detectionresults" +SIGNATURE_DURATION="3161" + +PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'" +./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ================================================= NEST THERMOSTAT ==================================================== +# Has no device side signature. + +# PHONE SIDE +ON_SIGNATURE="$SIGNATURES_BASE_DIR/nest-thermostat/signatures/nest-thermostat-onSignature-phone-side.sig" +OFF_SIGNATURE="$SIGNATURES_BASE_DIR/nest-thermostat/signatures/nest-thermostat-offSignature-phone-side.sig" +RESULTS_FILE="$OUTPUT_DIR/nest-thermostat/nest-thermostat.wlan1.detection.pcap___phone-side.detectionresults" +SIGNATURE_DURATION="1179" + +PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'" +./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ====================================================== ST PLUG ======================================================= +# Has no device side signature. + +# PHONE SIDE +ON_SIGNATURE="$SIGNATURES_BASE_DIR/st-plug/signatures/st-plug-onSignature-phone-side.sig" +OFF_SIGNATURE="$SIGNATURES_BASE_DIR/st-plug/signatures/st-plug-offSignature-phone-side.sig" +RESULTS_FILE="$OUTPUT_DIR/st-plug/st-plug.wlan1.detection.pcap___phone-side.detectionresults" +SIGNATURE_DURATION="2445" + +PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'" +./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ==================================================== TP-LINK BULB ==================================================== +# Has no device side signature. + +# PHONE SIDE +ON_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-bulb/signatures/tplink-bulb-onSignature-phone-side.sig" +OFF_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-bulb/signatures/tplink-bulb-offSignature-phone-side.sig" +RESULTS_FILE="$OUTPUT_DIR/tplink-bulb/tplink-bulb.wlan1.detection.pcap___phone-side.detectionresults" +SIGNATURE_DURATION="162" + +PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION' -onmacfilters 50:c7:bf:.* -offmacfilters 50:c7:bf:.*" +./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ==================================================== TP-LINK PLUG ==================================================== +# DEVICE SIDE (both the 112, 115 and 556, 1293 sequences) +ON_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-plug/signatures/tplink-plug-onSignature-device-side.sig" +OFF_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-plug/signatures/tplink-plug-offSignature-device-side.sig" +RESULTS_FILE="$OUTPUT_DIR/tplink-plug/tplink-plug.wlan1.detection.pcap___device-side.detectionresults" +SIGNATURE_DURATION="3660" + +PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION' -onmacfilters 50:c7:bf:.*;50:c7:bf:.* -offmacfilters 50:c7:bf:.*;50:c7:bf:.*" +./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" + +# DEVICE SIDE OUTBOUND (contains only those packets that go through the WAN port, i.e., only the 556, 1293 sequence) +ON_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-plug/signatures/tplink-plug-onSignature-device-side-outbound.sig" +OFF_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-plug/signatures/tplink-plug-offSignature-device-side-outbound.sig" +RESULTS_FILE="$OUTPUT_DIR/tplink-plug/tplink-plug.wlan1.detection.pcap___device-side-outbound.detectionresults" +SIGNATURE_DURATION="224" + +PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION' -onmacfilters 50:c7:bf:.* -offmacfilters 50:c7:bf:.*" +./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" + +# Phone side does not make sense as it is merely a subset of the device side and does not differentiate ONs from OFFs. +# ====================================================================================================================== + + + +# ================================================== WEMO INSIGHT PLUG ================================================= +# Has no device side signature. + +# PHONE SIDE +ON_SIGNATURE="$SIGNATURES_BASE_DIR/wemo-insight-plug/signatures/wemo-insight-plug-onSignature-phone-side.sig" +OFF_SIGNATURE="$SIGNATURES_BASE_DIR/wemo-insight-plug/signatures/wemo-insight-plug-offSignature-phone-side.sig" +RESULTS_FILE="$OUTPUT_DIR/wemo-insight-plug/wemo-insight-plug.wlan1.detection.pcap___phone-side.detectionresults" +SIGNATURE_DURATION="106" + +PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'" +./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" +# ====================================================================================================================== + + + +# ===================================================== WEMO PLUG ====================================================== +# Has no device side signature. + +# PHONE SIDE +ON_SIGNATURE="$SIGNATURES_BASE_DIR/wemo-plug/signatures/wemo-plug-onSignature-phone-side.sig" +OFF_SIGNATURE="$SIGNATURES_BASE_DIR/wemo-plug/signatures/wemo-plug-offSignature-phone-side.sig" +RESULTS_FILE="$OUTPUT_DIR/wemo-plug/wemo-plug.wlan1.detection.pcap___phone-side.detectionresults" +SIGNATURE_DURATION="147" + +PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'" +./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS" +# ====================================================================================================================== \ No newline at end of file diff --git a/Code/Projects/PacketLevelSignatureExtractor/execute_layer2_unsw_all_detection.sh b/Code/Projects/PacketLevelSignatureExtractor/execute_layer2_unsw_all_detection.sh new file mode 100755 index 0000000..9ac4569 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/execute_layer2_unsw_all_detection.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Arg1 should point to the folder with UNSW traces (PCAP files w/o any expected events). +UNSW_TRACES_DIR=$1 + +# Arg2 should point to the base directory for signature files (i.e., /some/local/path/experimental_result/standalone) +SIGNATURES_BASE_DIR=$2 +readonly SIGNATURES_BASE_DIR + +# Arg3 should point to base directory where the detection results for the UNSW trace are to be output. +# Subfolders will be created for each individual pcap file in UNSW_TRACES_DIR. +OUTPUT_DIR=$3 +readonly OUTPUT_DIR + +#set -x # echo invoked commands to std out + +for PCAP_FILE in $UNSW_TRACES_DIR/*.pcap; do + # skip non pcap files + [ -e "$PCAP_FILE" ] || continue + # make an output sub dir in the base output dir that is the filename minus extension + OUTPUT_SUB_DIR=$(basename "$PCAP_FILE" .pcap) + ./execute_layer2_unb_all_detection.sh $PCAP_FILE $SIGNATURES_BASE_DIR $OUTPUT_DIR/$OUTPUT_SUB_DIR +done + + diff --git a/Code/Projects/PacketLevelSignatureExtractor/gradle/wrapper/gradle-wrapper.jar b/Code/Projects/PacketLevelSignatureExtractor/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000..9411448 Binary files /dev/null and b/Code/Projects/PacketLevelSignatureExtractor/gradle/wrapper/gradle-wrapper.jar differ diff --git a/Code/Projects/PacketLevelSignatureExtractor/gradle/wrapper/gradle-wrapper.properties b/Code/Projects/PacketLevelSignatureExtractor/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000..a17f184 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Tue Aug 21 11:14:11 PDT 2018 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-4.9-bin.zip diff --git a/Code/Projects/PacketLevelSignatureExtractor/gradlew b/Code/Projects/PacketLevelSignatureExtractor/gradlew new file mode 100755 index 0000000..9d82f78 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/gradlew @@ -0,0 +1,160 @@ +#!/usr/bin/env bash + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS="" + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn ( ) { + echo "$*" +} + +die ( ) { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; +esac + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin, switch paths to Windows format before running java +if $cygwin ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=$((i+1)) + done + case $i in + (0) set -- ;; + (1) set -- "$args0" ;; + (2) set -- "$args0" "$args1" ;; + (3) set -- "$args0" "$args1" "$args2" ;; + (4) set -- "$args0" "$args1" "$args2" "$args3" ;; + (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules +function splitJvmOpts() { + JVM_OPTS=("$@") +} +eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS +JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME" + +exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@" diff --git a/Code/Projects/PacketLevelSignatureExtractor/gradlew.bat b/Code/Projects/PacketLevelSignatureExtractor/gradlew.bat new file mode 100644 index 0000000..aec9973 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/gradlew.bat @@ -0,0 +1,90 @@ +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS= + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windowz variants + +if not "%OS%" == "Windows_NT" goto win9xME_args +if "%@eval[2+2]" == "4" goto 4NT_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* +goto execute + +:4NT_args +@rem Get arguments from the 4NT Shell from JP Software +set CMD_LINE_ARGS=%$ + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/Code/Projects/PacketLevelSignatureExtractor/pcap/TP_LINK_LOCAL_OFF.pcap b/Code/Projects/PacketLevelSignatureExtractor/pcap/TP_LINK_LOCAL_OFF.pcap new file mode 100644 index 0000000..b30fad9 Binary files /dev/null and b/Code/Projects/PacketLevelSignatureExtractor/pcap/TP_LINK_LOCAL_OFF.pcap differ diff --git a/Code/Projects/PacketLevelSignatureExtractor/pcap/TP_LINK_LOCAL_ON.pcap b/Code/Projects/PacketLevelSignatureExtractor/pcap/TP_LINK_LOCAL_ON.pcap new file mode 100644 index 0000000..a85b153 Binary files /dev/null and b/Code/Projects/PacketLevelSignatureExtractor/pcap/TP_LINK_LOCAL_ON.pcap differ diff --git a/Code/Projects/PacketLevelSignatureExtractor/pcap/TP_LINK_LOCAL_ON_SUBSET.pcap b/Code/Projects/PacketLevelSignatureExtractor/pcap/TP_LINK_LOCAL_ON_SUBSET.pcap new file mode 100644 index 0000000..209bfbf Binary files /dev/null and b/Code/Projects/PacketLevelSignatureExtractor/pcap/TP_LINK_LOCAL_ON_SUBSET.pcap differ diff --git a/Code/Projects/PacketLevelSignatureExtractor/pcap/TP_LINK_REMOTE_CHARGING_ON.pcap b/Code/Projects/PacketLevelSignatureExtractor/pcap/TP_LINK_REMOTE_CHARGING_ON.pcap new file mode 100644 index 0000000..73a92d4 Binary files /dev/null and b/Code/Projects/PacketLevelSignatureExtractor/pcap/TP_LINK_REMOTE_CHARGING_ON.pcap differ diff --git a/Code/Projects/PacketLevelSignatureExtractor/pcap/TP_LINK_REMOTE_ON.pcap b/Code/Projects/PacketLevelSignatureExtractor/pcap/TP_LINK_REMOTE_ON.pcap new file mode 100644 index 0000000..d53625e Binary files /dev/null and b/Code/Projects/PacketLevelSignatureExtractor/pcap/TP_LINK_REMOTE_ON.pcap differ diff --git a/Code/Projects/PacketLevelSignatureExtractor/pcap/wlan1.local.dns.pcap b/Code/Projects/PacketLevelSignatureExtractor/pcap/wlan1.local.dns.pcap new file mode 100644 index 0000000..8c945a0 Binary files /dev/null and b/Code/Projects/PacketLevelSignatureExtractor/pcap/wlan1.local.dns.pcap differ diff --git a/Code/Projects/PacketLevelSignatureExtractor/pcap/wlan1.local.remote.dns.pcap b/Code/Projects/PacketLevelSignatureExtractor/pcap/wlan1.local.remote.dns.pcap new file mode 100644 index 0000000..7e96a6d Binary files /dev/null and b/Code/Projects/PacketLevelSignatureExtractor/pcap/wlan1.local.remote.dns.pcap differ diff --git a/Code/Projects/PacketLevelSignatureExtractor/pcap/wlan1.remote.dns.pcap b/Code/Projects/PacketLevelSignatureExtractor/pcap/wlan1.remote.dns.pcap new file mode 100644 index 0000000..8e99963 Binary files /dev/null and b/Code/Projects/PacketLevelSignatureExtractor/pcap/wlan1.remote.dns.pcap differ diff --git a/Code/Projects/PacketLevelSignatureExtractor/settings.gradle b/Code/Projects/PacketLevelSignatureExtractor/settings.gradle new file mode 100644 index 0000000..460cd63 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/settings.gradle @@ -0,0 +1,2 @@ +rootProject.name = 'PacketLevelSignatureExtractor' + diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/ConversationPair.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/ConversationPair.java new file mode 100644 index 0000000..b864ee5 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/ConversationPair.java @@ -0,0 +1,149 @@ +package edu.uci.iotproject; + +import org.pcap4j.core.PcapHandle; +import org.pcap4j.core.PcapPacket; + +import java.io.FileNotFoundException; +import java.io.PrintWriter; +import java.io.UnsupportedEncodingException; +import java.util.HashMap; +import java.util.Map; + +/** + * Models a (TCP) conversation/connection/session/flow (packet's belonging to the same session between a client and a + * server). + * Holds a pair of packet lengths from {@link PcapPacket}s identified as pertaining to the flow. + * Here we consider pairs of packet lengths, e.g., from device to cloud and cloud to device. + * We collect these pairs of data points as signatures that we can plot on a graph. + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class ConversationPair { + + /* Begin instance properties */ + /** + * The PrintWriter object that writes data points into file + */ + private PrintWriter pw; + + /** + * The direction of conversation + * true = device to server to device + */ + private Direction direction; + + /** + * If this is the first packet processed then the value is true (it is false otherwise). + */ + private boolean firstPacket; + + /** + * Count the frequencies of points + */ + private Map pointFreq; + private String dataPoint; + + /** + * Four possible directions of conversations. + * E.g., DEVICE_TO_SERVER means the conversation is started from + * a device-server packet and then a server-device as a response. + * SERVER_TO_DEVICE means the conversation is started from a + * server-device packet and then a device-server packet as a response. + * The same pattern applies to PHONE_TO_SERVER and SERVER_TO_PHONE + * directions. + */ + public enum Direction { + DEVICE_TO_SERVER, + SERVER_TO_DEVICE, + PHONE_TO_SERVER, + SERVER_TO_PHONE + } + + /** + * Constructs a ConversationPair object. + * @param fileName The file name to write data points into. + * @param direction The direction of the first packet of the pair. + */ + public ConversationPair(String fileName, Direction direction) { + try { + this.pw = new PrintWriter(fileName, "UTF-8"); + } catch(UnsupportedEncodingException | + FileNotFoundException e) { + e.printStackTrace(); + } + this.direction = direction; + this.firstPacket = true; + this.pointFreq = new HashMap<>(); + this.dataPoint = null; + } + + /** + * Writes conversation pair's packet lengths. + * @param packet The {@link PcapPacket} object that has packet information. + * @param fromClient If true then this packet comes from client, e.g., device. + * @param fromServer If true then this packet comes from server. + */ + public void writeConversationPair(PcapPacket packet, boolean fromClient, boolean fromServer) { + + // Write device data point first and then server + if (direction == Direction.DEVICE_TO_SERVER || direction == Direction.PHONE_TO_SERVER) { + if (fromClient && firstPacket) { // first packet + pw.print(packet.getTimestamp() + ", " + packet.getPayload().length() + ", "); + System.out.print(packet.getTimestamp() + ", " + packet.getPayload().length() + ", "); + dataPoint = Integer.toString(packet.getPayload().length()) + ", "; + firstPacket = false; + } else if (fromServer && !firstPacket) { // second packet + pw.println(packet.getPayload().length()); + System.out.println(packet.getPayload().length()); + dataPoint = dataPoint + Integer.toString(packet.getPayload().length()); + countFrequency(dataPoint); + firstPacket = true; + } + // Write server data point first and then device + } else if (direction == Direction.SERVER_TO_DEVICE || direction == Direction.SERVER_TO_PHONE) { + if (fromServer && firstPacket) { // first packet + pw.print(packet.getTimestamp() + ", " + packet.getPayload().length() + ", "); + dataPoint = Integer.toString(packet.getPayload().length()) + ", "; + firstPacket = false; + } else if (fromClient && !firstPacket) { // second packet + pw.println(packet.getPayload().length()); + dataPoint = dataPoint + Integer.toString(packet.getPayload().length()); + countFrequency(dataPoint); + firstPacket = true; + } + } + } + + /** + * Counts the frequencies of data points. + * @param dataPoint One data point for a conversation pair, e.g., 556, 1232. + */ + private void countFrequency(String dataPoint) { + + Integer freq = null; + if (pointFreq.containsKey(dataPoint)) { + freq = pointFreq.get(dataPoint); + } else { + freq = new Integer(0); + } + freq = freq + 1; + pointFreq.put(dataPoint, freq); + } + + /** + * Prints the frequencies of data points from the Map. + */ + public void printListFrequency() { + for(Map.Entry entry : pointFreq.entrySet()) { + System.out.println(entry.getKey() + " - " + entry.getValue()); + } + } + + /** + * Close the PrintWriter object. + */ + public void close() { + pw.close(); + } +} \ No newline at end of file diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/DnsMap.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/DnsMap.java new file mode 100644 index 0000000..0db01f8 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/DnsMap.java @@ -0,0 +1,111 @@ +package edu.uci.iotproject; + +import org.pcap4j.core.PacketListener; +import org.pcap4j.core.PcapPacket; +import org.pcap4j.packet.Packet; +import org.pcap4j.packet.DnsPacket; +import org.pcap4j.packet.DnsResourceRecord; +import org.pcap4j.packet.namednumber.DnsResourceRecordType; + + +import java.net.Inet4Address; +import java.net.UnknownHostException; +import java.util.*; + + +/** + * This is a class that does DNS mapping. + * Basically an IP address is mapped to its + * respective DNS hostnames. + * + * @author Rahmadi Trimananda (rtrimana@uci.edu) + * @version 0.1 + */ +public class DnsMap implements PacketListener { + + /* Class properties */ + private Map> ipToHostnameMap; + + /* Class constants */ + private static final Set EMPTY_SET = Collections.unmodifiableSet(new HashSet<>()); + + + /* Constructor */ + public DnsMap() { + ipToHostnameMap = new HashMap<>(); + } + + @Override + public void gotPacket(PcapPacket packet) { + try { + validateAndAddNewEntry(packet); + } catch (UnknownHostException e) { + e.printStackTrace(); + } + } + + /** + * Gets a packet and determine if this is a DNS packet + * + * @param packet Packet object + * @return DnsPacket object or null + */ + private DnsPacket getDnsPacket(Packet packet) { + DnsPacket dnsPacket = packet.get(DnsPacket.class); + return dnsPacket; + } + + /** + * Checks DNS packet and build the map data structure that + * maps IP addresses to DNS hostnames + * + * @param packet PcapPacket object + */ + public void validateAndAddNewEntry(PcapPacket packet) throws UnknownHostException { + // Make sure that this is a DNS packet + DnsPacket dnsPacket = getDnsPacket(packet); + if (dnsPacket != null) { + // We only care about DNS answers + if (dnsPacket.getHeader().getAnswers().size() != 0) { + String hostname = dnsPacket.getHeader().getQuestions().get(0).getQName().getName(); + for(DnsResourceRecord answer : dnsPacket.getHeader().getAnswers()) { + // We only care about type A records + if (!answer.getDataType().equals(DnsResourceRecordType.A)) + continue; + // Sanity check. For some reason the hostname appears to be the empty string in the answer . + // We hence have to assume that all answers correspond to a single question that holds the hostname as part of its object tree. + // Therefore, if there are more questions in one query-reply exchange, we are in trouble. + if (!answer.getName().getName().equals("") && !answer.getName().getName().equals(hostname)) + throw new RuntimeException("[DNS parser] mismatch between hostname in question and hostname in answer"); + // The IP in byte representation. + byte[] ipBytes = answer.getRData().getRawData(); + // Convert to string representation. + String ip = Inet4Address.getByAddress(ipBytes).getHostAddress(); + Set hostnameSet = new HashSet<>(); + hostnameSet.add(hostname); + // Update or insert depending on presence of key: + // Concat the existing set and the new set if ip already present as key, + // otherwise add an entry for ip pointing to new set. + ipToHostnameMap.merge(ip, hostnameSet, (v1, v2) -> { v1.addAll(v2); return v1; }); + } + } + } + } + + + /** + * Checks DNS packet and build the map data structure that + * maps IP addresses to DNS hostnames + * + * @param address Address to check + * @param hostname Hostname to check + */ + public boolean isRelatedToCloudServer(String address, String hostname) { + return ipToHostnameMap.getOrDefault(address, EMPTY_SET).contains(hostname); + } + + public Set getHostnamesForIp(String ip) { + Set hostnames = ipToHostnameMap.get(ip); + return hostnames != null ? Collections.unmodifiableSet(hostnames) : null; + } +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/FlowPattern.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/FlowPattern.java new file mode 100644 index 0000000..aca63ab --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/FlowPattern.java @@ -0,0 +1,215 @@ +package edu.uci.iotproject; + +import org.pcap4j.core.*; +import org.pcap4j.packet.*; +import org.pcap4j.packet.DnsPacket; +import org.pcap4j.packet.namednumber.DnsResourceRecordType; + +import java.io.EOFException; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.*; +import java.util.concurrent.TimeoutException; + +/** + * TODO add class documentation. + * TODO: At this point, this class is still in transition to having multiple hostnames and lists of packets + * + * @author Janus Varmarken + */ +public class FlowPattern { + + /** + * Class properties + */ + private final String mPatternId; + private final String hostname; // The hostname that this {@code FlowPattern} is associated with. + + /** + * The order of packet lengths that defines this {@link FlowPattern} + * TODO: this is a simplified representation, we should also include information about direction of each packet. + */ + private final List flowPacketOrder; + private final Map> mHostnameToPacketLengthsMap; + private final List mHostnameList; + private final PcapHandle mPcap; + + + /** + * Class constants + */ + + + /** + * Constructor #1 + */ + public FlowPattern(String mPatternId, String hostname, PcapHandle mPcap) { + this.mPatternId = mPatternId; + this.hostname = hostname; + this.mHostnameList = null; + this.mPcap = mPcap; + this.mHostnameToPacketLengthsMap = null; + this.flowPacketOrder = new ArrayList(); + processPcap(); + } + + + /** + * Process the PcapHandle to strip off unnecessary packets and just get the integer array of packet lengths + */ + private void processPcap() { + + PcapPacket packet; + try { + while ((packet = mPcap.getNextPacketEx()) != null) { + // For now, we only work support pattern search in TCP over IPv4. + IpV4Packet ipPacket = packet.get(IpV4Packet.class); + TcpPacket tcpPacket = packet.get(TcpPacket.class); + if (ipPacket == null || tcpPacket == null) + continue; + if (tcpPacket.getPayload() == null) // We skip non-payload control packets as these are less predictable + continue; + int packetLength = tcpPacket.getPayload().length(); + flowPacketOrder.add(packetLength); + } + } catch (EOFException eofe) { + System.out.println("[ FlowPattern ] Finished processing a training PCAP stream!"); + System.out.println("[ FlowPattern ] Pattern for " + mPatternId + ": " + Arrays.toString(flowPacketOrder.toArray())); + } catch (PcapNativeException | + TimeoutException | + NotOpenException ex) { + ex.printStackTrace(); + } + } + + + /** + * Process the PcapHandle to strip off unnecessary packets. + * We then map list of hostnames to their respective arrays of packet lengths + */ + private void processPcapToMap() { + + PcapPacket packet; + try { + int hostIndex = -1; + Set addressSet = new HashSet<>(); + while ((packet = mPcap.getNextPacketEx()) != null) { + // For now, we only work support pattern search in TCP over IPv4. + IpV4Packet ipPacket = packet.get(IpV4Packet.class); + TcpPacket tcpPacket = packet.get(TcpPacket.class); + if (ipPacket == null || tcpPacket == null) { + continue; + } + if (tcpPacket.getPayload() == null) { + // We skip non-payload control packets as these are less predictable + continue; + } + // We assume that if it is not a local address then it is a cloud server address + InetAddress srcAddress = ipPacket.getHeader().getSrcAddr(); + InetAddress dstAddress = ipPacket.getHeader().getDstAddr(); + boolean fromServer = !srcAddress.isSiteLocalAddress(); + boolean fromClient = !dstAddress.isSiteLocalAddress(); + if (!fromServer && !fromClient) { + // Packet not related to pattern, skip it + continue; + } else { + // We relate and assume that this address is from our cloud server + String cloudAddress = null; + if (fromClient) { + cloudAddress = dstAddress.getHostAddress(); + } else { // fromServer + cloudAddress = srcAddress.getHostAddress(); + } + //System.out.println("\nCloud address: " + cloudAddress); + if (!addressSet.contains(cloudAddress)) { + addressSet.add(cloudAddress); + hostIndex++; + } + + String hostname = mHostnameList.get(hostIndex); + List packetLengthsList = mHostnameToPacketLengthsMap.containsKey(hostname) ? + mHostnameToPacketLengthsMap.get(hostname) : new ArrayList<>(); + int packetLength = tcpPacket.getPayload().length(); + packetLengthsList.add(packetLength); + mHostnameToPacketLengthsMap.put(hostname, packetLengthsList); + } + } + } catch (EOFException eofe) { + System.out.println("[ FlowPattern ] Finished processing a training PCAP stream!"); + System.out.println("[ FlowPattern ] Pattern for " + mPatternId + ": " + Arrays.toString(mHostnameToPacketLengthsMap.entrySet().toArray())); + } catch (PcapNativeException | + TimeoutException | + NotOpenException ex) { + ex.printStackTrace(); + } + } + + + /** + * Constructor #2 + */ + public FlowPattern(String mPatternId, List mHostnameList, PcapHandle mPcap) { + this.mPatternId = mPatternId; + this.hostname = null; + this.mHostnameList = mHostnameList; + this.mPcap = mPcap; + this.flowPacketOrder = null; + this.mHostnameToPacketLengthsMap = new HashMap<>(); + processPcapToMap(); + } + + + public String getPatternId() { + return mPatternId; + } + + + public String getHostname() { + return hostname; + } + + + /** + * Get the sequence of packet lengths that defines this {@code FlowPattern}. + * @return the sequence of packet lengths that defines this {@code FlowPattern}. + */ + public List getPacketOrder() { + return flowPacketOrder; + } + + + /** + * Get the sequence of packet lengths based on input hostname. + * @return the sequence of packet lengths that defines this {@code FlowPattern}. + */ + public List getPacketOrder(String hostname) { + return mHostnameToPacketLengthsMap.get(hostname); + } + + + /** + * Get the list of associated hostnames. + * @return the associated hostnames that define this {@code FlowPattern}. + */ + public List getHostnameList() { + return mHostnameList; + } + + + /** + * Get the length of the List of {@code FlowPattern}. + * @return the length of the List of {@code FlowPattern}. + */ + public int getLength() { + return flowPacketOrder.size(); + } + + + /** + * Get the length of the List of {@code FlowPattern}. + * @return the length of the List of {@code FlowPattern}. + */ + public int getLength(String hostname) { + return mHostnameToPacketLengthsMap.get(hostname).size(); + } +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/FlowPatternFinder.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/FlowPatternFinder.java new file mode 100644 index 0000000..c384852 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/FlowPatternFinder.java @@ -0,0 +1,357 @@ +package edu.uci.iotproject; + +import edu.uci.iotproject.comparison.ComparisonFunctions; +import edu.uci.iotproject.comparison.CompleteMatchPatternComparisonResult; +import edu.uci.iotproject.comparison.PatternComparisonTask; +import edu.uci.iotproject.trafficreassembly.layer3.Conversation; +import org.pcap4j.core.NotOpenException; +import org.pcap4j.core.PcapHandle; +import org.pcap4j.core.PcapNativeException; +import org.pcap4j.core.PcapPacket; +import org.pcap4j.packet.DnsPacket; +import org.pcap4j.packet.IpV4Packet; +import org.pcap4j.packet.TcpPacket; + +import java.io.*; +import java.net.UnknownHostException; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.concurrent.*; + + +/** + *

Provides functionality for searching for the presence of a {@link FlowPattern} in a PCAP trace.

+ * + *

+ * The (entire) PCAP trace is traversed and parsed on one thread (specifically, the thread that calls + * {@link #findFlowPattern()}). This thread builds a {@link DnsMap} using the DNS packets present in the trace and uses + * that {@code DnsMap} to reassemble {@link Conversation}s that potentially match the provided + * {@link FlowPattern} (in that one end/party of said conversations matches the hostname(s) specified by the given + * {@code FlowPattern}). + * These potential matches are then examined on background worker thread(s) to determine if they are indeed a (complete) + * match of the provided {@code FlowPattern}. + *

+ * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class FlowPatternFinder { + + /* Begin class properties */ + /** + * {@link ExecutorService} responsible for parallelizing pattern searches. + * Declared as static to allow for reuse of threads across different instances of {@code FlowPatternFinder} and to + * avoid the overhead of initializing a new thread pool for each {@code FlowPatternFinder} instance. + */ + private static final ExecutorService EXECUTOR_SERVICE = Executors.newCachedThreadPool(); + /* End class properties */ + + /* Begin instance properties */ + /** + * Holds a set of {@link Conversation}s that potentially match {@link #mPattern} since each individual + * {@code Conversation} is communication with the hostname identified by {@code mPattern.getHostname()}. + * Note that due to limitations of the {@link Set} interface (specifically, there is no {@code get(T t)} method), + * we have to resort to a {@link Map} (in which keys map to themselves) to "mimic" a set with {@code get(T t)} + * functionality. + * + * @see this question on StackOverflow.com + */ + private final Map mConversations; + + /** + * Holds a list of trigger times. + */ + private final List mTriggerTimes; + private static int triggerListCounter; + + private final DnsMap mDnsMap; + private final PcapHandle mPcap; + private final FlowPattern mPattern; + private final ConversationPair mConvPair; + private final String FILE = "./devices/tplink_switch/datapoints.csv"; + //private final String REF_FILE = "./devices/dlink_switch/dlink-june-26-2018.timestamps"; + private final String REF_FILE = "./devices/tplink_switch/tplink-june-14-2018.timestamps"; + //private final String REF_FILE = "./devices/tplink_switch/tplink-feb-13-2018.timestamps"; + // Router time is in CET and we use PST for the trigger times + // Difference is 7 hours x 3600 x 1000ms = 25,200,000ms + private final long TIME_OFFSET = 25200000; + + private final List> mPendingComparisons = new ArrayList<>(); + /* End instance properties */ + + /** + * Constructs a new {@code FlowPatternFinder}. + * @param pcap an open {@link PcapHandle} that provides access to the trace that is to be examined. + * @param pattern the {@link FlowPattern} to search for. + */ + public FlowPatternFinder(PcapHandle pcap, FlowPattern pattern) { + this.mConversations = new HashMap<>(); + this.mTriggerTimes = readTriggerTimes(REF_FILE); + triggerListCounter = 0; + this.mDnsMap = new DnsMap(); + this.mPcap = Objects.requireNonNull(pcap, + String.format("Argument of type '%s' cannot be null", PcapHandle.class.getSimpleName())); + this.mPattern = Objects.requireNonNull(pattern, + String.format("Argument of type '%s' cannot be null", FlowPattern.class.getSimpleName())); + this.mConvPair = new ConversationPair(FILE, ConversationPair.Direction.DEVICE_TO_SERVER); + } + + + private List readTriggerTimes(String refFileName) { + + List listTriggerTimes = new ArrayList<>(); + try { + File file = new File(refFileName); + BufferedReader br = new BufferedReader(new FileReader(file)); + String s; + while ((s = br.readLine()) != null) { + listTriggerTimes.add(timeToMillis(s, false)); + } + } catch (IOException e) { + e.printStackTrace(); + } + System.out.println("List has: " + listTriggerTimes.size()); + + return listTriggerTimes; + } + + /** + * Starts the pattern search. + */ + public void start() { + + //findFlowPattern(); + findSignatureBasedOnTimestamp(); + } + + /** + * Find patterns based on the FlowPattern object (run by a thread) + */ + private void findFlowPattern() { + try { + PcapPacket packet; +// TODO: The new comparison method is pending +// TODO: For now, just compare using one hostname and one list per FlowPattern +// List hostnameList = mPattern.getHostnameList(); +// int hostIndex = 0; + while ((packet = mPcap.getNextPacketEx()) != null) { + // Let DnsMap handle DNS packets. + if (packet.get(DnsPacket.class) != null) { + // Check if this is a valid DNS packet + mDnsMap.validateAndAddNewEntry(packet); + continue; + } + // For now, we only work support pattern search in TCP over IPv4. + final IpV4Packet ipPacket = packet.get(IpV4Packet.class); + final TcpPacket tcpPacket = packet.get(TcpPacket.class); + if (ipPacket == null || tcpPacket == null) { + continue; + } + + String srcAddress = ipPacket.getHeader().getSrcAddr().getHostAddress(); + String dstAddress = ipPacket.getHeader().getDstAddr().getHostAddress(); + int srcPort = tcpPacket.getHeader().getSrcPort().valueAsInt(); + int dstPort = tcpPacket.getHeader().getDstPort().valueAsInt(); + // Is this packet related to the pattern; i.e. is it going to (or coming from) the cloud server? + boolean fromServer = mDnsMap.isRelatedToCloudServer(srcAddress, mPattern.getHostname()); + boolean fromClient = mDnsMap.isRelatedToCloudServer(dstAddress, mPattern.getHostname()); +// String currentHostname = hostnameList.get(hostIndex); +// boolean fromServer = mDnsMap.isRelatedToCloudServer(srcAddress, currentHostname); +// boolean fromClient = mDnsMap.isRelatedToCloudServer(dstAddress, currentHostname); + if (!fromServer && !fromClient) { + // Packet not related to pattern, skip it. + continue; + } + + // Conversations (connections/sessions) are identified by the four-tuple + // (clientIp, clientPort, serverIp, serverPort) (see Conversation Javadoc). + // Create "dummy" conversation for looking up an existing entry. + Conversation conversation = fromClient ? new Conversation(srcAddress, srcPort, dstAddress, dstPort) : + new Conversation(dstAddress, dstPort, srcAddress, srcPort); + // Add the packet so that the "dummy" conversation can be immediately added to the map if no entry + // exists for the conversation that the current packet belongs to. + if (tcpPacket.getHeader().getFin()) { + // Record FIN packets. + conversation.addFinPacket(packet); + } + if (tcpPacket.getPayload() != null) { + // Record regular payload packets. + conversation.addPacket(packet, true); + } + // Note: does not make sense to call attemptAcknowledgementOfFin here as the new packet has no FINs + // in its list, so if this packet is an ACK, it would not be added anyway. + // Need to retain a final reference to get access to the packet in the lambda below. + final PcapPacket finalPacket = packet; + // Add the new conversation to the map if an equal entry is not already present. + // If an existing entry is already present, the current packet is simply added to that conversation. + mConversations.merge(conversation, conversation, (existingEntry, toMerge) -> { + // toMerge may not have any payload packets if the current packet is a FIN packet. + if (toMerge.getPackets().size() > 0) { + existingEntry.addPacket(toMerge.getPackets().get(0), true); + } + if (toMerge.getFinAckPairs().size() > 0) { + // Add the FIN packet to the existing entry. + existingEntry.addFinPacket(toMerge.getFinAckPairs().get(0).getFinPacket()); + } + if (finalPacket.get(TcpPacket.class).getHeader().getAck()) { + existingEntry.attemptAcknowledgementOfFin(finalPacket); + } + return existingEntry; + }); + // Refresh reference to point to entry in map (in case packet was added to existing entry). + conversation = mConversations.get(conversation); + if (conversation.isGracefullyShutdown()) { + // Conversation terminated gracefully, so we can now start analyzing it. + // Remove the Conversation from the map and start the analysis. + // Any future packets identified by the same four tuple will be tied to a new Conversation instance. + mConversations.remove(conversation); + // Create comparison task and send to executor service. + PatternComparisonTask comparisonTask = + new PatternComparisonTask<>(conversation, mPattern, ComparisonFunctions.SUB_SEQUENCE_COMPLETE_MATCH); + mPendingComparisons.add(EXECUTOR_SERVICE.submit(comparisonTask)); + // Increment hostIndex to find the next + + } + } + } catch (EOFException eofe) { + // TODO should check for leftover conversations in map here and fire tasks for those. + // TODO [cont'd] such tasks may be present if connections did not terminate gracefully or if there are longlived connections. + System.out.println("[ findFlowPattern ] Finished processing entire PCAP stream!"); + System.out.println("[ findFlowPattern ] Now waiting for comparisons to finish..."); + // Wait for all comparisons to finish, then output their results to std.out. + for(Future comparisonTask : mPendingComparisons) { + try { + // Blocks until result is ready. + CompleteMatchPatternComparisonResult comparisonResult = comparisonTask.get(); + if (comparisonResult.getResult()) { + System.out.println(comparisonResult.getTextualDescription()); + } + } catch (InterruptedException|ExecutionException e) { + e.printStackTrace(); + } + } + } catch (UnknownHostException | + PcapNativeException | + NotOpenException | + TimeoutException ex) { + ex.printStackTrace(); + } + } + + /** + * Find patterns based on the FlowPattern object (run by a thread) + */ + private void findSignatureBasedOnTimestamp() { + try { + PcapPacket packet; +// TODO: The new comparison method is pending +// TODO: For now, just compare using one hostname and one list per FlowPattern + while ((packet = mPcap.getNextPacketEx()) != null) { + // Let DnsMap handle DNS packets. + if (packet.get(DnsPacket.class) != null) { + // Check if this is a valid DNS packet + mDnsMap.validateAndAddNewEntry(packet); + continue; + } + // For now, we only work support pattern search in TCP over IPv4. + final IpV4Packet ipPacket = packet.get(IpV4Packet.class); + final TcpPacket tcpPacket = packet.get(TcpPacket.class); + if (ipPacket == null || tcpPacket == null) { + continue; + } + + String srcAddress = ipPacket.getHeader().getSrcAddr().getHostAddress(); + String dstAddress = ipPacket.getHeader().getDstAddr().getHostAddress(); + int srcPort = tcpPacket.getHeader().getSrcPort().valueAsInt(); + int dstPort = tcpPacket.getHeader().getDstPort().valueAsInt(); + //System.out.println("Timestamp packet: " + packet.getTimestamp()); + // Is this packet related to the pattern; i.e. is it going to (or coming from) the cloud server? + boolean fromServer = mDnsMap.isRelatedToCloudServer(srcAddress, mPattern.getHostname()); + boolean fromClient = mDnsMap.isRelatedToCloudServer(dstAddress, mPattern.getHostname()); + if (!fromServer && !fromClient) { + // Packet not related to pattern, skip it. + continue; + } + // Record the conversation pairs + if (tcpPacket.getPayload() != null && checkTimeStamp(packet)) { + //if (tcpPacket.getPayload() != null) { + mConvPair.writeConversationPair(packet, fromClient, fromServer); + } + } + } catch (EOFException eofe) { + triggerListCounter = 0; + mConvPair.close(); + System.out.println("[ findFlowPattern ] ConversationPair writer closed!"); + System.out.println("[ findFlowPattern ] Frequencies of data points:"); + mConvPair.printListFrequency(); + } catch (UnknownHostException | + PcapNativeException | + NotOpenException | + TimeoutException ex) { + ex.printStackTrace(); + } + } + + private boolean checkTimeStamp(PcapPacket packet) { + + // Extract time from the packet's timestamp + String timeStamp = packet.getTimestamp().toString(); + String timeString = timeStamp.substring(timeStamp.indexOf("T") + 1, timeStamp.indexOf(".")); + // Timestamps are in CET (ahead of PST) so it should be deducted by TIME_OFFSET + long time = timeToMillis(timeString, true) - TIME_OFFSET; + //long time = timeToMillis(timeString, true); + + //System.out.println("Gets here: " + time + " trigger time: " + mTriggerTimes.get(triggerListCounter)); + + // We accept packets that are at most 3 seconds away from the trigger time + if ((mTriggerTimes.get(triggerListCounter) <= time) && + (time <= mTriggerTimes.get(triggerListCounter) + 3000)) { + //System.out.println("Gets here 1: " + timeString + " index: " + triggerListCounter); + return true; + } else { + // Handle the case that the timestamp is > 3000, but < next timestamp + // in the list. We ignore these packets. + if (time < mTriggerTimes.get(triggerListCounter)) { + // Timestamp is smaller than trigger, ignore! + //System.out.println("Gets here 2: " + timeString + " index: " + triggerListCounter); + return false; + } else { // Timestamp is greater than trigger, increment! + triggerListCounter = triggerListCounter + 1; + //System.out.println("Gets here 3: " + timeString + " index: " + triggerListCounter); + //return false; + return checkTimeStamp(packet); + } + } + + //System.out.println("Timestamp: " + timeToMillis(time, true)); + //String time2 = "21:38:08"; + //System.out.println("Timestamp: " + timeToMillis(time2, true)); + } + + /** + * A private function that returns time in milliseconds. + * @param time The time in the form of String. + * @param is24Hr If true, then this is in 24-hour format. + */ + private long timeToMillis(String time, boolean is24Hr) { + + String format = null; + if (is24Hr) { + format = "hh:mm:ss"; + } else { // 12 Hr format + format = "hh:mm:ss aa"; + } + DateFormat sdf = new SimpleDateFormat(format); + Date date = null; + try { + date = sdf.parse(time); + } catch(Exception e) { + e.printStackTrace(); + } + if (date == null) + return 0; + return date.getTime(); + } + +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/Main.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/Main.java new file mode 100644 index 0000000..f1056b0 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/Main.java @@ -0,0 +1,905 @@ +package edu.uci.iotproject; + +import static edu.uci.iotproject.analysis.UserAction.Type; + +import edu.uci.iotproject.analysis.*; +import edu.uci.iotproject.io.TriggerTimesFileReader; +import edu.uci.iotproject.trafficreassembly.layer3.Conversation; +import edu.uci.iotproject.trafficreassembly.layer3.TcpReassembler; +import edu.uci.iotproject.util.PcapPacketUtils; +import edu.uci.iotproject.util.PrintUtils; +import org.apache.commons.math3.stat.clustering.Cluster; +import org.apache.commons.math3.stat.clustering.DBSCANClusterer; +import org.pcap4j.core.*; +import org.pcap4j.packet.namednumber.DataLinkType; + +import java.io.EOFException; +import java.net.UnknownHostException; +import java.time.Duration; +import java.time.Instant; +import java.util.*; +import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * This is a system that reads PCAP files to compare + * patterns of DNS hostnames, packet sequences, and packet + * lengths with training data to determine certain events + * or actions for smart home devices. + * + * @author Janus Varmarken + * @author Rahmadi Trimananda (rtrimana@uci.edu) + * @version 0.1 + */ +public class Main { + + + public static void main(String[] args) throws PcapNativeException, NotOpenException, EOFException, TimeoutException, UnknownHostException { + // ------------------------------------------------------------------------------------------------------------- + // ------------ # Code for extracting traffic generated by a device within x seconds of a trigger # ------------ + // Paths to input and output files (consider supplying these as arguments instead) and IP of the device for + // which traffic is to be extracted: + String path = "/scratch/July-2018"; // Rahmadi +// String path = "/Users/varmarken/temp/UCI IoT Project/experiments"; // Janus + boolean verbose = true; + final String onPairsPath = "/scratch/July-2018/on.txt"; + final String offPairsPath = "/scratch/July-2018/off.txt"; + + // 1) D-Link July 26 experiment +// final String inputPcapFile = path + "/2018-07/dlink/dlink.wlan1.local.pcap"; +// final String outputPcapFile = path + "/2018-07/dlink/dlink-processed.pcap"; +// final String triggerTimesFile = path + "/2018-07/dlink/dlink-july-26-2018.timestamps"; +// final String deviceIp = "192.168.1.199"; // .246 == phone; .199 == dlink plug? + // Actual training +// final String inputPcapFile = path + "/2018-10/dlink-plug/dlink-plug.wlan1.local.pcap"; +// final String outputPcapFile = path + "/2018-10/dlink-plug/dlink-plug-processed.pcap"; +// final String triggerTimesFile = path + "/2018-10/dlink-plug/dlink-plug-oct-17-2018.timestamps"; +// final String deviceIp = "192.168.1.199"; // .246 == phone; .199 == dlink plug? + // TODO: EXPERIMENT - November 7, 2018 +// final String inputPcapFile = path + "/experimental_result/standalone/dlink-plug/wlan1/dlink-plug.wlan1.local.pcap"; +// final String outputPcapFile = path + "/experimental_result/standalone/dlink-plug/wlan1/dlink-plug-processed.pcap"; +// final String triggerTimesFile = path + "/experimental_result/standalone/dlink-plug/timestamps/dlink-plug-nov-7-2018.timestamps"; +// final String deviceIp = "192.168.1.199"; // .246 == phone; .199 == dlink plug? +//// final String deviceIp = "192.168.1.246"; // .246 == phone; .199 == dlink plug? + + // 2) TP-Link July 25 experiment +// final String inputPcapFile = path + "/2018-07/tplink/tplink.wlan1.local.pcap"; +// final String outputPcapFile = path + "/2018-07/tplink/tplink-processed.pcap"; +// final String triggerTimesFile = path + "/2018-07/tplink/tplink-july-25-2018.timestamps"; +// final String deviceIp = "192.168.1.159"; + // Actual training +// final String inputPcapFile = path + "/2018-10/tplink-plug/tplink-plug.wlan1.local.pcap"; +// final String outputPcapFile = path + "/2018-10/tplink-plug/tplink-plug-processed.pcap"; +// final String triggerTimesFile = path + "/2018-10/tplink-plug/tplink-plug-oct-17-2018.timestamps"; +// final String deviceIp = "192.168.1.159"; // .246 == phone; .159 == tplink plug + // TODO: EXPERIMENT - November 8, 2018 +// final String inputPcapFile = path + "/experimental_result/standalone/tplink-plug/wlan1/tplink-plug.wlan1.local.pcap"; +// final String outputPcapFile = path + "/experimental_result/standalone/tplink-plug/wlan1/tplink-plug-processed.pcap"; +// final String triggerTimesFile = path + "/experimental_result/standalone/tplink-plug/timestamps/tplink-plug-nov-8-2018.timestamps"; +// final String deviceIp = "192.168.1.159"; // .246 == phone; .159 == tplink plug +//// final String deviceIp = "192.168.1.246"; // .246 == phone; .159 == tplink plug + + // 2b) TP-Link July 25 experiment TRUNCATED: + // Only contains "true local" events, i.e., before the behavior changes to remote-like behavior. + // Last included event is at July 25 10:38:11; file filtered to only include packets with arrival time <= 10:38:27. +// final String inputPcapFile = path + "/2018-07/tplink/tplink.wlan1.local.truncated.pcap"; +// final String outputPcapFile = path + "/2018-07/tplink/tplink-processed.truncated.pcap"; +// final String triggerTimesFile = path + "/2018-07/tplink/tplink-july-25-2018.truncated.timestamps"; +// final String deviceIp = "192.168.1.159"; + + // 3) SmartThings Plug July 25 experiment +// final String inputPcapFile = path + "/2018-07/stplug/stplug.wlan1.local.pcap"; +// final String outputPcapFile = path + "/2018-07/stplug/stplug-processed.pcap"; +// final String triggerTimesFile = path + "/2018-07/stplug/smartthings-july-25-2018.timestamps"; +// final String deviceIp = "192.168.1.246"; // .246 == phone; .142 == SmartThings Hub (note: use eth0 capture for this!) + // October 18 +// final String inputPcapFile = path + "/2018-10/st-plug/st-plug.wlan1.local.pcap"; +// final String outputPcapFile = path + "/2018-10/st-plug/st-plug-processed.pcap"; +// final String triggerTimesFile = path + "/2018-10/st-plug/st-plug-oct-18-2018.timestamps"; +// final String deviceIp = "192.168.1.246"; // .246 == phone; .142 == SmartThings Hub (note: use eth0 capture for this!) + // TODO: EXPERIMENT - November 12, 2018 +// final String inputPcapFile = path + "/experimental_result/standalone/st-plug/wlan1/st-plug.wlan1.local.pcap"; +// final String outputPcapFile = path + "/experimental_result/standalone/st-plug/wlan1/st-plug-processed.pcap"; +//// final String inputPcapFile = path + "/experimental_result/standalone/st-plug/eth1/st-plug.eth1.local.pcap"; +//// final String outputPcapFile = path + "/experimental_result/standalone/st-plug/eth1/st-plug-processed.pcap"; +// final String triggerTimesFile = path + "/experimental_result/standalone/st-plug/timestamps/st-plug-nov-12-2018.timestamps"; +//// final String deviceIp = "192.168.1.142"; // .246 == phone; .142 == SmartThings Hub (note: use eth0 capture for this!) +// final String deviceIp = "192.168.1.246"; // .246 == phone; .142 == SmartThings Hub (note: use eth0 capture for this!) + + // 4) Wemo July 30 experiment +// final String inputPcapFile = path + "/2018-07/wemo/wemo.wlan1.local.pcap"; +// final String outputPcapFile = path + "/2018-07/wemo/wemo-processed.pcap"; +// final String triggerTimesFile = path + "/2018-07/wemo/wemo-july-30-2018.timestamps"; +// final String deviceIp = "192.168.1.145"; // .246 == phone; .145 == WeMo + // TODO: EXPERIMENT - November 20, 2018 +// final String inputPcapFile = path + "/experimental_result/standalone/wemo-plug/wlan1/wemo-plug.wlan1.local.pcap"; +// final String outputPcapFile = path + "/experimental_result/standalone/wemo-plug/wlan1/wemo-plug-processed.pcap"; +// final String triggerTimesFile = path + "/experimental_result/standalone/wemo-plug/timestamps/wemo-plug-nov-20-2018.timestamps"; +//// final String deviceIp = "192.168.1.145"; // .246 == phone; .145 == WeMo +// final String deviceIp = "192.168.1.246"; // .246 == phone; .145 == WeMo + + // 5) Wemo Insight July 31 experiment +// final String inputPcapFile = path + "/2018-07/wemoinsight/wemoinsight.wlan1.local.pcap"; +// final String outputPcapFile = path + "/2018-07/wemoinsight/wemoinsight-processed.pcap"; +// final String triggerTimesFile = path + "/2018-07/wemoinsight/wemo-insight-july-31-2018.timestamps"; +// final String deviceIp = "192.168.1.135"; + // TODO: EXPERIMENT - November 21, 2018 +// final String inputPcapFile = path + "/experimental_result/standalone/wemo-insight-plug/wlan1/wemo-insight-plug.wlan1.local.pcap"; +// final String outputPcapFile = path + "/experimental_result/standalone/wemo-insight-plug/wlan1/wemo-insight-plug-processed.pcap"; +// final String triggerTimesFile = path + "/experimental_result/standalone/wemo-insight-plug/timestamps/wemo-insight-plug-nov-21-2018.timestamps"; +//// final String deviceIp = "192.168.1.145"; // .246 == phone; .135 == WeMo Insight +// final String deviceIp = "192.168.1.246"; // .246 == phone; .135 == WeMo Insight + + // 6) TP-Link Bulb August 1 experiment +// final String inputPcapFile = path + "/2018-08/tplink-bulb/tplinkbulb.wlan1.local.pcap"; +// final String outputPcapFile = path + "/2018-08/tplink-bulb/tplinkbulb-processed.pcap"; +// final String triggerTimesFile = path + "/2018-08/tplink-bulb/tplink-bulb-aug-3-2018.timestamps"; +// final String deviceIp = "192.168.1.246"; // .246 == phone; .140 == TP-Link bulb + // TODO: EXPERIMENT - November 16, 2018 +// final String inputPcapFile = path + "/experimental_result/standalone/tplink-bulb/wlan1/tplink-bulb.wlan1.local.pcap"; +// final String outputPcapFile = path + "/experimental_result/standalone/tplink-bulb/wlan1/tplink-bulb-processed.pcap"; +//// final String inputPcapFile = path + "/experimental_result/standalone/tplink-bulb/eth0/tplink-bulb.eth1.local.pcap"; +//// final String outputPcapFile = path + "/experimental_result/standalone/tplink-bulb/eth0/tplink-bulb-processed.pcap"; +// final String triggerTimesFile = path + "/experimental_result/standalone/tplink-bulb/timestamps/tplink-bulb-nov-16-2018.timestamps"; +//// final String deviceIp = "192.168.1.140"; // .246 == phone; .140 == TP-Link bulb +// final String deviceIp = "192.168.1.246"; // .246 == phone; .140 == TP-Link bulb + + // 7) Kwikset Doorlock August 6 experiment +// final String inputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset-doorlock.data.wlan1.pcap"; +//// final String inputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset-doorlock.wlan1.local.pcap"; +// final String outputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset-doorlock-processed.pcap"; +//// final String triggerTimesFile = path + "/2018-08/kwikset-doorlock/kwikset-doorlock-aug-6-2018.timestamps"; +// final String triggerTimesFile = path + "/2018-08/kwikset-doorlock/kwikset-doorlock-8hr-data-oct-11-2018.timestamps"; +// final String deviceIp = "192.168.1.246"; // .246 == phone; .142 == SmartThings Hub (note: use eth0 capture for this!) + // TODO: EXPERIMENT - November 10, 2018 +// final String inputPcapFile = path + "/experimental_result/standalone/kwikset-doorlock/wlan1/kwikset-doorlock.wlan1.local.pcap"; +// final String outputPcapFile = path + "/experimental_result/standalone/kwikset-doorlock/wlan1/kwikset-doorlock-processed.pcap"; +//// final String inputPcapFile = path + "/experimental_result/standalone/kwikset-doorlock/eth1/kwikset-doorlock.eth1.local.pcap"; +//// final String outputPcapFile = path + "/experimental_result/standalone/kwikset-doorlock/eth1/kwikset-doorlock-processed.pcap"; +// final String triggerTimesFile = path + "/experimental_result/standalone/kwikset-doorlock/timestamps/kwikset-doorlock-nov-10-2018.timestamps"; +//// final String deviceIp = "192.168.1.142"; // .246 == phone; .142 == SmartThings Hub (note: use eth0 capture for this!) +// final String deviceIp = "192.168.1.246"; // .246 == phone; .142 == SmartThings Hub (note: use eth0 capture for this!) + + // September 12, 2018 - includes both wlan1 and eth1 interfaces +// final String inputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset3.wlan1.local.pcap"; +// //final String inputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset3.eth1.local.pcap"; +// final String outputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset3-processed.pcap"; +// final String triggerTimesFile = path + "/2018-08/kwikset-doorlock/kwikset-doorlock-sept-12-2018.timestamps"; +// final String deviceIp = "192.168.1.246"; // .246 == phone; .142 == SmartThings Hub (note: use eth0 capture for this!) + + // 8) Hue Bulb August 7 experiment +// final String inputPcapFile = path + "/2018-08/hue-bulb/hue-bulb.wlan1.local.pcap"; +// final String outputPcapFile = path + "/2018-08/hue-bulb/hue-bulb-processed.pcap"; +// final String triggerTimesFile = path + "/2018-08/hue-bulb/hue-bulb-aug-7-2018.timestamps"; +// final String deviceIp = "192.168.1.246"; + // October 30 experiment +// final String inputPcapFile = path + "/2018-10/hue-bulb/hue-bulb.wlan1.local.pcap"; +// final String outputPcapFile = path + "/2018-10/hue-bulb/hue-bulb-processed.pcap"; +// final String triggerTimesFile = path + "/2018-10/hue-bulb/hue-bulb-oct-30-2018.timestamps"; +// final String deviceIp = "192.168.1.246"; // .246 == phone; .100 == Hue hub + // TODO: EXPERIMENT - November 19, 2018 +// final String inputPcapFile = path + "/experimental_result/standalone/hue-bulb/wlan1/hue-bulb.wlan1.local.pcap"; +// final String outputPcapFile = path + "/experimental_result/standalone/hue-bulb/wlan1/hue-bulb-processed.pcap"; +// final String triggerTimesFile = path + "/experimental_result/standalone/hue-bulb/timestamps/hue-bulb-nov-19-2018.timestamps"; +//// final String deviceIp = "192.168.1.100"; // .246 == phone; .100 == Hue hub +// final String deviceIp = "192.168.1.246"; // .246 == phone; .100 == Hue hub + + // 9) Lifx Bulb August 8 experiment +// final String inputPcapFile = path + "/2018-08/lifx-bulb/lifx-bulb.wlan1.local.pcap"; +// final String outputPcapFile = path + "/2018-08/lifx-bulb/lifx-bulb-processed.pcap"; +// final String triggerTimesFile = path + "/2018-08/lifx-bulb/lifx-bulb-aug-8-2018.timestamps"; +// final String deviceIp = "192.168.1.246"; // .246 == phone; .231 == Lifx + // October 18 +// final String inputPcapFile = path + "/2018-10/lifx-bulb/lifx-bulb.wlan1.local.pcap"; +// final String outputPcapFile = path + "/2018-10/lifx-bulb/lifx-bulb-processed.pcap"; +// final String triggerTimesFile = path + "/2018-10/lifx-bulb/lifx-bulb-oct-18-2018.timestamps"; +// final String deviceIp = "192.168.1.231"; // .246 == phone; .231 == Lifx + // November 1 +// final String inputPcapFile = path + "/2018-10/lifx-bulb/lifx-bulb.wlan1.local.pcap"; +// final String outputPcapFile = path + "/2018-10/lifx-bulb/lifx-bulb-processed.pcap"; +// final String triggerTimesFile = path + "/2018-10/lifx-bulb/lifx-bulb-nov-1-2018.timestamps"; +// final String deviceIp = "192.168.1.231"; // .246 == phone; .231 == Lifx + + // 10) Amcrest Camera August 9 experiment +// final String inputPcapFile = path + "/2018-08/amcrest-camera/amcrest-camera.wlan1.local.pcap"; +// final String outputPcapFile = path + "/2018-08/amcrest-camera/amcrest-camera-processed.pcap"; +// final String triggerTimesFile = path + "/2018-08/amcrest-camera/amcrest-camera-aug-9-2018.timestamps"; +// final String deviceIp = "192.168.1.246"; // .246 == phone; .235 == camera + + // 11) Arlo Camera August 10 experiment +// final String inputPcapFile = path + "/2018-08/arlo-camera/arlo-camera.wlan1.local.pcap"; +// final String outputPcapFile = path + "/2018-08/arlo-camera/arlo-camera-processed.pcap"; +// final String triggerTimesFile = path + "/2018-08/arlo-camera/arlo-camera-aug-10-2018.timestamps"; +// final String deviceIp = "192.168.1.246"; // .246 == phone; .140 == camera + // TODO: EXPERIMENT - November 13, 2018 +// final String inputPcapFile = path + "/experimental_result/standalone/arlo-camera/wlan1/arlo-camera.wlan1.local.pcap"; +// final String outputPcapFile = path + "/experimental_result/standalone/arlo-camera/wlan1/arlo-camera-processed.pcap"; +//// final String inputPcapFile = path + "/experimental_result/standalone/arlo-camera/eth0/arlo-camera.eth1.local.pcap"; +//// final String outputPcapFile = path + "/experimental_result/standalone/arlo-camera/eth0/arlo-camera-processed.pcap"; +// final String triggerTimesFile = path + "/experimental_result/standalone/arlo-camera/timestamps/arlo-camera-nov-13-2018.timestamps"; +//// final String deviceIp = "192.168.1.140"; // .246 == phone; .140 == camera +// final String deviceIp = "192.168.1.246"; // .246 == phone; .140 == camera + + // 12) Blossom sprinkler August 13 experiment +// final String inputPcapFile = path + "/2018-08/blossom/blossom.wlan1.local.pcap"; +// final String outputPcapFile = path + "/2018-08/blossom/blossom-processed.pcap"; +// final String triggerTimesFile = path + "/2018-08/blossom/blossom-aug-13-2018.timestamps"; +// final String deviceIp = "192.168.1.229"; // .246 == phone; .229 == sprinkler +// // 2 November +// final String inputPcapFile = path + "/2018-10/blossom-sprinkler/blossom-sprinkler.wlan1.local.pcap"; +// final String outputPcapFile = path + "/2018-10/blossom-sprinkler/blossom-sprinkler-processed.pcap"; +// final String triggerTimesFile = path + "/2018-10/blossom-sprinkler/blossom-sprinkler-nov-2-2018.timestamps"; +// final String deviceIp = "192.168.1.229"; // .246 == phone; .229 == sprinkler + // January 9, 11, 13, 14 +// final String inputPcapFile = path + "/experimental_result/standalone/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.local.pcap"; +// final String outputPcapFile = path + "/experimental_result/standalone/blossom-sprinkler/wlan1/blossom-sprinkler-processed.pcap"; +// final String triggerTimesFile = path + "/experimental_result/standalone/blossom-sprinkler/timestamps/blossom-sprinkler-standalone-jan-14-2019.timestamps"; +//// final String triggerTimesFile = path + "/experimental_result/standalone/blossom-sprinkler/timestamps/blossom-sprinkler-standalone-jan-11-2019.timestamps"; +// final String deviceIp = "192.168.1.246"; // .246 == phone; .229 == sprinkler +//// final String deviceIp = "192.168.1.229"; // .246 == phone; .229 == sprinkler + +// // 13) DLink siren August 14 experiment +// final String inputPcapFile = path + "/2018-08/dlink-siren/dlink-siren.wlan1.local.pcap"; +// //final String inputPcapFile = path + "/evaluation/dlink-siren/dlink-siren.data.wlan1.pcap"; +// final String outputPcapFile = path + "/2018-08/dlink-siren/dlink-siren-processed.pcap"; +// final String triggerTimesFile = path + "/2018-08/dlink-siren/dlink-siren-oct-12-2018.timestamps"; +// //final String triggerTimesFile = path + "/2018-08/dlink-siren/dlink-siren-aug-14-2018.timestamps"; +// //final String triggerTimesFile = path + "/actual/timestamps/dlink-siren-8hr-data-oct-10-2018.timestamps"; +// final String deviceIp = "192.168.1.246"; // .246 == phone; .183 == siren + // TODO: EXPERIMENT - November 9, 2018 +// final String inputPcapFile = path + "/experimental_result/standalone/dlink-siren/wlan1/dlink-siren.wlan1.local.pcap"; +// final String outputPcapFile = path + "/experimental_result/standalone/dlink-siren/wlan1/dlink-siren-processed.pcap"; +// final String triggerTimesFile = path + "/experimental_result/standalone/dlink-siren/timestamps/dlink-siren-nov-9-2018.timestamps"; +//// final String deviceIp = "192.168.1.183"; // .246 == phone; .183 == siren +// final String deviceIp = "192.168.1.246"; // .246 == phone; .183 == siren + + // 14) Nest thermostat August 15 experiment +// final String inputPcapFile = path + "/2018-08/nest/nest.wlan1.local.pcap"; +// final String outputPcapFile = path + "/2018-08/nest/nest-processed.pcap"; +// final String triggerTimesFile = path + "/2018-08/nest/nest-aug-15-2018.timestamps"; +// final String deviceIp = "192.168.1.246"; // .246 == phone; .127 == Nest thermostat +// // TODO: EXPERIMENT - November 14, 2018 +// final String inputPcapFile = path + "/experimental_result/standalone/nest-thermostat/wlan1/nest-thermostat.wlan1.local.pcap"; +// final String outputPcapFile = path + "/experimental_result/standalone/nest-thermostat/wlan1/nest-thermostat-processed.pcap"; +//// final String inputPcapFile = path + "/experimental_result/standalone/nest-thermostat/eth0/nest-thermostat.eth1.local.pcap"; +//// final String outputPcapFile = path + "/experimental_result/standalone/nest-thermostat/eth0/nest-thermostat-processed.pcap"; +// final String triggerTimesFile = path + "/experimental_result/standalone/nest-thermostat/timestamps/nest-thermostat-nov-15-2018.timestamps"; +//// final String deviceIp = "192.168.1.127"; // .246 == phone; .127 == Nest thermostat +// final String deviceIp = "192.168.1.246"; // .246 == phone; .127 == Nest thermostat + + // 15) Alexa August 16 experiment +// final String inputPcapFile = path + "/2018-08/alexa/alexa.wlan1.local.pcap"; +// final String outputPcapFile = path + "/2018-08/alexa/alexa-processed.pcap"; +// final String triggerTimesFile = path + "/2018-08/alexa/alexa-aug-16-2018.timestamps"; +// final String deviceIp = "192.168.1.225"; // .246 == phone; .225 == Alexa + // August 17 +// final String inputPcapFile = path + "/2018-08/alexa/alexa2.wlan1.local.pcap"; +// final String outputPcapFile = path + "/2018-08/alexa/alexa2-processed.pcap"; +// final String triggerTimesFile = path + "/2018-08/alexa/alexa-aug-17-2018.timestamps"; +// final String deviceIp = "192.168.1.225"; // .246 == phone; .225 == Alexa + + // September 17 +// final String inputPcapFile = path + "/2018-08/noise/noise.eth1.pcap"; +// final String outputPcapFile = path + "/2018-08/noise/noise-processed.pcap"; +// final String triggerTimesFile = path + "/2018-08/noise/noise-sept-17-2018.timestamps"; +// final String deviceIp = "192.168.1.142"; // .142 == SmartThings Hub; .199 == dlink plug; .183 == siren + // September 26 - D-Link noise +// final String inputPcapFile = path + "/2018-08/noise/noise.dlink.wlan1.pcap"; +// final String outputPcapFile = path + "/2018-08/noise/noise-processed.pcap"; +// final String triggerTimesFile = path + "/2018-08/noise/dlink-noise-sept-26-2018.timestamps"; +// final String deviceIp = "192.168.1.183"; // .199 == dlink plug; .183 == siren + // September 27 - Kwikset noise +// final String inputPcapFile = path + "/2018-08/noise/noise.kwikset.eth1.pcap"; +// final String outputPcapFile = path + "/2018-08/noise/noise-processed.pcap"; +// final String triggerTimesFile = path + "/2018-08/noise/kwikset-doorlock-noise-sept-27-2018.timestamps"; +// final String deviceIp = "192.168.1.142"; // .142 == SmartThings Hub; + + // TODO: The below part is just for 15-second time sensitivity experiment + // TODO: The below part is just for 15-second time sensitivity experiment + // TODO: The below part is just for 15-second time sensitivity experiment + // D-Link plug +// final String triggerTimesFile = path + "/experimental_result/standalone/dlink-plug/timestamps/dlink-plug-nov-7-2018.timestamps"; +//// final String onSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-onSignature-phone-side.sig"; +//// final String offSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-offSignature-phone-side.sig"; +// final String onSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-onSignature-device-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-offSignature-device-side.sig"; + // TP-Link plug + final String triggerTimesFile = path + "/experimental_result/standalone/tplink-plug/timestamps/tplink-plug-nov-8-2018.timestamps"; +//// final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-phone-side.sig"; +//// final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-phone-side.sig"; +// final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-device-side-outbound.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-device-side-outbound.sig"; + final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-device-side.sig"; + final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-device-side.sig"; + + // D-Link siren +// final String triggerTimesFile = path + "/experimental_result/standalone/dlink-siren/timestamps/dlink-siren-nov-9-2018.timestamps"; +// final String onSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-offSignature-phone-side.sig"; + // Kwikset door lock +// final String triggerTimesFile = path + "/experimental_result/standalone/kwikset-doorlock/timestamps/kwikset-doorlock-nov-10-2018.timestamps"; +// final String onSignatureFile = path + "/experimental_result/standalone/kwikset-doorlock/signatures/kwikset-doorlock-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/kwikset-doorlock/signatures/kwikset-doorlock-offSignature-phone-side.sig"; + // SmartThings plug +// final String triggerTimesFile = path + "/experimental_result/standalone/st-plug/timestamps/st-plug-nov-12-2018.timestamps"; +// final String onSignatureFile = path + "/experimental_result/standalone/st-plug/signatures/st-plug-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/st-plug/signatures/st-plug-offSignature-phone-side.sig"; + // Arlo Q +// final String triggerTimesFile = path + "/experimental_result/standalone/arlo-camera/timestamps/arlo-camera-nov-13-2018.timestamps"; +// final String onSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig"; + // Nest thermostat +// final String triggerTimesFile = path + "/experimental_result/standalone/nest-thermostat/timestamps/nest-thermostat-nov-15-2018.timestamps"; +// final String onSignatureFile = path + "/experimental_result/standalone/nest-thermostat/signatures/nest-thermostat-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/nest-thermostat/signatures/nest-thermostat-offSignature-phone-side.sig"; + // Blossom sprinkler +// final String triggerTimesFile = path + "/experimental_result/standalone/blossom-sprinkler/timestamps/blossom-sprinkler-standalone-jan-14-2019.timestamps"; +// final String onSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-device-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-device-side.sig"; +// final String onSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-phone-side.sig"; + // TP-Link bulb +// final String triggerTimesFile = path + "/experimental_result/standalone/tplink-bulb/timestamps/tplink-bulb-nov-16-2018.timestamps"; +// final String onSignatureFile = path + "/experimental_result/standalone/tplink-bulb/signatures/tplink-bulb-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/tplink-bulb/signatures/tplink-bulb-offSignature-phone-side.sig"; + // Philips hue +// final String triggerTimesFile = path + "/2018-08/hue-bulb/hue-bulb-aug-7-2018.timestamps"; +// final String onSignatureFile = path + "/training/hue-bulb/signatures/hue-bulb-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/training/hue-bulb/signatures/hue-bulb-offSignature-phone-side.sig"; + // WeMo plug +// final String triggerTimesFile = path + "/experimental_result/standalone/wemo-plug/timestamps/wemo-plug-nov-20-2018.timestamps"; +// final String onSignatureFile = path + "/experimental_result/standalone/wemo-plug/signatures/wemo-plug-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/wemo-plug/signatures/wemo-plug-offSignature-phone-side.sig"; + // WeMo Insight plug +// final String triggerTimesFile = path + "/experimental_result/standalone/wemo-insight-plug/timestamps/wemo-insight-plug-nov-21-2018.timestamps"; +// final String onSignatureFile = path + "/experimental_result/standalone/wemo-insight-plug/signatures/wemo-insight-plug-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/wemo-insight-plug/signatures/wemo-insight-plug-offSignature-phone-side.sig"; + + + TriggerTimesFileReader ttfr = new TriggerTimesFileReader(); + List triggerTimes = ttfr.readTriggerTimes(triggerTimesFile, false); + + System.out.println("ON signature file in use is " + onSignatureFile); + System.out.println("OFF signature file in use is " + offSignatureFile); + + List>> onSignature = PrintUtils.deserializeSignatureFromFile(onSignatureFile); + List>> offSignature = PrintUtils.deserializeSignatureFromFile(offSignatureFile); + + List signatureTimestamps = new ArrayList<>(); + // Load ON signature last packet's timestamp + // Get the last only + List> lastListOn = onSignature.get(onSignature.size()-1); + for (List list : lastListOn) { + // Get timestamp Instant from the last packet + int lastPacketIndex = list.size()-1; + signatureTimestamps.add(list.get(lastPacketIndex).getTimestamp()); + } + // Load OFF signature last packet's timestamp + // Get the last only + List> lastListOff = offSignature.get(offSignature.size()-1); + for (List list : lastListOff) { + // Get timestamp Instant from the last packet + int lastPacketIndex = list.size()-1; + signatureTimestamps.add(list.get(lastPacketIndex).getTimestamp()); + } + // Sort the timestamps + signatureTimestamps.sort((p1, p2) -> { + return p1.compareTo(p2); + }); + + Iterator iterTrig = triggerTimes.iterator(); + Iterator iterSign = signatureTimestamps.iterator(); + System.out.println("Trigger to Last Packet:"); + while (iterTrig.hasNext() && iterSign.hasNext()) { + Instant trigInst = (Instant) iterTrig.next(); + Instant signInst = (Instant) iterSign.next(); + Duration dur = Duration.between(trigInst, signInst); + long duration = dur.toMillis(); + // Check duration --- should be below 15 seconds + if (duration >= 0 && duration <= 15000) { + System.out.println(dur.toMillis()); + } else if (duration > 15000) { + while (duration > 15000) { // that means we have to move to the next trigger + trigInst = (Instant) iterTrig.next(); + dur = Duration.between(trigInst, signInst); + duration = dur.toMillis(); + } + System.out.println(dur.toMillis()); + } else { // below 0 / negative --- that means we have to move to the next signature + while (duration < 0) { // that means we have to move to the next trigger + signInst = (Instant) iterSign.next(); + dur = Duration.between(trigInst, signInst); + duration = dur.toMillis(); + } + System.out.println(dur.toMillis()); + } + } + + + // ========================================================================== + List firstSignatureTimestamps = new ArrayList<>(); + List lastSignatureTimestamps = new ArrayList<>(); + List> firstListOnSign = onSignature.get(0); + List> lastListOnSign = onSignature.get(onSignature.size()-1); + // Load ON signature first and last packet's timestamps + for (List list : firstListOnSign) { + // Get timestamp Instant from the last packet + firstSignatureTimestamps.add(list.get(0).getTimestamp()); + } + for (List list : lastListOnSign) { + // Get timestamp Instant from the last packet + int lastPacketIndex = list.size()-1; + lastSignatureTimestamps.add(list.get(lastPacketIndex).getTimestamp()); + } + + List> firstListOffSign = offSignature.get(0); + List> lastListOffSign = offSignature.get(offSignature.size()-1); + // Load OFF signature first and last packet's timestamps + for (List list : firstListOffSign) { + // Get timestamp Instant from the last packet + firstSignatureTimestamps.add(list.get(0).getTimestamp()); + } + for (List list : lastListOffSign) { + // Get timestamp Instant from the last packet + int lastPacketIndex = list.size()-1; + lastSignatureTimestamps.add(list.get(lastPacketIndex).getTimestamp()); + } + // Sort the timestamps + firstSignatureTimestamps.sort((p1, p2) -> { + return p1.compareTo(p2); + }); + // Sort the timestamps + lastSignatureTimestamps.sort((p1, p2) -> { + return p1.compareTo(p2); + }); + + Iterator iterFirst = firstSignatureTimestamps.iterator(); + Iterator iterLast = lastSignatureTimestamps.iterator(); + System.out.println("First to Last Packet:"); + while (iterFirst.hasNext() && iterLast.hasNext()) { + Instant firstInst = (Instant) iterFirst.next(); + Instant lastInst = (Instant) iterLast.next(); + Duration dur = Duration.between(firstInst, lastInst); + long duration = dur.toMillis(); + // Check duration --- should be below 15 seconds + if (duration >= 0 && duration <= 15000) { + System.out.println(dur.toMillis()); + } else if (duration > 15000) { + while (duration > 15000) { // that means we have to move to the next trigger + firstInst = (Instant) iterFirst.next(); + dur = Duration.between(firstInst, lastInst); + duration = dur.toMillis(); + } + System.out.println(dur.toMillis()); + } else { // below 0 / negative --- that means we have to move to the next signature + while (duration < 0) { // that means we have to move to the next trigger + lastInst = (Instant) iterLast.next(); + dur = Duration.between(firstInst, lastInst); + duration = dur.toMillis(); + } + System.out.println(dur.toMillis()); + } + if (duration > 8000) { + break; + } + } + + // TODO: The above part is just for 15-second time sensitivity experiment + // TODO: The above part is just for 15-second time sensitivity experiment + // TODO: The above part is just for 15-second time sensitivity experiment + + + + +// TriggerTimesFileReader ttfr = new TriggerTimesFileReader(); +// List triggerTimes = ttfr.readTriggerTimes(triggerTimesFile, false); +// // Tag each trigger with "ON" or "OFF", assuming that the first trigger is an "ON" and that they alternate. +// List userActions = new ArrayList<>(); +// for (int i = 0; i < triggerTimes.size(); i++) { +// userActions.add(new UserAction(i % 2 == 0 ? Type.TOGGLE_ON : Type.TOGGLE_OFF, triggerTimes.get(i))); +// } +// TriggerTrafficExtractor tte = new TriggerTrafficExtractor(inputPcapFile, triggerTimes, deviceIp); +// final PcapDumper outputter = Pcaps.openDead(DataLinkType.EN10MB, 65536).dumpOpen(outputPcapFile); +// DnsMap dnsMap = new DnsMap(); +// TcpReassembler tcpReassembler = new TcpReassembler(); +// TrafficLabeler trafficLabeler = new TrafficLabeler(userActions); +// tte.performExtraction(pkt -> { +// try { +// outputter.dump(pkt); +// } catch (NotOpenException e) { +// e.printStackTrace(); +// } +// }, dnsMap, tcpReassembler, trafficLabeler); +// outputter.flush(); +// outputter.close(); +// +// if (tte.getPacketsIncludedCount() != trafficLabeler.getTotalPacketCount()) { +// // Sanity/debug check +// throw new AssertionError(String.format("mismatch between packet count in %s and %s", +// TriggerTrafficExtractor.class.getSimpleName(), TrafficLabeler.class.getSimpleName())); +// } +// +// // Extract all conversations present in the filtered trace. +// List allConversations = tcpReassembler.getTcpConversations(); +// // Group conversations by hostname. +// Map> convsByHostname = TcpConversationUtils.groupConversationsByHostname(allConversations, dnsMap); +// System.out.println("Grouped conversations by hostname."); +// // For each hostname, count the frequencies of packet lengths exchanged with that hostname. +// final Map> pktLenFreqsByHostname = new HashMap<>(); +// convsByHostname.forEach((host, convs) -> pktLenFreqsByHostname.put(host, TcpConversationUtils.countPacketLengthFrequencies(convs))); +// System.out.println("Counted frequencies of packet lengths exchanged with each hostname."); +// // For each hostname, count the frequencies of packet sequences (i.e., count how many conversations exchange a +// // sequence of packets of some specific lengths). +// final Map> pktSeqFreqsByHostname = new HashMap<>(); +// convsByHostname.forEach((host, convs) -> pktSeqFreqsByHostname.put(host, TcpConversationUtils.countPacketSequenceFrequencies(convs))); +// System.out.println("Counted frequencies of packet sequences exchanged with each hostname."); +// // For each hostname, count frequencies of packet pairs exchanged with that hostname across all conversations +// final Map> pktPairFreqsByHostname = +// TcpConversationUtils.countPacketPairFrequenciesByHostname(allConversations, dnsMap); +// System.out.println("Counted frequencies of packet pairs per hostname"); +// // For each user action, reassemble the set of TCP connections occurring shortly after +// final Map> userActionToConversations = trafficLabeler.getLabeledReassembledTcpTraffic(); +// final Map>> userActionsToConvsByHostname = trafficLabeler.getLabeledReassembledTcpTraffic(dnsMap); +// System.out.println("Reassembled TCP conversations occurring shortly after each user event"); +// +// +// +// /* +// * NOTE: no need to generate these more complex on/off maps that also contain mappings from hostname and +// * sequence identifiers as we do not care about hostnames and sequences during clustering. +// * We can simply use the UserAction->List map to generate ON/OFF groupings of conversations. +// */ +// +//// // Contains all ON events: hostname -> sequence identifier -> list of conversations with that sequence +//// Map>> ons = new HashMap<>(); +//// // Contains all OFF events: hostname -> sequence identifier -> list of conversations with that sequence +//// Map>> offs = new HashMap<>(); +//// userActionsToConvsByHostname.forEach((ua, hostnameToConvs) -> { +//// Map>> outer = ua.getType() == Type.TOGGLE_ON ? ons : offs; +//// hostnameToConvs.forEach((host, convs) -> { +//// Map> seqsToConvs = TcpConversationUtils. +//// groupConversationsByPacketSequence(convs, verbose); +//// outer.merge(host, seqsToConvs, (oldMap, newMap) -> { +//// newMap.forEach((sequence, cs) -> oldMap.merge(sequence, cs, (list1, list2) -> { +//// list1.addAll(list2); +//// return list1; +//// })); +//// return oldMap; +//// }); +//// }); +//// }); +//// +//// System.out.println("==== ON ===="); +//// // Print out all the pairs into a file for ON events +//// File fileOnEvents = new File(onPairsPath); +//// PrintWriter pwOn = null; +//// try { +//// pwOn = new PrintWriter(fileOnEvents); +//// } catch(Exception ex) { +//// ex.printStackTrace(); +//// } +//// for(Map.Entry>> entry : ons.entrySet()) { +//// Map> seqsToConvs = entry.getValue(); +//// for(Map.Entry> entryConv : seqsToConvs.entrySet()) { +//// List listConv = entryConv.getValue(); +//// // Just get the first Conversation because all Conversations in this group +//// // should have the same pairs of Application Data. +//// for(Conversation conv : listConv) { +//// // Process only if it is a TLS packet +//// if (conv.isTls()) { +//// List tlsAppDataList = TcpConversationUtils.extractTlsAppDataPacketPairs(conv); +//// for(PcapPacketPair pair: tlsAppDataList) { +//// System.out.println(PrintUtils.toCsv(pair, dnsMap)); +//// pwOn.println(PrintUtils.toCsv(pair, dnsMap)); +//// } +//// } else { // Non-TLS conversations +//// List packetList = TcpConversationUtils.extractPacketPairs(conv); +//// for(PcapPacketPair pair: packetList) { +//// System.out.println(PrintUtils.toCsv(pair, dnsMap)); +//// pwOn.println(PrintUtils.toCsv(pair, dnsMap)); +//// } +//// } +//// } +//// } +//// } +//// pwOn.close(); +//// +//// System.out.println("==== OFF ===="); +//// // Print out all the pairs into a file for ON events +//// File fileOffEvents = new File(offPairsPath); +//// PrintWriter pwOff = null; +//// try { +//// pwOff = new PrintWriter(fileOffEvents); +//// } catch(Exception ex) { +//// ex.printStackTrace(); +//// } +//// for(Map.Entry>> entry : offs.entrySet()) { +//// Map> seqsToConvs = entry.getValue(); +//// for(Map.Entry> entryConv : seqsToConvs.entrySet()) { +//// List listConv = entryConv.getValue(); +//// // Just get the first Conversation because all Conversations in this group +//// // should have the same pairs of Application Data. +//// for(Conversation conv : listConv) { +//// // Process only if it is a TLS packet +//// if (conv.isTls()) { +//// List tlsAppDataList = TcpConversationUtils.extractTlsAppDataPacketPairs(conv); +//// for(PcapPacketPair pair: tlsAppDataList) { +//// System.out.println(PrintUtils.toCsv(pair, dnsMap)); +//// pwOff.println(PrintUtils.toCsv(pair, dnsMap)); +//// } +//// } else { // Non-TLS conversations +//// List packetList = TcpConversationUtils.extractPacketPairs(conv); +//// for (PcapPacketPair pair : packetList) { +//// System.out.println(PrintUtils.toCsv(pair, dnsMap)); +//// pwOff.println(PrintUtils.toCsv(pair, dnsMap)); +//// } +//// } +//// } +//// } +//// } +//// pwOff.close(); +// +// +// // ================================================ CLUSTERING ================================================ +// // Note: no need to use the more convoluted on/off maps; can simply use the UserAction->List map +// // when don't care about hostnames and sequences (see comment earlier). +//// List onConversations = userActionToConversations.entrySet().stream(). +//// filter(e -> e.getKey().getType() == Type.TOGGLE_ON). // drop all OFF events from stream +//// map(e -> e.getValue()). // no longer interested in the UserActions +//// flatMap(List::stream). // flatten List> to a List +//// collect(Collectors.toList()); +//// List offConversations = userActionToConversations.entrySet().stream(). +//// filter(e -> e.getKey().getType() == Type.TOGGLE_OFF). +//// map(e -> e.getValue()). +//// flatMap(List::stream). +//// collect(Collectors.toList()); +//// //Collections.sort(onConversations, (c1, c2) -> c1.getPackets().) +//// +//// List onPairs = onConversations.stream(). +//// map(c -> c.isTls() ? TcpConversationUtils.extractTlsAppDataPacketPairs(c) : +//// TcpConversationUtils.extractPacketPairs(c)). +//// flatMap(List::stream). // flatten List> to List<> +//// collect(Collectors.toList()); +//// List offPairs = offConversations.stream(). +//// map(c -> c.isTls() ? TcpConversationUtils.extractTlsAppDataPacketPairs(c) : +//// TcpConversationUtils.extractPacketPairs(c)). +//// flatMap(List::stream). // flatten List> to List<> +//// collect(Collectors.toList()); +//// // Note: need to update the DnsMap of all PcapPacketPairs if we want to use the IP/hostname-sensitive distance. +//// Stream.concat(Stream.of(onPairs), Stream.of(offPairs)).flatMap(List::stream).forEach(p -> p.setDnsMap(dnsMap)); +//// // Perform clustering on conversation logged as part of all ON events. +////// DBSCANClusterer onClusterer = new DBSCANClusterer<>(10.0, 45); +//// DBSCANClusterer onClusterer = new DBSCANClusterer<>(2, 2); +//// //DBSCANClusterer onClusterer = new DBSCANClusterer<>(10.0, 10); +//// List> onClusters = onClusterer.cluster(onPairs); +//// // Perform clustering on conversation logged as part of all OFF events. +////// DBSCANClusterer offClusterer = new DBSCANClusterer<>(10.0, 45); +//// DBSCANClusterer offClusterer = new DBSCANClusterer<>(2, 2); +//// //DBSCANClusterer offClusterer = new DBSCANClusterer<>(10.0, 10); +//// List> offClusters = offClusterer.cluster(offPairs); +//// // Sort the conversations as reference +//// List sortedAllConversation = TcpConversationUtils.sortConversationList(allConversations); +//// // Output clusters +//// System.out.println("========================================"); +//// System.out.println(" Clustering results for ON "); +//// System.out.println(" Number of clusters: " + onClusters.size()); +//// int count = 0; +//// List>> ppListOfListReadOn = new ArrayList<>(); +//// List>> ppListOfListListOn = new ArrayList<>(); +//// for (Cluster c : onClusters) { +//// System.out.println(String.format("<<< Cluster #%02d (%03d points) >>>", ++count, c.getPoints().size())); +//// System.out.print(PrintUtils.toSummaryString(c)); +//// if(c.getPoints().size() > 45 && c.getPoints().size() < 55) { +//// //if(c.getPoints().size() > 25) { +//// // Print to file +//// List> ppListOfList = PcapPacketUtils.clusterToListOfPcapPackets(c); +//// ppListOfListListOn.add(ppListOfList); +//// } +//// } +//// // TODO: Merging test +//// ppListOfListListOn = PcapPacketUtils.mergeSignatures(ppListOfListListOn, sortedAllConversation); +//// // TODO: Need to remove sequence 550 567 for Blossom phone side since it is not a good signature (overlap)! +////// PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 1); +//// // TODO: Need to remove sequence 69 296 for Blossom device side since it is not a good signature (overlap)! +////// PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 2); +//// // TODO: Need to remove sequence number 2 for ST plug since it is not a good signature! +//// //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 2); +//// // TODO: Need to remove sequence number 0 for Arlo Camera since it is not a good signature! +//// //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 0); +//// // TODO: Need to remove sequence number 0 for TP-Link plug since it is not a good signature! +//// // TODO: This sequence actually belongs to the local communication between the plug and the phone +//// //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 0); +//// ppListOfListListOn = PcapPacketUtils.sortSignatures(ppListOfListListOn); +//// PcapPacketUtils.printSignatures(ppListOfListListOn); +//// //count = 0; +//// /*for (List> ll : ppListOfListListOn) { +//// PrintUtils.serializeClustersIntoFile("./onSignature" + ++count + ".sig", ll); +//// ppListOfListReadOn.add(PrintUtils.deserializeClustersFromFile("./onSignature" + count + ".sig")); +//// }*/ +//// PrintUtils.serializeSignatureIntoFile("./onSignature.sig", ppListOfListListOn); +//// ppListOfListReadOn = PrintUtils.deserializeSignatureFromFile("./onSignature.sig"); +//// +//// System.out.println("========================================"); +//// System.out.println(" Clustering results for OFF "); +//// System.out.println(" Number of clusters: " + offClusters.size()); +//// count = 0; +//// List>> ppListOfListReadOff = new ArrayList<>(); +//// List>> ppListOfListListOff = new ArrayList<>(); +//// for (Cluster c : offClusters) { +//// System.out.println(String.format("<<< Cluster #%03d (%06d points) >>>", ++count, c.getPoints().size())); +//// System.out.print(PrintUtils.toSummaryString(c)); +//// if(c.getPoints().size() > 45 && c.getPoints().size() < 55) { +//// //if(c.getPoints().size() > 25) { +//// // Print to file +//// List> ppListOfList = PcapPacketUtils.clusterToListOfPcapPackets(c); +//// ppListOfListListOff.add(ppListOfList); +//// } +//// } +//// // TODO: Merging test +//// ppListOfListListOff = PcapPacketUtils.mergeSignatures(ppListOfListListOff, sortedAllConversation); +//// // TODO: Need to remove sequence 69 296 for Blossom device side since it is not a good signature (overlap)! +////// PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 3); +//// // TODO: Need to remove sequence number 1 for Nest Thermostat since it is not a good signature! +//// //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 1); +//// // TODO: Need to remove sequence number 0 for Arlo Camera since it is not a good signature! +////// PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 1); +//// // TODO: Need to remove sequence number 2 for ST plug since it is not a good signature! +//// //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 2); +//// // TODO: Need to remove sequence number 0 for TP-Link plug since it is not a good signature! +//// // TODO: This sequence actually belongs to the local communication between the plug and the phone +//// //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 0); +//// ppListOfListListOff = PcapPacketUtils.sortSignatures(ppListOfListListOff); +//// PcapPacketUtils.printSignatures(ppListOfListListOff); +//// //count = 0; +//// /*for (List> ll : ppListOfListListOff) { +//// PrintUtils.serializeClustersIntoFile("./offSignature" + ++count + ".sig", ll); +//// ppListOfListReadOff.add(PrintUtils.deserializeClustersFromFile("./offSignature" + count + ".sig")); +//// }*/ +//// PrintUtils.serializeSignatureIntoFile("./offSignature.sig", ppListOfListListOff); +//// ppListOfListReadOff = PrintUtils.deserializeSignatureFromFile("./offSignature.sig"); +//// System.out.println("========================================"); +// // ============================================================================================================ +// +// // TODO: This part is just for DBSCAN sensitivity experiment +// // TODO: This part is just for DBSCAN sensitivity experiment +// // TODO: This part is just for DBSCAN sensitivity experiment +// // TODO: This part is just for DBSCAN sensitivity experiment +// // TODO: This part is just for DBSCAN sensitivity experiment +// List onConversations = userActionToConversations.entrySet().stream(). +// filter(e -> e.getKey().getType() == Type.TOGGLE_ON). // drop all OFF events from stream +// map(e -> e.getValue()). // no longer interested in the UserActions +// flatMap(List::stream). // flatten List> to a List +// collect(Collectors.toList()); +// List offConversations = userActionToConversations.entrySet().stream(). +// filter(e -> e.getKey().getType() == Type.TOGGLE_OFF). +// map(e -> e.getValue()). +// flatMap(List::stream). +// collect(Collectors.toList()); +// //Collections.sort(onConversations, (c1, c2) -> c1.getPackets().) +// +// List onPairs = onConversations.stream(). +// map(c -> c.isTls() ? TcpConversationUtils.extractTlsAppDataPacketPairs(c) : +// TcpConversationUtils.extractPacketPairs(c)). +// flatMap(List::stream). // flatten List> to List<> +// collect(Collectors.toList()); +// List offPairs = offConversations.stream(). +// map(c -> c.isTls() ? TcpConversationUtils.extractTlsAppDataPacketPairs(c) : +// TcpConversationUtils.extractPacketPairs(c)). +// flatMap(List::stream). // flatten List> to List<> +// collect(Collectors.toList()); +// // Note: need to update the DnsMap of all PcapPacketPairs if we want to use the IP/hostname-sensitive distance. +// Stream.concat(Stream.of(onPairs), Stream.of(offPairs)).flatMap(List::stream).forEach(p -> p.setDnsMap(dnsMap)); +// +// double eps = 10; // loop from eps 1-10 +// int minPts = 50; // loop from minPts 30-50 +// for(int epsCount = 7; epsCount <= eps; epsCount++) { +// for(int minPtsCount = 30; minPtsCount <= minPts; minPtsCount++) { +// System.out.println("Eps: " + epsCount + " --- minPts: " + minPtsCount); +// DBSCANClusterer onClusterer = new DBSCANClusterer<>(epsCount, minPtsCount); +// DBSCANClusterer offClusterer = new DBSCANClusterer<>(epsCount, minPtsCount); +// List> onClusters = onClusterer.cluster(onPairs); +// List> offClusters = offClusterer.cluster(offPairs); +// // Sort the conversations as reference +// List sortedAllConversation = TcpConversationUtils.sortConversationList(allConversations); +// // Output clusters +// System.out.println("========================================"); +// System.out.println(" Clustering results for ON "); +// System.out.println(" Number of clusters: " + onClusters.size()); +// int count = 0; +// List>> ppListOfListListOn = new ArrayList<>(); +// for (Cluster c : onClusters) { +// System.out.println(String.format("<<< Cluster #%02d (%03d points) >>>", ++count, c.getPoints().size())); +//// System.out.print(PrintUtils.toSummaryString(c)); +// if (c.getPoints().size() > 45 && c.getPoints().size() < 55) { +//// if(c.getPoints().size() > 25) { +// // Print to file +// List> ppListOfList = PcapPacketUtils.clusterToListOfPcapPackets(c); +// ppListOfListListOn.add(ppListOfList); +// } +// } +// PcapPacketUtils.printSignatures(ppListOfListListOn); +// +// System.out.println("========================================"); +// System.out.println(" Clustering results for OFF "); +// System.out.println(" Number of clusters: " + offClusters.size()); +// count = 0; +// List>> ppListOfListListOff = new ArrayList<>(); +// for (Cluster c : offClusters) { +// System.out.println(String.format("<<< Cluster #%03d (%06d points) >>>", ++count, c.getPoints().size())); +//// System.out.print(PrintUtils.toSummaryString(c)); +// if (c.getPoints().size() > 45 && c.getPoints().size() < 55) { +// //if(c.getPoints().size() > 25) { +// // Print to file +// List> ppListOfList = PcapPacketUtils.clusterToListOfPcapPackets(c); +// ppListOfListListOff.add(ppListOfList); +// } +// } +// PcapPacketUtils.printSignatures(ppListOfListListOff); +// System.out.println(); +// System.out.println(); +// System.out.println(); +// // ============================================================================================================ +// } +// } + + +// // ================================================================================================ +// // <<< Some work-in-progress/explorative code that extracts a "representative" sequence >>> +// // +// // Currently need to know relevant hostname in advance :( +// String hostname = "events.tplinkra.com"; +//// String hostname = "rfe-us-west-1.dch.dlink.com"; +// // Conversations with 'hostname' for ON events. +// List onsForHostname = new ArrayList<>(); +// // Conversations with 'hostname' for OFF events. +// List offsForHostname = new ArrayList<>(); +// // "Unwrap" sequence groupings in ons/offs maps. +// ons.get(hostname).forEach((k,v) -> onsForHostname.addAll(v)); +// offs.get(hostname).forEach((k,v) -> offsForHostname.addAll(v)); +// +// +// Map> onsForHostnameGroupedByTlsAppDataSequence = TcpConversationUtils.groupConversationsByTlsApplicationDataPacketSequence(onsForHostname); +// +// +// // Extract representative sequence for ON and OFF by providing the list of conversations with +// // 'hostname' observed for each event type (the training data). +// SequenceExtraction seqExtraction = new SequenceExtraction(); +//// ExtractedSequence extractedSequenceForOn = seqExtraction.extract(onsForHostname); +//// ExtractedSequence extractedSequenceForOff = seqExtraction.extract(offsForHostname); +// +// ExtractedSequence extractedSequenceForOn = seqExtraction.extractByTlsAppData(onsForHostname); +// ExtractedSequence extractedSequenceForOff = seqExtraction.extractByTlsAppData(offsForHostname); +// +// // Let's check how many ONs align with OFFs and vice versa (that is, how many times an event is incorrectly +// // labeled). +// int onsLabeledAsOff = 0; +// Integer[] representativeOnSeq = TcpConversationUtils.getPacketLengthSequence(extractedSequenceForOn.getRepresentativeSequence()); +// Integer[] representativeOffSeq = TcpConversationUtils.getPacketLengthSequence(extractedSequenceForOff.getRepresentativeSequence()); +// SequenceAlignment seqAlg = seqExtraction.getAlignmentAlgorithm(); +// for (Conversation c : onsForHostname) { +// Integer[] onSeq = TcpConversationUtils.getPacketLengthSequence(c); +// if (seqAlg.calculateAlignment(representativeOffSeq, onSeq) <= extractedSequenceForOff.getMaxAlignmentCost()) { +// onsLabeledAsOff++; +// } +// } +// int offsLabeledAsOn = 0; +// for (Conversation c : offsForHostname) { +// Integer[] offSeq = TcpConversationUtils.getPacketLengthSequence(c); +// if (seqAlg.calculateAlignment(representativeOnSeq, offSeq) <= extractedSequenceForOn.getMaxAlignmentCost()) { +// offsLabeledAsOn++; +// } +// } +// System.out.println(""); +// // ================================================================================================ +// +// +// // ------------------------------------------------------------------------------------------------------------- +// // ------------------------------------------------------------------------------------------------------------- + } + +} + + +// TP-Link MAC 50:c7:bf:33:1f:09 and usually IP 192.168.1.159 (remember to verify per file) +// frame.len >= 556 && frame.len <= 558 && ip.addr == 192.168.1.159 \ No newline at end of file diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/PcapPacketFilter.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/PcapPacketFilter.java new file mode 100644 index 0000000..529faf4 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/PcapPacketFilter.java @@ -0,0 +1,14 @@ +package edu.uci.iotproject.analysis; + +import org.pcap4j.core.PcapPacket; + +/** + * TODO add class documentation. + * + * @author Janus Varmarken + */ +public interface PcapPacketFilter { + + boolean shouldIncludePacket(PcapPacket packet); + +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/PcapPacketPair.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/PcapPacketPair.java new file mode 100644 index 0000000..2d6e9aa --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/PcapPacketPair.java @@ -0,0 +1,182 @@ +package edu.uci.iotproject.analysis; + +import edu.uci.iotproject.DnsMap; +import edu.uci.iotproject.util.PcapPacketUtils; +import org.apache.commons.math3.stat.clustering.Clusterable; +import org.pcap4j.core.PcapPacket; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Collection; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import static edu.uci.iotproject.util.PcapPacketUtils.getSourceIp; + +/** + *

+ * A simple wrapper for holding a pair of packets (e.g., a request and associated reply packet). + *

+ * + * Note: we use the deprecated version + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class PcapPacketPair implements Clusterable { + + /** + * If {@code true}, {@link #distanceFrom(PcapPacketPair)} will only consider if the sources of the two packets in + * the {@link PcapPacketPair}s being compared match in terms of whether the IP is a local or a remote IP. It will + * not check if the IPs/hostnames are actually the same. Set to {@code false} to make the comparison more + * strict, i.e., to enforce the requirement that the respective IPs (or hostnames) in the packets of the two + * {@link PcapPacketPair}s must be identical. + */ + private static final boolean SIMPLIFIED_SOURCE_COMPARISON = true; + + private final PcapPacket mFirst; + + private final Optional mSecond; + + /** + * IP to hostname mappings. + * Allows for grouping packets with different source IPs that map to the same hostname into one cluster. + */ + private DnsMap mDnsMap; // TODO implement and invoke setter + + public PcapPacketPair(PcapPacket first, PcapPacket second) { + mFirst = first; + mSecond = Optional.ofNullable(second); + } + + public PcapPacket getFirst() { return mFirst; } + + public boolean isFirstClient() { + String firstIp = PcapPacketUtils.getSourceIp(mFirst); + InetAddress ia = null; + try { + ia = InetAddress.getByName(firstIp); + } catch (UnknownHostException ex) { + ex.printStackTrace(); + } + return ia.isSiteLocalAddress(); + } + + public Optional getSecond() { return mSecond; } + + public boolean isSecondClient() { + // Return the value of the second source if it is not null + if (mSecond.isPresent()) { + String secondIp = PcapPacketUtils.getSourceIp(mSecond.get()); + InetAddress ia = null; + try { + ia = InetAddress.getByName(secondIp); + } catch (UnknownHostException ex) { + ex.printStackTrace(); + } + return ia.isSiteLocalAddress(); + } else { + // When it is null, we always return the opposite of the first source's status + return !isFirstClient(); + } + } + + /** + * Get the {@link DnsMap} that is queried for hostnames mappings when performing IP/hostname-sensitive clustering. + * @return the {@link DnsMap} that is queried for hostnames mappings when performing IP/hostname-sensitive clustering. + */ + public DnsMap getDnsMap() { + return mDnsMap; + } + + /** + * Set the {@link DnsMap} to be queried for hostnames mappings when performing IP/hostname-sensitive clustering. + * @param dnsMap a {@code DnsMap} to be queried for hostnames mappings when performing IP/hostname-sensitive clustering. + */ + public void setDnsMap(final DnsMap dnsMap) { + mDnsMap = dnsMap; + } + + @Override + public String toString() { + return String.format("%d, %s", + getFirst().getOriginalLength(), + getSecond().map(pkt -> Integer.toString(pkt.getOriginalLength())).orElse("null")); + } + + // ================================================================================================================= + // Begin implementation of org.apache.commons.math3.stat.clustering.Clusterable interface + @Override + public double distanceFrom(PcapPacketPair that) { + if (SIMPLIFIED_SOURCE_COMPARISON) { + // Direction of packets in terms of client-to-server or server-to-client must match, but we don't care about + // IPs and hostnames + if (this.isFirstClient() != that.isFirstClient() || this.isSecondClient() != that.isSecondClient()) { + // Distance is maximal if mismatch in direction of packets + return Double.MAX_VALUE; + } + } else { + // Strict mode enabled: IPs/hostnames must match! + // Extract src ips of both packets of each pair. + String thisSrc1 = getSourceIp(this.getFirst()); + String thisSrc2 = this.getSecond().map(pp -> getSourceIp(pp)).orElse(""); + String thatSrc1 = getSourceIp(that.getFirst()); + String thatSrc2 = that.getSecond().map(pp -> getSourceIp(pp)).orElse(""); + + // Replace IPs with hostnames if possible. + thisSrc1 = mapToHostname(thisSrc1); + thisSrc2 = mapToHostname(thisSrc2); + thatSrc1 = mapToHostname(thatSrc1); + thatSrc2 = mapToHostname(thatSrc2); + + if(!thisSrc1.equals(thatSrc1) || !thisSrc2.equals(thatSrc2)) { + // Distance is maximal if sources differ. + return Double.MAX_VALUE; + } + } + + // If the sources match, the distance is the Euclidean distance between each pair of packet lengths. + int thisLen1 = this.getFirst().getOriginalLength(); + // TODO should discard pairs w/o second packet from clustering; replace below with getSecond().get() when done. + int thisLen2 = this.getSecond().map(pp -> pp.getOriginalLength()).orElse(0); + int thatLen1 = that.getFirst().getOriginalLength(); + // TODO should discard pairs w/o second packet from clustering; replace below with getSecond().get() when done. + int thatLen2 = that.getSecond().map(pp -> pp.getOriginalLength()).orElse(0); + return Math.sqrt( + Math.pow(thisLen1 - thatLen1, 2) + + Math.pow(thisLen2 - thatLen2, 2) + ); + } + + @Override + public PcapPacketPair centroidOf(Collection p) { + // No notion of centroid in DBSCAN + throw new UnsupportedOperationException("Not implemented; no notion of a centroid in DBSCAN."); + } + // End implementation of org.apache.commons.math3.stat.clustering.Clusterable interface + // ================================================================================================================= + + private String mapToHostname(String ip) { + Set hostnames = mDnsMap.getHostnamesForIp(ip); + if (hostnames != null && hostnames.size() > 0) { + // append hostnames back-to-back separated by a delimiter if more than one item in set + // note: use sorted() to ensure that output remains consistent (as Set has no internal ordering of elements) + String result = hostnames.stream().sorted().collect(Collectors.joining(" ")); + if (hostnames.size() > 1) { + // One IP can map to multiple hostnames, although that is rare. For now just raise a warning. + String warningStr = String.format( + "%s.mapToHostname(): encountered an IP (%s) that maps to multiple hostnames (%s)", + getClass().getSimpleName(), ip, result); + System.err.println(warningStr); + } + return result; + } + // If unable to map to a hostname, return ip for ease of use; caller can overwrite input value, defaulting to + // the original value if no mapping is found: + // String src = ""; + // src = mapToHostname(src); // src is now either a hostname or the original ip. + return ip; + } + +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/TcpConversationUtils.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/TcpConversationUtils.java new file mode 100644 index 0000000..a4217cc --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/TcpConversationUtils.java @@ -0,0 +1,464 @@ +package edu.uci.iotproject.analysis; + +import edu.uci.iotproject.trafficreassembly.layer3.Conversation; +import edu.uci.iotproject.DnsMap; +import edu.uci.iotproject.util.PcapPacketUtils; +import org.pcap4j.core.PcapPacket; +import org.pcap4j.packet.IpV4Packet; +import org.pcap4j.packet.TcpPacket; + +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static edu.uci.iotproject.util.PcapPacketUtils.*; + +/** + * Utility functions for analyzing and structuring (sets of) {@link Conversation}s. + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class TcpConversationUtils { + + /** + * Identifies the adjacency type of the signature for merging. + */ + public enum SignaturePosition { + NOT_ADJACENT, + LEFT_ADJACENT, + RIGHT_ADJACENT + } + + /** + *

+ * Given a {@link Conversation}, extract its set of "packet pairs", i.e., pairs of request-reply packets. + * The extracted pairs are formed from the full set of payload-carrying TCP packets. + *

+ * + * Note: in the current implementation, if one endpoint sends multiple packets back-to-back with no + * interleaved reply packets from the other endpoint, such packets are converted to one-item pairs (i.e., instances + * of {@link PcapPacketPair} where {@link PcapPacketPair#getSecond()} is {@code null}). + * + * @param conv The {@code Conversation} for which packet pairs are to be extracted. + * @return The packet pairs extracted from {@code conv}. + */ + public static List extractPacketPairs(Conversation conv) { + return extractPacketPairs(conv.getPackets()); + } + + + /** + *

+ * Given a {@link Conversation}, extract its set of "packet pairs", i.e., pairs of request-reply packets. + * The extracted pairs are formed from the full set of TLS Application Data packets. + *

+ * + * Note: in the current implementation, if one endpoint sends multiple packets back-to-back with no + * interleaved reply packets from the other endpoint, such packets are converted to one-item pairs (i.e., instances + * of {@link PcapPacketPair} where {@link PcapPacketPair#getSecond()} is {@code null}). + * + * @param conv The {@code Conversation} for which packet pairs are to be extracted. + * @return The packet pairs extracted from {@code conv}. + */ + public static List extractTlsAppDataPacketPairs(Conversation conv) { + if (!conv.isTls()) { + throw new IllegalArgumentException(String.format("Provided %s argument is not a TLS session")); + } + return extractPacketPairs(conv.getTlsApplicationDataPackets()); + } + + // Helper method for implementing the public API of similarly named methods. + private static List extractPacketPairs(List packets) { + List pairs = new ArrayList<>(); +// for(PcapPacket pp : packets) { +// System.out.print(pp.length() + " "); +// } +// System.out.println(); + + int i = 0; + while (i < packets.size()) { + PcapPacket p1 = packets.get(i); + String p1SrcIp = p1.get(IpV4Packet.class).getHeader().getSrcAddr().getHostAddress(); + int p1SrcPort = p1.get(TcpPacket.class).getHeader().getSrcPort().valueAsInt(); + if (i+1 < packets.size()) { + PcapPacket p2 = packets.get(i+1); + if (PcapPacketUtils.isSource(p2, p1SrcIp, p1SrcPort)) { + // Two packets in a row going in the same direction -> create one item pair for p1 + pairs.add(new PcapPacketPair(p1, null)); + // Advance one packet as the following two packets may form a valid two-item pair. + i++; + } else { + // The two packets form a response-reply pair, create two-item pair. + pairs.add(new PcapPacketPair(p1, p2)); + // Advance two packets as we have already processed the packet at index i+1 in order to create the pair. + i += 2; + //i++; + } + } else { + // Last packet of conversation => one item pair + pairs.add(new PcapPacketPair(p1, null)); + // Advance i to ensure termination. + i++; + } + } + return pairs; + // TODO: what if there is long time between response and reply packet? Should we add a threshold and exclude those cases? + } + + /** + * Given a collection of TCP conversations and associated DNS mappings, groups the conversations by hostname. + * @param tcpConversations The collection of TCP conversations. + * @param ipHostnameMappings The associated DNS mappings. + * @return A map where each key is a hostname and its associated value is a list of conversations where one of the + * two communicating hosts is that hostname (i.e. its IP maps to the hostname). + */ + public static Map> groupConversationsByHostname(Collection tcpConversations, DnsMap ipHostnameMappings) { + HashMap> result = new HashMap<>(); + for (Conversation c : tcpConversations) { + if (c.getPackets().size() == 0) { + String warningStr = String.format("Detected a %s [%s] with no payload packets.", + c.getClass().getSimpleName(), c.toString()); + System.err.println(warningStr); + continue; + } + IpV4Packet firstPacketIp = c.getPackets().get(0).get(IpV4Packet.class); + String ipSrc = firstPacketIp.getHeader().getSrcAddr().getHostAddress(); + String ipDst = firstPacketIp.getHeader().getDstAddr().getHostAddress(); + // Check if src or dst IP is associated with one or more hostnames. + Set hostnames = ipHostnameMappings.getHostnamesForIp(ipSrc); + if (hostnames == null) { + // No luck with src ip (possibly because it's a client->srv packet), try dst ip. + hostnames = ipHostnameMappings.getHostnamesForIp(ipDst); + } + if (hostnames != null) { + // Put a reference to the conversation for each of the hostnames that the conversation's IP maps to. + for (String hostname : hostnames) { + List newValue = new ArrayList<>(); + newValue.add(c); + result.merge(hostname, newValue, (l1, l2) -> { l1.addAll(l2); return l1; }); + } + if (hostnames.size() > 1) { + // Print notice of IP mapping to multiple hostnames (debugging) + System.err.println(String.format("%s: encountered an IP that maps to multiple (%d) hostnames", + TcpConversationUtils.class.getSimpleName(), hostnames.size())); + } + } else { + // If no hostname mapping, store conversation under the key that is the concatenation of the two IPs. + // In order to ensure consistency when mapping conversations, use lexicographic order to select which IP + // goes first. + String delimiter = "_"; + // Note that the in case the comparison returns 0, the strings are equal, so it doesn't matter which of + // ipSrc and ipDst go first (also, this case should not occur in practice as it means that the device is + // communicating with itself!) + String key = ipSrc.compareTo(ipDst) <= 0 ? ipSrc + delimiter + ipDst : ipDst + delimiter + ipSrc; + List newValue = new ArrayList<>(); + newValue.add(c); + result.merge(key, newValue, (l1, l2) -> { l1.addAll(l2); return l1; }); + } + } + return result; + } + + public static Map countPacketSequenceFrequencies(Collection conversations) { + Map result = new HashMap<>(); + for (Conversation conv : conversations) { + if (conv.getPackets().size() == 0) { + // Skip conversations with no payload packets. + continue; + } + StringBuilder sb = new StringBuilder(); + for (PcapPacket pp : conv.getPackets()) { + sb.append(pp.length() + " "); + } + result.merge(sb.toString(), 1, (i1, i2) -> i1+i2); + } + return result; + } + + /** + * Given a {@link Collection} of {@link Conversation}s, builds a {@link Map} from {@link String} to {@link List} + * of {@link Conversation}s such that each key is the concatenation of the packet lengths of all payload packets + * (i.e., the set of packets returned by {@link Conversation#getPackets()}) separated by a delimiter of any + * {@link Conversation} pointed to by that key. In other words, what the {@link Conversation}s {@code cs} pointed to + * by the key {@code s} have in common is that they all contain exactly the same number of payload packets and + * these payload packets are identical across all {@code Conversation}s in {@code cs} in terms of packet + * length and packet order. For example, if the key is "152 440 550", this means that every individual + * {@code Conversation} in the list of {@code Conversation}s pointed to by that key contain exactly three payload + * packet of lengths 152, 440, and 550, and these three packets are ordered in the order prescribed by the key. + * + * @param conversations The collection of {@code Conversation}s to group by packet sequence. + * @param verbose If set to {@code true}, the grouping (and therefore the key) will also include SYN/SYNACK, + * FIN/FINACK, RST packets, and each payload-carrying packet will have an indication of the direction + * of the packet prepended. + * @return a {@link Map} from {@link String} to {@link List} of {@link Conversation}s such that each key is the + * concatenation of the packet lengths of all payload packets (i.e., the set of packets returned by + * {@link Conversation#getPackets()}) separated by a delimiter of any {@link Conversation} pointed to + * by that key. + */ + public static Map> groupConversationsByPacketSequence(Collection conversations, boolean verbose) { + return conversations.stream().collect(Collectors.groupingBy(c -> toSequenceString(c, verbose))); + } + + public static Map> groupConversationsByTlsApplicationDataPacketSequence(Collection conversations) { + return conversations.stream().collect(Collectors.groupingBy( + c -> c.getTlsApplicationDataPackets().stream().map(p -> Integer.toString(p.getOriginalLength())). + reduce("", (s1, s2) -> s1.length() == 0 ? s2 : s1 + " " + s2)) + ); + } + + /** + * Given a {@link Conversation}, counts the frequencies of each unique packet length seen as part of the + * {@code Conversation}. + * @param c The {@code Conversation} for which unique packet length frequencies are to be determined. + * @return A mapping from packet length to its frequency. + */ + public static Map countPacketLengthFrequencies(Conversation c) { + Map result = new HashMap<>(); + for (PcapPacket packet : c.getPackets()) { + result.merge(packet.length(), 1, (i1, i2) -> i1 + i2); + } + return result; + } + + /** + * Like {@link #countPacketLengthFrequencies(Conversation)}, but counts packet length frequencies for a collection + * of {@code Conversation}s, i.e., the frequency of a packet length becomes the total number of packets with that + * length across all {@code Conversation}s in {@code conversations}. + * @param conversations The collection of {@code Conversation}s for which packet length frequencies are to be + * counted. + * @return A mapping from packet length to its frequency. + */ + public static Map countPacketLengthFrequencies(Collection conversations) { + Map result = new HashMap<>(); + for (Conversation c : conversations) { + Map intermediateResult = countPacketLengthFrequencies(c); + for (Map.Entry entry : intermediateResult.entrySet()) { + result.merge(entry.getKey(), entry.getValue(), (i1, i2) -> i1 + i2); + } + } + return result; + } + + public static Map countPacketPairFrequencies(Collection pairs) { + Map result = new HashMap<>(); + for (PcapPacketPair ppp : pairs) { + result.merge(ppp.toString(), 1, (i1, i2) -> i1 + i2); + } + return result; + } + + public static Map> countPacketPairFrequenciesByHostname(Collection tcpConversations, DnsMap ipHostnameMappings) { + Map> convsByHostname = groupConversationsByHostname(tcpConversations, ipHostnameMappings); + HashMap> result = new HashMap<>(); + for (Map.Entry> entry : convsByHostname.entrySet()) { + // Merge all packet pairs exchanged during the course of all conversations with hostname into one list + List allPairsExchangedWithHostname = new ArrayList<>(); + entry.getValue().forEach(conversation -> allPairsExchangedWithHostname.addAll(extractPacketPairs(conversation))); + // Then count the frequencies of packet pairs exchanged with the hostname, irrespective of individual + // conversations + result.put(entry.getKey(), countPacketPairFrequencies(allPairsExchangedWithHostname)); + } + return result; + } + + /** + * Given a {@link Conversation}, extract its packet length sequence. + * @param c The {@link Conversation} from which a packet length sequence is to be extracted. + * @return An {@code Integer[]} that holds the packet lengths of all payload-carrying packets in {@code c}. The + * packet lengths in the returned array are ordered by packet timestamp. + */ + public static Integer[] getPacketLengthSequence(Conversation c) { + return getPacketLengthSequence(c.getPackets()); + } + + + /** + * Given a {@link Conversation}, extract its packet length sequence, but only include packet lengths of those + * packets that carry TLS Application Data. + * @param c The {@link Conversation} from which a TLS Application Data packet length sequence is to be extracted. + * @return An {@code Integer[]} that holds the packet lengths of all packets in {@code c} that carry TLS Application + * Data. The packet lengths in the returned array are ordered by packet timestamp. + */ + public static Integer[] getPacketLengthSequenceTlsAppDataOnly(Conversation c) { + if (!c.isTls()) { + throw new IllegalArgumentException("Provided " + c.getClass().getSimpleName() + " was not a TLS session"); + } + return getPacketLengthSequence(c.getTlsApplicationDataPackets()); + } + + /** + * Given a list of packets, extract the packet lengths and wrap them in an array such that the packet lengths in the + * resulting array appear in the same order as their corresponding packets in the input list. + * @param packets The list of packets for which the packet lengths are to be extracted. + * @return An array containing the packet lengths in the same order as their corresponding packets in the input list. + */ + private static Integer[] getPacketLengthSequence(List packets) { + return packets.stream().map(pkt -> pkt.getOriginalLength()).toArray(Integer[]::new); + } + + /** + * Builds a string representation of the sequence of packets exchanged as part of {@code c}. + * @param c The {@link Conversation} for which a string representation of the packet sequence is to be constructed. + * @param verbose {@code true} if set to true, the returned sequence string will also include SYN/SYNACK, + * FIN/FINACK, RST packets, as well as an indication of the direction of payload-carrying packets. + * @return a string representation of the sequence of packets exchanged as part of {@code c}. + */ + private static String toSequenceString(Conversation c, boolean verbose) { + // Payload-parrying packets are always included, but only prepend direction if verbose output is chosen. + Stream s = c.getPackets().stream().map(p -> verbose ? c.getDirection(p).toCompactString() + p.getOriginalLength() : Integer.toString(p.getOriginalLength())); + if (verbose) { + // In the verbose case, we also print SYN, FIN and RST packets. + // Convert the SYN packets to a string representation and prepend them in front of the payload packets. + s = Stream.concat(c.getSynPackets().stream().map(p -> isSyn(p) && isAck(p) ? "SYNACK" : "SYN"), s); + // Convert the FIN packets to a string representation and append them after the payload packets. + s = Stream.concat(s, c.getFinAckPairs().stream().map(f -> f.isAcknowledged() ? "FINACK" : "FIN")); + // Convert the RST packets to a string representation and append at the end. + s = Stream.concat(s, c.getRstPackets().stream().map(r -> "RST")); + } + /* + * Note: the collector internally uses a StringBuilder, which is more efficient than simply doing string + * concatenation as in the following example: + * s.reduce("", (s1, s2) -> s1.length() == 0 ? s2 : s1 + " " + s2); + * (above code is O(N^2) where N is the number of characters) + */ + return s.collect(Collectors.joining(" ")); + } + + /** + * Set of port numbers that we consider TLS traffic. + * Note: purposefully initialized as a {@link HashSet} to get O(1) {@code contains()} call. + */ + private static final Set TLS_PORTS = Stream.of(443, 8443, 41143). + collect(Collectors.toCollection(HashSet::new)); + + /** + * Check if a given port number is considered a TLS port. + * @param port The port number to check. + * @return {@code true} if the port number is considered a TLS port, {@code false} otherwise. + */ + public static boolean isTlsPort(int port) { + return TLS_PORTS.contains(port); + } + + /** + * Appends a space to {@code sb} iff {@code sb} already contains some content. + * @param sb A {@link StringBuilder} that should have a space appended iff it is not empty. + */ + private static void appendSpaceIfNotEmpty(StringBuilder sb) { + if (sb.length() != 0) { + sb.append(" "); + } + } + + /** + * Given a list of {@link Conversation} objects, sort them by timestamps. + * @param conversations The list of {@link Conversation} objects to be sorted. + * @return A sorted list of {@code Conversation} based on timestamps of the first + * packet in the {@code Conversation}. + */ + public static List sortConversationList(List conversations) { + // Get rid of Conversation objects with no packets. + conversations.removeIf(x -> x.getPackets().size() == 0); + // Sort the list based on the first packet's timestamp! + Collections.sort(conversations, (c1, c2) -> + c1.getPackets().get(0).getTimestamp().compareTo(c2.getPackets().get(0).getTimestamp())); + return conversations; + } + + /** + * Given a {@code List} of {@link Conversation} objects, find one that has the given {@code List} + * of {@code PcapPacket}. + * @param conversations The {@code List} of {@link Conversation} objects as reference. + * @param ppList The {@code List} of {@code PcapPacket} objects to search in the {@code List} of {@link Conversation}. + * @return A {@code Conversation} that contains the given {@code List} of {@code PcapPacket}. + */ + public static Conversation returnConversation(List ppList, List conversations) { + // TODO: This part of comparison takes into account that the list of conversations is not sorted + // TODO: We could optimize this to have a better performance by requiring a sorted-by-timestamp list + // TODO: as a parameter + // Find a Conversation that ppList is part of + for (Conversation c : conversations) { + // Figure out if c is the Conversation that ppList is in + if (isPartOfConversation(ppList, c)) { + return c; + } + } + // Return null if not found + return null; + } + + /** + * Given a {@link Conversation} objects, check if {@code List} of {@code PcapPacket} is part of it and return the + * adjacency label based on {@code SignaturePosition}. + * @param conversation The {@link Conversation} object as reference. + * @param ppListFirst The first {@code List} of {@code PcapPacket} objects in the {@link Conversation}. + * @param ppListSecond The second {@code List} of {@code PcapPacket} objects in the {@link Conversation} whose + * position will be observed in the {@link Conversation} with respect to ppListFirst. + * @return A {@code SignaturePosition} that represents the position of the signature against another signature + * in a {@link Conversation}. + */ + public static SignaturePosition isPartOfConversationAndAdjacent(List ppListFirst, + List ppListSecond, + Conversation conversation) { + // Take the first element in ppList and compare it + // The following elements in ppList are guaranteed to be in the same Conversation + // TODO: This part of comparison takes into account that the list of conversations is not sorted + // TODO: We could optimize this to have a better performance by requiring a sorted-by-timestamp list + // TODO: as a parameter + if (isPartOfConversation(ppListSecond, conversation)) { + // Compare the first element of ppListSecond with the last element of ppListFirst to know + // whether ppListSecond is RIGHT_ADJACENT relative to ppListFirst. + PcapPacket lastElOfFirstList = ppListFirst.get(ppListFirst.size() - 1); + PcapPacket firstElOfSecondList = ppListSecond.get(0); + // If the positions of the two are in order, then they are adjacent. + int indexOfLastElOfFirstList = returnIndexInConversation(lastElOfFirstList, conversation); + int indexOfFirstElOfSecondList = returnIndexInConversation(firstElOfSecondList, conversation); + if(indexOfLastElOfFirstList + 1 == indexOfFirstElOfSecondList) { + return SignaturePosition.RIGHT_ADJACENT; + } + // NOT RIGHT_ADJACENT, so check for LEFT_ADJACENT. + // Compare the first element of ppListRight with the last element of ppListSecond to know + // whether ppListSecond is LEFT_ADJACENT relative to ppListFirst. + PcapPacket firstElOfFirstList = ppListFirst.get(0); + PcapPacket lastElOfSecondList = ppListSecond.get(ppListSecond.size() - 1); + // If the positions of the two are in order, then they are adjacent. + int indexOfFirstElOfFirstList = returnIndexInConversation(firstElOfFirstList, conversation); + int indexOfLastElOfSecondList = returnIndexInConversation(lastElOfSecondList, conversation); + if(indexOfLastElOfSecondList + 1 == indexOfFirstElOfFirstList) { + return SignaturePosition.LEFT_ADJACENT; + } + } + // Return NOT_ADJACENT if not found. + return SignaturePosition.NOT_ADJACENT; + } + + /** + * Given a {@link Conversation} objects, check if {@code List} of {@code PcapPacket} is part of it. + * @param conversation The {@link Conversation} object as reference. + * @param ppList The {@code List} of {@code PcapPacket} objects to search in the {@link Conversation}. + * @return A {@code Boolean} value that represents the presence of the {@code List} of {@code PcapPacket} in + * the {@link Conversation}. + */ + private static boolean isPartOfConversation(List ppList, Conversation conversation) { + // Find the first element of ppList in conversation. + if (conversation.getPackets().contains(ppList.get(0))) + return true; + // Return false if not found. + return false; + } + + /** + * Given a {@link Conversation} objects, check the index of a {@code PcapPacket} in it. + * @param conversation The {@link Conversation} object as reference. + * @param pp The {@code PcapPacket} object to search in the {@link Conversation}. + * @return An {@code Integer} value that gives the index of the {@code PcapPacket} in the {@link Conversation}. + */ + private static int returnIndexInConversation(PcapPacket pp, Conversation conversation) { + // Find pp in conversation. + if (conversation.getPackets().contains(pp)) + return conversation.getPackets().indexOf(pp); + // Return -1 if not found. + return -1; + } +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/TrafficLabeler.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/TrafficLabeler.java new file mode 100644 index 0000000..983de12 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/TrafficLabeler.java @@ -0,0 +1,146 @@ +package edu.uci.iotproject.analysis; + +import edu.uci.iotproject.trafficreassembly.layer3.Conversation; +import edu.uci.iotproject.DnsMap; +import edu.uci.iotproject.trafficreassembly.layer3.TcpReassembler; +import org.pcap4j.core.PacketListener; +import org.pcap4j.core.PcapPacket; + +import java.time.Instant; +import java.util.*; +import java.util.function.Function; + +/** + * A {@link PacketListener} that marks network traffic as (potentially) related to a user's actions by comparing the + * timestamp of each packet to the timestamps of the provided list of user actions. + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class TrafficLabeler implements PacketListener { + + private final Map> mActionToTrafficMap; + private final List mActionsSorted; + /** + * The total number of packets labeled, i.e, the sum of the sizes of the values in {@link #mActionToTrafficMap}. + */ + private long mPackets = 0; + + public TrafficLabeler(List userActions) { + // Init map with empty lists (no packets have been mapped to UserActions at the onset). + mActionToTrafficMap = new HashMap<>(); + userActions.forEach(ua -> mActionToTrafficMap.put(ua, new ArrayList<>())); + // Sort list of UserActions by timestamp in order to facilitate fast Packet-to-UserAction mapping. + // For safety reasons, we create an internal copy of the list to prevent external code from changing the list's + // contents as that would render our assumptions about order of elements invalid. + // In addition, this also ensures that we do not break assumptions made by external code as we avoid reordering + // the elements of the list passed from the external code. + // If performance is to be favored over safety, assign userActions to mActionsSorted directly. + mActionsSorted = new ArrayList<>(); + mActionsSorted.addAll(userActions); + Collections.sort(mActionsSorted, (ua1, ua2) -> ua1.getTimestamp().compareTo(ua2.getTimestamp())); + } + + + @Override + public void gotPacket(PcapPacket packet) { + // Locate UserAction corresponding to packet, if any. + int index = Collections.binarySearch(mActionsSorted, new UserAction(null, packet.getTimestamp()), (listItem, key) -> { + // Start of inclusion interval is the time of the user action + Instant intervalStart = listItem.getTimestamp(); + // End of inclusion interval is some arbitrary number of milliseconds after the user action. + Instant intervalEnd = intervalStart.plusMillis(TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS); + if (key.getTimestamp().isAfter(intervalStart) && key.getTimestamp().isBefore(intervalEnd)) { + // Packet lies within specified interval after the current UserAction, so we're done. + // Communicate termination to binarySearch by returning 0 which indicates equality. + return 0; + } + // If packet lies outside inclusion interval of current list item, continue search in lower or upper half of + // list depending on whether the timestamp of the current list item is smaller or greater than that of the + // packet. + return listItem.getTimestamp().compareTo(key.getTimestamp()); + }); + if (index >= 0) { + // Associate the packet to the its corresponding user action (located during the binary search above). + mActionToTrafficMap.get(mActionsSorted.get(index)).add(packet); + mPackets++; + } + // Ignore packet if it is not found to be in temporal proximity of a user action. + } + + /** + * Get the total number of packets labeled by this {@code TrafficLabeler}. + * + * @return the total number of packets labeled by this {@code TrafficLabeler}. + */ + public long getTotalPacketCount() { + return mPackets; + } + + /** + * Get the labeled traffic. + * + * @return A {@link Map} in which a {@link UserAction} points to a {@link List} of {@link PcapPacket}s believed to + * be related (occurring as a result of) that {@code UserAction}. + */ + public Map> getLabeledTraffic() { + return Collections.unmodifiableMap(mActionToTrafficMap); + } + + /** + * Like {@link #getLabeledTraffic()}, but allows the caller to supply a mapping function that is applied to + * the traffic associated with each {@link UserAction} (the traffic label) before returning the labeled traffic. + * This may for example be useful for a caller who wishes to perform some postprocessing of labeled traffic, e.g., + * in order to perform additional filtering or to transform the representation of labeled traffic. + *

+ * An example usecase is provided in {@link #getLabeledReassembledTcpTraffic()} which uses this function to + * build a {@link Map} in which a {@link UserAction} points to the reassembled TCP connections believed to have + * occurred as a result of that {@code UserAction}. + *

+ * + * @param mappingFunction A mapping function that converts a {@link List} of {@link PcapPacket} into some other type + * {@code T}. + * @param The return type of {@code mappingFunction}. + * @return A {@link Map} in which a {@link UserAction} points to the result of applying {@code mappingFunction} to + * the set of packets believed to be related (occurring as a result of) that {@code UserAction}. + */ + public Map getLabeledTraffic(Function, T> mappingFunction) { + Map result = new HashMap<>(); + mActionToTrafficMap.forEach((ua, packets) -> result.put(ua, mappingFunction.apply(packets))); + return result; + } + + + /** + * Get the labeled traffic reassembled as TCP connections (note: discards all non-TCP traffic). + * + * @return A {@link Map} in which a {@link UserAction} points to a {@link List} of {@link Conversation}s believed to + * be related (occurring as a result of) that {@code UserAction}. + */ + public Map> getLabeledReassembledTcpTraffic() { + return getLabeledTraffic(packets -> { + TcpReassembler tcpReassembler = new TcpReassembler(); + packets.forEach(p -> tcpReassembler.gotPacket(p)); + return tcpReassembler.getTcpConversations(); + }); + } + + /** + * Like {@link #getLabeledReassembledTcpTraffic()}, but uses the provided {@code ipHostnameMappings} to group + * {@link Conversation}s by hostname. + * + * @param ipHostnameMappings A {@link DnsMap} with IP to hostname mappings used for reverse DNS lookup. + * @return A {@link Map} in which a {@link UserAction} points to the set of {@link Conversation}s believed to be + * related (occurring as a result of) that {@code UserAction}. More precisely, each {@code UserAction} in + * the returned {@code Map} points to another {@code Map} in which a hostname points to the set of + * {@code Conversation}s involving that hostname. + */ + public Map>> getLabeledReassembledTcpTraffic(DnsMap ipHostnameMappings) { + return getLabeledTraffic(packets -> { + TcpReassembler tcpReassembler = new TcpReassembler(); + packets.forEach(p -> tcpReassembler.gotPacket(p)); + return TcpConversationUtils.groupConversationsByHostname(tcpReassembler.getTcpConversations(), ipHostnameMappings); + }); + } + +} \ No newline at end of file diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/TriggerTrafficExtractor.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/TriggerTrafficExtractor.java new file mode 100644 index 0000000..0a22d63 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/TriggerTrafficExtractor.java @@ -0,0 +1,114 @@ +package edu.uci.iotproject.analysis; + +import edu.uci.iotproject.io.PcapHandleReader; +import org.pcap4j.core.*; + +import java.time.Instant; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeoutException; + +/** + * TODO add class documentation. + * + * @author Janus Varmarken + */ +public class TriggerTrafficExtractor implements PcapPacketFilter { + + private final String mPcapFilePath; + private final List mTriggerTimes; + private final String mDeviceIp; + + private int mTriggerIndex = 0; + + /** + * The total number of packets marked for inclusion during one run of {@link #performExtraction(PacketListener...)}. + */ + private long mIncludedPackets = 0; + + public static final int INCLUSION_WINDOW_MILLIS = 15_000; + + public TriggerTrafficExtractor(String pcapFilePath, List triggerTimes, String deviceIp) throws PcapNativeException, NotOpenException { + mPcapFilePath = pcapFilePath; + // Ensure that trigger times are sorted in ascending as we rely on this fact in the logic that works out if a + // packet is related to a trigger. + Collections.sort(triggerTimes, (i1, i2) -> { + if (i1.isBefore(i2)) return -1; + else if (i2.isBefore(i1)) return 1; + else return 0; + }); + mTriggerTimes = Collections.unmodifiableList(triggerTimes); + mDeviceIp = deviceIp; + } + + + public void performExtraction(PacketListener... extractedPacketsConsumers) throws PcapNativeException, NotOpenException, TimeoutException { + // Reset trigger index and packet counter in case client code chooses to rerun the extraction. + mTriggerIndex = 0; + mIncludedPackets = 0; + PcapHandle handle; + try { + handle = Pcaps.openOffline(mPcapFilePath, PcapHandle.TimestampPrecision.NANO); + } catch (PcapNativeException pne) { + handle = Pcaps.openOffline(mPcapFilePath); + } + // Use the native support for BPF to immediately filter irrelevant traffic. + handle.setFilter("ip host " + mDeviceIp, BpfProgram.BpfCompileMode.OPTIMIZE); + PcapHandleReader pcapReader = new PcapHandleReader(handle, this, extractedPacketsConsumers); + pcapReader.readFromHandle(); + + } + + /** + * Return the number of extracted packets (i.e., packets selected for inclusion) as a result of the most recent call + * to {@link #performExtraction(PacketListener...)}. + * + * @return the number of extracted packets (i.e., packets selected for inclusion) as a result of the most recent + * call to {@link #performExtraction(PacketListener...)}. + */ + public long getPacketsIncludedCount() { + return mIncludedPackets; + } + + @Override + public boolean shouldIncludePacket(PcapPacket packet) { + // New version. Simpler, but slower: the later a packet arrives, the more elements of mTriggerTimes will need to + // be traversed. + boolean include = mTriggerTimes.stream().anyMatch( + trigger -> trigger.isBefore(packet.getTimestamp()) && + packet.getTimestamp().isBefore(trigger.plusMillis(INCLUSION_WINDOW_MILLIS)) + ); + if (include) { + mIncludedPackets++; + } + return include; + + /* + // Old version. Faster, but more complex - is it correct? + if (mTriggerIndex >= mTriggerTimes.size()) { + // Don't include packet if we've exhausted the list of trigger times. + return false; + } + + // TODO hmm, is this correct? + Instant trigger = mTriggerTimes.get(mTriggerIndex); + if (trigger.isBefore(packet.getTimestamp()) && + packet.getTimestamp().isBefore(trigger.plusMillis(INCLUSION_WINDOW_MILLIS))) { + // Packet lies within INCLUSION_WINDOW_MILLIS after currently considered trigger, include it. + return true; + } else { + if (!trigger.isBefore(packet.getTimestamp())) { + // Packet is before currently considered trigger, so it shouldn't be included + return false; + } else { + // Packet is >= INCLUSION_WINDOW_MILLIS after currently considered trigger. + // Proceed to next trigger to see if it lies in range of that. + // Note that there's an assumption here that no two trigger intervals don't overlap! + mTriggerIndex++; + return shouldIncludePacket(packet); + } + } + */ + } + +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/UserAction.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/UserAction.java new file mode 100644 index 0000000..408d66a --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/UserAction.java @@ -0,0 +1,108 @@ +package edu.uci.iotproject.analysis; + +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; + +/** + * Models a user's action, such as toggling the smart plug on/off at a given time. + * + * @author Janus Varmarken + */ +public class UserAction { + + private static volatile DateTimeFormatter TIMESTAMP_FORMATTER = DateTimeFormatter.ISO_ZONED_DATE_TIME. + withZone(ZoneId.of("America/Los_Angeles")); + + /** + * Sets the {@link DateTimeFormatter} used when outputting a user action as a string and parsing a user action from + * a string. + * @param formatter The formatter to use for outputting and parsing. + */ + public static void setTimestampFormatter(DateTimeFormatter formatter) { + TIMESTAMP_FORMATTER = formatter; + } + + /** + * Instantiates a {@code UserAction} from a string that obeys the format used in {@link UserAction#toString()}. + * @param string The string that represents a {@code UserAction} + * @return A {@code UserAction} resulting from deserializing the string. + */ + public static UserAction fromString(String string) { + String[] parts = string.split("@"); + if (parts.length != 2) { + throw new IllegalArgumentException("Invalid string format"); + } + // If any of these two parses fail, an exception is thrown -- no need to check return values. + UserAction.Type actionType = UserAction.Type.valueOf(parts[0].trim()); + Instant timestamp = TIMESTAMP_FORMATTER.parse(parts[1].trim(), Instant::from); + return new UserAction(actionType, timestamp); + } + + + /** + * The specific type of action the user performed. + */ + private final Type mType; + + /** + * The time the action took place. + */ + private final Instant mTimestamp; + + public UserAction(Type typeOfAction, Instant timeOfAction) { + mType = typeOfAction; + mTimestamp = timeOfAction; + } + + /** + * Get the specific type of action performed by the user. + * @return the specific type of action performed by the user. + */ + public Type getType() { + return mType; + } + + /** + * Get the time at which the user performed this action. + * @return the time at which the user performed this action. + */ + public Instant getTimestamp() { + return mTimestamp; + } + + /** + * Enum for indicating what type of action the user performed. + */ + public enum Type { + TOGGLE_ON, TOGGLE_OFF + } + + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj instanceof UserAction) { + UserAction that = (UserAction) obj; + return this.mType == that.mType && this.mTimestamp.equals(that.mTimestamp); + } else { + return false; + } + } + + @Override + public int hashCode() { + final int prime = 31; + int hashCode = 17; + hashCode = prime * hashCode + mType.hashCode(); + hashCode = prime * hashCode + mTimestamp.hashCode(); + return hashCode; + } + + @Override + public String toString() { + return String.format("%s @ %s", mType.name(), TIMESTAMP_FORMATTER.format(mTimestamp)); + } +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/comparison/seqalignment/AlignmentPricer.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/comparison/seqalignment/AlignmentPricer.java new file mode 100644 index 0000000..0552279 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/comparison/seqalignment/AlignmentPricer.java @@ -0,0 +1,70 @@ +package edu.uci.iotproject.comparison.seqalignment; + +import java.util.function.ToIntBiFunction; +import java.util.function.ToIntFunction; + +/** + * Provides a generic implementation for the calculation of the cost of aligning two elements of a sequence as part of + * the sequence alignment algorithm (the algorithm is implemented in {@link SequenceAlignment}). + * + * @param The type of the elements that are being aligned. + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class AlignmentPricer { + + /** + * A function that provides the cost of aligning a {@link T} with a gap. + */ + private final ToIntFunction mGapCostFunction; + + /** + * A function that provides the cost of aligning a {@link T} with some other {@link T}. + */ + private final ToIntBiFunction mAlignmentCostFunction; + + /** + * Constructs a new {@link AlignmentPricer}. + * + * @param alignmentCostFunction A function that specifies the cost of aligning a {@link T} with some other {@link T} + * (e.g., based on the values of the properties of the two instances). + * @param gapCostFunction A function that specifies the cost of aligning a {@link T} with a gap. Note that the + * function is free to specify different gap costs for different {@link T}s. + */ + public AlignmentPricer(ToIntBiFunction alignmentCostFunction, ToIntFunction gapCostFunction) { + mAlignmentCostFunction = alignmentCostFunction; + mGapCostFunction = gapCostFunction; + } + + /** + * Calculate the cost of aligning {@code item1} with {@code item2}. If either of the two arguments is set to + * {@code null}, the cost of aligning the other argument with a gap will be returned. Note that both arguments + * cannot be {@code null} at the same time as that translates to aligning a gap with a gap, which is pointless. + * + * @param item1 The first of the two aligned objects. Set to {@code null} to calculate the cost of aligning + * {@code item2} with a gap. + * @param item2 The second of the two aligned objects. Set to {@code null} to calculate the cost of aligning + * {@code item2} with a gap. + * @return The cost of aligning {@code item1} with {@code item2}. + */ + public int alignmentCost(T item1, T item2) { + // If both arguments are null, the caller is aligning a gap with a gap which is pointless might as well remove + // both gaps in that case!) + if (item1 == null && item2 == null) { + throw new IllegalArgumentException("Both arguments cannot be null: you are aligning a gap with a gap!"); + } + // If one item is null, it means we're aligning an int with a gap. + // Invoke the provided gap cost function to get the gap cost. + if (item1 == null) { + return mGapCostFunction.applyAsInt(item2); + } + if (item2 == null) { + return mGapCostFunction.applyAsInt(item1); + } + // If both arguments are present, we simply delegate the task of calculating the cost of aligning the two items + // to the provided alignment cost function. + return mAlignmentCostFunction.applyAsInt(item1, item2); + } + +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/comparison/seqalignment/ExtractedSequence.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/comparison/seqalignment/ExtractedSequence.java new file mode 100644 index 0000000..2d193a9 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/comparison/seqalignment/ExtractedSequence.java @@ -0,0 +1,41 @@ +package edu.uci.iotproject.comparison.seqalignment; + +import edu.uci.iotproject.trafficreassembly.layer3.Conversation; +import org.pcap4j.core.PcapPacket; + +import java.util.List; + +/** + * TODO add class documentation. + * + * @author Janus Varmarken + */ +public class ExtractedSequence { + + private final Conversation mRepresentativeSequence; + + private final int mMaxAlignmentCost; + + private final String mSequenceString; + + public ExtractedSequence(Conversation sequence, int maxAlignmentCost, boolean tlsAppDataAlignment) { + mRepresentativeSequence = sequence; + mMaxAlignmentCost = maxAlignmentCost; + StringBuilder sb = new StringBuilder(); + List pkts = tlsAppDataAlignment ? sequence.getTlsApplicationDataPackets() : sequence.getPackets(); + pkts.forEach(p -> { + if (sb.length() != 0) sb.append(" "); + sb.append(p.getOriginalLength()); + }); + mSequenceString = sb.toString(); + } + + public Conversation getRepresentativeSequence() { + return mRepresentativeSequence; + } + + public int getMaxAlignmentCost() { + return mMaxAlignmentCost; + } + +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/comparison/seqalignment/SampleIntegerAlignmentPricer.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/comparison/seqalignment/SampleIntegerAlignmentPricer.java new file mode 100644 index 0000000..a09a10d --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/comparison/seqalignment/SampleIntegerAlignmentPricer.java @@ -0,0 +1,23 @@ +package edu.uci.iotproject.comparison.seqalignment; + +/** + * A sample {@link AlignmentPricer} for computing the cost of aligning integer values. In this sample implementation, + * the cost of aligning two integers {@code i1} and {@code i2} is {@code Math.abs(i1 - i2)}, i.e., it is the absolute + * value of the difference between {@code i1} and {@code i2}. The cost of aligning an integer {@code i} with a gap is + * simply {@code i}, i.e., the gap is essentially treated as a zero. + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class SampleIntegerAlignmentPricer extends AlignmentPricer { + + /** + * Constructs a new {@link SampleIntegerAlignmentPricer}. + */ + public SampleIntegerAlignmentPricer() { + // Cost of aligning integers i1 and i2 is the absolute value of their difference. + // Cost of aligning integer i with a gap is i (as it was aligned with 0). + super((i1,i2) -> Math.abs(i1 - i2) , (i) -> i); + } + +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/comparison/seqalignment/SequenceAlignment.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/comparison/seqalignment/SequenceAlignment.java new file mode 100644 index 0000000..005d7ff --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/comparison/seqalignment/SequenceAlignment.java @@ -0,0 +1,75 @@ +package edu.uci.iotproject.comparison.seqalignment; + +/** + * A generic implementation of the sequence alignment algorithm given in Kleinberg's and Tardos' "Algorithm Design". + * This implementation is the basic version. There is a more complex version which significantly reduces the space + * complexity at a slight cost to time complexity. + * + * @param The unit of the alignment, or, in other words, the granularity of the + * alignment. For example, for 'classical' string alignment (as in sequence alignment where we + * try to align two strings character by character -- the example most often used in books on + * algorithms) this would be a {@link Character}. As a second example, by specifying + * {@link String}, one can decrease the granularity so as to align blocks of characters + * (e.g., if one wants to align to two string arrays). + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class SequenceAlignment { + + + /** + * Provides the cost of aligning two {@link ALIGNMENT_UNIT}s with one another as well as the cost of aligning an + * {@link ALIGNMENT_UNIT} with a gap. + */ + private final AlignmentPricer mAlignmentPricer; + + /** + * Constructs a new {@link SequenceAlignment}. The new instance relies on the provided {@code alignmentPricer} to + * provide the cost of aligning two {@link ALIGNMENT_UNIT}s as well as the cost of aligning an + * {@link ALIGNMENT_UNIT} with a gap. + * + * @param alignmentPricer An {@link AlignmentPricer} that provides the cost of aligning two {@link ALIGNMENT_UNIT}s + * with one another as well as the cost of aligning an {@link ALIGNMENT_UNIT} with a gap. + */ + public SequenceAlignment(AlignmentPricer alignmentPricer) { + mAlignmentPricer = alignmentPricer; + } + + + /** + * Calculates the cost of aligning {@code sequence1} with {@code sequence2}. + * + * @param sequence1 A sequence that is to be aligned with {@code sequence2}. + * @param sequence2 A sequence that is to be aligned with {@code sequence1}. + * + * @return The cost of aligning {@code sequence1} with {@code sequence2}. + */ + public int calculateAlignment(ALIGNMENT_UNIT[] sequence1, ALIGNMENT_UNIT[] sequence2) { + int[][] costs = new int[sequence1.length + 1][sequence2.length +1]; + /* + * TODO: + * This is a homebrewn initialization; it is different from the one in the Kleinberg book - is it correct? + * It tries to add support for *different* gap costs depending on the input (e.g., such that one can say that + * matching a 'c' with a gap is more expensive than matching a 'b' with a gap). + */ + for (int i = 1; i <= sequence1.length; i++) { + costs[i][0] = mAlignmentPricer.alignmentCost(sequence1[i-1], null) + costs[i-1][0]; + } + for (int j = 1; j <= sequence2.length; j++) { + costs[0][j] = mAlignmentPricer.alignmentCost(sequence2[j-1], null) + costs[0][j-1]; + } + for (int j = 1; j <= sequence2.length; j++) { + for (int i = 1; i <= sequence1.length; i++) { + // The cost when current items of both sequences are aligned. + int costAligned = mAlignmentPricer.alignmentCost(sequence2[j-1], sequence1[i-1]) + costs[i-1][j-1]; + // The cost when current item from sequence1 is not aligned (it's matched with a gap) + int seq1ItemNotMached = mAlignmentPricer.alignmentCost(sequence1[i-1], null) + costs[i-1][j]; + // The cost when current item from sequence2 is not aligned (it's matched with a gap) + int seq2ItemNotMached = mAlignmentPricer.alignmentCost(sequence2[j-1], null) + costs[i][j-1]; + costs[i][j] = Math.min(costAligned, Math.min(seq1ItemNotMached, seq2ItemNotMached)); + } + } + return costs[sequence1.length][sequence2.length]; + } +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/comparison/seqalignment/SequenceExtraction.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/comparison/seqalignment/SequenceExtraction.java new file mode 100644 index 0000000..6aaa318 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/comparison/seqalignment/SequenceExtraction.java @@ -0,0 +1,152 @@ +package edu.uci.iotproject.comparison.seqalignment; + +import edu.uci.iotproject.trafficreassembly.layer3.Conversation; +import edu.uci.iotproject.analysis.TcpConversationUtils; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * TODO add class documentation. + * + * @author Janus Varmarken + */ +public class SequenceExtraction { + + + private final SequenceAlignment mAlignmentAlg; + + + public SequenceExtraction() { + mAlignmentAlg = new SequenceAlignment<>(new AlignmentPricer<>((i1,i2) -> Math.abs(i1-i2), i -> 10)); + } + + + public SequenceExtraction(SequenceAlignment alignmentAlgorithm) { + mAlignmentAlg = alignmentAlgorithm; + } + + /** + * Gets the {@link SequenceAlignment} used to perform the sequence extraction. + * @return the {@link SequenceAlignment} used to perform the sequence extraction. + */ + public SequenceAlignment getAlignmentAlgorithm() { + return mAlignmentAlg; + } + + // Initial +// /** +// * +// * @param convsForAction A set of {@link Conversation}s known to be associated with a single type of user action. +// */ +// public void extract(List convsForAction) { +// int maxDifference = 0; +// +// for (int i = 0; i < convsForAction.size(); i++) { +// for (int j = i+1; j < convsForAction.size(); i++) { +// Integer[] sequence1 = getPacketLengthSequence(convsForAction.get(i)); +// Integer[] sequence2 = getPacketLengthSequence(convsForAction.get(j)); +// int alignmentCost = mAlignmentAlg.calculateAlignment(sequence1, sequence2); +// if (alignmentCost > maxDifference) { +// maxDifference = alignmentCost; +// } +// } +// } +// +// } + + +// public void extract(Map> hostnameToConvs) { +// int maxDifference = 0; +// +// for (int i = 0; i < convsForAction.size(); i++) { +// for (int j = i+1; j < convsForAction.size(); i++) { +// Integer[] sequence1 = getPacketLengthSequence(convsForAction.get(i)); +// Integer[] sequence2 = getPacketLengthSequence(convsForAction.get(j)); +// int alignmentCost = mAlignmentAlg.calculateAlignment(sequence1, sequence2); +// if (alignmentCost > maxDifference) { +// maxDifference = alignmentCost; +// } +// } +// } +// +// } + + // Building signature from entire sequence + public ExtractedSequence extract(List convsForActionForHostname) { + // First group conversations by packet sequences. + // TODO: the introduction of SYN/SYNACK, FIN/FINACK and RST as part of the sequence ID may be undesirable here + // as it can potentially result in sequences that are equal in terms of payload packets to be considered + // different due to differences in how they are terminated. + Map> groupedBySequence = + TcpConversationUtils.groupConversationsByPacketSequence(convsForActionForHostname, false); + + // Then get a hold of one of the conversations that gave rise to the most frequent sequence. + Conversation mostFrequentConv = null; + int maxFrequency = 0; + for (Map.Entry> seqMapEntry : groupedBySequence.entrySet()) { + if (seqMapEntry.getValue().size() > maxFrequency) { + // Found a more frequent sequence + maxFrequency = seqMapEntry.getValue().size(); + // We just pick the first conversation as the representative conversation for this sequence type. + mostFrequentConv = seqMapEntry.getValue().get(0); + } else if (seqMapEntry.getValue().size() == maxFrequency) { + // This sequence has the same frequency as the max frequency seen so far. + // Break ties by choosing the longest sequence. + // First get an arbitrary representative of currently examined sequence; we just pick the first. + Conversation c = seqMapEntry.getValue().get(0); + mostFrequentConv = c.getPackets().size() > mostFrequentConv.getPackets().size() ? c : mostFrequentConv; + } + } + // Now find the maximum cost of aligning the most frequent (or, alternatively longest) conversation with the + // each of the rest of the conversations also associated with this action and hostname. + int maxCost = 0; + final Integer[] mostFrequentConvSeq = TcpConversationUtils.getPacketLengthSequence(mostFrequentConv); + for (Conversation c : convsForActionForHostname) { + if (c == mostFrequentConv) { + // Don't compute distance to self. + continue; + } + Integer[] cSeq = TcpConversationUtils.getPacketLengthSequence(c); + int alignmentCost = mAlignmentAlg.calculateAlignment(mostFrequentConvSeq, cSeq); + if (alignmentCost > maxCost) { + maxCost = alignmentCost; + } + } + return new ExtractedSequence(mostFrequentConv, maxCost, false); + } + + // Building signature from only TLS Application Data packets + public ExtractedSequence extractByTlsAppData(List convsForActionForHostname) { + // TODO: temporary hack to avoid 97-only conversations for dlink plug. We need some preprocessing/data cleaning. + convsForActionForHostname = convsForActionForHostname.stream().filter(c -> c.getTlsApplicationDataPackets().size() > 1).collect(Collectors.toList()); + + Map> groupedByTlsAppDataSequence = + TcpConversationUtils.groupConversationsByTlsApplicationDataPacketSequence(convsForActionForHostname); + // Get a Conversation representing the most frequent TLS application data sequence. + Conversation mostFrequentConv = groupedByTlsAppDataSequence.values().stream().max((l1, l2) -> { + // The frequency of a conversation with a specific packet sequence is the list size as that represents how + // many conversations exhibit that packet sequence. + // Hence, the difference between the list sizes can be used directly as the return value of the Comparator. + // Note: we break ties by choosing the one with the most TLS application data packets (i.e., the longest + // sequence) in case the frequencies are equal. + int diff = l1.size() - l2.size(); + return diff != 0 ? diff : l1.get(0).getTlsApplicationDataPackets().size() - l2.get(0).getTlsApplicationDataPackets().size(); + }).get().get(0); // Just pick the first as a representative of the most frequent sequence. + // Lengths of TLS Application Data packets in the most frequent (or most frequent and longest) conversation. + Integer[] mostFreqSeq = TcpConversationUtils.getPacketLengthSequenceTlsAppDataOnly(mostFrequentConv); + // Now find the maximum cost of aligning the most frequent (or, alternatively longest) conversation with the + // each of the rest of the conversations also associated with this action and hostname. + int maxCost = 0; + for (Conversation c : convsForActionForHostname) { + if (c == mostFrequentConv) continue; + int cost = mAlignmentAlg.calculateAlignment(mostFreqSeq, TcpConversationUtils.getPacketLengthSequenceTlsAppDataOnly(c)); + maxCost = cost > maxCost ? cost : maxCost; + } + return new ExtractedSequence(mostFrequentConv, maxCost, true); + // Now find the maximum cost of aligning the most frequent (or, alternatively longest) conversation with the + // each of the rest of the conversations also associated with this action and hostname. + } + +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/AbstractClusterMatcher.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/AbstractClusterMatcher.java new file mode 100644 index 0000000..45c6a55 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/AbstractClusterMatcher.java @@ -0,0 +1,72 @@ +package edu.uci.iotproject.detection; + +import org.pcap4j.core.PcapPacket; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +/** + * Base class for classes that search a traffic trace for sequences of packets that "belong to" a given cluster (in + * other words, classes that attempt to classify traffic as pertaining to a given cluster). + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +abstract public class AbstractClusterMatcher { + + /** + * The cluster that describes the sequence of packets that this {@link AbstractClusterMatcher} is trying to detect + * in the observed traffic. + */ + protected final List> mCluster; + + /** + * Observers registered for callbacks from this {@link AbstractClusterMatcher}. + */ + protected final List mObservers; + + protected AbstractClusterMatcher(List> cluster) { + // ===================== PRECONDITION SECTION ===================== + cluster = Objects.requireNonNull(cluster, "cluster cannot be null"); + if (cluster.isEmpty() || cluster.stream().anyMatch(inner -> inner.isEmpty())) { + throw new IllegalArgumentException("cluster is empty (or contains an empty inner List)"); + } + for (List clusterMember : cluster) { + if (clusterMember.size() != cluster.get(0).size()) { + throw new IllegalArgumentException("All sequences in cluster must contain the same number of packets"); + } + } + // ================================================================ + // Let the subclass prune the provided cluster + mCluster = pruneCluster(cluster); + mObservers = new ArrayList<>(); + } + + /** + * Register for callbacks from this cluster matcher. + * @param observer The target of the callbacks. + */ + public final void addObserver(ClusterMatcherObserver observer) { + mObservers.add(observer); + } + + /** + * Deregister for callbacks from this cluster matcher. + * @param observer The callback target that is to be deregistered. + */ + public final void removeObserver(ClusterMatcherObserver observer) { + mObservers.remove(observer); + } + + /** + * Allows subclasses to specify how to prune the input cluster provided to the constructor. + * @param cluster The input cluster provided to the constructor. + * @return The pruned cluster to use in place of the input cluster. + */ + abstract protected List> pruneCluster(List> cluster); + + // TODO: move Direction outside Conversation so that this is less confusing. +// abstract protected Conversation.Direction[] getPacketDirections(List packets); + +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/AbstractSignatureDetector.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/AbstractSignatureDetector.java new file mode 100644 index 0000000..b99116c --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/AbstractSignatureDetector.java @@ -0,0 +1,77 @@ +package edu.uci.iotproject.detection; + +import org.pcap4j.core.PcapPacket; + +import java.util.*; + +/** + * TODO add class documentation. + * + * @author Janus Varmarken + */ +public abstract class AbstractSignatureDetector implements ClusterMatcherObserver { + + + /** + * The signature that this {@link AbstractSignatureDetector} is searching for. + */ + private final List>> mSignature; + + /** + * The {@link AbstractClusterMatcher}s in charge of detecting each individual sequence of packets that together make + * up the the signature. + */ + private final List mClusterMatchers; + + /** + * For each {@code i} ({@code i >= 0 && i < pendingMatches.length}), {@code pendingMatches[i]} holds the matches + * found by the {@link AbstractClusterMatcher} at {@code mClusterMatchers.get(i)} that have yet to be "consumed", + * i.e., have yet to be included in a signature detected by this {@link AbstractSignatureDetector} (a signature can + * be encompassed of multiple packet sequences occurring shortly after one another on multiple connections). + */ + private final List>[] pendingMatches; + + /** + * Maps an {@link AbstractClusterMatcher} to its corresponding index in {@link #pendingMatches}. + */ + private final Map mClusterMatcherIds; + + public AbstractSignatureDetector(List>> searchedSignature) { + mSignature = Collections.unmodifiableList(searchedSignature); + List clusterMatchers = new ArrayList<>(); + for (List> cluster : mSignature) { + AbstractClusterMatcher clusterMatcher = constructClusterMatcher(cluster); + clusterMatcher.addObserver(this); + clusterMatchers.add(clusterMatcher); + } + mClusterMatchers = Collections.unmodifiableList(clusterMatchers); + pendingMatches = new List[mClusterMatchers.size()]; + for (int i = 0; i < pendingMatches.length; i++) { + pendingMatches[i] = new ArrayList<>(); + } + Map clusterMatcherIds = new HashMap<>(); + for (int i = 0; i < mClusterMatchers.size(); i++) { + clusterMatcherIds.put(mClusterMatchers.get(i), i); + } + mClusterMatcherIds = Collections.unmodifiableMap(clusterMatcherIds); + } + + abstract protected AbstractClusterMatcher constructClusterMatcher(List> cluster); + + /** + * Encapsulates a {@code List} so as to allow the list to be used as a vertex in a graph while avoiding + * the expensive {@link AbstractList#equals(Object)} calls when adding vertices to the graph. + * Using this wrapper makes the incurred {@code equals(Object)} calls delegate to {@link Object#equals(Object)} + * instead of {@link AbstractList#equals(Object)}. The net effect is a faster implementation, but the graph will not + * recognize two lists that contain the same items--from a value and not reference point of view--as the same + * vertex. However, this is fine for our purposes -- in fact restricting it to reference equality seems more + * appropriate. + */ + private static class Vertex { + private final List sequence; + private Vertex(List wrappedSequence) { + sequence = wrappedSequence; + } + } + +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/ClusterMatcherObserver.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/ClusterMatcherObserver.java new file mode 100644 index 0000000..d67c520 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/ClusterMatcherObserver.java @@ -0,0 +1,26 @@ +package edu.uci.iotproject.detection; + +import org.pcap4j.core.PcapPacket; + +import java.util.List; + +/** + * Interface used by client code to register for receiving a notification whenever an {@link AbstractClusterMatcher} + * detects traffic that matches an element of its associated cluster. + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public interface ClusterMatcherObserver { + + /** + * Callback that is invoked by an {@link AbstractClusterMatcher} whenever it detects traffic that matches an element + * of its associated cluster. + * + * @param clusterMatcher The {@link AbstractClusterMatcher} that detected a match (i.e., classified traffic as + * pertaining to its associated cluster). + * @param match The traffic that was deemed to match the cluster associated with {@code clusterMatcher}. + */ + void onMatch(AbstractClusterMatcher clusterMatcher, List match); + +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/SignatureDetectorObserver.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/SignatureDetectorObserver.java new file mode 100644 index 0000000..c2d8e09 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/SignatureDetectorObserver.java @@ -0,0 +1,22 @@ +package edu.uci.iotproject.detection; + +import org.pcap4j.core.PcapPacket; + +import java.util.List; + +/** + * Used for registering for notifications from a signature detector. + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public interface SignatureDetectorObserver { + + /** + * Invoked when the signature detector has detected the presence of a signature in the traffic that it's examining. + * @param searchedSignature The signature that the signature detector reporting the match is searching for. + * @param matchingTraffic The actual traffic trace that matches the searched signature. + */ + void onSignatureDetected(List>> searchedSignature, List> matchingTraffic); + +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2ClusterMatcher.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2ClusterMatcher.java new file mode 100644 index 0000000..5021c31 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2ClusterMatcher.java @@ -0,0 +1,153 @@ +package edu.uci.iotproject.detection.layer2; + +import edu.uci.iotproject.trafficreassembly.layer2.Layer2FlowReassembler; +import edu.uci.iotproject.trafficreassembly.layer2.Layer2Flow; +import edu.uci.iotproject.trafficreassembly.layer2.Layer2FlowReassemblerObserver; +import edu.uci.iotproject.detection.AbstractClusterMatcher; +import edu.uci.iotproject.trafficreassembly.layer2.Layer2FlowObserver; +import org.pcap4j.core.*; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +/** + * Attempts to detect members of a cluster (packet sequence mutations) in layer 2 flows. + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class Layer2ClusterMatcher extends AbstractClusterMatcher implements Layer2FlowReassemblerObserver, Layer2FlowObserver { + + /** + * Maps from a flow to a table of {@link Layer2SequenceMatcher}s for that particular flow. The table {@code t} is + * structured such that {@code t[i][j]} is a {@link Layer2SequenceMatcher} that attempts to match member {@code i} + * of {@link #mCluster} and has so far matched {@code j} packets of that particular sequence. + */ + private final Map mPerFlowSeqMatchers = new HashMap<>(); + + private final Function mFlowFilter; + + /** + * Create a new {@link Layer2ClusterMatcher} that attempts to find occurrences of {@code cluster}'s members. + * @param cluster The sequence mutations that the new {@link Layer2ClusterMatcher} should search for. + */ + public Layer2ClusterMatcher(List> cluster) { + // Consider all flows if no flow filter specified. + this(cluster, flow -> true); + } + + /** + * Create a new {@link Layer2ClusterMatcher} that attempts to find occurrences of {@code cluster}'s members. + * @param cluster The sequence mutations that the new {@link Layer2ClusterMatcher} should search for. + * @param flowFilter A filter that defines what {@link Layer2Flow}s the new {@link Layer2ClusterMatcher} should + * search for {@code cluster}'s members in. If {@code flowFilter} returns {@code true}, the flow + * will be included (searched). Note that {@code flowFilter} is only queried once for each flow, + * namely when the {@link Layer2FlowReassembler} notifies the {@link Layer2ClusterMatcher} about + * the new flow. This functionality may for example come in handy when one only wants to search + * for matches in the subset of flows that involves a specific (range of) MAC(s). + */ + public Layer2ClusterMatcher(List> cluster, Function flowFilter) { + super(cluster); + mFlowFilter = flowFilter; + } + + @Override + public void onNewPacket(Layer2Flow flow, PcapPacket newPacket) { + if (mPerFlowSeqMatchers.get(flow) == null) { + // If this is the first time we encounter this flow, we need to set up sequence matchers for it. + // All sequences of the cluster have the same length, so we only need to compute the length of the nested + // arrays once. We want to make room for a cluster matcher in each state, including the initial empty state + // but excluding the final "full match" state (as there is no point in keeping a terminated sequence matcher + // around), so the length of the inner array is simply the sequence length. + Layer2SequenceMatcher[][] matchers = new Layer2SequenceMatcher[mCluster.size()][mCluster.get(0).size()]; + // Prepare a "state 0" sequence matcher for each sequence variation in the cluster. + for (int i = 0; i < matchers.length; i++) { + matchers[i][0] = new Layer2SequenceMatcher(mCluster.get(i)); + } + // Associate the new sequence matcher table with the new flow + mPerFlowSeqMatchers.put(flow, matchers); + } + // Fetch table that contains sequence matchers for this flow. + Layer2SequenceMatcher[][] matchers = mPerFlowSeqMatchers.get(flow); + // Present the packet to all sequence matchers. + for (int i = 0; i < matchers.length; i++) { + // Present packet to the sequence matchers that has advanced the most first. This is to prevent discarding + // the sequence matchers that have advanced the most in the special case where the searched sequence + // contains two packets of the same length going in the same direction. + for (int j = matchers[i].length - 1; j >= 0 ; j--) { + Layer2SequenceMatcher sm = matchers[i][j]; + if (sm == null) { + // There is currently no sequence matcher that has managed to match j packets. + continue; + } + boolean matched = sm.matchPacket(newPacket); + if (matched) { + if (sm.getMatchedPacketsCount() == sm.getTargetSequencePacketCount()) { + // Sequence matcher has a match. Report it to observers. + mObservers.forEach(o -> o.onMatch(this, sm.getMatchedPackets())); + // Remove the now terminated sequence matcher. + matchers[i][j] = null; + } else { + // Sequence matcher advanced one step, so move it to its corresponding new position iff the + // packet that advanced it has a later timestamp than that of the last matched packet of the + // sequence matcher at the new index, if any. In most traces, a small amount of the packets + // appear out of order (with regards to their timestamp), which is why this check is required. + // Obviously it would not be needed if packets where guaranteed to be processed in timestamp + // order here. + if (matchers[i][j+1] == null || + newPacket.getTimestamp().isAfter(matchers[i][j+1].getLastPacket().getTimestamp())) { + matchers[i][j+1] = sm; + } + } + // We always want to have a sequence matcher in state 0, regardless of if the one that advanced + // from state zero completed its matching or if it replaced a different one in state 1 or not. + if (sm.getMatchedPacketsCount() == 1) { + matchers[i][j] = new Layer2SequenceMatcher(sm.getTargetSequence()); + } + } + } + } + } + + + @Override + protected List> pruneCluster(List> cluster) { + // Note: we assume that all sequences in the input cluster are of the same length and that their packet + // directions are identical. + List> prunedCluster = new ArrayList<>(); + for (List originalClusterSeq : cluster) { + boolean alreadyPresent = prunedCluster.stream().anyMatch(pcPkts -> { + for (int i = 0; i < pcPkts.size(); i++) { + if (pcPkts.get(i).getOriginalLength() != originalClusterSeq.get(i).getOriginalLength()) { + return false; + } + } + return true; + }); + if (!alreadyPresent) { + // Add the sequence if not already present in the pruned cluster. + prunedCluster.add(originalClusterSeq); + } + } + return prunedCluster; + } + + private static final boolean DEBUG = false; + + @Override + public void onNewFlow(Layer2FlowReassembler reassembler, Layer2Flow newFlow) { + // New flow detected. Check if we should consider it when searching for cluster member matches. + if (mFlowFilter.apply(newFlow)) { + if (DEBUG) { + System.out.println(">>> ACCEPTING FLOW: " + newFlow + " <<<"); + } + // Subscribe to the new flow to get updates whenever a new packet pertaining to the flow is processed. + newFlow.addFlowObserver(this); + } else if (DEBUG) { + System.out.println(">>> IGNORING FLOW: " + newFlow + " <<<"); + } + } +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SequenceMatcher.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SequenceMatcher.java new file mode 100644 index 0000000..672fb72 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SequenceMatcher.java @@ -0,0 +1,173 @@ +package edu.uci.iotproject.detection.layer2; + +import edu.uci.iotproject.analysis.TriggerTrafficExtractor; +import edu.uci.iotproject.util.PcapPacketUtils; +import org.pcap4j.core.PcapPacket; +import org.pcap4j.util.MacAddress; + +import java.util.ArrayList; +import java.util.List; + +/** + * Attempts to detect the presence of a specific packet sequence in the set of packets provided through multiple calls + * to {@link #matchPacket(PcapPacket)}, considering only layer 2 information. + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class Layer2SequenceMatcher { + + /** + * The sequence this {@link Layer2SequenceMatcher} is searching for. + */ + private final List mSequence; + + /** + * Buffer of actual packets seen so far that match the searched sequence (i.e., constitutes a subsequence of the + * searched sequence). + */ + private final List mMatchedPackets = new ArrayList<>(); + + /** + * Models the directions of packets in {@link #mSequence}. As the sequence matcher assumes that it is only presented + * with packet from a single flow (packets exchanged between two devices), we can model the packet directions with a + * single bit. We don't have any notion "phone to device" or "device to phone" as we don't know the MAC addresses + * of devices in advance during matching. + */ + private final boolean[] mPacketDirections; + + /** + * Create a {@code Layer2SequenceMatcher}. + * @param sequence The sequence to match against (search for). + */ + public Layer2SequenceMatcher(List sequence) { + mSequence = sequence; + // Compute packet directions for sequence. + mPacketDirections = new boolean[sequence.size()]; + for (int i = 0; i < sequence.size(); i++) { + if (i == 0) { + // No previous packet; boolean parameter is ignored in this special case. + mPacketDirections[i] = getPacketDirection(null, true, sequence.get(i)); + } else { + // Base direction marker on direction of previous packet. + PcapPacket prevPkt = mSequence.get(i-1); + boolean prevPktDirection = mPacketDirections[i-1]; + mPacketDirections[i] = getPacketDirection(prevPkt, prevPktDirection, sequence.get(i)); + } + } + } + + /** + * Attempt to advance this {@code Layer2SequenceMatcher} by matching {@code packet} against the packet that this + * {@code Layer2SequenceMatcher} expects as the next packet of the sequence it is searching for. + * @param packet + * @return {@code true} if this {@code Layer2SequenceMatcher} could advance by adding {@code packet} to its set of + * matched packets, {@code false} otherwise. + */ + public boolean matchPacket(PcapPacket packet) { + if (getMatchedPacketsCount() == getTargetSequencePacketCount()) { + // We already matched the entire sequence, so we can't match any more packets. + return false; + } + + // Verify that new packet pertains to same flow as previously matched packets, if any. + if (getMatchedPacketsCount() > 0) { + MacAddress pktSrc = PcapPacketUtils.getEthSrcAddr(packet); + MacAddress pktDst = PcapPacketUtils.getEthDstAddr(packet); + MacAddress earlierPktSrc = PcapPacketUtils.getEthSrcAddr(mMatchedPackets.get(0)); + MacAddress earlierPktDst = PcapPacketUtils.getEthDstAddr(mMatchedPackets.get(0)); + if (!(pktSrc.equals(earlierPktSrc) && pktDst.equals(earlierPktDst) || + pktSrc.equals(earlierPktDst) && pktDst.equals(earlierPktSrc))) { + return false; + } + } + + // Get representative of the packet we expect to match next. + PcapPacket expected = mSequence.get(mMatchedPackets.size()); + // First verify if the received packet has the length we're looking for. + if (packet.getOriginalLength() == expected.getOriginalLength()) { + // If this is the first packet, we only need to verify that its length is correct. Time constraints are + // obviously satisfied as there are no previous packets. Furthermore, direction matches by definition as we + // don't know the MAC of the device (or phone) in advance, so we can't enforce a rule saying "first packet + // must originate from this particular MAC". + if (getMatchedPacketsCount() == 0) { + // Store packet as matched and advance. + mMatchedPackets.add(packet); + return true; + } + // Check if direction of packet matches expected direction. + boolean actualDirection = getPacketDirection(mMatchedPackets.get(getMatchedPacketsCount()-1), + mPacketDirections[getMatchedPacketsCount()-1], packet); + boolean expectedDirection = mPacketDirections[getMatchedPacketsCount()]; + if (actualDirection != expectedDirection) { + return false; + } + // Next apply timing constraints: + // 1: to be a match, the packet must have a later timestamp than any other packet currently matched + // 2: does adding the packet cause the max allowed time between first packet and last packet to be exceeded? + if (!packet.getTimestamp().isAfter(mMatchedPackets.get(getMatchedPacketsCount()-1).getTimestamp())) { + return false; + } + if (packet.getTimestamp().isAfter(mMatchedPackets.get(0).getTimestamp(). + plusMillis(TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS))) { + return false; + } + // If we made it here, it means that this packet has the expected length, direction, and obeys the timing + // constraints, so we store it and advance. + mMatchedPackets.add(packet); + if (mMatchedPackets.size() == mSequence.size()) { + // TODO report (to observers?) that we are done? + } + return true; + } + return false; + } + + public int getMatchedPacketsCount() { + return mMatchedPackets.size(); + } + + public int getTargetSequencePacketCount() { + return mSequence.size(); + } + + public List getTargetSequence() { + return mSequence; + } + + public List getMatchedPackets() { + return mMatchedPackets; + } + + /** + * Utility for {@code getMatchedPackets().get(getMatchedPackets().size()-1)}. + * @return The last matched packet, or {@code null} if no packets have been matched yet. + */ + public PcapPacket getLastPacket() { + return mSequence.size() > 0 ? mSequence.get(mSequence.size()-1) : null; + } + + /** + * Compute the direction of a packet based on the previous packet. If no previous packet is provided, the direction + * of {@code currPkt} is {@code true} by definition. + * @param prevPkt The previous packet, if any. + * @param prevPktDirection The computed direction of the previous packet + * @param currPkt The current packet for which the direction is to be determined. + * @return The direction of {@code currPkt}. + */ + private boolean getPacketDirection(PcapPacket prevPkt, boolean prevPktDirection, PcapPacket currPkt) { + if (prevPkt == null) { + // By definition, use true as direction marker for first packet + return true; + } + if (PcapPacketUtils.getEthSrcAddr(prevPkt).equals(PcapPacketUtils.getEthSrcAddr(currPkt))) { + // Current packet goes in same direction as previous packet. + return prevPktDirection; + } else { + // Current packet goes in opposite direction of previous packet. + return !prevPktDirection; + } + } + + +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SignatureDetector.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SignatureDetector.java new file mode 100644 index 0000000..a721914 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SignatureDetector.java @@ -0,0 +1,352 @@ +package edu.uci.iotproject.detection.layer2; + +import edu.uci.iotproject.analysis.TriggerTrafficExtractor; +import edu.uci.iotproject.analysis.UserAction; +import edu.uci.iotproject.detection.AbstractClusterMatcher; +import edu.uci.iotproject.detection.ClusterMatcherObserver; +import edu.uci.iotproject.detection.SignatureDetectorObserver; +import edu.uci.iotproject.io.PcapHandleReader; +import edu.uci.iotproject.io.PrintWriterUtils; +import edu.uci.iotproject.trafficreassembly.layer2.Layer2Flow; +import edu.uci.iotproject.trafficreassembly.layer2.Layer2FlowReassembler; +import edu.uci.iotproject.util.PrintUtils; +import org.jgrapht.GraphPath; +import org.jgrapht.alg.shortestpath.DijkstraShortestPath; +import org.jgrapht.graph.DefaultWeightedEdge; +import org.jgrapht.graph.SimpleDirectedWeightedGraph; +import org.pcap4j.core.*; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintWriter; +import java.time.Duration; +import java.util.*; +import java.util.function.Function; +import java.util.regex.Pattern; + +/** + * Performs layer 2 signature detection. + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class Layer2SignatureDetector implements PacketListener, ClusterMatcherObserver { + + /** + * If set to {@code true}, output written to the results file is also dumped to standard out. + */ + private static boolean DUPLICATE_OUTPUT_TO_STD_OUT = true; + + private static List> parseSignatureMacFilters(String filtersString) { + List> filters = new ArrayList<>(); + String[] filterRegexes = filtersString.split(";"); + for (String filterRegex : filterRegexes) { + final Pattern regex = Pattern.compile(filterRegex); + // Create a filter that includes all flows where one of the two MAC addresses match the regex. + filters.add(flow -> regex.matcher(flow.getEndpoint1().toString()).matches() || regex.matcher(flow.getEndpoint2().toString()).matches()); + } + return filters; + } + + public static void main(String[] args) throws PcapNativeException, NotOpenException, IOException { + // Parse required parameters. + if (args.length < 5) { + String errMsg = String.format("Usage: %s inputPcapFile onSignatureFile offSignatureFile resultsFile" + + "\n inputPcapFile: the target of the detection" + + "\n onSignatureFile: the file that contains the ON signature to search for" + + "\n offSignatureFile: the file that contains the OFF signature to search for" + + "\n resultsFile: where to write the results of the detection" + + "\n signatureDuration: the maximum duration of signature detection", + Layer2SignatureDetector.class.getSimpleName()); + System.out.println(errMsg); + String optParamsExplained = "Above are the required, positional arguments. In addition to these, the " + + "following options and associated positional arguments may be used:\n" + + " '-onmacfilters ;;...;' which specifies that sequence matching should ONLY" + + " be performed on flows where the MAC of one of the two endpoints matches the given regex. Note " + + "that you MUST specify a regex for each cluster of the signature. This is to facilitate more " + + "aggressive filtering on parts of the signature (e.g., the communication that involves the " + + "smart home device itself as one can drop all flows that do not include an endpoint with a MAC " + + "that matches the vendor's prefix).\n" + + " '-offmacfilters ;;...;' works exactly the same as onmacfilters, but " + + "applies to the OFF signature instead of the ON signature.\n" + + " '-sout ' true/false literal indicating if output should also be printed to std out; default is true."; + System.out.println(optParamsExplained); + return; + } + final String pcapFile = args[0]; + final String onSignatureFile = args[1]; + final String offSignatureFile = args[2]; + final String resultsFile = args[3]; + final int signatureDuration = Integer.parseInt(args[4]); + + // Parse optional parameters. + List> onSignatureMacFilters = null, offSignatureMacFilters = null; + final int optParamsStartIdx = 5; + if (args.length > optParamsStartIdx) { + for (int i = optParamsStartIdx; i < args.length; i++) { + if (args[i].equalsIgnoreCase("-onMacFilters")) { + // Next argument is the cluster-wise MAC filters (separated by semicolons). + onSignatureMacFilters = parseSignatureMacFilters(args[i+1]); + } else if (args[i].equalsIgnoreCase("-offMacFilters")) { + // Next argument is the cluster-wise MAC filters (separated by semicolons). + offSignatureMacFilters = parseSignatureMacFilters(args[i+1]); + } else if (args[i].equalsIgnoreCase("-sout")) { + // Next argument is a boolean true/false literal. + DUPLICATE_OUTPUT_TO_STD_OUT = Boolean.parseBoolean(args[i+1]); + } + } + } + + // Prepare file outputter. + File outputFile = new File(resultsFile); + outputFile.getParentFile().mkdirs(); + final PrintWriter resultsWriter = new PrintWriter(new FileWriter(outputFile)); + // Include metadata as comments at the top + PrintWriterUtils.println("# Detection results for:", resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println("# - inputPcapFile: " + pcapFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println("# - onSignatureFile: " + onSignatureFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println("# - offSignatureFile: " + offSignatureFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); + resultsWriter.flush(); + + // Create signature detectors and add observers that output their detected events. + List>> onSignature = PrintUtils.deserializeSignatureFromFile(onSignatureFile); + List>> offSignature = PrintUtils.deserializeSignatureFromFile(offSignatureFile); + Layer2SignatureDetector onDetector = onSignatureMacFilters == null ? + new Layer2SignatureDetector(onSignature) : new Layer2SignatureDetector(onSignature, onSignatureMacFilters, signatureDuration); + Layer2SignatureDetector offDetector = offSignatureMacFilters == null ? + new Layer2SignatureDetector(offSignature) : new Layer2SignatureDetector(offSignature, offSignatureMacFilters, signatureDuration); + onDetector.addObserver((signature, match) -> { + UserAction event = new UserAction(UserAction.Type.TOGGLE_ON, match.get(0).get(0).getTimestamp()); + PrintWriterUtils.println(event, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); + }); + offDetector.addObserver((signature, match) -> { + UserAction event = new UserAction(UserAction.Type.TOGGLE_OFF, match.get(0).get(0).getTimestamp()); + PrintWriterUtils.println(event, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); + }); + + // Load the PCAP file + PcapHandle handle; + try { + handle = Pcaps.openOffline(pcapFile, PcapHandle.TimestampPrecision.NANO); + } catch (PcapNativeException pne) { + handle = Pcaps.openOffline(pcapFile); + } + PcapHandleReader reader = new PcapHandleReader(handle, p -> true, onDetector, offDetector); + // Parse the file + reader.readFromHandle(); + + // Flush output to results file and close it. + resultsWriter.flush(); + resultsWriter.close(); + } + + /** + * The signature that this {@link Layer2SignatureDetector} is searching for. + */ + private final List>> mSignature; + + /** + * The {@link Layer2ClusterMatcher}s in charge of detecting each individual sequence of packets that together make + * up the the signature. + */ + private final List mClusterMatchers; + + /** + * For each {@code i} ({@code i >= 0 && i < mPendingMatches.length}), {@code mPendingMatches[i]} holds the matches + * found by the {@link Layer2ClusterMatcher} at {@code mClusterMatchers.get(i)} that have yet to be "consumed", + * i.e., have yet to be included in a signature detected by this {@link Layer2SignatureDetector} (a signature can + * be encompassed of multiple packet sequences occurring shortly after one another on multiple connections). + */ + private final List>[] mPendingMatches; + + /** + * Maps a {@link Layer2ClusterMatcher} to its corresponding index in {@link #mPendingMatches}. + */ + private final Map mClusterMatcherIds; + + /** + * In charge of reassembling layer 2 packet flows. + */ + private final Layer2FlowReassembler mFlowReassembler = new Layer2FlowReassembler(); + + private final List mObservers = new ArrayList<>(); + + private int mInclusionTimeMillis; + + public Layer2SignatureDetector(List>> searchedSignature) { + this(searchedSignature, null, 0); + } + + public Layer2SignatureDetector(List>> searchedSignature, List> flowFilters, int inclusionTimeMillis) { + if (flowFilters != null && flowFilters.size() != searchedSignature.size()) { + throw new IllegalArgumentException("If flow filters are used, there must be a flow filter for each cluster of the signature."); + } + mSignature = Collections.unmodifiableList(searchedSignature); + List clusterMatchers = new ArrayList<>(); + for (int i = 0; i < mSignature.size(); i++) { + List> cluster = mSignature.get(i); + Layer2ClusterMatcher clusterMatcher = flowFilters == null ? + new Layer2ClusterMatcher(cluster) : new Layer2ClusterMatcher(cluster, flowFilters.get(i)); + clusterMatcher.addObserver(this); + clusterMatchers.add(clusterMatcher); + } + mClusterMatchers = Collections.unmodifiableList(clusterMatchers); + mPendingMatches = new List[mClusterMatchers.size()]; + for (int i = 0; i < mPendingMatches.length; i++) { + mPendingMatches[i] = new ArrayList<>(); + } + Map clusterMatcherIds = new HashMap<>(); + for (int i = 0; i < mClusterMatchers.size(); i++) { + clusterMatcherIds.put(mClusterMatchers.get(i), i); + } + mClusterMatcherIds = Collections.unmodifiableMap(clusterMatcherIds); + // Register all cluster matchers to receive a notification whenever a new flow is encountered. + mClusterMatchers.forEach(cm -> mFlowReassembler.addObserver(cm)); + mInclusionTimeMillis = + inclusionTimeMillis == 0 ? TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS : inclusionTimeMillis; + } + + @Override + public void gotPacket(PcapPacket packet) { + // Forward packet processing to the flow reassembler that in turn notifies the cluster matchers as appropriate + mFlowReassembler.gotPacket(packet); + } + + @Override + public void onMatch(AbstractClusterMatcher clusterMatcher, List match) { + // TODO: a cluster matcher found a match + if (clusterMatcher instanceof Layer2ClusterMatcher) { + // Add the match at the corresponding index + mPendingMatches[mClusterMatcherIds.get(clusterMatcher)].add(match); + checkSignatureMatch(); + } + } + + public void addObserver(SignatureDetectorObserver observer) { + mObservers.add(observer); + } + + public boolean removeObserver(SignatureDetectorObserver observer) { + return mObservers.remove(observer); + } + + + @SuppressWarnings("Duplicates") + private void checkSignatureMatch() { + // << Graph-based approach using Balint's idea. >> + // This implementation assumes that the packets in the inner lists (the sequences) are ordered by asc timestamp. + + // There cannot be a signature match until each Layer3ClusterMatcher has found a match of its respective sequence. + if (Arrays.stream(mPendingMatches).noneMatch(l -> l.isEmpty())) { + // Construct the DAG + final SimpleDirectedWeightedGraph graph = + new SimpleDirectedWeightedGraph<>(DefaultWeightedEdge.class); + // Add a vertex for each match found by all cluster matchers. + // And maintain an array to keep track of what cluster matcher each vertex corresponds to + final List[] vertices = new List[mPendingMatches.length]; + for (int i = 0; i < mPendingMatches.length; i++) { + vertices[i] = new ArrayList<>(); + for (List sequence : mPendingMatches[i]) { + Vertex v = new Vertex(sequence); + vertices[i].add(v); // retain reference for later when we are to add edges + graph.addVertex(v); // add to vertex to graph + } + } + // Add dummy source and sink vertices to facilitate search. + final Vertex source = new Vertex(null); + final Vertex sink = new Vertex(null); + graph.addVertex(source); + graph.addVertex(sink); + // The source is connected to all vertices that wrap the sequences detected by cluster matcher at index 0. + // Note: zero cost edges as this is just a dummy link to facilitate search from a common start node. + for (Vertex v : vertices[0]) { + DefaultWeightedEdge edge = graph.addEdge(source, v); + graph.setEdgeWeight(edge, 0.0); + } + // Similarly, all vertices that wrap the sequences detected by the last cluster matcher of the signature + // are connected to the sink node. + for (Vertex v : vertices[vertices.length-1]) { + DefaultWeightedEdge edge = graph.addEdge(v, sink); + graph.setEdgeWeight(edge, 0.0); + } + // Now link sequences detected by the cluster matcher at index i to sequences detected by the cluster + // matcher at index i+1 if they obey the timestamp constraint (i.e., that the latter is later in time than + // the former). + for (int i = 0; i < vertices.length; i++) { + int j = i + 1; + if (j < vertices.length) { + for (Vertex iv : vertices[i]) { + PcapPacket ivLast = iv.sequence.get(iv.sequence.size()-1); + for (Vertex jv : vertices[j]) { + PcapPacket jvFirst = jv.sequence.get(jv.sequence.size()-1); + if (ivLast.getTimestamp().isBefore(jvFirst.getTimestamp())) { + DefaultWeightedEdge edge = graph.addEdge(iv, jv); + // The weight is the duration of the i'th sequence plus the duration between the i'th + // and i+1'th sequence. + Duration d = Duration. + between(iv.sequence.get(0).getTimestamp(), jvFirst.getTimestamp()); + // Unfortunately weights are double values, so must convert from long to double. + // TODO: need nano second precision? If so, use d.toNanos(). + // TODO: risk of overflow when converting from long to double..? + graph.setEdgeWeight(edge, Long.valueOf(d.toMillis()).doubleValue()); + } + // Alternative version if we cannot assume that sequences are ordered by timestamp: +// if (iv.sequence.stream().max(Comparator.comparing(PcapPacket::getTimestamp)).get() +// .getTimestamp().isBefore(jv.sequence.stream().min( +// Comparator.comparing(PcapPacket::getTimestamp)).get().getTimestamp())) { +// +// } + } + } + } + } + // Graph construction complete, run shortest-path to find a (potential) signature match. + DijkstraShortestPath dijkstra = new DijkstraShortestPath<>(graph); + GraphPath shortestPath = dijkstra.getPath(source, sink); + if (shortestPath != null) { + // The total weight is the duration between the first packet of the first sequence and the last packet + // of the last sequence, so we simply have to compare the weight against the timeframe that we allow + // the signature to span. For now we just use the inclusion window we defined for training purposes. + // Note however, that we must convert back from double to long as the weight is stored as a double in + // JGraphT's API. + if (((long)shortestPath.getWeight()) < mInclusionTimeMillis) { + // There's a signature match! + // Extract the match from the vertices + List> signatureMatch = new ArrayList<>(); + for(Vertex v : shortestPath.getVertexList()) { + if (v == source || v == sink) { + // Skip the dummy source and sink nodes. + continue; + } + signatureMatch.add(v.sequence); + // As there is a one-to-one correspondence between vertices[] and pendingMatches[], we know that + // the sequence we've "consumed" for index i of the matched signature is also at index i in + // pendingMatches. We must remove it from pendingMatches so that we don't use it to construct + // another signature match in a later call. + mPendingMatches[signatureMatch.size()-1].remove(v.sequence); + } + // Declare success: notify observers + mObservers.forEach(obs -> obs.onSignatureDetected(mSignature, + Collections.unmodifiableList(signatureMatch))); + } + } + } + } + + /** + * Encapsulates a {@code List} so as to allow the list to be used as a vertex in a graph while avoiding + * the expensive {@link AbstractList#equals(Object)} calls when adding vertices to the graph. + * Using this wrapper makes the incurred {@code equals(Object)} calls delegate to {@link Object#equals(Object)} + * instead of {@link AbstractList#equals(Object)}. The net effect is a faster implementation, but the graph will not + * recognize two lists that contain the same items--from a value and not reference point of view--as the same + * vertex. However, this is fine for our purposes -- in fact restricting it to reference equality seems more + * appropriate. + */ + private static class Vertex { + private final List sequence; + private Vertex(List wrappedSequence) { + sequence = wrappedSequence; + } + } +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3ClusterMatcher.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3ClusterMatcher.java new file mode 100644 index 0000000..b9584ff --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3ClusterMatcher.java @@ -0,0 +1,334 @@ +package edu.uci.iotproject.detection.layer3; + +import edu.uci.iotproject.detection.AbstractClusterMatcher; +import edu.uci.iotproject.detection.ClusterMatcherObserver; +import edu.uci.iotproject.trafficreassembly.layer3.Conversation; +import edu.uci.iotproject.trafficreassembly.layer3.TcpReassembler; +import edu.uci.iotproject.analysis.TcpConversationUtils; +import edu.uci.iotproject.io.PcapHandleReader; +import edu.uci.iotproject.util.PrintUtils; +import org.pcap4j.core.*; + +import java.time.ZoneId; +import java.util.*; +import java.util.stream.Collectors; + +import static edu.uci.iotproject.util.PcapPacketUtils.*; + +/** + * Searches a traffic trace for sequences of packets "belong to" a given cluster (in other words, attempts to classify + * traffic as pertaining to a given cluster). + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class Layer3ClusterMatcher extends AbstractClusterMatcher implements PacketListener { + + // Test client + public static void main(String[] args) throws PcapNativeException, NotOpenException { + +// String path = "/scratch/July-2018"; // Rahmadi + String path = "/Users/varmarken/temp/UCI IoT Project/experiments"; // Janus + final String inputPcapFile = path + "/2018-07/dlink/dlink.wlan1.local.pcap"; + final String signatureFile = path + "/2018-07/dlink/offSignature1.sig"; + + List> signature = PrintUtils.deserializeClustersFromFile(signatureFile); + Layer3ClusterMatcher clusterMatcher = new Layer3ClusterMatcher(signature, null, + (sig, match) -> System.out.println( + String.format("[ !!! SIGNATURE DETECTED AT %s !!! ]", + match.get(0).getTimestamp().atZone(ZoneId.of("America/Los_Angeles"))) + ) + ); + + PcapHandle handle; + try { + handle = Pcaps.openOffline(inputPcapFile, PcapHandle.TimestampPrecision.NANO); + } catch (PcapNativeException pne) { + handle = Pcaps.openOffline(inputPcapFile); + } + PcapHandleReader reader = new PcapHandleReader(handle, p -> true, clusterMatcher); + reader.readFromHandle(); + clusterMatcher.performDetection(); + } + + /** + * The ordered directions of packets in the sequences that make up {@link #mCluster}. + */ + private final Conversation.Direction[] mClusterMemberDirections; + + /** + * For reassembling the observed traffic into TCP connections. + */ + private final TcpReassembler mTcpReassembler = new TcpReassembler(); + + /** + * IP of the router's WAN port (if analyzed traffic is captured at the ISP's point of view). + */ + private final String mRouterWanIp; + + /** + * Create a {@link Layer3ClusterMatcher}. + * @param cluster The cluster that traffic is matched against. + * @param routerWanIp The router's WAN IP if examining traffic captured at the ISP's point of view (used for + * determining the direction of packets). + * @param detectionObservers Client code that wants to get notified whenever the {@link Layer3ClusterMatcher} detects that + * (a subset of) the examined traffic is similar to the traffic that makes up + * {@code cluster}, i.e., when the examined traffic is classified as pertaining to + * {@code cluster}. + */ + public Layer3ClusterMatcher(List> cluster, String routerWanIp, + ClusterMatcherObserver... detectionObservers) { + super(cluster); + Objects.requireNonNull(detectionObservers, "detectionObservers cannot be null"); + for (ClusterMatcherObserver obs : detectionObservers) { + addObserver(obs); + } + // Build the cluster members' direction sequence. + // Note: assumes that the provided cluster was captured within the local network (routerWanIp is set to null). + mClusterMemberDirections = getPacketDirections(cluster.get(0), null); + /* + * Enforce restriction on cluster members: all representatives must exhibit the same direction pattern and + * contain the same number of packets. Note that this is a somewhat heavy operation, so it may be disabled later + * on in favor of performance. However, it is only run once (at instantiation), so the overhead may be warranted + * in order to ensure correctness, especially during the development/debugging phase. + */ + if (mCluster.stream(). + anyMatch(inner -> !Arrays.equals(mClusterMemberDirections, getPacketDirections(inner, null)))) { + throw new IllegalArgumentException( + "cluster members must contain the same number of packets and exhibit the same packet direction " + + "pattern" + ); + } + mRouterWanIp = routerWanIp; + } + + @Override + public void gotPacket(PcapPacket packet) { + // Present packet to TCP reassembler so that it can be mapped to a connection (if it is a TCP packet). + mTcpReassembler.gotPacket(packet); + } + + /** + * Get the cluster that describes the packet sequence that this {@link Layer3ClusterMatcher} is searching for. + * @return the cluster that describes the packet sequence that this {@link Layer3ClusterMatcher} is searching for. + */ + public List> getCluster() { + return mCluster; + } + + public void performDetection() { + /* + * Let's start out simple by building a version that only works for signatures that do not span across multiple + * TCP conversations... + */ + for (Conversation c : mTcpReassembler.getTcpConversations()) { + if (c.isTls() && c.getTlsApplicationDataPackets().isEmpty() || !c.isTls() && c.getPackets().isEmpty()) { + // Skip empty conversations. + continue; + } + for (List signatureSequence : mCluster) { + if (isTlsSequence(signatureSequence) != c.isTls()) { + // We consider it a mismatch if one is a TLS application data sequence and the other is not. + continue; + } + // Fetch set of packets to examine based on TLS or not. + List cPkts = c.isTls() ? c.getTlsApplicationDataPackets() : c.getPackets(); + /* + * Note: we embed the attempt to detect the signature sequence in a loop in order to capture those cases + * where the same signature sequence appears multiple times in one Conversation. + * + * Note: since we expect all sequences that together make up the signature to exhibit the same direction + * pattern, we can simply pass the precomputed direction array for the signature sequence so that it + * won't have to be recomputed internally in each call to findSubsequenceInSequence(). + */ + Optional> match; + while ((match = findSubsequenceInSequence(signatureSequence, cPkts, mClusterMemberDirections, null)). + isPresent()) { + List matchSeq = match.get(); + // Notify observers about the match. + mObservers.forEach(o -> o.onMatch(Layer3ClusterMatcher.this, matchSeq)); + /* + * Get the index in cPkts of the last packet in the sequence of packets that matches the searched + * signature sequence. + */ + int matchSeqEndIdx = cPkts.indexOf(matchSeq.get(matchSeq.size()-1)); + // We restart the search for the signature sequence immediately after that index, so truncate cPkts. + cPkts = cPkts.stream().skip(matchSeqEndIdx + 1).collect(Collectors.toList()); + } + } + /* + * TODO: + * if no item in cluster matches, also perform a distance-based matching to cover those cases where we did + * not manage to capture every single mutation of the sequence during training. + * + * Need to compute average/centroid of cluster to do so...? Compute within-cluster variance, then check if + * distance between input conversation and cluster average/centroid is smaller than or equal to the computed + * variance? + */ + } + } + + /** + * Checks if {@code sequence} is a sequence of TLS packets. Note: the current implementation relies on inspection + * of the port numbers when deciding between TLS vs. non-TLS. Therefore, only the first packet of {@code sequence} + * is examined as it is assumed that all packets in {@code sequence} pertain to the same {@link Conversation} and + * hence share the same set of two src/dst port numbers (albeit possibly alternating between which one is the src + * and which one is the dst, as packets in {@code sequence} may be in alternating directions). + * @param sequence The sequence of packets for which it is to be determined if it is a sequence of TLS packets or + * non-TLS packets. + * @return {@code true} if {@code sequence} is a sequence of TLS packets, {@code false} otherwise. + */ + private boolean isTlsSequence(List sequence) { + // NOTE: Assumes ALL packets in sequence pertain to the same TCP connection! + PcapPacket firstPkt = sequence.get(0); + int srcPort = getSourcePort(firstPkt); + int dstPort = getDestinationPort(firstPkt); + return TcpConversationUtils.isTlsPort(srcPort) || TcpConversationUtils.isTlsPort(dstPort); + } + + /** + * Examine if a given sequence of packets ({@code sequence}) contains a given shorter sequence of packets + * ({@code subsequence}). Note: the current implementation actually searches for a substring as it does not allow + * for interleaving packets in {@code sequence} that are not in {@code subsequence}; for example, if + * {@code subsequence} consists of packet lengths [2, 3, 5] and {@code sequence} consists of packet lengths + * [2, 3, 4, 5], the result will be that there is no match (because of the interleaving 4). If we are to allow + * interleaving packets, we need a modified version of + * this. + * + * @param subsequence The sequence to search for. + * @param sequence The sequence to search. + * @param subsequenceDirections The directions of packets in {@code subsequence} such that for all {@code i}, + * {@code subsequenceDirections[i]} is the direction of the packet returned by + * {@code subsequence.get(i)}. May be set to {@code null}, in which this call will + * internally compute the packet directions. + * @param sequenceDirections The directions of packets in {@code sequence} such that for all {@code i}, + * {@code sequenceDirections[i]} is the direction of the packet returned by + * {@code sequence.get(i)}. May be set to {@code null}, in which this call will internally + * compute the packet directions. + * + * @return An {@link Optional} containing the part of {@code sequence} that matches {@code subsequence}, or an empty + * {@link Optional} if no part of {@code sequence} matches {@code subsequence}. + */ + private Optional> findSubsequenceInSequence(List subsequence, + List sequence, + Conversation.Direction[] subsequenceDirections, + Conversation.Direction[] sequenceDirections) { + if (sequence.size() < subsequence.size()) { + // If subsequence is longer, it cannot be contained in sequence. + return Optional.empty(); + } + if (isTlsSequence(subsequence) != isTlsSequence(sequence)) { + // We consider it a mismatch if one is a TLS application data sequence and the other is not. + return Optional.empty(); + } + // If packet directions have not been precomputed by calling code, we need to construct them. + if (subsequenceDirections == null) { + subsequenceDirections = getPacketDirections(subsequence, mRouterWanIp); + } + if (sequenceDirections == null) { + sequenceDirections = getPacketDirections(sequence, mRouterWanIp); + } + int subseqIdx = 0; + int seqIdx = 0; + while (seqIdx < sequence.size()) { + PcapPacket subseqPkt = subsequence.get(subseqIdx); + PcapPacket seqPkt = sequence.get(seqIdx); + // We only have a match if packet lengths and directions match. + if (subseqPkt.getOriginalLength() == seqPkt.getOriginalLength() && + subsequenceDirections[subseqIdx] == sequenceDirections[seqIdx]) { + // A match; advance both indices to consider next packet in subsequence vs. next packet in sequence. + subseqIdx++; + seqIdx++; + if (subseqIdx == subsequence.size()) { + // We managed to match the entire subsequence in sequence. + // Return the sublist of sequence that matches subsequence. + /* + * TODO: + * ASSUMES THE BACKING LIST (i.e., 'sequence') IS _NOT_ STRUCTURALLY MODIFIED, hence may not work + * for live traces! + */ + return Optional.of(sequence.subList(seqIdx - subsequence.size(), seqIdx)); + } + } else { + // Mismatch. + if (subseqIdx > 0) { + /* + * If we managed to match parts of subsequence, we restart the search for subsequence in sequence at + * the index of sequence where the current mismatch occurred. I.e., we must reset subseqIdx, but + * leave seqIdx untouched. + */ + subseqIdx = 0; + } else { + /* + * First packet of subsequence didn't match packet at seqIdx of sequence, so we move forward in + * sequence, i.e., we continue the search for subsequence in sequence starting at index seqIdx+1 of + * sequence. + */ + seqIdx++; + } + } + } + return Optional.empty(); + } + + /** + * Given a cluster, produces a pruned version of that cluster. In the pruned version, there are no duplicate cluster + * members. Two cluster members are considered identical if their packets lengths and packet directions are + * identical. The resulting pruned cluster is unmodifiable (this applies to both the outermost list as well as the + * nested lists) in order to preserve its integrity when exposed to external code (e.g., through + * {@link #getCluster()}). + * + * @param cluster A cluster to prune. + * @return The resulting pruned cluster. + */ + @Override + protected List> pruneCluster(List> cluster) { + List> prunedCluster = new ArrayList<>(); + for (List originalClusterSeq : cluster) { + boolean alreadyPresent = false; + for (List prunedClusterSeq : prunedCluster) { + Optional> duplicate = findSubsequenceInSequence(originalClusterSeq, prunedClusterSeq, + mClusterMemberDirections, mClusterMemberDirections); + if (duplicate.isPresent()) { + alreadyPresent = true; + break; + } + } + if (!alreadyPresent) { + prunedCluster.add(Collections.unmodifiableList(originalClusterSeq)); + } + } + return Collections.unmodifiableList(prunedCluster); + } + + /** + * Given a {@code List}, generate a {@code Conversation.Direction[]} such that each entry in the + * resulting {@code Conversation.Direction[]} specifies the direction of the {@link PcapPacket} at the corresponding + * index in the input list. + * @param packets The list of packets for which to construct a corresponding array of packet directions. + * @param routerWanIp The IP of the router's WAN port. This is used for determining the direction of packets when + * the traffic is captured just outside the local network (at the ISP side of the router). Set to + * {@code null} if {@code packets} stem from traffic captured within the local network. + * @return A {@code Conversation.Direction[]} specifying the direction of the {@link PcapPacket} at the + * corresponding index in {@code packets}. + */ + private static Conversation.Direction[] getPacketDirections(List packets, String routerWanIp) { + Conversation.Direction[] directions = new Conversation.Direction[packets.size()]; + for (int i = 0; i < packets.size(); i++) { + PcapPacket pkt = packets.get(i); + if (getSourceIp(pkt).equals(getDestinationIp(pkt))) { + // Sanity check: we shouldn't be processing loopback traffic + throw new AssertionError("loopback traffic detected"); + } + if (isSrcIpLocal(pkt) || getSourceIp(pkt).equals(routerWanIp)) { + directions[i] = Conversation.Direction.CLIENT_TO_SERVER; + } else if (isDstIpLocal(pkt) || getDestinationIp(pkt).equals(routerWanIp)) { + directions[i] = Conversation.Direction.SERVER_TO_CLIENT; + } else { + //throw new IllegalArgumentException("no local IP or router WAN port IP found, can't detect direction"); + } + } + return directions; + } + +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/SignatureDetector.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/SignatureDetector.java new file mode 100644 index 0000000..0c4324d --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/SignatureDetector.java @@ -0,0 +1,666 @@ +package edu.uci.iotproject.detection.layer3; + +import edu.uci.iotproject.analysis.TriggerTrafficExtractor; +import edu.uci.iotproject.analysis.UserAction; +import edu.uci.iotproject.detection.AbstractClusterMatcher; +import edu.uci.iotproject.detection.ClusterMatcherObserver; +import edu.uci.iotproject.io.PcapHandleReader; +import edu.uci.iotproject.util.PrintUtils; +import org.jgrapht.GraphPath; +import org.jgrapht.alg.shortestpath.DijkstraShortestPath; +import org.jgrapht.graph.DefaultWeightedEdge; +import org.jgrapht.graph.SimpleDirectedWeightedGraph; +import org.pcap4j.core.*; + +import java.time.Duration; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.time.format.FormatStyle; +import java.util.*; +import java.util.function.Consumer; + +/** + * Detects an event signature that spans one or multiple TCP connections. + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class SignatureDetector implements PacketListener, ClusterMatcherObserver { + + // Test client + public static void main(String[] args) throws PcapNativeException, NotOpenException { +// if (args.length < 3) { +// String errMsg = String.format("Usage: %s inputPcapFile onSignatureFile offSignatureFile", +// SignatureDetector.class.getSimpleName()); +// System.out.println(errMsg); +// return; +// } +// final String inputPcapFile = args[0]; +// final String onSignatureFile = args[1]; +// final String offSignatureFile = args[2]; + + String path = "/scratch/July-2018"; // Rahmadi +// String path = "/Users/varmarken/temp/UCI IoT Project/experiments"; // Janus +// String path = "/home/jvarmark/iot_project/datasets"; // Hera (server) +// String path = "/raid/varmarken/iot_project/datasets"; // Zeus (server) + + // No activity test + //final String inputPcapFile = path + "/evaluation/no-activity/no-activity.wlan1.pcap"; + + // D-Link Siren experiment +// final String inputPcapFile = path + "/evaluation/dlink-siren/dlink-siren.data.wlan1.pcap"; +// final String inputPcapFile = path + "/evaluation/dlink-siren/dlink-siren.eth0.local.pcap"; + // D-Link Siren DEVICE signatures +// final String onSignatureFile = path + "/2018-08/dlink-siren/onSignature-DLink-Siren-device.sig"; +// final String offSignatureFile = path + "/2018-08/dlink-siren/offSignature-DLink-Siren-device.sig"; + // D-Link Siren PHONE signatures +// final String onSignatureFile = path + "/2018-08/dlink-siren/onSignature-DLink-Siren-phone.sig"; +// final String offSignatureFile = path + "/2018-08/dlink-siren/offSignature-DLink-Siren-phone.sig"; + // TODO: EXPERIMENT - November 19, 2018 + // Hue Bulb experiment +// final String inputPcapFile = path + "/2018-08/hue-bulb/hue-bulb.wlan1.local.pcap"; + // Hue Bulb PHONE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/hue-bulb/signatures/hue-bulb-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/hue-bulb/signatures/hue-bulb-offSignature-phone-side.sig"; + + /* + // Kwikset Doorlock Sep 12 experiment +// final String inputPcapFile = path + "/evaluation/kwikset-doorlock/kwikset-doorlock.data.wlan1.pcap"; + final String inputPcapFile = path + "/evaluation/kwikset-doorlock/kwikset-doorlock.data.eth0.pcap"; +// // Kwikset Doorlock PHONE signatures + final String onSignatureFile = path + "/2018-08/kwikset-doorlock/onSignature-Kwikset-Doorlock-phone-new.sig"; + final String offSignatureFile = path + "/2018-08/kwikset-doorlock/offSignature-Kwikset-Doorlock-phone-new.sig"; + */ + + // D-Link Plug experiment + //final String inputPcapFile = path + "/evaluation/dlink/dlink-plug.data.wlan1.pcap"; +// final String inputPcapFile = path + "/evaluation/dlink/dlink-plug.data.eth0.pcap"; + + // D-Link Plug DEVICE signatures +// final String onSignatureFile = path + "/2018-07/dlink/onSignature-DLink-Plug-device.sig"; +// final String offSignatureFile = path + "/2018-07/dlink/offSignature-DLink-Plug-device.sig"; + // D-Link Plug PHONE signatures +// final String onSignatureFile = path + "/2018-07/dlink/onSignature-DLink-Plug-phone.sig"; +// final String offSignatureFile = path + "/2018-07/dlink/offSignature-DLink-Plug-phone.sig"; + + // TODO: The following are negative tests against the PCAP file from UNSW +// final String inputPcapFile = path + "/UNSW/16-10-04.pcap"; // TODO: Seems to be broken! Zero-payload! +// final String inputPcapFile = path + "/UNSW/16-10-12.pcap"; + +// final String inputPcapFile = path + "/UNSW/16-09-28.pcap"; // TODO: Seems to be broken! Zero-payload! +// final String inputPcapFile = path + "/UNSW/16-10-02.pcap"; // TODO: Seems to be broken! +// final String inputPcapFile = path + "/UNSW/16-10-03.pcap"; // TODO: Seems to be broken! +// final String inputPcapFile = path + "/UNSW/16-10-04-a.pcap"; // TODO: Seems to be broken! Zero-payload! +// final String inputPcapFile = path + "/UNSW/16-10-04-b.pcap"; // TODO: Seems to be broken! Zero-payload! +// final String inputPcapFile = path + "/UNSW/16-10-07.pcap"; // TODO: Seems to be broken! +// final String inputPcapFile = path + "/UNSW/16-10-08.pcap"; // TODO: Seems to be broken! +// final String inputPcapFile = path + "/UNSW/16-10-09.pcap"; // TODO: Seems to be broken! +// final String inputPcapFile = path + "/UNSW/16-10-10.pcap"; // TODO: Seems to be broken! +// final String inputPcapFile = path + "/UNSW/16-10-11.pcap"; // TODO: Seems to be broken! + // TODO: The following one is very long!!! - Split into smaller files! +// final String inputPcapFile = path + "/UNSW/16-10-12-a.pcap"; +// final String inputPcapFile = path + "/UNSW/16-10-12-b.pcap"; +// final String inputPcapFile = path + "/UNSW/16-10-12-c.pcap"; +// final String inputPcapFile = path + "/UNSW/16-10-12-d.pcap"; + +// final String inputPcapFile = path + "/UNSW/16-09-23.pcap"; +// final String inputPcapFile = path + "/UNSW/16-09-24.pcap"; +// final String inputPcapFile = path + "/UNSW/16-09-25.pcap"; +// final String inputPcapFile = path + "/UNSW/16-09-26.pcap"; +// final String inputPcapFile = path + "/UNSW/16-09-27.pcap"; +// final String inputPcapFile = path + "/UNSW/16-09-29.pcap"; +// final String inputPcapFile = path + "/UNSW/16-10-01.pcap"; +// final String inputPcapFile = path + "/UNSW/16-10-06.pcap"; + // Negative test: dataset from UNB +// final String inputPcapFile = path + "/evaluation/negative-datasets/UNB/Monday-WorkingHours_one-local-endpoint-001.pcap"; + + // TODO: The following are tests for signatures against training data + + // D-Link Plug experiment +// final String inputPcapFile = path + "/training/dlink-plug/wlan1/dlink-plug.wlan1.local.pcap"; + // D-Link Plug DEVICE signatures +// final String onSignatureFile = path + "/training/dlink-plug/signatures/dlink-plug-onSignature-device-side.sig"; +// final String offSignatureFile = path + "/training/dlink-plug/signatures/dlink-plug-offSignature-device-side.sig"; + // D-Link Plug PHONE signatures +// final String onSignatureFile = path + "/training/dlink-plug/signatures/dlink-plug-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/training/dlink-plug/signatures/dlink-plug-offSignature-phone-side.sig"; + + // TODO: EXPERIMENT - November 7, 2018 + // D-Link Plug experiment + //final String inputPcapFile = path + "/experimental_result/standalone/dlink-plug/wlan1/dlink-plug.wlan1.local.pcap"; + //final String inputPcapFile = path + "/experimental_result/smarthome/dlink-plug/wlan1/dlink-plug.wlan1.detection.pcap"; + //final String inputPcapFile = path + "/experimental_result/smarthome/dlink-plug/eth0/dlink-plug.eth0.detection.pcap"; + // D-Link Plug DEVICE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-onSignature-device-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-offSignature-device-side.sig"; + // D-Link Plug PHONE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-offSignature-phone-side.sig"; + + // TODO: EXPERIMENT - November 9, 2018 + // D-Link Siren experiment + //final String inputPcapFile = path + "/experimental_result/standalone/dlink-siren/wlan1/dlink-siren.wlan1.local.pcap"; + //final String inputPcapFile = path + "/experimental_result/smarthome/dlink-siren/wlan1/dlink-siren.wlan1.detection.pcap"; +// final String inputPcapFile = path + "/experimental_result/smarthome/dlink-siren/eth0/dlink-siren.eth0.detection.pcap"; + // D-Link Siren DEVICE signatures + // TODO: The device signature does not have pairs---only one packet which is 216, so we don't consider this as a signature +// final String onSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-onSignature-device-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-offSignature-device-side.sig"; + // D-Link Siren PHONE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-offSignature-phone-side.sig"; +// final String onSignatureFile = path + "/training/signatures/dlink-siren/dlink-siren-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/training/signatures/dlink-siren/dlink-siren-offSignature-phone-side.sig"; + + // TP-Link Plug experiment +//// final String inputPcapFile = path + "/training/tplink-plug/wlan1/tplink-plug.wlan1.local.pcap"; +//// final String inputPcapFile = path + "/experimental_result/wifi-Sniffer/tests2/airtool_2019-01-04_11.08.45.AM.pcap"; +// final String inputPcapFile = path + "/experimental_result/wifi-Sniffer/tests2/command-frames-only.pcap"; +// // TP-Link Plug DEVICE signatures +// final String onSignatureFile = path + "/training/tplink-plug/signatures/tplink-plug-onSignature-device-side.sig"; +// final String offSignatureFile = path + "/training/tplink-plug/signatures/tplink-plug-offSignature-device-side.sig"; + // TODO: EXPERIMENT - November 8, 2018 + // TP-Link Plug experiment +// final String inputPcapFile = path + "/experimental_result/standalone/tplink-plug/wlan1/tplink-plug.wlan1.local.pcap"; +// final String inputPcapFile = path + "/experimental_result/standalone/tplink-plug/eth0/tplink-plug.eth0.local.pcap"; +// final String inputPcapFile = path + "/experimental_result/smarthome/tplink-plug/wlan1/tplink-plug.wlan1.detection.pcap"; + //final String inputPcapFile = path + "/experimental_result/smarthome/tplink-plug/eth0/tplink-plug.eth0.detection.pcap"; + // TP-Link Plug DEVICE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-device-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-device-side.sig"; +// final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-device-side-outbound.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-device-side-outbound.sig"; + // TP-Link Plug PHONE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-phone-side.sig"; + + // Arlo camera experiment +// final String inputPcapFile = path + "/training/arlo-camera/wlan1/arlo-camera.wlan1.local.pcap"; +//// // TP-Link Plug DEVICE signatures +// final String onSignatureFile = path + "/training/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/training/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig"; + // TODO: EXPERIMENT - November 13, 2018 + // Arlo Camera experiment +// final String inputPcapFile = path + "/experimental_result/standalone/arlo-camera/wlan1/arlo-camera.wlan1.local.pcap"; +// final String inputPcapFile = path + "/experimental_result/standalone/arlo-camera/eth0/arlo-camera.eth0.local.pcap"; +// final String inputPcapFile = path + "/experimental_result/smarthome/arlo-camera/wlan1/arlo-camera.wlan1.detection.pcap"; +// final String inputPcapFile = path + "/experimental_result/smarthome/arlo-camera/eth0/arlo-camera.eth0.detection.pcap"; +// final String inputPcapFile = path + "/training/arlo-camera/eth0/arlo-camera.eth0.local.pcap"; + // Arlo Camera PHONE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig"; + + // Amazon Alexa experiment +// final String inputPcapFile = path + "/training/amazon-alexa/wlan1/alexa2.wlan1.local.pcap"; +// // TP-Link Plug DEVICE signatures +// final String onSignatureFile = path + "/training/amazon-alexa/signatures/amazon-alexa-onSignature-device-side.sig"; +// final String offSignatureFile = path + "/training/amazon-alexa/signatures/amazon-alexa-offSignature-device-side.sig"; + + // SmartThings Plug experiment +// final String inputPcapFile = path + "/training/st-plug/wlan1/st-plug.wlan1.local.pcap"; +// // SmartThings Plug DEVICE signatures +// //final String onSignatureFile = path + "/training/st-plug/signatures/st-plug-onSignature-device-side.sig"; +// //final String offSignatureFile = path + "/training/st-plug/signatures/st-plug-offSignature-device-side.sig"; +// // SmartThings Plug PHONE signatures +// final String onSignatureFile = path + "/training/st-plug/signatures/st-plug-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/training/st-plug/signatures/st-plug-offSignature-phone-side.sig"; + // TODO: EXPERIMENT - November 12, 2018 + // SmartThings Plug experiment +// final String inputPcapFile = path + "/experimental_result/standalone/st-plug/wlan1/st-plug.wlan1.local.pcap"; +// final String inputPcapFile = path + "/experimental_result/standalone/st-plug/eth0/st-plug.eth0.local.pcap"; +// //final String inputPcapFile = path + "/experimental_result/smarthome/st-plug/wlan1/st-plug.wlan1.detection.pcap"; +// final String inputPcapFile = path + "/experimental_result/smarthome/st-plug/eth0/st-plug.eth0.detection.pcap"; +// // SmartThings Plug PHONE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/st-plug/signatures/st-plug-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/st-plug/signatures/st-plug-offSignature-phone-side.sig"; +// final String onSignatureFile = path + "/training/signatures/st-plug/st-plug-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/training/signatures/st-plug/st-plug-offSignature-phone-side.sig"; + + // TODO: EXPERIMENT - January 9, 2018 + // Blossom Sprinkler experiment +// final String inputPcapFile = path + "/experimental_result/standalone/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.local.pcap"; + final String inputPcapFile = path + "/experimental_result/smarthome/blossom-sprinkler/eth0/blossom-sprinkler.eth0.detection.pcap"; +// final String inputPcapFile = path + "/experimental_result/smarthome/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.detection.pcap"; + // Blossom Sprinkler DEVICE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-device-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-device-side.sig"; + // Blossom Sprinkler PHONE signatures + final String onSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-phone-side.sig"; + final String offSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-phone-side.sig"; + + // LiFX Bulb experiment +// final String inputPcapFile = path + "/training/lifx-bulb/wlan1/lifx-bulb.wlan1.local.pcap"; +// // LiFX Bulb DEVICE signatures +// final String onSignatureFile = path + "/training/lifx-bulb/signatures/lifx-bulb-onSignature-device-side.sig"; +// final String offSignatureFile = path + "/training/lifx-bulb/signatures/lifx-bulb-offSignature-device-side.sig"; + // LiFX Bulb PHONE signatures +// final String onSignatureFile = path + "/training/lifx-bulb/signatures/lifx-bulb-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/training/lifx-bulb/signatures/lifx-bulb-offSignature-phone-side.sig"; + + // Blossom Sprinkler experiment +// //final String inputPcapFile = path + "/training/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.local.pcap"; +// final String inputPcapFile = path + "/2018-08/blossom/blossom.wlan1.local.pcap"; +// //final String inputPcapFile = path + "/training/blossom-sprinkler/eth0/blossom-sprinkler.eth0.local.pcap"; +// // Blossom Sprinkler DEVICE signatures +// final String onSignatureFile = path + "/training/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-device-side.sig"; +// final String offSignatureFile = path + "/training/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-device-side.sig"; + + // Nest Thermostat experiment +// final String inputPcapFile = path + "/training/nest-thermostat/wlan1/nest-thermostat.wlan1.local.pcap"; +// // Nest Thermostat DEVICE signatures +//// final String onSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-onSignature-device-side.sig"; +//// final String offSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-offSignature-device-side.sig"; +// // Nest Thermostat PHONE signatures +// final String onSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-offSignature-phone-side.sig"; + // TODO: EXPERIMENT - November 15, 2018 + // Nest Thermostat experiment +// final String inputPcapFile = path + "/experimental_result/standalone/nest-thermostat/wlan1/nest-thermostat.wlan1.local.pcap"; +//// final String inputPcapFile = path + "/experimental_result/standalone/nest-thermostat/eth0/nest-thermostat.eth0.local.pcap"; +//// final String inputPcapFile = path + "/experimental_result/smarthome/nest-thermostat/wlan1/nest-thermostat.wlan1.detection.pcap"; +// final String inputPcapFile = path + "/experimental_result/smarthome/nest-thermostat/eth0/nest-thermostat.eth0.detection.pcap"; +//// // Nest Thermostat PHONE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/nest-thermostat/signatures/nest-thermostat-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/nest-thermostat/signatures/nest-thermostat-offSignature-phone-side.sig"; + + /* + // Hue Bulb experiment + final String inputPcapFile = path + "/training/hue-bulb/wlan1/hue-bulb.wlan1.local.pcap"; + // Hue Bulb PHONE signatures + final String onSignatureFile = path + "/training/hue-bulb/signatures/hue-bulb-onSignature-phone-side.sig"; + final String offSignatureFile = path + "/training/hue-bulb/signatures/hue-bulb-offSignature-phone-side.sig"; + */ + + + + // TP-Link Bulb experiment +// final String inputPcapFile = path + "/training/tplink-bulb/wlan1/tplink-bulb.wlan1.local.pcap"; +// // TP-Link Bulb PHONE signatures +// final String onSignatureFile = path + "/training/tplink-bulb/signatures/tplink-bulb-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/training/tplink-bulb/signatures/tplink-bulb-offSignature-phone-side.sig"; + // TODO: EXPERIMENT - November 16, 2018 + // TP-Link Bulb experiment +// final String inputPcapFile = path + "/experimental_result/standalone/tplink-bulb/wlan1/tplink-bulb.wlan1.local.pcap"; +// final String inputPcapFile = path + "/experimental_result/standalone/tplink-bulb/eth0/tplink-bulb.eth0.local.pcap"; +// final String inputPcapFile = path + "/experimental_result/smarthome/tplink-bulb/wlan1/tplink-bulb.wlan1.detection.pcap"; +//// final String inputPcapFile = path + "/experimental_result/smarthome/tplink-bulb/eth0/tplink-bulb.eth0.detection.pcap"; +// // TP-Link Bulb PHONE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/tplink-bulb/signatures/tplink-bulb-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/tplink-bulb/signatures/tplink-bulb-offSignature-phone-side.sig"; + + /* + // WeMo Plug experiment + final String inputPcapFile = path + "/training/wemo-plug/wlan1/wemo-plug.wlan1.local.pcap"; + // WeMo Plug PHONE signatures + final String onSignatureFile = path + "/training/wemo-plug/signatures/wemo-plug-onSignature-device-side.sig"; + final String offSignatureFile = path + "/training/wemo-plug/signatures/wemo-plug-offSignature-device-side.sig"; + */ + // TODO: EXPERIMENT - November 20, 2018 + // WeMo Plug experiment +// final String inputPcapFile = path + "/experimental_result/standalone/wemo-plug/wlan1/wemo-plug.wlan1.local.pcap"; +// final String inputPcapFile = path + "/experimental_result/standalone/wemo-plug/eth0/wemo-plug.eth0.local.pcap"; + // TODO: WE HAVE 4 ADDITIONAL EVENTS (TRIGGERED MANUALLY), SO WE JUST IGNORE THEM BECAUSE THEY HAPPENED BEFORE + // TODO: THE ACTUAL TRIGGERS +// final String inputPcapFile = path + "/experimental_result/smarthome/wemo-plug/wlan1/wemo-plug.wlan1.detection.pcap"; +//// final String inputPcapFile = path + "/experimental_result/smarthome/wemo-plug/eth0/wemo-plug.eth0.detection.pcap"; +// // WeMo Plug PHONE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/wemo-plug/signatures/wemo-plug-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/wemo-plug/signatures/wemo-plug-offSignature-phone-side.sig"; + + /* + // WeMo Insight Plug experiment + final String inputPcapFile = path + "/training/wemo-insight-plug/wlan1/wemo-insight-plug.wlan1.local.pcap"; + // WeMo Insight Plug PHONE signatures + final String onSignatureFile = path + "/training/wemo-insight-plug/signatures/wemo-insight-plug-onSignature-device-side.sig"; + final String offSignatureFile = path + "/training/wemo-insight-plug/signatures/wemo-insight-plug-offSignature-device-side.sig"; + */ + // TODO: EXPERIMENT - November 21, 2018 + // WeMo Insight Plug experiment +// final String inputPcapFile = path + "/experimental_result/standalone/wemo-insight-plug/wlan1/wemo-insight-plug.wlan1.local.pcap"; +// final String inputPcapFile = path + "/experimental_result/standalone/wemo-insight-plug/eth0/wemo-insight-plug.eth0.local.pcap"; + // TODO: WE HAVE 1 ADDITIONAL EVENT (FROM WEMO PLUG) +// final String inputPcapFile = path + "/experimental_result/smarthome/wemo-insight-plug/wlan1/wemo-insight-plug.wlan1.detection.pcap"; +// final String inputPcapFile = path + "/experimental_result/smarthome/wemo-insight-plug/eth0/wemo-insight-plug.eth0.detection.pcap"; + // WeMo Insight Plug PHONE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/wemo-insight-plug/signatures/wemo-insight-plug-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/wemo-insight-plug/signatures/wemo-insight-plug-offSignature-phone-side.sig"; + + + // Kwikset Doorlock Sep 12 experiment +// final String inputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset3.wlan1.local.pcap"; +// // Kwikset Doorlock PHONE signatures +// final String onSignatureFile = path + "/2018-08/kwikset-doorlock/onSignature-Kwikset-Doorlock-phone.sig"; +// final String offSignatureFile = path + "/2018-08/kwikset-doorlock/offSignature-Kwikset-Doorlock-phone.sig"; + // TODO: EXPERIMENT - November 10, 2018 + // Kwikset Door lock experiment +// final String inputPcapFile = path + "/experimental_result/standalone/kwikset-doorlock/wlan1/kwikset-doorlock.wlan1.local.pcap"; +// //final String inputPcapFile = path + "/experimental_result/smarthome/kwikset-doorlock/wlan1/kwikset-doorlock.wlan1.detection.pcap"; +// final String inputPcapFile = path + "/experimental_result/smarthome/kwikset-doorlock/eth0/kwikset-doorlock.eth0.detection.pcap"; +//// // Kwikset Door lock PHONE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/kwikset-doorlock/signatures/kwikset-doorlock-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/kwikset-doorlock/signatures/kwikset-doorlock-offSignature-phone-side.sig"; +// final String onSignatureFile = path + "/training/signatures/kwikset-doorlock/kwikset-doorlock-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/training/signatures/kwikset-doorlock/kwikset-doorlock-offSignature-phone-side.sig"; + + + + // D-Link Siren experiment +// final String inputPcapFile = path + "/2018-08/dlink-siren/dlink-siren.wlan1.local.pcap"; + // D-Link Siren DEVICE signatures + //final String onSignatureFile = path + "/2018-08/dlink-siren/onSignature-DLink-Siren-device.sig"; + //final String offSignatureFile = path + "/2018-08/dlink-siren/offSignature-DLink-Siren-device.sig"; + // D-Link Siren PHONE signatures +// final String onSignatureFile = path + "/2018-08/dlink-siren/onSignature-DLink-Siren-phone.sig"; +// final String offSignatureFile = path + "/2018-08/dlink-siren/offSignature-DLink-Siren-phone.sig"; + + + // Output file names used (to make it easy to catch if one forgets to change them) + System.out.println("ON signature file in use is " + onSignatureFile); + System.out.println("OFF signature file in use is " + offSignatureFile); + System.out.println("PCAP file that is the target of detection is " + inputPcapFile); + + List>> onSignature = PrintUtils.deserializeSignatureFromFile(onSignatureFile); + List>> offSignature = PrintUtils.deserializeSignatureFromFile(offSignatureFile); + + // LAN +// SignatureDetector onDetector = new SignatureDetector(onSignature, null); +// SignatureDetector offDetector = new SignatureDetector(offSignature, null); + // WAN + SignatureDetector onDetector = new SignatureDetector(onSignature, "128.195.205.105", 0); + SignatureDetector offDetector = new SignatureDetector(offSignature, "128.195.205.105", 0); + + final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofLocalizedDateTime(FormatStyle.MEDIUM). + withLocale(Locale.US).withZone(ZoneId.of("America/Los_Angeles")); + + // Outputs information about a detected event to std.out + final Consumer outputter = ua -> { + String eventDescription; + switch (ua.getType()) { + case TOGGLE_ON: + eventDescription = "ON"; + break; + case TOGGLE_OFF: + eventDescription = "OFF"; + break; + default: + throw new AssertionError("unhandled event type"); + } + //String output = String.format("[ !!! %s SIGNATURE DETECTED at %s !!! ]", + // eventDescription, dateTimeFormatter.format(ua.getTimestamp())); + String output = String.format("%s", + dateTimeFormatter.format(ua.getTimestamp())); + System.out.println(output); + }; + + // Let's create observers that construct a UserAction representing the detected event. + final List detectedEvents = new ArrayList<>(); + onDetector.addObserver((searched, match) -> { + PcapPacket firstPkt = match.get(0).get(0); + detectedEvents.add(new UserAction(UserAction.Type.TOGGLE_ON, firstPkt.getTimestamp())); + }); + offDetector.addObserver((searched, match) -> { + PcapPacket firstPkt = match.get(0).get(0); + detectedEvents.add(new UserAction(UserAction.Type.TOGGLE_OFF, firstPkt.getTimestamp())); + }); + + PcapHandle handle; + try { + handle = Pcaps.openOffline(inputPcapFile, PcapHandle.TimestampPrecision.NANO); + } catch (PcapNativeException pne) { + handle = Pcaps.openOffline(inputPcapFile); + } + PcapHandleReader reader = new PcapHandleReader(handle, p -> true, onDetector, offDetector); + reader.readFromHandle(); + + // TODO: need a better way of triggering detection than this... + onDetector.mClusterMatchers.forEach(cm -> cm.performDetection()); + offDetector.mClusterMatchers.forEach(cm -> cm.performDetection()); + + // Sort the list of detected events by timestamp to make it easier to compare it line-by-line with the trigger + // times file. + Collections.sort(detectedEvents, Comparator.comparing(UserAction::getTimestamp)); + + // Output the detected events + detectedEvents.forEach(outputter); + + System.out.println("Number of detected events of type " + UserAction.Type.TOGGLE_ON + ": " + + detectedEvents.stream().filter(ua -> ua.getType() == UserAction.Type.TOGGLE_ON).count()); + System.out.println("Number of detected events of type " + UserAction.Type.TOGGLE_OFF + ": " + + detectedEvents.stream().filter(ua -> ua.getType() == UserAction.Type.TOGGLE_OFF).count()); + + + // TODO: Temporary clean up until we clean the pipeline +// List cleanedDetectedEvents = SignatureDetector.removeDuplicates(detectedEvents); +// cleanedDetectedEvents.forEach(outputter); + } + + /** + * The signature that this {@link SignatureDetector} is searching for. + */ + private final List>> mSignature; + + /** + * The {@link Layer3ClusterMatcher}s in charge of detecting each individual sequence of packets that together make up the + * the signature. + */ + private final List mClusterMatchers; + + /** + * For each {@code i} ({@code i >= 0 && i < pendingMatches.length}), {@code pendingMatches[i]} holds the matches + * found by the {@link Layer3ClusterMatcher} at {@code mClusterMatchers.get(i)} that have yet to be "consumed", i.e., + * have yet to be included in a signature detected by this {@link SignatureDetector} (a signature can be encompassed + * of multiple packet sequences occurring shortly after one another on multiple connections). + */ + private final List>[] pendingMatches; + + /** + * Maps a {@link Layer3ClusterMatcher} to its corresponding index in {@link #pendingMatches}. + */ + private final Map mClusterMatcherIds; + + private final List mObservers = new ArrayList<>(); + + private int mInclusionTimeMillis; + + /** + * Remove duplicates in {@code List} of {@code UserAction} objects. We need to clean this up for user actions + * that appear multiple times. + * TODO: This static method is probably just for temporary and we could get rid of this after we clean up + * TODO: the pipeline + * + * @param listUserAction A {@link List} of {@code UserAction}. + * + */ + public static List removeDuplicates(List listUserAction) { + + // Iterate and check for duplicates (check timestamps) + Set epochSecondSet = new HashSet<>(); + // Create a target list for cleaned up list + List listUserActionClean = new ArrayList<>(); + for(UserAction userAction : listUserAction) { + // Don't insert if any duplicate is found + if(!epochSecondSet.contains(userAction.getTimestamp().getEpochSecond())) { + listUserActionClean.add(userAction); + epochSecondSet.add(userAction.getTimestamp().getEpochSecond()); + } + } + return listUserActionClean; + } + + public SignatureDetector(List>> searchedSignature, String routerWanIp, int inclusionTimeMillis) { + // note: doesn't protect inner lists from changes :'( + mSignature = Collections.unmodifiableList(searchedSignature); + // Generate corresponding/appropriate ClusterMatchers based on the provided signature + List clusterMatchers = new ArrayList<>(); + for (List> cluster : mSignature) { + clusterMatchers.add(new Layer3ClusterMatcher(cluster, routerWanIp, this)); + } + mClusterMatchers = Collections.unmodifiableList(clusterMatchers); + + // < exploratory > + pendingMatches = new List[mClusterMatchers.size()]; + for (int i = 0; i < pendingMatches.length; i++) { + pendingMatches[i] = new ArrayList<>(); + } + Map clusterMatcherIds = new HashMap<>(); + for (int i = 0; i < mClusterMatchers.size(); i++) { + clusterMatcherIds.put(mClusterMatchers.get(i), i); + } + mClusterMatcherIds = Collections.unmodifiableMap(clusterMatcherIds); + mInclusionTimeMillis = + inclusionTimeMillis == 0 ? TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS : inclusionTimeMillis; + } + + public void addObserver(SignatureDetectionObserver observer) { + mObservers.add(observer); + } + + public boolean removeObserver(SignatureDetectionObserver observer) { + return mObservers.remove(observer); + } + + @Override + public void gotPacket(PcapPacket packet) { + // simply delegate packet reception to all ClusterMatchers. + mClusterMatchers.forEach(cm -> cm.gotPacket(packet)); + } + + @Override + public void onMatch(AbstractClusterMatcher clusterMatcher, List match) { + // Add the match at the corresponding index + pendingMatches[mClusterMatcherIds.get(clusterMatcher)].add(match); + checkSignatureMatch(); + } + + private void checkSignatureMatch() { + // << Graph-based approach using Balint's idea. >> + // This implementation assumes that the packets in the inner lists (the sequences) are ordered by asc timestamp. + + // There cannot be a signature match until each Layer3ClusterMatcher has found a match of its respective sequence. + if (Arrays.stream(pendingMatches).noneMatch(l -> l.isEmpty())) { + // Construct the DAG + final SimpleDirectedWeightedGraph graph = + new SimpleDirectedWeightedGraph<>(DefaultWeightedEdge.class); + // Add a vertex for each match found by all ClusterMatchers + // And maintain an array to keep track of what cluster matcher each vertex corresponds to + final List[] vertices = new List[pendingMatches.length]; + for (int i = 0; i < pendingMatches.length; i++) { + vertices[i] = new ArrayList<>(); + for (List sequence : pendingMatches[i]) { + Vertex v = new Vertex(sequence); + vertices[i].add(v); // retain reference for later when we are to add edges + graph.addVertex(v); // add to vertex to graph + } + } + // Add dummy source and sink vertices to facilitate search. + final Vertex source = new Vertex(null); + final Vertex sink = new Vertex(null); + graph.addVertex(source); + graph.addVertex(sink); + // The source is connected to all vertices that wrap the sequences detected by Layer3ClusterMatcher at index 0. + // Note: zero cost edges as this is just a dummy link to facilitate search from a common start node. + for (Vertex v : vertices[0]) { + DefaultWeightedEdge edge = graph.addEdge(source, v); + graph.setEdgeWeight(edge, 0.0); + } + // Similarly, all vertices that wrap the sequences detected by the last Layer3ClusterMatcher of the signature + // are connected to the sink node. + for (Vertex v : vertices[vertices.length-1]) { + DefaultWeightedEdge edge = graph.addEdge(v, sink); + graph.setEdgeWeight(edge, 0.0); + } + // Now link sequences detected by Layer3ClusterMatcher at index i to sequences detected by Layer3ClusterMatcher at index + // i+1 if they obey the timestamp constraint (i.e., that the latter is later in time than the former). + for (int i = 0; i < vertices.length; i++) { + int j = i + 1; + if (j < vertices.length) { + for (Vertex iv : vertices[i]) { + PcapPacket ivLast = iv.sequence.get(iv.sequence.size()-1); + for (Vertex jv : vertices[j]) { + PcapPacket jvFirst = jv.sequence.get(jv.sequence.size()-1); + if (ivLast.getTimestamp().isBefore(jvFirst.getTimestamp())) { + DefaultWeightedEdge edge = graph.addEdge(iv, jv); + // The weight is the duration of the i'th sequence plus the duration between the i'th + // and i+1'th sequence. + Duration d = Duration. + between(iv.sequence.get(0).getTimestamp(), jvFirst.getTimestamp()); + // Unfortunately weights are double values, so must convert from long to double. + // TODO: need nano second precision? If so, use d.toNanos(). + // TODO: risk of overflow when converting from long to double..? + graph.setEdgeWeight(edge, Long.valueOf(d.toMillis()).doubleValue()); + } + // Alternative version if we cannot assume that sequences are ordered by timestamp: +// if (iv.sequence.stream().max(Comparator.comparing(PcapPacket::getTimestamp)).get() +// .getTimestamp().isBefore(jv.sequence.stream().min( +// Comparator.comparing(PcapPacket::getTimestamp)).get().getTimestamp())) { +// +// } + } + } + } + } + // Graph construction complete, run shortest-path to find a (potential) signature match. + DijkstraShortestPath dijkstra = new DijkstraShortestPath<>(graph); + GraphPath shortestPath = dijkstra.getPath(source, sink); + if (shortestPath != null) { + // The total weight is the duration between the first packet of the first sequence and the last packet + // of the last sequence, so we simply have to compare the weight against the timeframe that we allow + // the signature to span. For now we just use the inclusion window we defined for training purposes. + // Note however, that we must convert back from double to long as the weight is stored as a double in + // JGraphT's API. + if (((long)shortestPath.getWeight()) < mInclusionTimeMillis) { + // There's a signature match! + // Extract the match from the vertices + List> signatureMatch = new ArrayList<>(); + for(Vertex v : shortestPath.getVertexList()) { + if (v == source || v == sink) { + // Skip the dummy source and sink nodes. + continue; + } + signatureMatch.add(v.sequence); + // As there is a one-to-one correspondence between vertices[] and pendingMatches[], we know that + // the sequence we've "consumed" for index i of the matched signature is also at index i in + // pendingMatches. We must remove it from pendingMatches so that we don't use it to construct + // another signature match in a later call. + pendingMatches[signatureMatch.size()-1].remove(v.sequence); + } + // Declare success: notify observers + mObservers.forEach(obs -> obs.onSignatureDetected(mSignature, + Collections.unmodifiableList(signatureMatch))); + } + } + } + } + + /** + * Used for registering for notifications of signatures detected by a {@link SignatureDetector}. + */ + interface SignatureDetectionObserver { + + /** + * Invoked when the {@link SignatureDetector} detects the presence of a signature in the traffic that it's + * examining. + * @param searchedSignature The signature that the {@link SignatureDetector} reporting the match is searching + * for. + * @param matchingTraffic The actual traffic trace that matches the searched signature. + */ + void onSignatureDetected(List>> searchedSignature, + List> matchingTraffic); + } + + /** + * Encapsulates a {@code List} so as to allow the list to be used as a vertex in a graph while avoiding + * the expensive {@link AbstractList#equals(Object)} calls when adding vertices to the graph. + * Using this wrapper makes the incurred {@code equals(Object)} calls delegate to {@link Object#equals(Object)} + * instead of {@link AbstractList#equals(Object)}. The net effect is a faster implementation, but the graph will not + * recognize two lists that contain the same items--from a value and not reference point of view--as the same + * vertex. However, this is fine for our purposes -- in fact restricting it to reference equality seems more + * appropriate. + */ + private static class Vertex { + private final List sequence; + private Vertex(List wrappedSequence) { + sequence = wrappedSequence; + } + } +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/evaluation/DetectionResultsAnalyzer.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/evaluation/DetectionResultsAnalyzer.java new file mode 100644 index 0000000..34063d5 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/evaluation/DetectionResultsAnalyzer.java @@ -0,0 +1,108 @@ +package edu.uci.iotproject.evaluation; + +import edu.uci.iotproject.analysis.TriggerTrafficExtractor; +import edu.uci.iotproject.analysis.UserAction; +import edu.uci.iotproject.io.PrintWriterUtils; +import edu.uci.iotproject.io.TriggerTimesFileReader; + +import java.io.*; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +/** + * Utility for comparing detected events to logged (actual) events. + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class DetectionResultsAnalyzer { + + private static boolean DUPLICATE_OUTPUT_TO_STD_OUT = true; + + public static void main(String[] args) throws IOException { + if (args.length < 3) { + String errMsg = String.format("Usage: %s triggerTimesFile detectionOutputFile [stdOut]" + + "\n - triggerTimesFile: the file that contains the timestamps for the user actions" + + "\n - detectionOutputFile: the file that contains the detected events" + + "\n - analysisResultsFile: where to write the results of the detection analysis" + + "\n - stdOut: optional true/false literal indicating if output should also be printed to std out; default is true", + DetectionResultsAnalyzer.class.getSimpleName()); + return; + } + String triggerTimesFile = args[0]; + File detectionOutputFile = new File(args[1]); + String analysisResultsFile = args[2]; + if (args.length > 3) { + DUPLICATE_OUTPUT_TO_STD_OUT = Boolean.parseBoolean(args[3]); + } + + // -------------------------------------- Parse the input files -------------------------------------- + + // Read the trigger times. + // The trigger times file does not contain event types as we initially assumed that we would just be alternating + // between ON and OFF. + List triggerTimestamps = new TriggerTimesFileReader().readTriggerTimes(triggerTimesFile, false); + // Now generate user actions based on this alternating ON/OFF pattern. + List triggers = new ArrayList<>(); + for (int i = 0; i < triggerTimestamps.size(); i++) { + // NOTE: assumes triggers alternate between ON and OFF + UserAction.Type actionType = i % 2 == 0 ? UserAction.Type.TOGGLE_ON : UserAction.Type.TOGGLE_OFF; + triggers.add(new UserAction(actionType, triggerTimestamps.get(i))); + } + // Read the detection output file, assuming a format as specified in UserAction.toString() + List detectedEvents = new ArrayList<>(); + try (BufferedReader br = new BufferedReader(new FileReader(detectionOutputFile))) { + String s; + while ((s = br.readLine()) != null) { + if (s.startsWith("#")) { + // Ignore comments. + continue; + } + detectedEvents.add(UserAction.fromString(s)); + } + } + + // ----------------- Now ready to compare the detected events with the logged events ----------------- + + // To contain all detected events that could be mapped to a trigger + List truePositives = new ArrayList<>(); + for (UserAction detectedEvent : detectedEvents) { + Optional matchingTrigger = triggers.stream() + .filter(t -> t.getType() == detectedEvent.getType() && + t.getTimestamp().isBefore(detectedEvent.getTimestamp()) && + t.getTimestamp().plusMillis(TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS). + isAfter(detectedEvent.getTimestamp()) + ).findFirst(); + matchingTrigger.ifPresent(mt -> { + // We've consumed the trigger (matched it with a detected event), so remove it so we don't match with + // another detected event. + triggers.remove(mt); + // The current detected event was a true positive as we could match it with a trigger. + truePositives.add(detectedEvent); + }); + } + // Now the false positives are those elements in detectedEvents that are not in truePositives + List falsePositives = new ArrayList<>(); + falsePositives.addAll(detectedEvents); + falsePositives.removeAll(truePositives); + + // Output the results... + PrintWriter outputter = new PrintWriter(new FileWriter(analysisResultsFile)); + PrintWriterUtils.println("---------- False negatives (events that where not detected) ----------", outputter, DUPLICATE_OUTPUT_TO_STD_OUT); + for (UserAction missing : triggers) { + PrintWriterUtils.println(missing, outputter, DUPLICATE_OUTPUT_TO_STD_OUT); + } + PrintWriterUtils.println("Total of " + Integer.toString(triggers.size()), outputter, DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.printEmptyLine(outputter, DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println("---------- False positives (detected, but no matching trigger) ----------", outputter, DUPLICATE_OUTPUT_TO_STD_OUT); + for (UserAction fp : falsePositives) { + PrintWriterUtils.println(fp, outputter, DUPLICATE_OUTPUT_TO_STD_OUT); + } + PrintWriterUtils.println("Total of " + Integer.toString(falsePositives.size()), outputter, DUPLICATE_OUTPUT_TO_STD_OUT); + outputter.flush(); + outputter.close(); + } + +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/evaluation/SanitySignatureGenerator.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/evaluation/SanitySignatureGenerator.java new file mode 100644 index 0000000..060387a --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/evaluation/SanitySignatureGenerator.java @@ -0,0 +1,121 @@ +package edu.uci.iotproject.evaluation; + +import edu.uci.iotproject.trafficreassembly.layer3.Conversation; +import edu.uci.iotproject.trafficreassembly.layer3.TcpReassembler; +import edu.uci.iotproject.io.PcapHandleReader; +import edu.uci.iotproject.util.PrintUtils; +import org.pcap4j.core.*; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +/** + * Hacky utility for producing a sanity signature for negative test sets. + *

+ * More precisely, given information about packet lengths and packet directions known to be present in an input trace, + * this class locates the first occurrence of a matching sequence in the input trace and outputs it to a file in the + * signature format (i.e., a {@code List>>}. + *

+ *

+ * Note: can only produce simplistic signatures, i.e., a signature that is a single packet sequence that + * occurs on a single TCP connection. + *

+ * + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class SanitySignatureGenerator { + + public static void main(String[] args) throws PcapNativeException, NotOpenException { + // The pcap file + final String pcapPath = "/Users/varmarken/temp/UCI IoT Project/experiments/evaluation/negative-datasets/UNB/Monday-WorkingHours_one-local-endpoint.pcap"; + final String sigOutputPath = "/Users/varmarken/temp/UCI IoT Project/experiments/evaluation/negative-datasets/UNB/Monday-WorkingHours_one-local-endpoint_sanity.sig"; + // The sequence of packet lengths known to be present in the trace + final List pktLengths = new ArrayList<>(); + pktLengths.add(340); + pktLengths.add(295); + // ...and their corresponding directions + final List pktDirections = new ArrayList<>(); + pktDirections.add(Conversation.Direction.CLIENT_TO_SERVER); + pktDirections.add(Conversation.Direction.SERVER_TO_CLIENT); + // Is the signature a TLS sequence? + final boolean tlsSequence = false; + + + PcapHandle handle; + try { + handle = Pcaps.openOffline(pcapPath, PcapHandle.TimestampPrecision.NANO); + } catch (PcapNativeException pne) { + handle = Pcaps.openOffline(pcapPath); + } + SequenceFinder seqFinder = new SequenceFinder(pktLengths, pktDirections, tlsSequence, sigOutputPath); + final PcapHandleReader reader = new PcapHandleReader(handle, p -> true, seqFinder); + seqFinder.setPcapHandleReader(reader); + reader.readFromHandle(); + } + + + private static class SequenceFinder implements PacketListener { + private final TcpReassembler mTcpReassembler = new TcpReassembler(); + private final List mPktLengths; + private final List mPktDirections; + private final boolean mTlsSequence; + private PcapHandleReader mReader; + private final String mSignatureOutputPath; + + private SequenceFinder(List pktLengths, + List pktDirections, + boolean tlsSequence, + String sigOutputPath) { + mPktLengths = pktLengths; + mPktDirections = pktDirections; + mTlsSequence = tlsSequence; + mSignatureOutputPath = sigOutputPath; + } + + @Override + public void gotPacket(PcapPacket packet) { + // Skip packets not matching expected length + if (!mPktLengths.contains(packet.getOriginalLength())) { + return; + } + // Otherwise forward to TCP reassembler. + mTcpReassembler.gotPacket(packet); + // We are done as soon as we have one conversation that has the expected number of packets with the expected + // directions. + Optional match = mTcpReassembler.getTcpConversations().stream().filter(c -> { + List cPkts = mTlsSequence ? c.getTlsApplicationDataPackets() : c.getPackets(); + if (cPkts.size() != mPktLengths.size()) { + return false; + } + for (int i = 0; i < cPkts.size(); i++) { + if (c.getDirection(cPkts.get(i)) != mPktDirections.get(i) || + cPkts.get(i).getOriginalLength() != mPktLengths.get(i)) { + return false; + } + } + return true; + }).findFirst(); + if (match.isPresent()) { + System.out.println("match found"); + // Terminate reader; no need to process the full file as we already have the data to produce the signature. + mReader.stopReading(); + // Convert sequence to signature format. + List>> signature = new ArrayList<>(); + List> cluster = new ArrayList<>(); + List sequence = mTlsSequence ? match.get().getTlsApplicationDataPackets() : match.get().getPackets(); + cluster.add(sequence); + signature.add(cluster); + // Output the signature to a file. + PrintUtils.serializeSignatureIntoFile(mSignatureOutputPath, signature); + } + } + + private void setPcapHandleReader(PcapHandleReader reader) { + mReader = reader; + } + } + +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/io/LiveCapture.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/io/LiveCapture.java new file mode 100644 index 0000000..2539415 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/io/LiveCapture.java @@ -0,0 +1,102 @@ +package edu.uci.iotproject.io; + +import org.pcap4j.core.*; +import org.pcap4j.packet.namednumber.DataLinkType; +import org.pcap4j.util.NifSelector; + +import java.io.IOException; +import java.util.Objects; + +/** + * Utility methods for setting up a {@link PcapHandleReader} that reads live traffic from a network interface card. + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class LiveCapture { + + // This main method is just for experimental purposes! + public static void main(String[] args) throws PcapNativeException, NotOpenException, InterruptedException { + // ================================================ EXAMPLE USE ================================================ + final String outputPcapFile = System.getProperty("user.home") + "/temp/livecapture42.pcap"; + final PcapDumper outputter = Pcaps.openDead(DataLinkType.EN10MB, 65536).dumpOpen(outputPcapFile); + // Prompt user to select what interface we should be listening to; dump packets to a file. + PcapHandleReader reader = fromCliNicSelection( + p -> { + try { + outputter.dump(p); + } catch (NotOpenException noe) { + noe.printStackTrace(); + } + } + ); + + // Read on separate thread so that we can get a chance to terminate the reader on this thread. + Thread readerThread = new Thread(() -> { + try { + reader.readFromHandle(); + } catch (PcapNativeException e) { + e.printStackTrace(); + } catch (NotOpenException e) { + e.printStackTrace(); + } + }); + readerThread.start(); + + // Pause to let reader read some packets before we terminate it. + Thread.sleep(30_000); + + // Shutdown reader. + reader.stopReading(); + System.out.println("Waiting for " + reader.getClass().getSimpleName() + " to terminate..."); + while (!reader.hasTerminated()); + // remember to flush any buffered output + outputter.flush(); + System.out.println(reader.getClass().getSimpleName() + " terminated."); + // ============================================================================================================= + } + + /** + * Prompts the user to pick a Network Interface Card (NIC) for which live traffic is to be captured, then creates a + * {@link PcapHandleReader} that is ready to start capturing live traffic from that NIC. + * + * @param listeners One or more {@link PacketListener}s to which packets read from the NIC will be delivered. + * + * @return A {@link PcapHandleReader} that is ready to start capturing live traffic from the selected NIC or + * {@code null} if no NICs can be found. + * + * @throws PcapNativeException if an error occurs in the pcap native library. + */ + public static PcapHandleReader fromCliNicSelection(PacketListener... listeners) throws PcapNativeException { + PcapNetworkInterface networkInterface = null; + try { + networkInterface = new NifSelector().selectNetworkInterface(); + } catch (IOException ioe) { + System.err.println("No network interfaces found."); + ioe.printStackTrace(); + } + return networkInterface != null ? fromNic(networkInterface, listeners) : null; + } + + /** + * Creates a {@link PcapHandleReader} that is ready to start capturing live traffic from the provided Network + * Interface Card (NIC). + * + * @param networkInterface The target NIC. + * @param listeners One or more {@link PacketListener}s to which packets read from the NIC will be delivered. + * + * @return A {@link PcapHandleReader} that is ready to start capturing live traffic from the provided NIC. + * + * @throws PcapNativeException if an error occurs in the pcap native library. + */ + public static PcapHandleReader fromNic(PcapNetworkInterface networkInterface, PacketListener... listeners) + throws PcapNativeException { + Objects.requireNonNull(networkInterface); + int snapshotLength = 65536; // in bytes + int readTimeout = 10000; // 0 is infinite on all systems but Solaris + PcapHandle handle = networkInterface.openLive(snapshotLength, PcapNetworkInterface.PromiscuousMode.PROMISCUOUS, readTimeout); + // Supply a filter that accepts all packets (p -> true) as we want to examine all traffic. + return new PcapHandleReader(handle, p -> true, listeners); + } + +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/io/PcapHandleReader.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/io/PcapHandleReader.java new file mode 100644 index 0000000..e52ce25 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/io/PcapHandleReader.java @@ -0,0 +1,116 @@ +package edu.uci.iotproject.io; + +import edu.uci.iotproject.analysis.PcapPacketFilter; +import org.pcap4j.core.*; + +import java.io.EOFException; +import java.util.concurrent.TimeoutException; + +/** + * Reads packets from a {@link PcapHandle} (online or offline) and delivers those packets that pass the test exercised + * by the provided {@link PcapPacketFilter} onto the provided {@link PacketListener}s. + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class PcapHandleReader { + + private final PcapPacketFilter mPacketFilter; + private final PcapHandle mHandle; + private final PacketListener[] mPacketListeners; + private volatile boolean mTerminated = false; + + /** + * Create a {@code PcapHandleReader}. + * @param handle An open {@link PcapHandle} that packets will be read from. + * @param packetFilter A {@link PcapPacketFilter} that dictates which of the packets read from {@code handle} should + * be delivered to {@code packetListeners}. Note that while a value of {@code null} is not + * permitted here, the caller can instead simply provide an implementation that always returns + * {@code true} if they want to include all packets read from {@code handle}. + * @param packetListeners One or more {@link PacketListener}s to which those packets read from {@code handle} that + * pass through {@code packetFilter} are delivered. + */ + public PcapHandleReader(PcapHandle handle, PcapPacketFilter packetFilter, PacketListener... packetListeners) { + mHandle = handle; + mPacketFilter = packetFilter; + mPacketListeners = packetListeners; + } + + + /** + * Start reading (and filtering) packets from the provided {@link PcapHandle}. + * @throws PcapNativeException if an error occurs in the pcap native library. + * @throws NotOpenException if the provided {@code PcapHandle} is not open. + */ + public void readFromHandle() throws PcapNativeException, NotOpenException { + int outOfOrderPackets = 0; + try { + PcapPacket prevPacket = null; + PcapPacket packet = null; + + while (!mTerminated) { + try { + packet = mHandle.getNextPacketEx(); + } catch (TimeoutException te) { + System.err.println("timeout occurred while reading from network interface"); + // No need to check termination flag here. Can defer it to the loop condition as it is the next + // instruction anyway. + continue; + } + + if (packet == null) { + System.err.println("null-packet read from handle"); + continue; + } + + if (prevPacket != null && packet.getTimestamp().isBefore(prevPacket.getTimestamp())) { + outOfOrderPackets++; + /* + // Fail early if assumption doesn't hold. + mHandle.close(); + throw new AssertionError("Packets not in ascending temporal order"); + */ + } + if (mPacketFilter.shouldIncludePacket(packet)) { + // Packet accepted for inclusion; deliver it to observing client code. + for (PacketListener consumer : mPacketListeners) { + consumer.gotPacket(packet); + } + } + prevPacket = packet; + } + } catch (EOFException eof) { + // Reached end of file. All good. + System.out.println(String.format("%s: finished reading pcap file", getClass().getSimpleName())); + } + if (outOfOrderPackets > 0) { + System.err.println( + String.format("[[[ %s: %d packets appeared out of order (with regards to their timestamps) ]]]", + getClass().getSimpleName(), outOfOrderPackets)); + } + mHandle.close(); + } + + /** + * Stop reading from the wrapped {@link PcapHandle}. Note that this call only initiates the shutdown by + * setting a termination flag. Shutdown will be deferred until the time at which this flag can be checked by + * {@link #readFromHandle()}. For example, if {@link #readFromHandle()} is currently in the middle of a blocking + * call to {@link PcapHandle#getNextPacketEx()}, shutdown will not occur until the next packet is returned from the + * wrapped {@link PcapHandle} or its read timeout expires. Use {@link #hasTerminated()} to check if the shutdown + * has completed. + */ + public void stopReading() { + mTerminated = true; + } + + /** + * Checks if this {@link PcapHandleReader} has gracefully terminated, i.e., that the wrapped {@link PcapHandle} has + * been closed. + * + * @return {@code true} if this {@link PcapHandleReader} has terminated, {@code false} otherwise. + */ + public boolean hasTerminated() { + return mTerminated && !mHandle.isOpen(); + } + +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/io/PrintWriterUtils.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/io/PrintWriterUtils.java new file mode 100644 index 0000000..167993d --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/io/PrintWriterUtils.java @@ -0,0 +1,43 @@ +package edu.uci.iotproject.io; + +import java.io.PrintWriter; + +/** + * Utility methods for (jointly) printing to a {@link PrintWriter} (and standard output). + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public final class PrintWriterUtils { + + private PrintWriterUtils() { + // Disallow instantiation. Static-only class. + } + + /** + * Invoke {@link PrintWriter#println(Object)} passing {@code line} as argument while also printing {@code line} to + * standard output if {@code duplicateToStdOut} is {@code true}. + * @param line The line to be printed. + * @param writer The {@link PrintWriter} that is to print {@code line}. + * @param duplicateToStdOut Set to {@code true} if {@code line} should also be printed in standard output. + */ + public static void println(Object line, PrintWriter writer, boolean duplicateToStdOut) { + if (duplicateToStdOut) { + System.out.println(line); + } + writer.println(line); + } + + /** + * Make writer (and standard output, if {@code duplicateToStdOut} is {@code true}) print an empty line. + * @param writer The writer that {@link PrintWriter#println()} is to be invoked on. + * @param duplicateToStdOut If {@code true}, prints an empty line to standard output. + */ + public static void printEmptyLine(PrintWriter writer, boolean duplicateToStdOut) { + if (duplicateToStdOut) { + System.out.println(); + } + writer.println(); + } + +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/io/TriggerTimesFileReader.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/io/TriggerTimesFileReader.java new file mode 100644 index 0000000..75d06ec --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/io/TriggerTimesFileReader.java @@ -0,0 +1,67 @@ +package edu.uci.iotproject.io; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.*; + +/** + * Parses a file to obtain the timestamps at which the smart plug was toggled on/off. + * + * @author Rahmadi Trimananda {@literal } + * @author Janus Varmarken {@literal } + */ +public class TriggerTimesFileReader { + + public static final ZoneId ZONE_ID_LOS_ANGELES = ZoneId.of("America/Los_Angeles"); + public static final ZoneId ZONE_ID_BUDAPEST = ZoneId.of("Europe/Budapest"); + + /** + * Reads a file with trigger timestamps and parses the timestamps into {@link Instant}s using the rules specified + * by {@link #parseTriggerTimestamp(String, boolean)}. + * @param fileName The absolute path to the file with trigger timestamps. + * @param _24hFormat {@code true} if the timestamps in the file are in 24 hour format, {@code false} if they are in + * AM/PM format. + * @return A containing the trigger timestamps represented as {@code Instant}s. + */ + public List readTriggerTimes(String fileName, boolean _24hFormat) { + List listTriggerTimes = new ArrayList<>(); + File file = new File(fileName); + try (BufferedReader br = new BufferedReader(new FileReader(file))) { + String s; + while ((s = br.readLine()) != null) { + listTriggerTimes.add(parseTriggerTimestamp(s, _24hFormat)); + } + } catch (IOException e) { + e.printStackTrace(); + } + System.out.println("List has: " + listTriggerTimes.size()); + return listTriggerTimes; + } + + /** + * Parses a timestamp string to an {@link Instant} (UTC). Assumes timestamps are LA time. + * Format is expected to be either "MM/dd/uuuu HH:mm:ss" or "MM/dd/uuuu h:mm:ss a". + * + * @param timestampStr The string containing a date-time timestamp for LA's timezone. + * @param _24hFormat {@code true} if the time in {@code timestampStr} is given in 24 hour format, {@code false} if + * it is given in AM/PM format. + * @return An {@code Instant} representation of the parsed timestamp. Note that the {@code Instant} marks a point on + * the timeline in UTC. Use {@link Instant#atZone(ZoneId)} to convert to the corresponding time in a given + * timezone. + */ + public Instant parseTriggerTimestamp(String timestampStr, boolean _24hFormat) { + // Note: only one 'h' when not prefixed with leading 0 for 1-9; and only one 'a' for AM/PM marker in Java 8 time + String format = _24hFormat ? "MM/dd/uuuu HH:mm:ss" : "MM/dd/uuuu h:mm:ss a"; + LocalDateTime localDateTime = LocalDateTime.parse(timestampStr, DateTimeFormatter.ofPattern(format, Locale.US)); + ZonedDateTime laZonedDateTime = localDateTime.atZone(ZONE_ID_LOS_ANGELES); + return laZonedDateTime.toInstant(); + } + +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/maclayer/MacLayerFlowPattern.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/maclayer/MacLayerFlowPattern.java new file mode 100644 index 0000000..7cff733 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/maclayer/MacLayerFlowPattern.java @@ -0,0 +1,46 @@ +package edu.uci.iotproject.maclayer; + +import java.util.Collections; +import java.util.List; + +/** + * TODO create base class for FlowPattern and derive MacLayer, TCP/IP layer versions from that. + * + * @author Janus Varmarken + */ +public class MacLayerFlowPattern { + + private final List mPacketLengthSequence; + private final String mMacPrefix; + private final String mPatternId; + private final byte[] mMacPreixBytes; + + public MacLayerFlowPattern(String patternId, String macPrefix, List packetLengthSequence) { + mMacPrefix = macPrefix; + mPatternId = patternId; + mPacketLengthSequence = packetLengthSequence; + // Conversion provided by https://stackoverflow.com/a/10839361/1214974 + String[] addressParts = macPrefix.split(":"); + mMacPreixBytes = new byte[addressParts.length]; + for(int i = 0; i < mMacPreixBytes.length; i++) { + Integer hex = Integer.parseInt(addressParts[i], 16); + mMacPreixBytes[i] = hex.byteValue(); + } + } + + public String getPatternId() { + return mPatternId; + } + + public byte[] getMacPrefixRawBytes() { + return mMacPreixBytes; + } + + public List getPacketLengthSequence() { + return Collections.unmodifiableList(mPacketLengthSequence); + } + + public int getLength() { + return mPacketLengthSequence.size(); + } +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/maclayer/MacLayerFlowPatternFinder.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/maclayer/MacLayerFlowPatternFinder.java new file mode 100644 index 0000000..6cca3ad --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/maclayer/MacLayerFlowPatternFinder.java @@ -0,0 +1,102 @@ +package edu.uci.iotproject.maclayer; + +import edu.uci.iotproject.FlowPattern; +import org.pcap4j.core.NotOpenException; +import org.pcap4j.core.PcapHandle; +import org.pcap4j.core.PcapNativeException; +import org.pcap4j.core.PcapPacket; +import org.pcap4j.packet.RadiotapPacket; + +import java.io.EOFException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.TimeoutException; + +/** + * Performs a search for {@link FlowPattern} + * TODO: May want to create an abstract FlowPatternFinder and then derive MacLayer, TcpipLayer FlowPatternFinders from that one. + * + * @author Janus Varmarken + */ +public class MacLayerFlowPatternFinder { + + private final MacLayerFlowPattern mPattern; + private final PcapHandle mPcap; + + public MacLayerFlowPatternFinder(PcapHandle pcap, MacLayerFlowPattern pattern) { + this.mPcap = Objects.requireNonNull(pcap, + String.format("Argument of type '%s' cannot be null", PcapHandle.class.getSimpleName())); + this.mPattern = Objects.requireNonNull(pattern, + String.format("Argument of type '%s' cannot be null", FlowPattern.class.getSimpleName())); + } + + public void findFlowPattern() { + PcapPacket packet; + try { + // Packets matched to flow pattern searched for. + List patternPackets = new ArrayList<>(); + while ((packet = mPcap.getNextPacketEx()) != null) { + RadiotapPacket radiotapPacket; + try { + // Some packets throw an IAE with message "msi must be between 0 and 6 but is actually: 7" + // when accessing the RadiotapPacket. + radiotapPacket = packet.get(RadiotapPacket.class); + } catch (IllegalArgumentException iae) { + System.out.println(iae.getMessage()); + continue; + } + if (radiotapPacket == null) { + continue; + } + // Restart search if pattern not found within reasonable time frame (hardcoded for now). + if (patternPackets.size() > 0 && packet.getTimestamp().getEpochSecond() - + patternPackets.get(patternPackets.size()-1).getTimestamp().getEpochSecond() > 2) { + patternPackets = new ArrayList<>(); + } + + byte[] rawData = radiotapPacket.getPayload().getRawData(); + // Search rawData for MAC of FlowPattern in sender/receiver section + // [TODO needs verification that this section is actually the sender/receiver section] + if (rawData.length < 16) { + continue; + } + int prefixLength = mPattern.getMacPrefixRawBytes().length; + byte[] mac1 = Arrays.copyOfRange(rawData, 4, prefixLength < 6 ? 4 + prefixLength : 10); + byte[] mac2 = Arrays.copyOfRange(rawData, 10, prefixLength < 6 ? 10 + prefixLength : 16); + if (!Arrays.equals(mac1, mPattern.getMacPrefixRawBytes()) && !Arrays.equals(mac2, mPattern.getMacPrefixRawBytes())) { + // MAC prefix not present in raw data. + continue; + } + // Packet related to device associated with the pattern we are looking for. + int expectedLength = mPattern.getPacketLengthSequence().get(patternPackets.size()); + if (packet.length() == expectedLength) { + patternPackets.add(packet); + if (patternPackets.size() == mPattern.getLength()) { + // Full pattern found, declare success if packets are within some reasonable amount of time of + // one another. + // For now, we use a hardcoded value. + if (patternPackets.get(patternPackets.size()-1).getTimestamp().getEpochSecond() - + patternPackets.get(0).getTimestamp().getEpochSecond() < 5) { + System.out.println(String.format("[ find ] Detected a COMPLETE MATCH of pattern '%s' at %s!", + mPattern.getPatternId(), patternPackets.get(0).getTimestamp().toString())); + } + // Reset search by resetting list. + patternPackets = new ArrayList<>(); + } + } else { + // Discard packet, not relevant to pattern. + continue; + } + } + } catch (EOFException e) { + // TODO wait for, and print, results. + } catch (PcapNativeException|TimeoutException|NotOpenException e) { + e.printStackTrace(); + } + } + +} + + diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2Flow.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2Flow.java new file mode 100644 index 0000000..f8c3237 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2Flow.java @@ -0,0 +1,115 @@ +package edu.uci.iotproject.trafficreassembly.layer2; + +import org.pcap4j.core.PcapPacket; +import org.pcap4j.packet.EthernetPacket; +import org.pcap4j.util.MacAddress; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Models a layer 2 flow: groups packets exchanged between two specific endpoints (MAC addresses). + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class Layer2Flow { + + /** + * The first endpoint of this layer 2 flow. + */ + private final MacAddress mEndpoint1; + + /** + * The second endpoint of this layer 2 flow. + */ + private final MacAddress mEndpoint2; + + /** + * Clients observing for changes to this layer 2 flow. + */ + private final List mFlowObservers = new ArrayList<>(); + + public Layer2Flow(MacAddress endpoint1, MacAddress endpoint2) { + mEndpoint1 = endpoint1; + mEndpoint2 = endpoint2; + } + + /** + * Get the first endpoint of this flow. + * @return the first endpoint of this flow. + */ + public MacAddress getEndpoint1() { + return mEndpoint1; + } + + /** + * Get the second endpoint of this flow. + * @return the second endpoint of this flow. + */ + public MacAddress getEndpoint2() { + return mEndpoint2; + } + + /** + * Register as an observer of this flow. + * @param observer The client that is to be notified whenever this flow changes (has new packets added). + */ + public void addFlowObserver(Layer2FlowObserver observer) { + mFlowObservers.add(observer); + } + + /** + * Deregister as an observer of this flow. + * @param observer The client that no longer wishes to be notified whenever this flow changes. + */ + public void removeFlowObserver(Layer2FlowObserver observer) { + mFlowObservers.remove(observer); + } + + /** + * The packets in the flow. + */ + private final List mPackets = new ArrayList<>(); + + /** + * Add a packet to this flow. + * @param packet The packet that is to be added to the flow. + */ + public void addPacket(PcapPacket packet) { + verifyAddresses(packet); + mPackets.add(packet); + // Notify flow observers of the new packet + mFlowObservers.forEach(o -> o.onNewPacket(this, packet)); + } + + /** + * Get the packets pertaining to this flow. + * @return The packets pertaining to this flow. + */ + public List getPackets() { + return Collections.unmodifiableList(mPackets); + } + + /** + * Verify that a packet pertains to this flow. + * @param packet The packet that is to be verified. + */ + private void verifyAddresses(PcapPacket packet) { + EthernetPacket ethPkt = packet.get(EthernetPacket.class); + MacAddress srcAddr = ethPkt.getHeader().getSrcAddr(); + MacAddress dstAddr = ethPkt.getHeader().getDstAddr(); + if ((mEndpoint1.equals(srcAddr) && mEndpoint2.equals(dstAddr)) || + (mEndpoint1.equals(dstAddr) && mEndpoint2.equals(srcAddr))) { + // All is good. + return; + } + throw new IllegalArgumentException("Mismatch in MACs: packet does not pertain to this flow"); + } + + @Override + public String toString() { + return getClass().getSimpleName() + String.format(" with mEndpoint1=%s and mEndpoint2=%s", mEndpoint1, mEndpoint2); + } +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowObserver.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowObserver.java new file mode 100644 index 0000000..e1648ba --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowObserver.java @@ -0,0 +1,20 @@ +package edu.uci.iotproject.trafficreassembly.layer2; + +import org.pcap4j.core.PcapPacket; + +/** + * Interface for observing a {@link Layer2Flow}. + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public interface Layer2FlowObserver { + + /** + * Invoked when a new packet is added to the observed flow. + * @param flow The observed flow. + * @param newPacket The packet that was added to the flow. + */ + void onNewPacket(Layer2Flow flow, PcapPacket newPacket); + +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowReassembler.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowReassembler.java new file mode 100644 index 0000000..e7b7304 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowReassembler.java @@ -0,0 +1,86 @@ +package edu.uci.iotproject.trafficreassembly.layer2; + +import edu.uci.iotproject.trafficreassembly.layer2.Layer2Flow; +import edu.uci.iotproject.trafficreassembly.layer2.Layer2FlowReassemblerObserver; +import org.pcap4j.core.PacketListener; +import org.pcap4j.core.PcapPacket; +import org.pcap4j.packet.EthernetPacket; +import org.pcap4j.util.MacAddress; + +import java.util.*; + +/** + * Reassembles traffic flows at layer 2, i.e., for each combination of hosts, creates a list of packets exchanged + * between said hosts. + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class Layer2FlowReassembler implements PacketListener { + + /** + * Maps a pair of MAC addresses to the packets exchanged between the two hosts. + * The key is the concatenation of the two MAC addresses in hex string format, where the lexicographically smaller + * MAC is at the front of the string. + */ + private final Map mFlows = new HashMap<>(); + + private final List mObservers = new ArrayList<>(); + + @Override + public void gotPacket(PcapPacket packet) { + // TODO: update to 802.11 packet...? + EthernetPacket ethPkt = packet.get(EthernetPacket.class); + + MacAddress srcAddr = ethPkt.getHeader().getSrcAddr(); + MacAddress dstAddr = ethPkt.getHeader().getDstAddr(); + + String key = keyFromAddresses(srcAddr, dstAddr); + // Create a new list if this pair of MAC addresses where not previously encountered and add packet to that list, + // or simply add to an existing list if one is present. + mFlows.computeIfAbsent(key, k -> { + Layer2Flow newFlow = new Layer2Flow(srcAddr, dstAddr); + // Inform observers of the new flow + mObservers.forEach(o -> o.onNewFlow(this, newFlow)); + return newFlow; + }).addPacket(packet); + } + + public void addObserver(Layer2FlowReassemblerObserver observer) { + mObservers.add(observer); + } + + public void removeObserver(Layer2FlowReassemblerObserver observer) { + mObservers.remove(observer); + } + + /** + * Get the traffic flow between two local endpoints ({@link MacAddress}es). + * @param addr1 The first endpoint. + * @param addr2 The second endpoint + * @return The traffic exchanged between the two endpoints. + */ + public Layer2Flow getFlowForAddresses(MacAddress addr1, MacAddress addr2) { + return mFlows.get(keyFromAddresses(addr1, addr2)); + } + + /** + * Get all traffic flows, i.e., a traffic flow for each unique pair of endpoints (MAC addresses). + * @return All traffic flows. + */ + public Collection getFlows() { + return mFlows.values(); + } + + /** + * Given two {@link MacAddress}es, generates the corresponding key string used in {@link #mFlows}. + * @param addr1 The first address. + * @param addr2 The second address. + * @return the key string used in {@link #mFlows} corresponding to the two addresses. + */ + private String keyFromAddresses(MacAddress addr1, MacAddress addr2) { + String addr1Str = addr1.toString(); + String addr2Str = addr2.toString(); + return addr1Str.compareTo(addr2Str) < 0 ? addr1Str + addr2Str : addr2Str + addr1Str; + } +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowReassemblerObserver.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowReassemblerObserver.java new file mode 100644 index 0000000..a71cd19 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowReassemblerObserver.java @@ -0,0 +1,19 @@ +package edu.uci.iotproject.trafficreassembly.layer2; + +/** + * For observing a {@link Layer2FlowReassembler}. + * + * @author Janus Varmarken + */ +public interface Layer2FlowReassemblerObserver { + + /** + * Invoked when when a {@link Layer2FlowReassembler} detects a new flow (i.e., when it encounters traffic between two + * MAC addresses that has not previously communicated in the traffic trace). + * + * @param reassembler The {@link Layer2FlowReassembler} that detected the new flow. + * @param newFlow The new flow. + */ + void onNewFlow(Layer2FlowReassembler reassembler, Layer2Flow newFlow); + +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/Conversation.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/Conversation.java new file mode 100644 index 0000000..e89e81b --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/Conversation.java @@ -0,0 +1,585 @@ +package edu.uci.iotproject.trafficreassembly.layer3; + +import edu.uci.iotproject.analysis.TcpConversationUtils; +import edu.uci.iotproject.util.PcapPacketUtils; +import org.pcap4j.core.PcapPacket; +import org.pcap4j.packet.IpV4Packet; +import org.pcap4j.packet.Packet; +import org.pcap4j.packet.TcpPacket; + +import java.util.*; + +/** + * Models a (TCP) conversation/connection/session/flow (packet's belonging to the same session between a client and a + * server). + * Holds a list of {@link PcapPacket}s identified as pertaining to the flow. Note that this list is not + * considered when determining equality of two {@code Conversation} instances in order to allow for a + * {@code Conversation} to function as a key in data structures such as {@link java.util.Map} and {@link java.util.Set}. + * See {@link #equals(Object)} for the definition of equality. + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class Conversation { + + /* Begin instance properties */ + /** + * The IP of the host that is considered the client (i.e. the host that initiates the conversation) + * in this conversation. + */ + private final String mClientIp; + + /** + * The port number used by the host that is considered the client in this conversation. + */ + private final int mClientPort; + + /** + * The IP of the host that is considered the server (i.e. is the responder) in this conversation. + */ + private final String mServerIp; + + /** + * The port number used by the server in this conversation. + */ + private final int mServerPort; + + /** + * The list of packets (with payload) pertaining to this conversation. + */ + private final List mPackets; + + /** + * If {@link #isTls()} is {@code true}, this list contains the subset of {@link #mPackets} which are TLS Application + * Data packets. + */ + private final List mTlsApplicationDataPackets; + + /** + * Contains the sequence numbers used thus far by the host that is considered the client in this + * {@code Conversation}. + * Used for filtering out retransmissions. + */ + private final Set mSeqNumbersClient; + + /** + * Contains the sequence numbers used thus far by the host that is considered the server in this + * {@code Conversation}. + * Used for filtering out retransmissions. + */ + private final Set mSeqNumbersSrv; + + /** + * List of SYN packets pertaining to this conversation. + */ + private final List mSynPackets; + + /** + * List of pairs FINs and their corresponding ACKs associated with this conversation. + */ + private final List mFinPackets; + + /** + * List of RST packets associated with this conversation. + */ + private final List mRstPackets; + + /** + * Boolean to mark the packet as Application Data based on the previous packet that reaches MTU + */ + private boolean mApplicationData; + /* End instance properties */ + + /** + * Factory method for creating a {@code Conversation} from a {@link PcapPacket}. + * @param pcapPacket The {@code PcapPacket} that wraps a TCP segment for which a {@code Conversation} is to be initiated. + * @param clientIsSrc If {@code true}, the source address and source port found in the IP datagram and TCP segment + * wrapped in the {@code PcapPacket} are regarded as pertaining to the client, and the destination + * address and destination port are regarded as pertaining to the server---and vice versa if set + * to {@code false}. + * @return A {@code Conversation} initiated with ip:port for client and server according to the direction of the packet. + */ + public static Conversation fromPcapPacket(PcapPacket pcapPacket, boolean clientIsSrc) { + IpV4Packet ipPacket = pcapPacket.get(IpV4Packet.class); + TcpPacket tcpPacket = pcapPacket.get(TcpPacket.class); + String clientIp = clientIsSrc ? ipPacket.getHeader().getSrcAddr().getHostAddress() : + ipPacket.getHeader().getDstAddr().getHostAddress(); + String srvIp = clientIsSrc ? ipPacket.getHeader().getDstAddr().getHostAddress() : + ipPacket.getHeader().getSrcAddr().getHostAddress(); + int clientPort = clientIsSrc ? tcpPacket.getHeader().getSrcPort().valueAsInt() : + tcpPacket.getHeader().getDstPort().valueAsInt(); + int srvPort = clientIsSrc ? tcpPacket.getHeader().getDstPort().valueAsInt() : + tcpPacket.getHeader().getSrcPort().valueAsInt(); + return new Conversation(clientIp, clientPort, srvIp, srvPort); + } + + /** + * Constructs a new {@code Conversation}. + * @param clientIp The IP of the host that is considered the client (i.e. the host that initiates the conversation) + * in the conversation. + * @param clientPort The port number used by the client for the conversation. + * @param serverIp The IP of the host that is considered the server (i.e. is the responder) in the conversation. + * @param serverPort The port number used by the server for the conversation. + */ + public Conversation(String clientIp, int clientPort, String serverIp, int serverPort) { + this.mClientIp = clientIp; + this.mClientPort = clientPort; + this.mServerIp = serverIp; + this.mServerPort = serverPort; + this.mPackets = new ArrayList<>(); + this.mTlsApplicationDataPackets = new ArrayList<>(); + this.mSeqNumbersClient = new HashSet<>(); + this.mSeqNumbersSrv = new HashSet<>(); + this.mSynPackets = new ArrayList<>(); + this.mFinPackets = new ArrayList<>(); + this.mRstPackets = new ArrayList<>(); + this.mApplicationData = false; + } + + /** + * Add a packet to the list of packets associated with this conversation. + * @param packet The packet that is to be added to (associated with) this conversation. + * @param ignoreRetransmissions Boolean value indicating if retransmissions should be ignored. + * If set to {@code true}, {@code packet} will not be added to the + * internal list of packets pertaining to this {@code Conversation} + * iff the sequence number of {@code packet} was already + * seen in a previous packet. + */ + public void addPacket(PcapPacket packet, boolean ignoreRetransmissions) { + // Precondition: verify that packet does indeed pertain to conversation. + onAddPrecondition(packet); + if (ignoreRetransmissions && isRetransmission(packet)) { + // Packet is a retransmission. Ignore it. + return; + } + // Select direction-dependent set of sequence numbers seen so far and update it with sequence number of new packet. + addSeqNumber(packet); + // Finally add packet to list of packets pertaining to this conversation. + mPackets.add(packet); + // Preserve order of packets in list: sort according to timestamp. + if (mPackets.size() > 1 && + mPackets.get(mPackets.size()-1).getTimestamp().isBefore(mPackets.get(mPackets.size()-2).getTimestamp())) { + Collections.sort(mPackets, (o1, o2) -> { + if (o1.getTimestamp().isBefore(o2.getTimestamp())) { return -1; } + else if (o2.getTimestamp().isBefore(o1.getTimestamp())) { return 1; } + else { return 0; } + }); + } + // If TLS, inspect packet to see if it's a TLS Application Data packet, and if so add it to the list of TLS + // Application Data packets. + if (isTls()) { + TcpPacket tcpPacket = packet.get(TcpPacket.class); + Packet tcpPayload = tcpPacket.getPayload(); + if (tcpPayload == null) { + return; + } + byte[] rawPayload = tcpPayload.getRawData(); + // The SSL record header is at the front of the payload and is 5 bytes long. + // The SSL record header type field (the first byte) is set to 23 if it is an Application Data packet. + if (rawPayload != null && rawPayload.length >= 5) { + if (rawPayload[0] == 23) { + mTlsApplicationDataPackets.add(packet); + // Consider the following packet a data packet if this packet's size == MTU size 1448 + if (rawPayload.length >= 1448) + mApplicationData = true; + } else if (rawPayload[0] == 20) { + // Do nothing for now - CHANGE_CIPHER_SPEC + } else if (rawPayload[0] == 21) { + // Do nothing for now - ALERT + } else if (rawPayload[0] == 22) { + // Do nothing for now - HANDSHAKE + } else { + // If it is TLS with payload, but rawPayload[0] != 23 + if (mApplicationData == true) { + // It is a continuation of the previous packet if the previous packet reaches MTU size 1448 and + // it is not either type 20, 21, or 22 + mTlsApplicationDataPackets.add(packet); + if (rawPayload.length < 1448) + mApplicationData = false; + } + } + } + } + } + + /** + * Get a list of packets pertaining to this {@code Conversation}. + * The returned list is a read-only list. + * @return the list of packets pertaining to this {@code Conversation}. + */ + public List getPackets() { + // Return read-only view to prevent external code from manipulating internal state (preserve invariant). + return Collections.unmodifiableList(mPackets); + } + + /** + * Records a TCP SYN packet as pertaining to this conversation (adds it to the the internal list). + * Attempts to add duplicate SYN packets will be ignored, and the caller is made aware of the attempt to add a + * duplicate by the return value being {@code false}. + * + * @param synPacket A {@link PcapPacket} wrapping a TCP SYN packet. + * @return {@code true} if the packet was successfully added to this {@code Conversation}, {@code false} otherwise. + */ + public boolean addSynPacket(PcapPacket synPacket) { + onAddPrecondition(synPacket); + final IpV4Packet synPacketIpSection = synPacket.get(IpV4Packet.class); + final TcpPacket synPacketTcpSection = synPacket.get(TcpPacket.class); + if (synPacketTcpSection == null || !synPacketTcpSection.getHeader().getSyn()) { + throw new IllegalArgumentException("Not a SYN packet."); + } + // We are only interested in recording one copy of the two SYN packets (one SYN packet in each direction), i.e., + // we want to discard retransmitted SYN packets. + if (mSynPackets.size() >= 2) { + return false; + } + // Check the set of recorded SYN packets to see if we have already recorded a SYN packet going in the same + // direction as the packet given in the argument. + boolean matchingPrevSyn = mSynPackets.stream().anyMatch(p -> { + IpV4Packet pIp = p.get(IpV4Packet.class); + TcpPacket pTcp = p.get(TcpPacket.class); + boolean srcAddrMatch = synPacketIpSection.getHeader().getSrcAddr().getHostAddress(). + equals(pIp.getHeader().getSrcAddr().getHostAddress()); + boolean dstAddrMatch = synPacketIpSection.getHeader().getDstAddr().getHostAddress(). + equals(pIp.getHeader().getDstAddr().getHostAddress()); + boolean srcPortMatch = synPacketTcpSection.getHeader().getSrcPort().valueAsInt() == + pTcp.getHeader().getSrcPort().valueAsInt(); + boolean dstPortMatch = synPacketTcpSection.getHeader().getDstPort().valueAsInt() == + pTcp.getHeader().getDstPort().valueAsInt(); + return srcAddrMatch && dstAddrMatch && srcPortMatch && dstPortMatch; + }); + if (matchingPrevSyn) { + return false; + } + // Update direction-dependent set of sequence numbers and record/log packet. + addSeqNumber(synPacket); + return mSynPackets.add(synPacket); + + /* + mSynPackets.stream().anyMatch(p -> { + IpV4Packet pIp = p.get(IpV4Packet.class); + TcpPacket pTcp = p.get(TcpPacket.class); + boolean srcAddrMatch = synPacketIpSection.getHeader().getSrcAddr().getHostAddress(). + equals(pIp.getHeader().getSrcAddr().getHostAddress()); + boolean dstAddrMatch = synPacketIpSection.getHeader().getDstAddr().getHostAddress(). + equals(pIp.getHeader().getDstAddr().getHostAddress()); + boolean srcPortMatch = synPacketTcpSection.getHeader().getSrcPort().valueAsInt() == + pTcp.getHeader().getSrcPort().valueAsInt(); + boolean dstPortMatch = synPacketTcpSection.getHeader().getDstPort().value() == + pTcp.getHeader().getDstPort().value(); + + boolean fourTupleMatch = srcAddrMatch && dstAddrMatch && srcPortMatch && dstPortMatch; + + boolean seqNoMatch = synPacketTcpSection.getHeader().getSequenceNumber() == + pTcp.getHeader().getSequenceNumber(); + + if (fourTupleMatch && !seqNoMatch) { + // If the four tuple that identifies the conversation matches, but the sequence number is different, + // it means that this SYN packet is, in fact, an attempt to establish a **new** connection, and hence + // the given packet is NOT part of this conversation, even though the ip:port combinations are (by + // chance) selected such that they match this conversation. + throw new IllegalArgumentException("Attempt to add SYN packet that belongs to a different conversation " + + "(which is identified by the same four tuple as this conversation)"); + } + return fourTupleMatch && seqNoMatch; + }); + */ + } + + /** + * Get a list of SYN packets pertaining to this {@code Conversation}. + * The returned list is a read-only list. + * @return the list of SYN packets pertaining to this {@code Conversation}. + */ + public List getSynPackets() { + return Collections.unmodifiableList(mSynPackets); + } + + /** + * Adds a TCP FIN packet to the list of TCP FIN packets associated with this conversation. + * @param finPacket The TCP FIN packet that is to be added to (associated with) this conversation. + */ + public void addFinPacket(PcapPacket finPacket) { + // Precondition: verify that packet does indeed pertain to conversation. + onAddPrecondition(finPacket); + // TODO: should call addSeqNumber here? + addSeqNumber(finPacket); + mFinPackets.add(new FinAckPair(finPacket)); + } + + /** + * Attempt to ACK any FIN packets held by this conversation. + * @param ackPacket The ACK for a FIN previously added to this conversation. + */ + public void attemptAcknowledgementOfFin(PcapPacket ackPacket) { + // Precondition: verify that the packet pertains to this conversation. + onAddPrecondition(ackPacket); + // Mark unack'ed FIN(s) that this ACK matches as ACK'ed (there might be more than one in case of retransmissions..?) + mFinPackets.replaceAll(finAckPair -> !finAckPair.isAcknowledged() && finAckPair.isCorrespondingAckPacket(ackPacket) ? new FinAckPair(finAckPair.getFinPacket(), ackPacket) : finAckPair); + } + + /** + * Retrieves an unmodifiable view of the list of {@link FinAckPair}s associated with this {@code Conversation}. + * @return an unmodifiable view of the list of {@link FinAckPair}s associated with this {@code Conversation}. + */ + public List getFinAckPairs() { + return Collections.unmodifiableList(mFinPackets); + } + + /** + * Get if this {@code Conversation} is considered to have been gracefully shut down. + * A {@code Conversation} has been gracefully shut down if it contains a FIN+ACK pair for both directions + * (client to server, and server to client). + * @return {@code true} if the connection has been gracefully shut down, false otherwise. + */ + public boolean isGracefullyShutdown() { + // The conversation has been gracefully shut down if we have recorded a FIN from both the client and the server which have both been ack'ed. + return mFinPackets.stream().anyMatch(finAckPair -> finAckPair.isAcknowledged() && PcapPacketUtils.isSource(finAckPair.getFinPacket(), mClientIp, mClientPort)) && + mFinPackets.stream().anyMatch(finAckPair -> finAckPair.isAcknowledged() && PcapPacketUtils.isSource(finAckPair.getFinPacket(), mServerIp, mServerPort)); + } + + /** + * Add a TCP segment for which the RST flag is set to this {@code Conversation}. + * @param packet A {@link PcapPacket} wrapping a TCP segment pertaining to this {@code Conversation} for which the + * RST flag is set. + */ + public void addRstPacket(PcapPacket packet) { + /* + * TODO: + * When now also keeping track of RST packets, should we also...? + * 1) Prevent later packets from being added once a RST segment has been added? + * 2) Extend 'isGracefullyShutdown()' to also consider RST segments, or add another method, 'isShutdown()' that + * both considers FIN/ACK (graceful) as well as RST (abrupt/"ungraceful") shutdown? + * 3) Should it be impossible to associate more than one RST segment with each Conversation? + */ + onAddPrecondition(packet); + TcpPacket tcpPacket = packet.get(TcpPacket.class); + if (tcpPacket == null || !tcpPacket.getHeader().getRst()) { + throw new IllegalArgumentException("not a RST packet"); + } + mRstPackets.add(packet); + } + + /** + * Get the TCP segments pertaining to this {@code Conversation} for which it was detected that the RST flag is set. + * @return the TCP segments pertaining to this {@code Conversation} for which it was detected that the RST flag is + * set. + */ + public List getRstPackets() { + return Collections.unmodifiableList(mRstPackets); + } + + // ========================================================================================================= + // We simply reuse equals and hashCode methods of String.class to be able to use this class as a key + // in a Map. + + /** + * Note: currently, equality is determined based on pairwise equality of the elements of the four tuple + * ({@link #mClientIp}, {@link #mClientPort}, {@link #mServerIp}, {@link #mServerPort}) for {@code this} and + * {@code obj}. + * @param obj The object to test for equality with {@code this}. + * @return {@code true} if {@code obj} is considered equal to {@code this} based on the definition of equality given above. + */ + @Override + public boolean equals(Object obj) { + return obj instanceof Conversation && this.toString().equals(obj.toString()); + } + + @Override + public int hashCode() { + return toString().hashCode(); + } + // ========================================================================================================= + + @Override + public String toString() { + return String.format("%s:%d %s:%d", mClientIp, mClientPort, mServerIp, mServerPort); + } + + /** + * Invoke to verify that the precondition holds when a caller attempts to add a packet to this {@code Conversation}. + * An {@link IllegalArgumentException} is thrown if the precondition is violated. + * @param packet the packet to be added to this {@code Conversation} + */ + private void onAddPrecondition(PcapPacket packet) { + // Apply precondition to preserve class invariant: all packets in mPackets must match the 4 tuple that + // defines the conversation. + IpV4Packet ipPacket = Objects.requireNonNull(packet.get(IpV4Packet.class)); + // For now we only support TCP flows. + TcpPacket tcpPacket = Objects.requireNonNull(packet.get(TcpPacket.class)); + String ipSrc = ipPacket.getHeader().getSrcAddr().getHostAddress(); + String ipDst = ipPacket.getHeader().getDstAddr().getHostAddress(); + int srcPort = tcpPacket.getHeader().getSrcPort().valueAsInt(); + int dstPort = tcpPacket.getHeader().getDstPort().valueAsInt(); + String clientIp, serverIp; + int clientPort, serverPort; + if (ipSrc.equals(mClientIp)) { + clientIp = ipSrc; + clientPort = srcPort; + serverIp = ipDst; + serverPort = dstPort; + } else { + clientIp = ipDst; + clientPort = dstPort; + serverIp = ipSrc; + serverPort = srcPort; + } + if (!(clientIp.equals(mClientIp) && clientPort == mClientPort && + serverIp.equals(mServerIp) && serverPort == mServerPort)) { + throw new IllegalArgumentException( + String.format("Attempt to add packet that does not pertain to %s", + Conversation.class.getSimpleName())); + } + } + + /** + *

+ * Determines if the TCP packet contained in {@code packet} is a retransmission of a previously seen (logged) + * packet. + *

+ * + * + * TODO: + * the current implementation, which uses a set of previously seen sequence numbers, will consider a segment + * with a reused sequence number---occurring as a result of sequence number wrap around for a very long-lived + * connection---as a retransmission (and may therefore end up discarding it even though it is in fact NOT a + * retransmission). Ideas? + * + * + * @param packet The packet. + * @return {@code true} if {@code packet} was determined to be a retransmission, {@code false} otherwise. + */ + public boolean isRetransmission(PcapPacket packet) { + // Extract sequence number. + int seqNo = packet.get(TcpPacket.class).getHeader().getSequenceNumber(); + switch (getDirection(packet)) { + case CLIENT_TO_SERVER: + return mSeqNumbersClient.contains(seqNo); + case SERVER_TO_CLIENT: + return mSeqNumbersSrv.contains(seqNo); + default: + throw new AssertionError(String.format("Unexpected value of enum '%s'", + Direction.class.getSimpleName())); + } + } + + /** + *

+ * Is this {@code Conversation} a TLS session? + *

+ * + * Note: the current implementation simply examines the port number(s) for 443; it does not verify if the + * application data is indeed encrypted. + * + * @return {@code true} if this {@code Conversation} is interpreted as a TLS session, {@code false} otherwise. + */ + public boolean isTls() { + /* + * TODO: + * - may want to change this to be "return mServerPort == 443 || mClientPort == 443;" in order to also detect + * TLS in those cases where it is not possible to correctly label who is the client and who is the server, + * i.e., when the trace does not contain the SYN/SYNACK exchange. + * - current implementation relies on the server using the conventional TLS port number; may instead want to + * inspect the first 4 bytes of each potential TLS packet to see if they match the SSL record header. + * + * 08/31/18: Added unconvetional TLS ports used by WeMo plugs and LiFX bulb. + * 09/20/18: Moved hardcoded ports to other class to allow other classes to query the set of TLS ports. + */ + return TcpConversationUtils.isTlsPort(mServerPort); + } + + /** + * If this {@code Conversation} is backing a TLS session (i.e., if the value of {@link #isTls()} is {@code true}), + * get the packets labeled as TLS Application Data packets. This is a subset of the full set of payload-carrying + * packets (as returned by {@link #getPackets()}). An exception is thrown if this method is invoked on a + * {@code Conversation} for which {@link #isTls()} returns {@code false}. + * + * @return A list containing exactly those packets that could be identified as TLS Application Data packets (through + * inspecting of the SSL record header). The list may be empty, if no TLS application data packets have been + * recorded for this {@code Conversation}. + */ + public List getTlsApplicationDataPackets() { + if (!isTls()) { + throw new NoSuchElementException("cannot get TLS Application Data packets for non-TLS TCP conversation"); + } + return Collections.unmodifiableList(mTlsApplicationDataPackets); + } + + /** + * Extracts the TCP sequence number from {@code packet} and adds it to the proper set of sequence numbers by + * analyzing the direction of the packet. + * @param packet A TCP packet (wrapped in a {@code PcapPacket}) that was added to this conversation and whose + * sequence number is to be recorded as seen. + */ + private void addSeqNumber(PcapPacket packet) { + // Note: below check is redundant if client code is correct as the call to check the precondition should already + // have been made by the addXPacket method that invokes this method. As such, the call below may be removed in + // favor of speed, but the improvement will be minor, hence the added safety may be worth it. + onAddPrecondition(packet); + // Extract sequence number. + int seqNo = packet.get(TcpPacket.class).getHeader().getSequenceNumber(); + // Determine direction of packet and add packet's sequence number to corresponding set of sequence numbers. + switch (getDirection(packet)) { + case CLIENT_TO_SERVER: + // Client to server packet. + mSeqNumbersClient.add(seqNo); + break; + case SERVER_TO_CLIENT: + // Server to client packet. + mSeqNumbersSrv.add(seqNo); + break; + default: + throw new AssertionError(String.format("Unexpected value of enum '%s'", + Direction.class.getSimpleName())); + } + } + + /** + * Determine the direction of {@code packet}. An {@link IllegalArgumentException} is thrown if {@code packet} does + * not pertain to this conversation. + * + * @param packet The packet whose direction is to be determined. + * @return A {@link Direction} indicating the direction of the packet. + */ + public Direction getDirection(PcapPacket packet) { + IpV4Packet ipPacket = packet.get(IpV4Packet.class); + String ipSrc = ipPacket.getHeader().getSrcAddr().getHostAddress(); + String ipDst = ipPacket.getHeader().getDstAddr().getHostAddress(); + // Determine direction of packet. + if (ipSrc.equals(mClientIp) && ipDst.equals(mServerIp)) { + // Client to server packet. + return Direction.CLIENT_TO_SERVER; + } else if (ipSrc.equals(mServerIp) && ipDst.equals(mClientIp)) { + // Server to client packet. + return Direction.SERVER_TO_CLIENT; + } else { + throw new IllegalArgumentException("getDirection: packet not related to " + getClass().getSimpleName()); + } + } + + /** + * Utility enum for expressing the direction of a packet pertaining to this {@code Conversation}. + */ + public enum Direction { + + CLIENT_TO_SERVER { + @Override + public String toCompactString() { + return "*"; + } + }, + SERVER_TO_CLIENT { + @Override + public String toCompactString() { + return ""; + } + }; + + /** + * Get a compact string representation of this {@code Direction}. + * @return a compact string representation of this {@code Direction}. + */ + abstract public String toCompactString(); + + } + +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/FinAckPair.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/FinAckPair.java new file mode 100644 index 0000000..fe4f032 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/FinAckPair.java @@ -0,0 +1,134 @@ +package edu.uci.iotproject.trafficreassembly.layer3; + +import org.pcap4j.core.PcapPacket; +import org.pcap4j.packet.IpV4Packet; +import org.pcap4j.packet.TcpPacket; + +/** + * Groups a FIN packet and its corresponding ACK packet. Immutable and thread safe. + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class FinAckPair { + + private final PcapPacket mFinPacket; + private final PcapPacket mCorrespondingAckPacket; + + /** + * Constructs a {@code FinAckPair} given a FIN packet. + * The corresponding ACK packet field is set to {@code null}. + * @param finPacket A FIN packet. + */ + public FinAckPair(PcapPacket finPacket) { + if (!finPacket.get(TcpPacket.class).getHeader().getFin()) { + throw new IllegalArgumentException("not a FIN packet"); + } + mFinPacket = finPacket; + mCorrespondingAckPacket = null; + } + + /** + * Constructs a {@code FinAckPair} given a FIN and an ACK packet. + * @param finPacket A FIN packet. + * @param correspondingAckPacket The ACK packet corresponding to {@code finPacket}. + */ + public FinAckPair(PcapPacket finPacket, PcapPacket correspondingAckPacket) { + // Enforce class invariant, i.e. that the FIN and ACK are related. + // Note that it is indirectly checked whether finPacket is indeed a FIN packet + // as isCorrespondingAckPacket calls the single parameter constructor. + if (!FinAckPair.isCorrespondingAckPacket(finPacket, correspondingAckPacket)) { + throw new IllegalArgumentException("FIN and ACK not related"); + } + mFinPacket = finPacket; + mCorrespondingAckPacket = correspondingAckPacket; + } + + /** + * Get the FIN packet of this pair. + * @return the FIN packet of this pair. + */ + public PcapPacket getFinPacket() { + return mFinPacket; + } + + /** + * Get the corresponding ACK packet of this pair, if any. + * @return the corresponding ACK packet of this pair, if any. + */ + public PcapPacket getCorrespondingAckPacket() { + return mCorrespondingAckPacket; + } + + /** + * Was the FIN in this {@code FinAckPair} acknowledged? + * + * @return {@code true} if the corresponding ACK has been set in this {@code FinAckPair}. + */ + public boolean isAcknowledged() { + return mFinPacket != null && mCorrespondingAckPacket != null; + } + + /** + * Checks if a given packet is an ACK corresponding to the FIN packet in this {@code FinAckPair}. + * @return {@code true} if {@code packet} is an ACK that corresponds to the FIN in this pair, {@code false} otherwise. + */ + public boolean isCorrespondingAckPacket(PcapPacket packet) { + IpV4Packet inputIpPacket = packet.get(IpV4Packet.class); + TcpPacket inputTcpPacket = packet.get(TcpPacket.class); + if (inputIpPacket == null || inputTcpPacket == null || !inputTcpPacket.getHeader().getAck()) { + return false; + } + + IpV4Packet finIpPacket = mFinPacket.get(IpV4Packet.class); + TcpPacket finTcpPacket = mFinPacket.get(TcpPacket.class); + + // Extract (srcIp:port,dstIp:port) for input and member (FIN) packets. + String inputPacketIpSrc = inputIpPacket.getHeader().getSrcAddr().getHostAddress(); + String inputPacketIpDst = inputIpPacket.getHeader().getDstAddr().getHostAddress(); + int inputPacketPortSrc = inputTcpPacket.getHeader().getSrcPort().valueAsInt(); + int inputPacketPortDst = inputTcpPacket.getHeader().getDstPort().valueAsInt(); + String finPacketIpSrc = finIpPacket.getHeader().getSrcAddr().getHostAddress(); + String finPacketIpDst = finIpPacket.getHeader().getDstAddr().getHostAddress(); + int finPacketPortSrc = finTcpPacket.getHeader().getSrcPort().valueAsInt(); + int finPacketPortDst = finTcpPacket.getHeader().getDstPort().valueAsInt(); + + // For the two packets to be related, the dst of one must be the src of the other. + // Split into multiple if statements for readability. First check IP fields, then ports. + if (!(inputPacketIpDst.equals(finPacketIpSrc) && finPacketIpDst.equals(inputPacketIpSrc))) { + return false; + } + if (!(inputPacketPortDst == finPacketPortSrc && finPacketPortDst == inputPacketPortSrc)) { + return false; + } + + // Packets are (most likely) related (part of same conversation/stream). + // Now all that is left for us to check is if the sequence numbers match. + // Note: recall that the FIN packet advances the seq numbers by 1, + // so the ACK number will be one larger than the seq. number in the FIN packet. + return inputTcpPacket.getHeader().getAcknowledgmentNumber() == finTcpPacket.getHeader().getSequenceNumber() + 1; + } + + /** + * Static method to check if two given packets are a FIN and the corresponding ACK packet. + * The purpose of this method is a workaround to enforce the class invariant in the two parameter constructor. + * Specifically, the following should be avoided: + * 
+     *     public FinAckPair(PcapPacket finPacket, PcapPacket correspondingAckPacket) {
+     *         mFinPacket = finPacket;
+     *         // Below line is considered bad practice as the object has not been fully initialized at this stage.
+     *         if (!this.isCorrespondingAckPacket(correspondingAckPacket)) {
+     *             // ... throw exception
+     *         }
+     *     }
+     *
+ * @param finPacket The FIN packet. + * @param ackPacket The ACK packet that is to be checked if it corresponds to the given FIN packet. + * @return {@code true} if the ACK corresponds to the FIN, {@code false} otherwise. + */ + private static boolean isCorrespondingAckPacket(PcapPacket finPacket, PcapPacket ackPacket) { + FinAckPair tmp = new FinAckPair(finPacket); + return tmp.isCorrespondingAckPacket(ackPacket); + } + +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/TcpReassembler.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/TcpReassembler.java new file mode 100644 index 0000000..e150875 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/TcpReassembler.java @@ -0,0 +1,260 @@ +package edu.uci.iotproject.trafficreassembly.layer3; + +import org.pcap4j.core.PacketListener; +import org.pcap4j.core.PcapPacket; +import org.pcap4j.packet.*; + +import java.util.*; + +/** + * Reassembles TCP conversations (streams). + * Note: current version only supports TCP over IPv4. + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class TcpReassembler implements PacketListener { + + /** + * Holds open {@link Conversation}s, i.e., {@code Conversation}s that have not been detected as + * (gracefully) terminated based on the set of packets observed thus far. + * A {@link Conversation} is moved to {@link #mTerminatedConversations} if it can be determined that it is has + * terminated. Termination can be detected by a) observing two {@link FinAckPair}s, one in each direction, (graceful + * termination, see {@link Conversation#isGracefullyShutdown()}) or b) by observing a SYN packet that matches the + * four tuple of an existing {@code Conversation}, but which holds a different sequence number than the + * same-direction SYN packet recorded for the {@code Conversation}. + *

+ * Note that due to limitations of the {@link Set} interface (specifically, there is no {@code get(T t)} method), + * we have to resort to a {@link Map} (in which keys map to themselves) to "mimic" a set with {@code get(T t)} + * functionality. + * + * @see this question on StackOverflow.com + */ + private final Map mOpenConversations = new HashMap<>(); + + /** + * Holds terminated {@link Conversation}s. + */ + private final List mTerminatedConversations = new ArrayList<>(); + + @Override + public void gotPacket(PcapPacket pcapPacket) { + IpV4Packet ipPacket = pcapPacket.get(IpV4Packet.class); + TcpPacket tcpPacket = pcapPacket.get(TcpPacket.class); + + if (ipPacket == null || tcpPacket == null) { + return; + } + // ... TODO? + processPacket(pcapPacket); +// Class clazz = pcapPacket.getClass(); +// RadiotapPacket radiotapPacket = pcapPacket.get(RadiotapPacket.class); +// Dot11ManagementPacket dot11ManagementPacket = pcapPacket.get(Dot11ManagementPacket.class); +// if (dot11ManagementPacket != null) { +// return; +// } +// if (radiotapPacket != null) { +// processRadiotapPacket(pcapPacket); +// } + } + + /** + * Get the reassembled TCP connections. Note that if this is called while packets are still being processed (by + * calls to {@link #gotPacket(PcapPacket)}), the behavior is undefined and the returned list may be inconsistent. + * @return The reassembled TCP connections. + */ + public List getTcpConversations() { + ArrayList combined = new ArrayList<>(); + combined.addAll(mTerminatedConversations); + combined.addAll(mOpenConversations.values()); + return combined; + } + + private void processRadiotapPacket(PcapPacket pcapPacket) { + RadiotapPacket radiotapPacket = pcapPacket.get(RadiotapPacket.class); + + RadiotapPacket.RadiotapHeader header = radiotapPacket.getHeader(); + short length = header.getLength(); + ArrayList radiotapData = header.getDataFields(); + // TODO: We can handle this 802.11 QoS data by creating our own class + // TODO: We only need to handle the first few bytes for source, destination, receiver, and transmitter + // TODO: addresses + Packet dataPacket = radiotapPacket.getPayload(); + int dataLength = dataPacket.length(); + } + + private void processPacket(PcapPacket pcapPacket) { + TcpPacket tcpPacket = pcapPacket.get(TcpPacket.class); + // Handle client connection initiation attempts. + if (tcpPacket.getHeader().getSyn() && !tcpPacket.getHeader().getAck()) { + // A segment with the SYN flag set, but no ACK flag indicates that a client is attempting to initiate a new + // connection. + processNewConnectionRequest(pcapPacket); + return; + } + // Handle server connection initiation acknowledgement + if (tcpPacket.getHeader().getSyn() && tcpPacket.getHeader().getAck()) { + // A segment with both the SYN and ACK flags set indicates that the server has accepted the client's request + // to initiate a new connection. + processNewConnectionAck(pcapPacket); + return; + } + // Handle resets + if (tcpPacket.getHeader().getRst()) { + processRstPacket(pcapPacket); + return; + } + // Handle FINs + if (tcpPacket.getHeader().getFin()) { + // Handle FIN packet. + processFinPacket(pcapPacket); + } + // Handle ACKs (currently only ACKs of FINS) + if (tcpPacket.getHeader().getAck()) { + processAck(pcapPacket); + } + // Handle packets that carry payload (application data). + if (tcpPacket.getPayload() != null) { + processPayloadPacket(pcapPacket); + } + } + + private void processNewConnectionRequest(PcapPacket clientSynPacket) { + // A SYN w/o ACK always originates from the client. + Conversation conv = Conversation.fromPcapPacket(clientSynPacket, true); + conv.addSynPacket(clientSynPacket); + // Is there an ongoing conversation for the same four tuple (clientIp, clientPort, serverIp, serverPort) as + // found in the new SYN packet? + Conversation ongoingConv = mOpenConversations.get(conv); + if (ongoingConv != null) { + if (ongoingConv.isRetransmission(clientSynPacket)) { + // SYN retransmission detected, do nothing. + return; + // TODO: the way retransmission detection is implemented may cause a bug for connections where we have + // not recorded the initial SYN, but only the SYN ACK, as retransmission is determined by comparing the + // sequence numbers of initial SYNs -- and if no initial SYN is present for the Conversation, the new + // SYN will be interpreted as a retransmission. Possible fix: let isRentransmission ALWAYS return false + // when presented with a SYN packet when the Conversation already holds a SYN ACK packet? + } else { + // New SYN has different sequence number than SYN recorded for ongoingConv, so this must be an attempt + // to establish a new conversation with the same four tuple as ongoingConv. + // Mark existing connection as terminated. + // TODO: is this 100% theoretically correct, e.g., if many connection attempts are made back to back? And RST packets? + mTerminatedConversations.add(ongoingConv); + mOpenConversations.remove(ongoingConv); + } + } + // Finally, update the map of open connections with the new connection. + mOpenConversations.put(conv, conv); + } + + + /* + * TODO a problem across the board for all processXPacket methods below: + * if we start the capture in the middle of a TCP connection, we will not have an entry for the conversation in the + * map as we have not seen the initial SYN packet. + * Two ways we can address this: + * a) Perform null-checks and ignore packets for which we have not seen SYN + * + easy to get correct + * - we discard data (issue for long-lived connections!) + * b) Add a corresponding conversation entry whenever we encounter a packet that does not map to a conversation + * + we consider all data + * - not immediately clear if this will introduce bugs (incorrectly mapping packets to wrong conversations?) + * + * [[[ I went with option b) for now; see getOngoingConversationOrCreateNew(PcapPacket pcapPacket). ]]] + */ + + private void processNewConnectionAck(PcapPacket srvSynPacket) { + // Find the corresponding ongoing connection, if any (if we start the capture just *after* the initial SYN, no + // ongoing conversation entry will exist, so it must be created in that case). +// Conversation conv = mOpenConversations.get(Conversation.fromPcapPacket(srvSynPacket, false)); + Conversation conv = getOngoingConversationOrCreateNew(srvSynPacket); + // Note: exploits &&'s short-circuit operation: only attempts to add non-retransmissions. + if (!conv.isRetransmission(srvSynPacket) && !conv.addSynPacket(srvSynPacket)) { + // For safety/debugging: if NOT a retransmission and add fails, + // something has gone terribly wrong/invariant is broken. +// throw new AssertionError("Attempt to add SYN ACK packet that was NOT a retransmission failed." + +// Conversation.class.getSimpleName() + " invariant broken."); + } + } + + private void processRstPacket(PcapPacket rstPacket) { + Conversation conv = getOngoingConversationOrCreateNew(rstPacket); + // Add RST packet to conversation. + conv.addRstPacket(rstPacket); + // Move conversation to set of terminated conversations. + mTerminatedConversations.add(conv); + mOpenConversations.remove(conv, conv); + } + + private void processFinPacket(PcapPacket finPacket) { +// getOngoingConversationForPacket(finPacket).addFinPacket(finPacket); + getOngoingConversationOrCreateNew(finPacket).addFinPacket(finPacket); + } + + private void processAck(PcapPacket ackPacket) { +// getOngoingConversationForPacket(ackPacket).attemptAcknowledgementOfFin(ackPacket); + // Note that unlike the style for SYN, FIN, and payload packets, for "ACK only" packets, we want to avoid + // creating a new conversation. + Conversation conv = getOngoingConversationForPacket(ackPacket); + if (conv != null) { + // The ACK may be an ACK of a FIN, so attempt to mark the FIN as ack'ed. + conv.attemptAcknowledgementOfFin(ackPacket); + if (conv.isGracefullyShutdown()) { + // Move conversation to set of terminated conversations. + mTerminatedConversations.add(conv); + mOpenConversations.remove(conv); + } + } + // Note: add (additional) processing of ACKs (that are not ACKs of FINs) as necessary here... + } + + private void processPayloadPacket(PcapPacket pcapPacket) { +// getOngoingConversationForPacket(pcapPacket).addPacket(pcapPacket, true); + getOngoingConversationOrCreateNew(pcapPacket).addPacket(pcapPacket, true); + } + + /** + * Locates an ongoing conversation (if any) that {@code pcapPacket} pertains to. + * @param pcapPacket The packet that is to be mapped to an ongoing {@code Conversation}. + * @return The {@code Conversation} matching {@code pcapPacket} or {@code null} if there is no match. + */ + private Conversation getOngoingConversationForPacket(PcapPacket pcapPacket) { + // We cannot know if this is a client-to-server or server-to-client packet without trying both options... + Conversation conv = mOpenConversations.get(Conversation.fromPcapPacket(pcapPacket, true)); + if (conv == null) { + conv = mOpenConversations.get(Conversation.fromPcapPacket(pcapPacket, false)); + } + return conv; + } + + /** + * Like {@link #getOngoingConversationForPacket(PcapPacket)}, but creates and inserts a new {@code Conversation} + * into {@link #mOpenConversations} if no open conversation is found (i.e., in the case that + * {@link #getOngoingConversationForPacket(PcapPacket)} returns {@code null}). + * + * @param pcapPacket The packet that is to be mapped to an ongoing {@code Conversation}. + * @return The existing, ongoing {@code Conversation} matching {@code pcapPacket} or the newly created one in case + * no match was found. + */ + private Conversation getOngoingConversationOrCreateNew(PcapPacket pcapPacket) { + Conversation conv = getOngoingConversationForPacket(pcapPacket); + if (conv == null) { + TcpPacket tcpPacket = pcapPacket.get(TcpPacket.class); + if (tcpPacket.getHeader().getSyn() && tcpPacket.getHeader().getAck()) { + // A SYN ACK packet always originates from the server (it is a reply to the initial SYN packet from the client) + conv = Conversation.fromPcapPacket(pcapPacket, false); + } else { + // TODO: can we do anything else but arbitrarily select who is designated as the server in this case? + // We can check if the IP prefix matches a local IP when handling traffic observed inside the local + // network, but that obviously won't be a useful strategy for an observer at the WAN port. + String srcIp = pcapPacket.get(IpV4Packet.class).getHeader().getSrcAddr().getHostAddress(); + // TODO: REPLACE THE ROUTER'S IP WITH A PARAMETER!!! + boolean clientIsSrc = srcIp.startsWith("10.") || srcIp.startsWith("192.168.") || srcIp.equals("128.195.205.105"); + conv = Conversation.fromPcapPacket(pcapPacket, clientIsSrc); + } + mOpenConversations.put(conv, conv); + } + return conv; + } +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java new file mode 100644 index 0000000..067af93 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java @@ -0,0 +1,484 @@ +package edu.uci.iotproject.util; + +import edu.uci.iotproject.trafficreassembly.layer3.Conversation; +import edu.uci.iotproject.analysis.PcapPacketPair; +import edu.uci.iotproject.analysis.TcpConversationUtils; +import edu.uci.iotproject.analysis.TriggerTrafficExtractor; +import org.apache.commons.math3.stat.clustering.Cluster; +import org.pcap4j.core.PcapPacket; +import org.pcap4j.packet.EthernetPacket; +import org.pcap4j.packet.IpV4Packet; +import org.pcap4j.packet.TcpPacket; +import org.pcap4j.util.MacAddress; + +import java.util.*; + +/** + * Utility methods for inspecting {@link PcapPacket} properties. + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public final class PcapPacketUtils { + + /** + * This is the threshold value for a signature's number of members + * If after a merging the number of members of a signature falls below this threshold, then we can boldly + * get rid of that signature. + */ + private static final int SIGNATURE_MERGE_THRESHOLD = 5; + + /** + * This is an overlap counter (we consider overlaps between signatures if it happens more than once) + */ + private static int mOverlapCounter = 0; + + + /** + * Gets the source address of the Ethernet part of {@code packet}. + * @param packet The packet for which the Ethernet source address is to be extracted. + * @return The source address of the Ethernet part of {@code packet}. + */ + public static MacAddress getEthSrcAddr(PcapPacket packet) { + return getEthernetPacketOrThrow(packet).getHeader().getSrcAddr(); + } + + /** + * Gets the destination address of the Ethernet part of {@code packet}. + * @param packet The packet for which the Ethernet destination address is to be extracted. + * @return The destination address of the Ethernet part of {@code packet}. + */ + public static MacAddress getEthDstAddr(PcapPacket packet) { + return getEthernetPacketOrThrow(packet).getHeader().getDstAddr(); + } + + /** + * Determines if a given {@link PcapPacket} wraps a {@link TcpPacket}. + * @param packet The {@link PcapPacket} to inspect. + * @return {@code true} if {@code packet} wraps a {@link TcpPacket}, {@code false} otherwise. + */ + public static boolean isTcp(PcapPacket packet) { + return packet.get(TcpPacket.class) != null; + } + + /** + * Gets the source IP (in decimal format) of an IPv4 packet. + * @param packet The packet for which the IPv4 source address is to be extracted. + * @return The decimal representation of the source IP of {@code packet} iff {@code packet} wraps an + * {@link IpV4Packet}. + * @throws NullPointerException if {@code packet} does not encapsulate an {@link IpV4Packet}. + */ + public static String getSourceIp(PcapPacket packet) { + return getIpV4PacketOrThrow(packet).getHeader().getSrcAddr().getHostAddress(); + } + + /** + * Gets the destination IP (in decimal format) of an IPv4 packet. + * @param packet The packet for which the IPv4 source address is to be extracted. + * @return The decimal representation of the destination IP of {@code packet} iff {@code packet} wraps an + * {@link IpV4Packet}. + * @throws NullPointerException if {@code packet} does not encapsulate an {@link IpV4Packet}. + */ + public static String getDestinationIp(PcapPacket packet) { + return getIpV4PacketOrThrow(packet).getHeader().getDstAddr().getHostAddress(); + } + + /** + * Gets the source port of a TCP packet. + * @param packet The packet for which the source port is to be extracted. + * @return The source port of the {@link TcpPacket} encapsulated by {@code packet}. + * @throws IllegalArgumentException if {@code packet} does not encapsulate a {@link TcpPacket}. + */ + public static int getSourcePort(PcapPacket packet) { + TcpPacket tcpPacket = packet.get(TcpPacket.class); + if (tcpPacket == null) { + throw new IllegalArgumentException("not a TCP packet"); + } + return tcpPacket.getHeader().getSrcPort().valueAsInt(); + } + + /** + * Gets the destination port of a TCP packet. + * @param packet The packet for which the destination port is to be extracted. + * @return The destination port of the {@link TcpPacket} encapsulated by {@code packet}. + * @throws IllegalArgumentException if {@code packet} does not encapsulate a {@link TcpPacket}. + */ + public static int getDestinationPort(PcapPacket packet) { + TcpPacket tcpPacket = packet.get(TcpPacket.class); + if (tcpPacket == null) { + throw new IllegalArgumentException("not a TCP packet"); + } + return tcpPacket.getHeader().getDstPort().valueAsInt(); + } + + /** + * Helper method to determine if the given combination of IP and port matches the source of the given packet. + * @param packet The packet to check. + * @param ip The IP to look for in the ip.src field of {@code packet}. + * @param port The port to look for in the tcp.port field of {@code packet}. + * @return {@code true} if the given ip+port match the corresponding fields in {@code packet}. + */ + public static boolean isSource(PcapPacket packet, String ip, int port) { + IpV4Packet ipPacket = Objects.requireNonNull(packet.get(IpV4Packet.class)); + // For now we only support TCP flows. + TcpPacket tcpPacket = Objects.requireNonNull(packet.get(TcpPacket.class)); + String ipSrc = ipPacket.getHeader().getSrcAddr().getHostAddress(); + int srcPort = tcpPacket.getHeader().getSrcPort().valueAsInt(); + return ipSrc.equals(ip) && srcPort == port; + } + + /** + * Helper method to determine if the given combination of IP and port matches the destination of the given packet. + * @param packet The packet to check. + * @param ip The IP to look for in the ip.dst field of {@code packet}. + * @param port The port to look for in the tcp.dstport field of {@code packet}. + * @return {@code true} if the given ip+port match the corresponding fields in {@code packet}. + */ + public static boolean isDestination(PcapPacket packet, String ip, int port) { + IpV4Packet ipPacket = Objects.requireNonNull(packet.get(IpV4Packet.class)); + // For now we only support TCP flows. + TcpPacket tcpPacket = Objects.requireNonNull(packet.get(TcpPacket.class)); + String ipDst = ipPacket.getHeader().getDstAddr().getHostAddress(); + int dstPort = tcpPacket.getHeader().getDstPort().valueAsInt(); + return ipDst.equals(ip) && dstPort == port; + } + + /** + * Checks if the source IP address of the {@link IpV4Packet} contained in {@code packet} is a local address, i.e., + * if it pertains to subnet 10.0.0.0/8, 172.16.0.0/16, or 192.168.0.0/16. + * @param packet The packet for which the source IP address is to be examined. + * @return {@code true} if {@code packet} wraps a {@link IpV4Packet} for which the source IP address is a local IP + * address, {@code false} otherwise. + * @throws NullPointerException if {@code packet} does not encapsulate an {@link IpV4Packet}. + */ + public static boolean isSrcIpLocal(PcapPacket packet) { + return getIpV4PacketOrThrow(packet).getHeader().getSrcAddr().isSiteLocalAddress(); + } + + /** + * Checks if the destination IP address of the {@link IpV4Packet} contained in {@code packet} is a local address, + * i.e., if it pertains to subnet 10.0.0.0/8, 172.16.0.0/16, or 192.168.0.0/16. + * @param packet The packet for which the destination IP address is to be examined. + * @return {@code true} if {@code packet} wraps a {@link IpV4Packet} for which the destination IP address is a local + * IP address, {@code false} otherwise. + * @throws NullPointerException if {@code packet} does not encapsulate an {@link IpV4Packet}. + */ + public static boolean isDstIpLocal(PcapPacket packet) { + return getIpV4PacketOrThrow(packet).getHeader().getDstAddr().isSiteLocalAddress(); + } + + /** + * Checks if {@code packet} wraps a TCP packet that has the SYN flag set. + * @param packet A {@link PcapPacket} that is suspected to contain a {@link TcpPacket} for which the SYN flag is set. + * @return {@code true} iff {@code packet} contains a {@code TcpPacket} for which the SYN flag is set, + * {@code false} otherwise. + */ + public static boolean isSyn(PcapPacket packet) { + TcpPacket tcp = packet.get(TcpPacket.class); + return tcp != null && tcp.getHeader().getSyn(); + } + + /** + * Checks if {@code packet} wraps a TCP packet th at has the ACK flag set. + * @param packet A {@link PcapPacket} that is suspected to contain a {@link TcpPacket} for which the ACK flag is set. + * @return {@code true} iff {@code packet} contains a {@code TcpPacket} for which the ACK flag is set, + * {@code false} otherwise. + */ + public static boolean isAck(PcapPacket packet) { + TcpPacket tcp = packet.get(TcpPacket.class); + return tcp != null && tcp.getHeader().getAck(); + } + + /** + * Transform a {@code Cluster} of {@code PcapPacketPair} objects into a {@code List} of {@code List} of + * {@code PcapPacket} objects. + * @param cluster A {@link Cluster} of {@link PcapPacketPair} objects that needs to be transformed. + * @return A {@link List} of {@link List} of {@link PcapPacket} objects as the result of the transformation. + */ + public static List> clusterToListOfPcapPackets(Cluster cluster) { + List> ppListOfList = new ArrayList<>(); + for (PcapPacketPair ppp: cluster.getPoints()) { + // Create a list of PcapPacket objects (list of two members). + List ppList = new ArrayList<>(); + ppList.add(ppp.getFirst()); + if(ppp.getSecond().isPresent()) + ppList.add(ppp.getSecond().get()); + else + ppList.add(null); + // Create a list of list of PcapPacket objects. + ppListOfList.add(ppList); + } + // Sort the list of lists based on the first packet's timestamp! + Collections.sort(ppListOfList, (p1, p2) -> p1. get(0).getTimestamp().compareTo(p2.get(0).getTimestamp())); + return ppListOfList; + } + + /** + * Merge signatures in {@code List} of {@code List} of {@code List} of {@code PcapPacket} objects. + * We cross-check these with {@code List} of {@code Conversation} objects to see + * if two {@code List} of {@code PcapPacket} objects actually belong to the same {@code Conversation}. + * @param signatures A {@link List} of {@link List} of {@link List} of + * {@link PcapPacket} objects that needs to be checked and merged. + * @param conversations A {@link List} of {@link Conversation} objects as reference for merging. + * @return A {@link List} of {@link List} of {@link List} of + * {@link PcapPacket} objects as the result of the merging. + */ + public static List>> + mergeSignatures(List>> signatures, List conversations) { + + // TODO: THIS IS NOT A DEEP COPY; IT BASICALLY CREATES A REFERENCE TO THE SAME LIST OBJECT + // List>> copySignatures = new ArrayList<>(signatures); + // Make a deep copy first. + List>> copySignatures = new ArrayList<>(); + listDeepCopy(copySignatures, signatures); + // Traverse and look into the pairs of signatures. + for (int first = 0; first < signatures.size(); first++) { + List> firstList = signatures.get(first); + for (int second = first+1; second < signatures.size(); second++) { + int maxSignatureEl = 0; // Number of maximum signature elements. + List> secondList = signatures.get(second); + int initialSecondListMembers = secondList.size(); + // Iterate over the signatures in the first list. + for (List signature : firstList) { + signature.removeIf(el -> el == null); // Clean up null elements. + // Return the Conversation that the signature is part of. + Conversation conv = TcpConversationUtils.returnConversation(signature, conversations); + // Find the element of the second list that is a match for that Conversation. + for (List ppList : secondList) { + ppList.removeIf(el -> el == null); // Clean up null elements. + // Check if they are part of a Conversation and are adjacent to the first signature. + // If yes then merge into the first list. + TcpConversationUtils.SignaturePosition position = + TcpConversationUtils.isPartOfConversationAndAdjacent(signature, ppList, conv); + if (position == TcpConversationUtils.SignaturePosition.LEFT_ADJACENT) { + // Merge to the left side of the first signature. + ppList.addAll(signature); + signature = ppList; + maxSignatureEl = signature.size() > maxSignatureEl ? signature.size() : maxSignatureEl; + secondList.remove(ppList); // Remove as we merge. + break; + } else if (position == TcpConversationUtils.SignaturePosition.RIGHT_ADJACENT) { + // Merge to the right side of the first signature. + signature.addAll(ppList); + maxSignatureEl = signature.size() > maxSignatureEl ? signature.size() : maxSignatureEl; + secondList.remove(ppList); // Remove as we merge. + break; + } // TcpConversationUtils.SignaturePosition.NOT_ADJACENT. + } + } + // Call it a successful merging if there are only less than 5 elements from the second list that + // cannot be merged. + if (secondList.size() < SIGNATURE_MERGE_THRESHOLD) { + // Prune the unsuccessfully merged signatures (i.e., these will have size() < maxSignatureEl). + final int maxNumOfEl = maxSignatureEl; + // TODO: DOUBLE CHECK IF WE REALLY NEED TO PRUNE FAILED BINDINGS + // TODO: SOMETIMES THE SEQUENCES ARE JUST INCOMPLETE + // TODO: AND BOTH THE COMPLETE AND INCOMPLETE SEQUENCES ARE VALID SIGNATURES! + firstList.removeIf(el -> el.size() < maxNumOfEl); + // Remove the merged set of signatures when successful. + signatures.remove(secondList); + } else if (secondList.size() < initialSecondListMembers) { + // If only some of the signatures from the second list are merged, this means UNSUCCESSFUL merging. + // Return the original copy of the signatures object. + return copySignatures; + } + } + } + return signatures; + } + + /** + * Deep copy to create an entirely new {@link List} of {@link List} of {@link List} of {@link PcapPacket} objects. + * @param destList A {@link List} of {@link List} of {@link List} of + * {@link PcapPacket} objects that will be the final container of the deep copy + * @param sourceList A {@link List} of {@link List} of {@link List} of + * {@link PcapPacket} objects that will be the source of the deep copy. + */ + private static void listDeepCopy(List>> destList, List>> sourceList) { + + for(List> llPcapPacket : sourceList) { + List> tmpListOfList = new ArrayList<>(); + for(List lPcapPacket : llPcapPacket) { + List tmpList = new ArrayList<>(); + for(PcapPacket pcapPacket : lPcapPacket) { + tmpList.add(pcapPacket); + } + tmpListOfList.add(tmpList); + } + destList.add(tmpListOfList); + } + } + + /** + * Sort the signatures in the {@code List} of {@code List} of {@code List} of {@code PcapPacket} objects. + * The purpose of this is to sort the order of signatures in the signature list. For detection purposes, we need + * to know if one signature occurs earlier/later in time with respect to the other signatures for more confidence + * in detecting the occurrence of an event. + * @param signatures A {@code List} of {@code List} of {@code List} of {@code PcapPacket} objects that needs sorting. + * We assume that innermost {@code List} of {@code PcapPacket} objects have been sorted ascending + * by timestamps. By the time we use this method, we should have sorted it when calling the + * {@code clusterToListOfPcapPackets} method. + * @return A sorted {@code List} of {@code List} of {@code List} of {@code PcapPacket} objects. + */ + public static List>> sortSignatures(List>> signatures) { + // TODO: This is the simplest solution!!! Might not cover all corner cases. + // TODO: Sort the list of lists based on the first packet's timestamps! +// Collections.sort(signatures, (p1, p2) -> { +// //return p1.get(0).get(0).getTimestamp().compareTo(p2.get(0).get(0).getTimestamp()); +// int compare = p1.get(0).get(0).getTimestamp().compareTo(p2.get(0).get(0).getTimestamp()); +// return compare; +// }); + // TODO: The following is a more complete solution that covers corner cases. + // Sort the list of lists based on one-to-one comparison between timestamps of signatures on both lists. + // This also takes into account the fact that the number of signatures in the two lists could be different. + // Additionally, this code forces the comparison between two signatures only if they occur in the + // INCLUSION_WINDOW_MILLIS window; otherwise, it tries to find the right pair of signatures in the time window. + Collections.sort(signatures, (p1, p2) -> { + int compare = 0; + int comparePrev = 0; + int count1 = 0; + int count2 = 0; + // Need to make sure that both are not out of bound! + while (count1 + 1 < p1.size() && count2 + 1 < p2.size()) { + long timestamp1 = p1.get(count1).get(0).getTimestamp().toEpochMilli(); + long timestamp2 = p2.get(count2).get(0).getTimestamp().toEpochMilli(); + // The two timestamps have to be within a 15-second window! + if (Math.abs(timestamp1 - timestamp2) < TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS) { + // If these two are within INCLUSION_WINDOW_MILLIS window then compare! + compare = p1.get(count1).get(0).getTimestamp().compareTo(p2.get(count2).get(0).getTimestamp()); +// if (comparePrev != 0) { // First time since it is 0 +// if (Integer.signum(compare) != Integer.signum(comparePrev)) { +// // Throw an exception if the order of the two signatures is not consistent, +// // E.g., 111, 222, 333 in one occassion and 222, 333, 111 in the other. +// throw new Error("OVERLAP WARNING: " + "" + +// "Please remove one of the sequences: " + +// p1.get(0).get(0).length() + "... OR " + +// p2.get(0).get(0).length() + "..."); +// } +// } + overlapChecking(compare, comparePrev, p1.get(count1), p2.get(count2)); + comparePrev = compare; + count1++; + count2++; + } else { + // If not within INCLUSION_WINDOW_MILLIS window then find the correct pair + // by incrementing one of them. + if (timestamp1 < timestamp2) + count1++; + else + count2++; + } + } + return compare; + }); + return signatures; + } + + /** + * Checks for overlapping between two packet sequences. + * @param compare Current comparison value between packet sequences p1 and p2 + * @param comparePrev Previous comparison value between packet sequences p1 and p2 + * @param sequence1 The packet sequence ({@link List} of {@link PcapPacket} objects). + * @param sequence2 The packet sequence ({@link List} of {@link PcapPacket} objects). + */ + private static void overlapChecking(int compare, int comparePrev, List sequence1, List sequence2) { + + // Check if p1 occurs before p2 but both have same overlap + if (comparePrev != 0) { // First time since it is 0 + if (Integer.signum(compare) != Integer.signum(comparePrev)) { + // Throw an exception if the order of the two signatures is not consistent, + // E.g., 111, 222, 333 in one occassion and 222, 333, 111 in the other. + throw new Error("OVERLAP WARNING: " + "" + + "Two sequences have some overlap. Please remove one of the sequences: " + + sequence1.get(0).length() + "... OR " + + sequence2.get(0).length() + "..."); + } + } + // Check if p1 is longer than p2 and p2 occurs during the occurrence of p1 + int lastIndexOfSequence1 = sequence1.size() - 1; + int lastIndexOfSequence2 = sequence2.size() - 1; + int compareLast = + sequence1.get(lastIndexOfSequence1).getTimestamp().compareTo(sequence2.get(lastIndexOfSequence2).getTimestamp()); + // Check the signs of compare and compareLast + if ((compare <= 0 && compareLast > 0) || + (compareLast <= 0 && compare > 0)) { + mOverlapCounter++; + // TODO: Probably not the best approach but we consider overlap if it happens more than once + if (mOverlapCounter > 1) { + throw new Error("OVERLAP WARNING: " + "" + + "One sequence is in the other. Please remove one of the sequences: " + + sequence1.get(0).length() + "... OR " + + sequence2.get(0).length() + "..."); + } + } + + } + + /** + * Gets the {@link IpV4Packet} contained in {@code packet}, or throws a {@link NullPointerException} if + * {@code packet} does not contain an {@link IpV4Packet}. + * @param packet A {@link PcapPacket} that is expected to contain an {@link IpV4Packet}. + * @return The {@link IpV4Packet} contained in {@code packet}. + * @throws NullPointerException if {@code packet} does not encapsulate an {@link IpV4Packet}. + */ + private static IpV4Packet getIpV4PacketOrThrow(PcapPacket packet) { + return Objects.requireNonNull(packet.get(IpV4Packet.class), "not an IPv4 packet"); + } + + /** + * Gets the {@link EthernetPacket} contained in {@code packet}, or throws a {@link NullPointerException} if + * {@code packet} does not contain an {@link EthernetPacket}. + * @param packet A {@link PcapPacket} that is expected to contain an {@link EthernetPacket}. + * @return The {@link EthernetPacket} contained in {@code packet}. + * @throws NullPointerException if {@code packet} does not encapsulate an {@link EthernetPacket}. + */ + private static final EthernetPacket getEthernetPacketOrThrow(PcapPacket packet) { + return Objects.requireNonNull(packet.get(EthernetPacket.class), "not an Ethernet packet"); + } + + /** + * Print signatures in {@code List} of {@code List} of {@code List} of {@code PcapPacket} objects. + * + * @param signatures A {@link List} of {@link List} of {@link List} of + * {@link PcapPacket} objects that needs to be printed. + */ + public static void printSignatures(List>> signatures) { + + // Iterate over the list of all clusters/sequences + int sequenceCounter = 0; + for(List> listListPcapPacket : signatures) { + // Iterate over every member of a cluster/sequence + System.out.print("====== SEQUENCE " + ++sequenceCounter); + System.out.println(" - " + listListPcapPacket.size() + " MEMBERS ======"); + for(List listPcapPacket : listListPcapPacket) { + // Print out packet lengths in a sequence + int packetCounter = 0; + for(PcapPacket pcapPacket : listPcapPacket) { + if(pcapPacket != null) { + System.out.print(pcapPacket.length()); + } + if(packetCounter < listPcapPacket.size() - 1) { + System.out.print(" "); // Provide space if not last packet + } else { + System.out.println(); // Newline if last packet + } + packetCounter++; + } + } + } + } + + /** + * Remove a sequence in a signature object. + * + * @param signatures A {@link List} of {@link List} of {@link List} of + * {@link PcapPacket} objects. + * @param sequenceIndex An index for a sequence that consists of {{@link List} of {@link List} of + * {@link PcapPacket} objects. + */ + public static void removeSequenceFromSignature(List>> signatures, int sequenceIndex) { + + // Sequence index starts from 0 + signatures.remove(sequenceIndex); + } +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/util/PrintUtils.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/util/PrintUtils.java new file mode 100644 index 0000000..3d3a3be --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/util/PrintUtils.java @@ -0,0 +1,221 @@ +package edu.uci.iotproject.util; + +import edu.uci.iotproject.DnsMap; +import edu.uci.iotproject.analysis.PcapPacketPair; +import org.apache.commons.math3.stat.clustering.Cluster; + +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import org.pcap4j.core.PcapPacket; + +/** + * Utility methods for generating (output) strings. + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class PrintUtils { + + /** + * This is the path for writing the list of list of packet pairs {@code List>} into a file. + * The packet pairs are the pairs in one cluster, so the list represents a cluster that has been derived through + * the DBSCAN algorithm. + * + * E.g., this file could contain a list like the following: + * + * [[1109, 613],[1111, 613],[1115, 613],...] + * + * This list has lists of PcapPacket pairs as its members. We do not maintain the pairs in the form of + * {@code Cluster} objects because there might be a situation where we could combine multiple + * PcapPacketPair objects into a longer signature, i.e., a string of PcapPacket objects and not just a pair. + */ + private static final String SERIALIZABLE_FILE_PATH = "./signature.sig"; + + private PrintUtils() { /* private constructor to prevent instantiation */ } + + /** + * Write the list of list of packet pairs {@code List>} into a file. + * + * After the DBSCAN algorithm derives the clusters from pairs, we save the signature in the form of list of + * packet pairs. We harvest the pairs and transform them back into a list of PcapPacket objects. + * We do not maintain the pairs in the form of {@code Cluster} objects because there might be + * a situation where we could combine multiple PcapPacketPair objects into a longer signature, i.e., a string of + * PcapPacket objects and not just a pair. + * + * @param fileName The path of the file in {@link String}. We could leave this one {@code null} if we wanted the + * default file name {@code SERIALIZABLE_FILE_PATH}. + * @param clusterPackets The {@link Cluster} objects in the form of list of {@code PcapPacket} objects. + */ + public static void serializeClustersIntoFile(String fileName, List> clusterPackets) { + if (fileName == null) + fileName = SERIALIZABLE_FILE_PATH; + try (ObjectOutputStream oos = + new ObjectOutputStream(new FileOutputStream(fileName))) { + oos.writeObject(clusterPackets); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + /** + * Write the signature {@code List>>} into a file. + * + * After the DBSCAN algorithm derives the clusters from pairs, we save the signature in the form of list of + * packet pairs. We harvest the pairs and transform them back into a list of PcapPacket objects. + * We do not maintain the pairs in the form of {@code Cluster} objects because there might be + * a situation where we could combine multiple PcapPacketPair objects into a longer signature, i.e., a string of + * PcapPacket objects and not just a pair. + * + * @param fileName The path of the file in {@link String}. We could leave this one {@code null} if we wanted the + * default file name {@code SERIALIZABLE_FILE_PATH}. + * @param signature The {@link Cluster} objects in the form of list of {@code PcapPacket} objects. + */ + public static void serializeSignatureIntoFile(String fileName, List>> signature) { + if (fileName == null) + fileName = SERIALIZABLE_FILE_PATH; + try (ObjectOutputStream oos = + new ObjectOutputStream(new FileOutputStream(fileName))) { + oos.writeObject(signature); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + /** + * Read the list of list of packet pairs {@code List>} from a file. + * + * After the DBSCAN algorithm derives the clusters from pairs, we save the signature in the form of list of + * packet pairs. We harvest the pairs and transform them back into a list of PcapPacket objects. + * We do not maintain the pairs in the form of {@code Cluster} objects because there might be + * a situation where we could combine multiple PcapPacketPair objects into a longer signature, i.e., a string of + * PcapPacket objects and not just a pair. + * + * @param fileName The path of the file in {@link String}. We could leave this one {@code null} if we wanted the + * default file name {@code SERIALIZABLE_FILE_PATH}. + * @return The list of list of {@link Cluster} objects ({@code List>}) that is read from file. + */ + public static List> deserializeClustersFromFile(String fileName) { + if (fileName == null) + fileName = SERIALIZABLE_FILE_PATH; + List> ppListOfList = null; + try (ObjectInputStream ois = + new ObjectInputStream(new FileInputStream(fileName))) { + ppListOfList = (List>) ois.readObject(); + } catch (Exception ex) { + ex.printStackTrace(); + } + + return ppListOfList; + } + + /** + * Read the list of list of packet pairs {@code List>>} from a file. + * + * After the DBSCAN algorithm derives the clusters from pairs, we save the signature in the form of list of + * packet pairs. We harvest the pairs and transform them back into a list of PcapPacket objects. + * We do not maintain the pairs in the form of {@code Cluster} objects because there might be + * a situation where we could combine multiple PcapPacketPair objects into a longer signature, i.e., a string of + * PcapPacket objects and not just a pair. + * + * @param fileName The path of the file in {@link String}. We could leave this one {@code null} if we wanted the + * default file name {@code SERIALIZABLE_FILE_PATH}. + * @return The list of list of list of {@link Cluster} objects ({@code List>>}) + * that is read from file. + */ + public static List>> deserializeSignatureFromFile(String fileName) { + if (fileName == null) + fileName = SERIALIZABLE_FILE_PATH; + List>> ppListOfListOfList = null; + try (ObjectInputStream ois = + new ObjectInputStream(new FileInputStream(fileName))) { + ppListOfListOfList = (List>>) ois.readObject(); + } catch (Exception ex) { + ex.printStackTrace(); + } + + return ppListOfListOfList; + } + + /** + * Converts a {@code PcapPacketPair} into a CSV string containing the packet lengths of the two packets in the pair. + * + * For example, the resulting string will be "123, 456" if the first packet of the pair has a length of 123 and the + * second packet of the pair has a length of 456. + * + * Note: if the {@link PcapPacketPair} has no second element, 0 is printed as the length of the second packet + * in the pair. + * + * @return a CSV string containing the packet lengths of the two packets of the given {@code PcapPacketPair}. + */ + public static String toCsv(PcapPacketPair packetPair) { + return String.format("%d, %d", packetPair.getFirst().getOriginalLength(), + packetPair.getSecond().map(pp -> pp.getOriginalLength()).orElse(0)); + } + + /** + * Converts a {@code PcapPacketPair} into a CSV string containing the packet lengths of the two packets in the pair + * followed by the source of each packet. The source will be a (set of) hostname(s) if the source IP can be resolved + * to a (set of) hostname(s) using the provided {@link DnsMap}. + * + * For example, the resulting string will be "123, 456, 192.168.1.42, domain.com" if the first packet of the pair + * has a length of 123, the second packet of the pair has a length of 456, the first packet of the pair the pair has + * a source IP of '192.168.1.42' that cannot be resolved to a hostname, and the second packet of the pair has an IP + * that resolves to 'domain.com'. + * + * Note: if the {@link PcapPacketPair} has no second element, 0 is printed as the length of the second packet + * in the pair, and null is printed for its source. + * + * @return a CSV string containing the packet lengths of the two packets of the given {@code PcapPacketPair} as well + * as their respective sources. + */ + public static String toCsv(PcapPacketPair packetPair, DnsMap ipHostnameMappings) { + // First obtain source IPs + String firstSrc = PcapPacketUtils.getSourceIp(packetPair.getFirst()); + // Note: use optional for second item in pair as there might not be one. + Optional secondSrc = packetPair.getSecond().map(pkt -> PcapPacketUtils.getSourceIp(pkt)); + + // If possible, map source IPs to hostnames. + Set firstHostnames = ipHostnameMappings.getHostnamesForIp(firstSrc); + Optional> secondHostnames = secondSrc.map(src -> ipHostnameMappings.getHostnamesForIp(src)); + final String delimiter = " "; + if (firstHostnames != null) { + // If one IP maps to multiple hostnames, we concatenate the hostnames (separated by a delimiter). + firstSrc = firstHostnames.stream().collect(Collectors.joining(delimiter)); + } + // If one IP maps to multiple hostnames, we concatenate the hostnames (separated by a delimiter). + Optional hostnames = secondHostnames.map(hostnameSet -> hostnameSet.stream().collect(Collectors.joining(delimiter))); + // Fall back to IP if we couldn't second pair is present, but we couldn't map to (a) hostname(s). + secondSrc = hostnames.isPresent() ? hostnames : secondSrc; + + // Check if the first source is C (client) or S (server). + String firstSrcCorS = packetPair.isFirstClient() ? "C" : "S"; + String secondSrcCorS = packetPair.isSecondClient() ? "C" : "S"; + + return String.format("%d, %d, %s, %s, %s, %s", packetPair.getFirst().getOriginalLength(), + packetPair.getSecond().map(pp -> pp.getOriginalLength()).orElse(0), + firstSrc, + secondSrc.orElse("null"), + firstSrcCorS, + secondSrcCorS); + } + + /** + * Generate a string that summarizes/describes {@code cluster}. + * @param cluster The {@link Cluster} to summarize/describe. + * @return A string that summarizes/describes {@code cluster}. + */ + public static String toSummaryString(Cluster cluster) { + StringBuilder sb = new StringBuilder(); + for (PcapPacketPair ppp : cluster.getPoints()) { + sb.append(toCsv(ppp, ppp.getDnsMap()) + System.lineSeparator()); + } + return sb.toString(); + } +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/test/java/edu/uci/iotproject/test/SequenceAlignmentTest.java b/Code/Projects/PacketLevelSignatureExtractor/src/test/java/edu/uci/iotproject/test/SequenceAlignmentTest.java new file mode 100644 index 0000000..90e8eab --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/test/java/edu/uci/iotproject/test/SequenceAlignmentTest.java @@ -0,0 +1,279 @@ +package edu.uci.iotproject.test; + +import edu.uci.iotproject.comparison.seqalignment.AlignmentPricer; +import edu.uci.iotproject.comparison.seqalignment.SequenceAlignment; +import org.junit.Before; +import org.junit.Test; + +import java.util.function.ToIntBiFunction; +import java.util.function.ToIntFunction; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Tests the implementation of {@link SequenceAlignment}. + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class SequenceAlignmentTest { + + private char[] lowercaseVowels; + private char[] lowercaseConsonants; + + private Character[] meanChars; + private Character[] nameChars; + + /** + * Cost function for the alignment of letters in the example execution of the sequence alignment algorithm in + * Kleinberg's and Tardos' "Algorithm Design", where 'mean' and 'name' are aligned. + */ + private ToIntBiFunction kleinbergExampleAlignmentCostFunc; + + /** + * Cost function for the alignment of letters with gaps in the example execution of the sequence alignment algorithm + * in Kleinberg's and Tardos' "Algorithm Design", where 'mean' and 'name' are aligned. Gap cost is set to 2, + * regardless of input character. + */ + private ToIntFunction kleinbergExampleGapCostFunc; + + /** + * Calculates the cost of aligning a letter with another letter or a letter with a gap according to the cost recipe + * used in the example in Kleinberg & Tardos. + */ + private AlignmentPricer kleinbergAlignmentPricer; + + /** + * Executes the sequence alignment algorithm using the cost function defined in the example in Kleinberg & Tardos, + * i.e., {@link #kleinbergAlignmentPricer}. + */ + private SequenceAlignment kleinbergSequenceAligner; + + @Before + public void initialize() { + // We consider 'y' a vowel for the sake of simplicity. + // Note: we assume an all lowercase string! + lowercaseVowels = new char[] { 'a', 'e', 'i', 'o', 'u', 'y' }; + lowercaseConsonants = new char[] { 'b', 'c', 'd', 'f', 'g', 'h', 'j', 'k', 'l', 'm', 'n', 'p', 'q', 'r', 's', + 't', 'v', 'w', 'x', 'z' }; + kleinbergExampleAlignmentCostFunc = (c1, c2) -> { + // Unbox to primitive type for the sake of brevity in the statements to follow. + final char char1 = c1.charValue(); + final char char2 = c2.charValue(); + + // If char1 and char2 are the same characters, the cost of aligning them is 0. + if (char1 == char2) return 0; + + final boolean char1IsVowel = isVowel(char1); + final boolean char1IsConsonant = isConsonant(char1); + final boolean char2IsVowel = isVowel(char2); + final boolean char2IsConsonant = isConsonant(char2); + + // Alignment cost is undefined for non alphabet characters. + if (!char1IsVowel && !char1IsConsonant) fail("not an alphabet letter: " + char1); + if (!char2IsVowel && !char2IsConsonant) fail("not an alphabet letter: " + char2); + + // If char1 and char2 are both vowels or both consonants, the cost is 1. + if (char1IsVowel && char2IsVowel || char1IsConsonant && char2IsConsonant) return 1; + + // If one of char1 and char2 is a consonant, while the other is a vowel, the cost is 3. + return 3; + }; + // The cost of a gap is 2, regardless of what letter is aligned with the gap. + kleinbergExampleGapCostFunc = c -> 2; + + // char[] -> Character[] conversion courtesy of https://stackoverflow.com/a/27690990/1214974 + meanChars = "mean".chars().mapToObj(c -> (char)c).toArray(Character[]::new); + nameChars = "name".chars().mapToObj(c -> (char)c).toArray(Character[]::new); + + kleinbergAlignmentPricer = new AlignmentPricer<>(kleinbergExampleAlignmentCostFunc, + kleinbergExampleGapCostFunc); + + kleinbergSequenceAligner = new SequenceAlignment<>(kleinbergAlignmentPricer); + } + + @Test + public void kleinbergExampleOptAlignmentCostShouldBe6() { + // Cost of the optimal alignment of the two words + final int optAlignmentCost = kleinbergSequenceAligner.calculateAlignment(meanChars, nameChars); + final int expectedAlignmentCost = 6; + String msg = String.format("Kleinberg example: computed opt != expected opt (computed=%d expected=%d)", + optAlignmentCost, expectedAlignmentCost); + assertTrue(msg, optAlignmentCost == expectedAlignmentCost); + } + + + @Test + public void meanAlignedWithEmptyStringShouldBe8() { + final int optAlignmentCost = kleinbergSequenceAligner.calculateAlignment(meanChars, new Character[0]); + // 'mean' aligned with the empty string equals paying four gap costs, so total cost is: 4 * 2 = 8. + final int expectedAlignmentCost = 8; + String msg = String.format("'mean' aligned with empty string: computed opt != expected opt (computed=%d expected=%d)", + optAlignmentCost, expectedAlignmentCost); + assertTrue(msg, optAlignmentCost == expectedAlignmentCost); + } + + @Test + public void mAlignedWithNameShouldBe6() { + /* + * Note: this also uses the cost function specified in Kleinberg & Tardos. + * Best alignment should be: + * n a m e + * _ _ m _ + * This should have a cost of 3 * gapCost = 6 + */ + final int optAlignmentCost = kleinbergSequenceAligner.calculateAlignment(new Character[] { 'm' }, nameChars); + final int expectedAlignmentCost = 6; + String msg = String.format("'m' aligned with 'name': computed opt != expected opt (computed=%d expected=%d)", + optAlignmentCost, expectedAlignmentCost); + assertTrue(msg, optAlignmentCost == expectedAlignmentCost); + } + + @Test + public void meAlignedWithNameShouldBe4() { + /* + * Note: this also uses the cost function specified in Kleinberg & Tardos. + * Best alignment should be: + * n a m e + * _ _ m e + * This should have a cost of 2 * gapCost = 4 + */ + final int optAlignmentCost = kleinbergSequenceAligner.calculateAlignment(new Character[] { 'm', 'e' }, nameChars); + final int expectedAlignmentCost = 4; + String msg = String.format("'me' aligned with 'name': computed opt != expected opt (computed=%d expected=%d)", + optAlignmentCost, expectedAlignmentCost); + assertTrue(msg, optAlignmentCost == expectedAlignmentCost); + // Check that order of arguments doesn't matter + final int optAlignmentCostReversed = kleinbergSequenceAligner.calculateAlignment(nameChars, new Character[] { 'm', 'e' }); + msg = "'me' aligned with 'name': different order of arguments unexpectedly produced different result"; + assertTrue(msg, optAlignmentCostReversed == optAlignmentCost && optAlignmentCostReversed == expectedAlignmentCost); + } + + @Test + public void ameAlignedWithNameShouldBe2() { + /* + * Note: this also uses the cost function specified in Kleinberg & Tardos. + * Best alignment should be: + * n a m e + * _ a m e + * This should have a cost of 1 * gapCost = 2 + */ + final int optAlignmentCost = kleinbergSequenceAligner.calculateAlignment(new Character[] { 'a', 'm', 'e' }, nameChars); + final int expectedAlignmentCost = 2; + String msg = String.format("'ame' aligned with 'name': computed opt != expected opt (computed=%d expected=%d)", + optAlignmentCost, expectedAlignmentCost); + assertTrue(msg, optAlignmentCost == expectedAlignmentCost); + } + + @Test + public void fameAlignedWithNameShouldBe1() { + /* + * Note: this also uses the cost function specified in Kleinberg & Tardos. + * Best alignment should be: + * n a m e + * f a m e + * This should have a cost of 1 * consonantMatchedWithConsonantCost = 1 + */ + final int optAlignmentCost = kleinbergSequenceAligner.calculateAlignment(new Character[] { 'f', 'a', 'm', 'e' }, + nameChars); + final int expectedAlignmentCost = 1; + String msg = String.format("'fame' aligned with 'name': computed opt != expected opt (computed=%d expected=%d)", + optAlignmentCost, expectedAlignmentCost); + assertTrue(msg, optAlignmentCost == expectedAlignmentCost); + } + + @Test + public void nameAlignedWithNameShouldBe0() { + /* + * Note: this also uses the cost function specified in Kleinberg & Tardos. + * Best alignment should be: + * n a m e + * n a m e + * This should have a cost of 0. + */ + final int optAlignmentCost = kleinbergSequenceAligner.calculateAlignment(new Character[] { 'n', 'a', 'm', 'e' }, + nameChars); + final int expectedAlignmentCost = 0; + String msg = String.format("'name' aligned with 'name': computed opt != expected opt (computed=%d expected=%d)", + optAlignmentCost, expectedAlignmentCost); + assertTrue(msg, optAlignmentCost == expectedAlignmentCost); + } + + @Test + public void emanAlignedWithNameShouldBe6() { + /* + * Note: this also uses the cost function specified in Kleinberg & Tardos. + * Best alignment should be: + * + * _ n a m e + * e m a n _ + * + * or + * + * n a m e _ + * _ e m a n + * + * This should have a cost of 2 * gapCost + 2 * consonantMatchedWithConsonantCost = 2 * 2 + 2 * 1 = 6. + */ + final int optAlignmentCost = kleinbergSequenceAligner.calculateAlignment(new Character[] { 'e', 'm', 'a', 'n' }, + nameChars); + final int expectedAlignmentCost = 6; + String msg = String.format("'eman' aligned with 'name': computed opt != expected opt (computed=%d expected=%d)", + optAlignmentCost, expectedAlignmentCost); + assertTrue(msg, optAlignmentCost == expectedAlignmentCost); + } + + @Test + public void naemAlignedWithNameShouldBe4() { + /* + * Note: this also uses the cost function specified in Kleinberg & Tardos. + * Best alignment should be: + * + * n a _ m e + * n a e m _ + * + * or + * + * n a m e _ + * n a _ e m + * + * This should have a cost of 2 * gapCost = 4. + */ + final int optAlignmentCost = kleinbergSequenceAligner.calculateAlignment(new Character[] { 'n', 'a', 'e', 'm' }, + nameChars); + final int expectedAlignmentCost = 4; + String msg = String.format("'naem' aligned with 'name': computed opt != expected opt (computed=%d expected=%d)", + optAlignmentCost, expectedAlignmentCost); + assertTrue(msg, optAlignmentCost == expectedAlignmentCost); + } + + + /** + * Checks if {@code letter} is a lowercase vowel. Note: for simplicity, 'y' is considered a vowel. + * @param letter A {@code char} expected to be a vowel. + * @return {@code true} if {@code letter} is a vowel, {@code false} otherwise. + */ + private boolean isVowel(char letter) { + for (char vowel : lowercaseVowels) { + if (letter == vowel) { + return true; + } + } + return false; + } + + /** + * Checks if {@code letter} is a lowercase consonant. Note: for simplicity, 'y' is considered a vowel. + * @param letter A {@code char} expected to be a consonant. + * @return {@code true} if {@code letter} is a consonant, {@code false} otherwise. + */ + private boolean isConsonant(char letter) { + for (char consonant : lowercaseConsonants) { + if (letter == consonant) { + return true; + } + } + return false; + } +}