import edu.uci.iotproject.io.PrintWriterUtils;
import edu.uci.iotproject.trafficreassembly.layer2.Layer2Flow;
import edu.uci.iotproject.trafficreassembly.layer2.Layer2FlowReassembler;
+import edu.uci.iotproject.util.PcapPacketUtils;
import edu.uci.iotproject.util.PrintUtils;
import org.jgrapht.GraphPath;
import org.jgrapht.alg.shortestpath.DijkstraShortestPath;
*/
private static boolean DUPLICATE_OUTPUT_TO_STD_OUT = true;
+ /**
+ * Router's MAC.
+ * This is only useful for the filter for direction when it is a WAN signature (Phone-Cloud or Device-Cloud).
+ * Phone-Device signatures do not have router MAC address in it.
+ */
+ // TODO: We can remove the following constants if we do remove router's MAC filtering for directions
+ private static String TRAINING_ROUTER_WLAN_MAC = null;
+ private static String ROUTER_WLAN_MAC = null;
+ //private static String TRAINING_ROUTER_WLAN_MAC = "b0:b9:8a:73:69:8e";
+ //private static String ROUTER_WLAN_MAC = "00:c1:b1:14:eb:31";
+
private static List<Function<Layer2Flow, Boolean>> parseSignatureMacFilters(String filtersString) {
List<Function<Layer2Flow, Boolean>> filters = new ArrayList<>();
String[] filterRegexes = filtersString.split(";");
public static void main(String[] args) throws PcapNativeException, NotOpenException, IOException {
// Parse required parameters.
- if (args.length < 5) {
- String errMsg = String.format("Usage: %s inputPcapFile onSignatureFile offSignatureFile resultsFile" +
+ if (args.length < 8) {
+ String errMsg = String.format("SPECTO version 1.0\n" +
+ "Copyright (C) 2018-2019 Janus Varmarken and Rahmadi Trimananda.\n" +
+ "University of California, Irvine.\n" +
+ "All rights reserved.\n\n" +
+ "Usage: %s inputPcapFile onAnalysisFile offAnalysisFile onSignatureFile offSignatureFile " +
+ "resultsFile signatureDuration eps onMaxSkippedPackets offMaxSkippedPackets" +
"\n inputPcapFile: the target of the detection" +
+ "\n onAnalysisFile: the file that contains the ON clusters analysis" +
+ "\n offAnalysisFile: the file that contains the OFF clusters analysis" +
"\n onSignatureFile: the file that contains the ON signature to search for" +
"\n offSignatureFile: the file that contains the OFF signature to search for" +
"\n resultsFile: where to write the results of the detection" +
- "\n signatureDuration: the maximum duration of signature detection",
+ "\n signatureDuration: the maximum duration of signature detection" +
+ "\n eps: the epsilon value for the DBSCAN algorithm",
Layer2SignatureDetector.class.getSimpleName());
System.out.println(errMsg);
String optParamsExplained = "Above are the required, positional arguments. In addition to these, the " +
"that matches the vendor's prefix).\n" +
" '-offmacfilters <regex>;<regex>;...;<regex>' works exactly the same as onmacfilters, but " +
"applies to the OFF signature instead of the ON signature.\n" +
- " '-sout <boolean literal>' true/false literal indicating if output should also be printed to std out; default is true.";
+ " '-sout <boolean literal>' true/false literal indicating if output should also be printed to std out; default is true.\n" +
+ " '-vpn <router mac>' router's MAC address; this is to simulate a VPN that combines all flows even when the traffic is not a VPN traffic.\n" +
+ " '-onskipped <max duration of on-signature>' the maximum duration of ON signature detection.\n" +
+ " '-offskipped <max duration of off-signature>' the maximum duration of OFF signature detection.\n";
System.out.println(optParamsExplained);
return;
}
final String pcapFile = args[0];
- final String onSignatureFile = args[1];
- final String offSignatureFile = args[2];
- final String resultsFile = args[3];
- final int signatureDuration = Integer.parseInt(args[4]);
+ final String onClusterAnalysisFile = args[1];
+ final String offClusterAnalysisFile = args[2];
+ final String onSignatureFile = args[3];
+ final String offSignatureFile = args[4];
+ final String resultsFile = args[5];
+ final int signatureDuration = Integer.parseInt(args[6]);
+ final double eps = Double.parseDouble(args[7]);
// Parse optional parameters.
List<Function<Layer2Flow, Boolean>> onSignatureMacFilters = null, offSignatureMacFilters = null;
- final int optParamsStartIdx = 5;
+ String vpnClientMacAddress = null;
+ int onMaxSkippedPackets = -1;
+ int offMaxSkippedPackets = -1;
+ final int optParamsStartIdx = 8;
if (args.length > optParamsStartIdx) {
for (int i = optParamsStartIdx; i < args.length; i++) {
if (args[i].equalsIgnoreCase("-onMacFilters")) {
} else if (args[i].equalsIgnoreCase("-sout")) {
// Next argument is a boolean true/false literal.
DUPLICATE_OUTPUT_TO_STD_OUT = Boolean.parseBoolean(args[i+1]);
+ } else if (args[i].equalsIgnoreCase("-vpn")) {
+ vpnClientMacAddress = args[i+1];
+ } else if (args[i].equalsIgnoreCase("-onskipped")) {
+ if (i+2 > args.length - 1 || !args[i+2].equalsIgnoreCase("-offskipped")) {
+ throw new Error("Please make sure that the -onskipped and -offskipped options are both used at the same time...");
+ }
+ if (args[i+2].equalsIgnoreCase("-offskipped")) {
+ onMaxSkippedPackets = Integer.parseInt(args[i+1]);
+ offMaxSkippedPackets = Integer.parseInt(args[i+3]);
+ }
}
}
}
// Include metadata as comments at the top
PrintWriterUtils.println("# Detection results for:", resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
PrintWriterUtils.println("# - inputPcapFile: " + pcapFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
+ PrintWriterUtils.println("# - onAnalysisFile: " + onClusterAnalysisFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
+ PrintWriterUtils.println("# - offAnalysisFile: " + offClusterAnalysisFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
PrintWriterUtils.println("# - onSignatureFile: " + onSignatureFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
PrintWriterUtils.println("# - offSignatureFile: " + offSignatureFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
resultsWriter.flush();
- // TODO: IMPLEMENT THE RANGE-BASED DETECTION HERE
- boolean isRangeBased = true;
-
// Create signature detectors and add observers that output their detected events.
List<List<List<PcapPacket>>> onSignature = PrintUtils.deserializeFromFile(onSignatureFile);
List<List<List<PcapPacket>>> offSignature = PrintUtils.deserializeFromFile(offSignatureFile);
+ // Load signature analyses
+ List<List<List<PcapPacket>>> onClusterAnalysis = PrintUtils.deserializeFromFile(onClusterAnalysisFile);
+ List<List<List<PcapPacket>>> offClusterAnalysis = PrintUtils.deserializeFromFile(offClusterAnalysisFile);
+ // TODO: FOR NOW WE DECIDE PER SIGNATURE AND THEN WE OR THE BOOLEANS
+ // TODO: SINCE WE ONLY HAVE 2 SIGNATURES FOR NOW (ON AND OFF), THEN IT IS USUALLY EITHER RANGE-BASED OR
+ // TODO: STRICT MATCHING
+ // Check if we should use range-based matching
+ boolean isRangeBasedForOn = PcapPacketUtils.isRangeBasedMatching(onSignature, eps, offSignature);
+ boolean isRangeBasedForOff = PcapPacketUtils.isRangeBasedMatching(offSignature, eps, onSignature);
+ // TODO: WE DON'T DO RANGE-BASED FOR NOW BECAUSE THE RESULTS ARE TERRIBLE FOR LAYER 2 MATCHING
+ // TODO: THIS WOULD ONLY WORK FOR SIGNATURES LONGER THAN 2 PACKETS
+// boolean isRangeBasedForOn = false;
+// boolean isRangeBasedForOff = false;
+ // Update the signature with ranges if it is range-based
+ if (isRangeBasedForOn) {
+ onSignature = PcapPacketUtils.useRangeBasedMatching(onSignature, onClusterAnalysis);
+ }
+ if (isRangeBasedForOff) {
+ offSignature = PcapPacketUtils.useRangeBasedMatching(offSignature, offClusterAnalysis);
+ }
Layer2SignatureDetector onDetector = onSignatureMacFilters == null ?
- new Layer2SignatureDetector(onSignature) :
- new Layer2SignatureDetector(onSignature, onSignatureMacFilters, signatureDuration, isRangeBased);
+ new Layer2SignatureDetector(onSignature, TRAINING_ROUTER_WLAN_MAC, ROUTER_WLAN_MAC, signatureDuration,
+ isRangeBasedForOn, eps, onMaxSkippedPackets, vpnClientMacAddress) :
+ new Layer2SignatureDetector(onSignature, TRAINING_ROUTER_WLAN_MAC, ROUTER_WLAN_MAC,
+ onSignatureMacFilters, signatureDuration, isRangeBasedForOn, eps, onMaxSkippedPackets,
+ vpnClientMacAddress);
Layer2SignatureDetector offDetector = offSignatureMacFilters == null ?
- new Layer2SignatureDetector(offSignature) :
- new Layer2SignatureDetector(offSignature, offSignatureMacFilters, signatureDuration, isRangeBased);
+ new Layer2SignatureDetector(offSignature, TRAINING_ROUTER_WLAN_MAC, ROUTER_WLAN_MAC, signatureDuration,
+ isRangeBasedForOff, eps, offMaxSkippedPackets, vpnClientMacAddress) :
+ new Layer2SignatureDetector(offSignature, TRAINING_ROUTER_WLAN_MAC, ROUTER_WLAN_MAC, offSignatureMacFilters,
+ signatureDuration, isRangeBasedForOff, eps, offMaxSkippedPackets, vpnClientMacAddress);
+ final List<UserAction> detectedEvents = new ArrayList<>();
onDetector.addObserver((signature, match) -> {
UserAction event = new UserAction(UserAction.Type.TOGGLE_ON, match.get(0).get(0).getTimestamp());
PrintWriterUtils.println(event, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
+ detectedEvents.add(event);
});
offDetector.addObserver((signature, match) -> {
UserAction event = new UserAction(UserAction.Type.TOGGLE_OFF, match.get(0).get(0).getTimestamp());
PrintWriterUtils.println(event, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
+ detectedEvents.add(event);
});
// Load the PCAP file
// Parse the file
reader.readFromHandle();
+ String resultOn = "# Number of detected events of type " + UserAction.Type.TOGGLE_ON + ": " +
+ detectedEvents.stream().filter(ua -> ua.getType() == UserAction.Type.TOGGLE_ON).count();
+ String resultOff = "# Number of detected events of type " + UserAction.Type.TOGGLE_OFF + ": " +
+ detectedEvents.stream().filter(ua -> ua.getType() == UserAction.Type.TOGGLE_OFF).count();
+ String onMaximumSkippedPackets = "# Maximum number of skipped packets in ON signature " +
+ Integer.toString(onDetector.getMaxSkippedPackets());
+ String offMaximumSkippedPackets = "# Maximum number of skipped packets in OFF signature " +
+ Integer.toString(offDetector.getMaxSkippedPackets());
+ PrintWriterUtils.println(resultOn, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
+ PrintWriterUtils.println(resultOff, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
+ // Perform the skipped packet analysis if needed
+ if (onMaxSkippedPackets != -1 && offMaxSkippedPackets != -1) {
+ PrintWriterUtils.println(onMaximumSkippedPackets, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
+ for (Integer skippedPackets : onDetector.getSkippedPackets()) {
+ PrintWriterUtils.println(skippedPackets, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
+ }
+ PrintWriterUtils.println(offMaximumSkippedPackets, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
+ for (Integer skippedPackets : offDetector.getSkippedPackets()) {
+ PrintWriterUtils.println(skippedPackets, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
+ }
+ }
// Flush output to results file and close it.
resultsWriter.flush();
resultsWriter.close();
/**
* In charge of reassembling layer 2 packet flows.
*/
- private final Layer2FlowReassembler mFlowReassembler = new Layer2FlowReassembler();
+ private Layer2FlowReassembler mFlowReassembler;
private final List<SignatureDetectorObserver> mObservers = new ArrayList<>();
private int mInclusionTimeMillis;
- public Layer2SignatureDetector(List<List<List<PcapPacket>>> searchedSignature) {
- this(searchedSignature, null, 0, false);
+ /**
+ * Skipped-packet analysis.
+ */
+ private int mMaxSkippedPackets;
+ private List<Integer> mSkippedPackets;
+
+
+
+ public Layer2SignatureDetector(List<List<List<PcapPacket>>> searchedSignature, String trainingRouterWlanMac,
+ String routerWlanMac, int signatureDuration, boolean isRangeBased, double eps,
+ int limitSkippedPackets, String vpnClientMacAddress) {
+ this(searchedSignature, trainingRouterWlanMac, routerWlanMac, null, signatureDuration, isRangeBased,
+ eps, limitSkippedPackets, vpnClientMacAddress);
}
- public Layer2SignatureDetector(List<List<List<PcapPacket>>> searchedSignature, List<Function<Layer2Flow,
- Boolean>> flowFilters, int inclusionTimeMillis, boolean isRangeBased) {
+ public Layer2SignatureDetector(List<List<List<PcapPacket>>> searchedSignature, String trainingRouterWlanMac,
+ String routerWlanMac, List<Function<Layer2Flow, Boolean>> flowFilters,
+ int inclusionTimeMillis, boolean isRangeBased, double eps, int limitSkippedPackets,
+ String vpnClientMacAddress) {
if (flowFilters != null && flowFilters.size() != searchedSignature.size()) {
- throw new IllegalArgumentException("If flow filters are used, there must be a flow filter for each cluster of the signature.");
+ throw new IllegalArgumentException("If flow filters are used, there must be a flow filter for each cluster " +
+ "of the signature.");
}
mSignature = Collections.unmodifiableList(searchedSignature);
List<Layer2ClusterMatcher> clusterMatchers = new ArrayList<>();
for (int i = 0; i < mSignature.size(); i++) {
List<List<PcapPacket>> cluster = mSignature.get(i);
Layer2ClusterMatcher clusterMatcher = flowFilters == null ?
- new Layer2ClusterMatcher(cluster) : new Layer2ClusterMatcher(cluster, flowFilters.get(i), isRangeBased);
+ new Layer2ClusterMatcher(cluster, trainingRouterWlanMac, routerWlanMac, inclusionTimeMillis,
+ isRangeBased, eps, limitSkippedPackets) :
+ new Layer2ClusterMatcher(cluster, trainingRouterWlanMac, routerWlanMac, flowFilters.get(i),
+ inclusionTimeMillis, isRangeBased, eps, limitSkippedPackets);
clusterMatcher.addObserver(this);
clusterMatchers.add(clusterMatcher);
}
}
mClusterMatcherIds = Collections.unmodifiableMap(clusterMatcherIds);
// Register all cluster matchers to receive a notification whenever a new flow is encountered.
+ if (vpnClientMacAddress != null) {
+ mFlowReassembler = new Layer2FlowReassembler(vpnClientMacAddress);
+ } else {
+ mFlowReassembler = new Layer2FlowReassembler();
+ }
mClusterMatchers.forEach(cm -> mFlowReassembler.addObserver(cm));
mInclusionTimeMillis =
inclusionTimeMillis == 0 ? TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS : inclusionTimeMillis;
+ mMaxSkippedPackets = 0;
+ mSkippedPackets = new ArrayList<>();
+ }
+
+ public int getMaxSkippedPackets() {
+ return mMaxSkippedPackets;
+ }
+
+ public List<Integer> getSkippedPackets() {
+ for (Layer2ClusterMatcher matcher : mClusterMatchers) {
+ mSkippedPackets.addAll(matcher.getSkippedPackets());
+ }
+ return mSkippedPackets;
}
@Override
// Add the match at the corresponding index
mPendingMatches[mClusterMatcherIds.get(clusterMatcher)].add(match);
checkSignatureMatch();
+ // Update maximum number of skipped packets
+ if (mMaxSkippedPackets < ((Layer2ClusterMatcher) clusterMatcher).getMaxSkippedPackets()) {
+ mMaxSkippedPackets = ((Layer2ClusterMatcher) clusterMatcher).getMaxSkippedPackets();
+ }
}
}