4 Script that takes a file (output by wireshark/tshark, in JSON format) with DNS traffic
5 and constructs a map (dictionary) in which a hostname points to a set that contains the
6 IP addresses that is associated with that hostname.
11 from collections import defaultdict
14 ROUTER_MAC = "b0:b9:8a:73:69:8e"
16 JSON_KEY_SOURCE = "_source"
17 JSON_KEY_LAYERS = "layers"
19 JSON_KEY_QUERIES = "Queries"
20 JSON_KEY_ANSWERS = "Answers"
21 JSON_KEY_DNS_RESP_TYPE = "dns.resp.type"
22 JSON_KEY_DNS_A = "dns.a" # Key for retrieving IP. 'a' for type A DNS record.
23 JSON_KEY_DNS_RESP_NAME = "dns.resp.name"
24 JSON_KEY_DNS_CNAME = "dns.cname"
26 JSON_KEY_ETH_DST = "eth.dst"
27 JSON_KEY_FRAME = "frame"
28 JSON_KEY_FRAME_TIME_EPOCH = "frame.time_epoch"
32 print "Usage: python", sys.argv[0], "input_file"
34 mac_to_ddm = parse_json_dns(sys.argv[1])
35 for mac in mac_to_ddm:
38 # maps_tuple = parse_json_dns(sys.argv[1])
40 # # print hostname to ip map
41 # hn_ip_map = maps_tuple[0]
42 # for hn in hn_ip_map.keys():
43 # print "====================================================================="
44 # print hn, "maps to:"
45 # for ip in hn_ip_map[hn]:
47 # print "====================================================================="
51 # # print ip to hostname map
52 # ip_hn_map = maps_tuple[1]
53 # for ip in ip_hn_map.keys():
54 # print "====================================================================="
55 # print ip, "maps to:"
56 # for hn in ip_hn_map[ip]:
58 # print "====================================================================="
61 def __init__(self, mac_address):
62 # MAC address of device
63 self.mac = mac_address
64 # Maps an external IP to a list of (timestamp,hostname) tuples.
65 # Entries in the list should be interpreted as follows:
66 # the timestamp indicates WHEN this device mapped the given ip (key in dict) to the hostname.
67 self.ip_mappings = defaultdict(list)
69 def hostname_for_ip_at_time(self, ip, timestamp):
70 # Does device have a mapping for the given IP?
71 if not ip in self.ip_mappings:
73 if not self.ip_mappings[ip]:
74 # If list of (timestamp,hostname) tuples is empty, there is no mapping to report.
76 # Best fit mapping: the mapping immediately BEFORE timestamp parameter.
77 # Start with random pick (element 0).
78 best_fit = self.ip_mappings[ip][0]
79 for t in self.ip_mappings[ip]:
80 # t is a (timestamp,hostname) tuple
81 if t[0] < timestamp and t[0] > best_fit[0]:
82 # t is a better fit if it happened BEFORE the input timestamp
83 # and is LATER than the current best_fit
85 # return the matching hostname
88 def add_mapping(self, ip, timestamp_hostname_tuple):
89 self.ip_mappings[ip].append(timestamp_hostname_tuple)
91 def print_mappings(self):
93 print "### Mappings for MAC = ", self.mac, "###"
94 for ip in self.ip_mappings:
95 print "--- IP ", ip, " maps to: ---"
96 for t in self.ip_mappings[ip]:
97 print t[1], "at epoch time =", t[0]
99 print "### Total of", count, "mappings for", self.mac, "###"
101 # --------------------------------------------------------------------------
102 # Define eq and hash such that instances of the class can be used as keys in dictionaries.
103 # Equality is based on MAC as a MAC uniquely identifies the device.
104 def __eq__(self, another):
105 return hasattr(another, 'mac') and self.mac == another.mac
107 return hash(self.data)
108 # --------------------------------------------------------------------------
111 def parse_json_dns(file_path):
112 # Our end output: dictionary of MAC addresses with DeviceDNSMaps as values.
113 # Each DeviceDNSMap contains DNS lookups performed by the device with the corresponding MAC.
114 result = defaultdict()
115 with open(file_path) as jf:
117 # data becomes reference to root JSON object (or in our case json array)
119 # Loop through json objects in data
120 # Each entry is a pcap entry (request/response (packet) and associated metadata)
122 # p is a JSON object, not an index
123 # Drill down to DNS part: _source->layers->dns
124 layers = p[JSON_KEY_SOURCE][JSON_KEY_LAYERS]
125 dns = layers.get(JSON_KEY_DNS, None)
126 # Skip any non DNS traffic
128 #print "[ WARNING: Non DNS traffic ]"
130 # We only care about DNS responses as these also contain a copy of the query that they answer
131 answers = dns.get(JSON_KEY_ANSWERS, None)
134 ## Now that we know that it is an answer, the queries should also be available.
135 queries = dns.get(JSON_KEY_QUERIES)
136 if len(queries.keys()) > 1:
137 # Unclear if script will behave correctly for DNS lookups with multiple queries
138 print "[ WARNING: Multi query DNS lookup ]"
139 # Get ethernet information for identifying the device performing the DNS lookup.
140 eth = layers.get(JSON_KEY_ETH, None)
142 print "[ WARNING: eth data not found ]"
144 # As this is a response to a DNS query, the IoT device is the destination.
145 # Get the device MAC of that device.
146 device_mac = eth.get(JSON_KEY_ETH_DST, None)
147 if device_mac is None:
148 print "[ WARNING: eth.dst data not found ]"
150 # Get the router's timestamp for this packet
151 # so that we can mark when the DNS mapping occurred
152 timestamp = Decimal(layers[JSON_KEY_FRAME][JSON_KEY_FRAME_TIME_EPOCH])
153 for ak in answers.keys():
155 # We are looking for type A records as these are the ones that contain the IP.
157 if a[JSON_KEY_DNS_RESP_TYPE] == "1":
159 ip = a[JSON_KEY_DNS_A]
160 # The answer may be the canonical name.
161 # Now trace back the answer stack, looking for any higher level aliases.
162 hostname = find_alias_hostname(answers, a[JSON_KEY_DNS_RESP_NAME])
163 # Create the tuple that indicates WHEN the ip to hostname mapping occurred
164 timestamp_hostname_tuple = (timestamp,hostname)
165 if device_mac in result:
166 # If we already have DNS data for the device with this MAC:
167 # Add the mapping to the DeviceDNSMap that is already present in the dict.
168 result[device_mac].add_mapping(ip, timestamp_hostname_tuple)
170 # No DNS data for this device yet:
171 # Create a new DeviceDNSMap, add the mapping, and at it to the dict.
172 ddm = DeviceDNSMap(device_mac)
173 ddm.add_mapping(ip, timestamp_hostname_tuple)
174 result[device_mac] = ddm
177 # Recursively traverse set of answers trying to find the top most alias for a canonical name
178 def find_alias_hostname(answers, hostname):
179 for ak in answers.keys():
181 cname = a.get(JSON_KEY_DNS_CNAME, None)
182 # We only care about type=CNAME records
185 if cname == hostname:
186 # Located the right answer, perform recursive search for higher level aliases.
187 return find_alias_hostname(answers, a[JSON_KEY_DNS_RESP_NAME])
190 if __name__ == '__main__':
193 # ================================================================================================
194 # Notes/brainstorming how to do ip to host mappings.
196 # Maps IPs to hostnames. Uses a dictionary of dictionaries.
197 # IP lookup in the outer dictionary returns a dictionary that has hostnames as keys.
198 # Looking up a hostname in the inner dictionary returns a set of timestamps.
199 # Each timestamp indicate the time at which the IP<->hostname mapping was determined by a DNS query.
200 # Note that the keyset of the inner dictionary will be of size 1 in most cases.
201 # When this is the case, the value (the set of timestamps) can be ignored.
202 # The values are only relevant when one IP maps to more than 1 hostname.
203 # When this the case, the timestamps must be considered to find the most recent mapping.
204 # ip_host_mappings = defaultdict(defaultdict(set))
206 # ================================================================================================