2 * Copyright(C) 2016 Linaro Limited. All rights reserved.
3 * Author: Tor Jeremiassen <tor.jeremiassen@linaro.org>
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 as published by
7 * the Free Software Foundation.
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * You should have received a copy of the GNU General Public License along with
15 * this program. If not, see <http://www.gnu.org/licenses/>.
18 #include <linux/err.h>
19 #include <linux/kernel.h>
20 #include <linux/types.h>
21 #include <linux/bitops.h>
22 #include <linux/log2.h>
25 #include "thread_map.h"
27 #include "thread-stack.h"
28 #include "callchain.h"
33 #include "util/intlist.h"
36 #include "cs-etm-decoder/cs-etm-decoder.h"
41 #define KiB(x) ((x) * 1024)
42 #define MiB(x) ((x) * 1024 * 1024)
43 #define MAX_TIMESTAMP (~0ULL)
45 struct cs_etm_auxtrace {
46 struct auxtrace auxtrace;
47 struct auxtrace_queues queues;
48 struct auxtrace_heap heap;
51 struct perf_session *session;
52 struct machine *machine;
53 struct perf_evsel *switch_evsel;
54 struct thread *unknown_thread;
56 bool timeless_decoding;
61 bool synth_needs_swap;
62 int have_sched_switch;
64 bool sample_instructions;
65 u64 instructions_sample_type;
66 u64 instructions_sample_period;
68 struct itrace_synth_opts synth_opts;
73 struct cs_etm_auxtrace *etm;
75 struct auxtrace_buffer *buffer;
76 const struct cs_etm_state *state;
77 struct ip_callchain *chain;
78 union perf_event *event_buf;
80 bool step_through_buffers;
81 bool use_buffer_pid_tid;
84 struct thread *thread;
88 struct cs_etm_decoder *decoder;
94 static int cs_etm__get_trace(struct cs_etm_buffer *buff, struct cs_etm_queue *etmq);
95 static int cs_etm__update_queues(struct cs_etm_auxtrace *);
96 static int cs_etm__process_queues(struct cs_etm_auxtrace *, u64);
97 static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *, pid_t, u64);
98 static uint32_t cs_etm__mem_access(struct cs_etm_queue *, uint64_t , size_t , uint8_t *);
100 static void cs_etm__packet_dump(const char *pkt_string)
102 const char *color = PERF_COLOR_BLUE;
104 color_fprintf(stdout,color, " %s\n", pkt_string);
108 static void cs_etm__dump_event(struct cs_etm_auxtrace *etm,
109 struct auxtrace_buffer *buffer)
111 const char *color = PERF_COLOR_BLUE;
112 struct cs_etm_decoder_params d_params;
113 struct cs_etm_trace_params *t_params;
114 struct cs_etm_decoder *decoder;
115 size_t buffer_used = 0;
118 fprintf(stdout,"\n");
119 color_fprintf(stdout, color,
120 ". ... CoreSight ETM Trace data: size %zu bytes\n",
123 t_params = zalloc(sizeof(struct cs_etm_trace_params) * etm->num_cpu);
124 for (i = 0; i < etm->num_cpu; ++i) {
125 t_params[i].protocol = CS_ETM_PROTO_ETMV4i;
126 t_params[i].reg_idr0 = etm->metadata[i][CS_ETMV4_TRCIDR0];
127 t_params[i].reg_idr1 = etm->metadata[i][CS_ETMV4_TRCIDR1];
128 t_params[i].reg_idr2 = etm->metadata[i][CS_ETMV4_TRCIDR2];
129 t_params[i].reg_idr8 = etm->metadata[i][CS_ETMV4_TRCIDR8];
130 t_params[i].reg_configr = etm->metadata[i][CS_ETMV4_TRCCONFIGR];
131 t_params[i].reg_traceidr = etm->metadata[i][CS_ETMV4_TRCTRACEIDR];
132 //[CS_ETMV4_TRCAUTHSTATUS] = " TRCAUTHSTATUS %"PRIx64"\n",
134 d_params.packet_printer = cs_etm__packet_dump;
135 d_params.operation = CS_ETM_OPERATION_PRINT;
136 d_params.formatted = true;
137 d_params.fsyncs = false;
138 d_params.hsyncs = false;
139 d_params.frame_aligned = true;
141 decoder = cs_etm_decoder__new(etm->num_cpu,&d_params, t_params);
145 if (decoder == NULL) {
150 cs_etm_decoder__process_data_block(decoder,buffer->offset,&(((uint8_t *)buffer->data)[buffer_used]),buffer->size - buffer_used, &consumed);
151 buffer_used += consumed;
152 } while(buffer_used < buffer->size);
153 cs_etm_decoder__free(decoder);
156 static int cs_etm__flush_events(struct perf_session *session, struct perf_tool *tool){
157 struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
158 struct cs_etm_auxtrace,
166 if (!tool->ordered_events)
169 ret = cs_etm__update_queues(etm);
174 if (etm->timeless_decoding)
175 return cs_etm__process_timeless_queues(etm,-1,MAX_TIMESTAMP - 1);
177 return cs_etm__process_queues(etm, MAX_TIMESTAMP);
180 static void cs_etm__set_pid_tid_cpu(struct cs_etm_auxtrace *etm,
181 struct auxtrace_queue *queue)
183 struct cs_etm_queue *etmq = queue->priv;
185 if ((queue->tid == -1) || (etm->have_sched_switch)) {
186 etmq->tid = machine__get_current_tid(etm->machine, etmq->cpu);
187 thread__zput(etmq->thread);
190 if ((!etmq->thread) && (etmq->tid != -1)) {
191 etmq->thread = machine__find_thread(etm->machine,-1,etmq->tid);
195 etmq->pid = etmq->thread->pid_;
196 if (queue->cpu == -1) {
197 etmq->cpu = etmq->thread->cpu;
202 static void cs_etm__free_queue(void *priv)
204 struct cs_etm_queue *etmq = priv;
209 thread__zput(etmq->thread);
210 cs_etm_decoder__free(etmq->decoder);
211 zfree(&etmq->event_buf);
216 static void cs_etm__free_events(struct perf_session *session)
218 struct cs_etm_auxtrace *aux = container_of(session->auxtrace,
219 struct cs_etm_auxtrace,
222 struct auxtrace_queues *queues = &(aux->queues);
226 for (i = 0; i < queues->nr_queues; ++i) {
227 cs_etm__free_queue(queues->queue_array[i].priv);
228 queues->queue_array[i].priv = 0;
231 auxtrace_queues__free(queues);
235 static void cs_etm__free(struct perf_session *session)
239 struct int_node *inode, *tmp;
240 struct cs_etm_auxtrace *aux = container_of(session->auxtrace,
241 struct cs_etm_auxtrace,
243 auxtrace_heap__free(&aux->heap);
244 cs_etm__free_events(session);
245 session->auxtrace = NULL;
247 /* First remove all traceID/CPU# nodes from the RB tree */
248 intlist__for_each_safe(inode, tmp, traceid_list)
249 intlist__remove(traceid_list, inode);
250 /* Then the RB tree itself */
251 intlist__delete(traceid_list);
253 //thread__delete(aux->unknown_thread);
254 for (i = 0; i < aux->num_cpu; ++i) {
255 zfree(&aux->metadata[i]);
257 zfree(&aux->metadata);
261 static void cs_etm__use_buffer_pid_tid(struct cs_etm_queue *etmq,
262 struct auxtrace_queue *queue,
263 struct auxtrace_buffer *buffer)
265 if ((queue->cpu == -1) && (buffer->cpu != -1)) {
266 etmq->cpu = buffer->cpu;
269 etmq->pid = buffer->pid;
270 etmq->tid = buffer->tid;
272 thread__zput(etmq->thread);
274 if (etmq->tid != -1) {
275 if (etmq->pid != -1) {
276 etmq->thread = machine__findnew_thread(etmq->etm->machine,
280 etmq->thread = machine__findnew_thread(etmq->etm->machine,
288 static int cs_etm__get_trace(struct cs_etm_buffer *buff, struct cs_etm_queue *etmq)
290 struct auxtrace_buffer *aux_buffer = etmq->buffer;
291 struct auxtrace_buffer *old_buffer = aux_buffer;
292 struct auxtrace_queue *queue;
299 queue = &etmq->etm->queues.queue_array[etmq->queue_nr];
301 aux_buffer = auxtrace_buffer__next(queue,aux_buffer);
305 auxtrace_buffer__drop_data(old_buffer);
311 etmq->buffer = aux_buffer;
313 if (!aux_buffer->data) {
314 int fd = perf_data_file__fd(etmq->etm->session->file);
316 aux_buffer->data = auxtrace_buffer__get_data(aux_buffer, fd);
317 if (!aux_buffer->data)
322 auxtrace_buffer__drop_data(old_buffer);
324 if (aux_buffer->use_data) {
325 buff->offset = aux_buffer->offset;
326 buff->len = aux_buffer->use_size;
327 buff->buf = aux_buffer->use_data;
329 buff->offset = aux_buffer->offset;
330 buff->len = aux_buffer->size;
331 buff->buf = aux_buffer->data;
335 buff->len = sizeof(cstrace);
339 buff->ref_timestamp = aux_buffer->reference;
341 if (etmq->use_buffer_pid_tid &&
342 ((etmq->pid != aux_buffer->pid) ||
343 (etmq->tid != aux_buffer->tid))) {
344 cs_etm__use_buffer_pid_tid(etmq,queue,aux_buffer);
347 if (etmq->step_through_buffers)
353 static struct cs_etm_queue *cs_etm__alloc_queue(struct cs_etm_auxtrace *etm,
354 unsigned int queue_nr)
356 struct cs_etm_decoder_params d_params;
357 struct cs_etm_trace_params *t_params;
358 struct cs_etm_queue *etmq;
361 etmq = zalloc(sizeof(struct cs_etm_queue));
365 if (etm->synth_opts.callchain) {
366 size_t sz = sizeof(struct ip_callchain);
368 sz += etm->synth_opts.callchain_sz * sizeof(u64);
369 etmq->chain = zalloc(sz);
376 etmq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
377 if (!etmq->event_buf)
381 etmq->queue_nr = queue_nr;
386 etmq->kernel_mapped = false;
388 t_params = zalloc(sizeof(struct cs_etm_trace_params)*etm->num_cpu);
390 for (i = 0; i < etm->num_cpu; ++i) {
391 t_params[i].reg_idr0 = etm->metadata[i][CS_ETMV4_TRCIDR0];
392 t_params[i].reg_idr1 = etm->metadata[i][CS_ETMV4_TRCIDR1];
393 t_params[i].reg_idr2 = etm->metadata[i][CS_ETMV4_TRCIDR2];
394 t_params[i].reg_idr8 = etm->metadata[i][CS_ETMV4_TRCIDR8];
395 t_params[i].reg_configr = etm->metadata[i][CS_ETMV4_TRCCONFIGR];
396 t_params[i].reg_traceidr = etm->metadata[i][CS_ETMV4_TRCTRACEIDR];
397 t_params[i].protocol = CS_ETM_PROTO_ETMV4i;
399 d_params.packet_printer = cs_etm__packet_dump;
400 d_params.operation = CS_ETM_OPERATION_DECODE;
401 d_params.formatted = true;
402 d_params.fsyncs = false;
403 d_params.hsyncs = false;
404 d_params.frame_aligned = true;
405 d_params.data = etmq;
407 etmq->decoder = cs_etm_decoder__new(etm->num_cpu,&d_params,t_params);
421 zfree(&etmq->event_buf);
427 static int cs_etm__setup_queue(struct cs_etm_auxtrace *etm,
428 struct auxtrace_queue *queue,
429 unsigned int queue_nr)
431 struct cs_etm_queue *etmq = queue->priv;
433 if (list_empty(&(queue->head)))
437 etmq = cs_etm__alloc_queue(etm,queue_nr);
445 if (queue->cpu != -1) {
446 etmq->cpu = queue->cpu;
449 etmq->tid = queue->tid;
451 if (etm->sampling_mode) {
452 if (etm->timeless_decoding)
453 etmq->step_through_buffers = true;
454 if (etm->timeless_decoding || !etm->have_sched_switch)
455 etmq->use_buffer_pid_tid = true;
459 if (!etmq->on_heap &&
460 (!etm->sync_switch)) {
461 const struct cs_etm_state *state;
464 if (etm->timeless_decoding)
467 //cs_etm__log("queue %u getting timestamp\n",queue_nr);
468 //cs_etm__log("queue %u decoding cpu %d pid %d tid %d\n",
469 //queue_nr, etmq->cpu, etmq->pid, etmq->tid);
474 state = cs_etm_decoder__decode(etmq->decoder);
476 if (state->err == CS_ETM_ERR_NODATA) {
477 //cs_etm__log("queue %u has no timestamp\n",
483 if (state->timestamp)
487 etmq->timestamp = state->timestamp;
488 //cs_etm__log("queue %u timestamp 0x%"PRIx64 "\n",
489 //queue_nr, etmq->timestamp);
491 etmq->have_sample = true;
492 //cs_etm__sample_flags(etmq);
493 ret = auxtrace_heap__add(&etm->heap, queue_nr, etmq->timestamp);
496 etmq->on_heap = true;
504 static int cs_etm__setup_queues(struct cs_etm_auxtrace *etm)
509 for (i = 0; i < etm->queues.nr_queues; i++) {
510 ret = cs_etm__setup_queue(etm, &(etm->queues.queue_array[i]),i);
518 struct cs_etm_cache_entry {
519 struct auxtrace_cache_entry entry;
524 static size_t cs_etm__cache_divisor(void)
526 static size_t d = 64;
531 static size_t cs_etm__cache_size(struct dso *dso,
532 struct machine *machine)
536 size = dso__data_size(dso,machine);
537 size /= cs_etm__cache_divisor();
542 if (size > (1 << 21))
545 return 32 - __builtin_clz(size);
548 static struct auxtrace_cache *cs_etm__cache(struct dso *dso,
549 struct machine *machine)
551 struct auxtrace_cache *c;
554 if (dso->auxtrace_cache)
555 return dso->auxtrace_cache;
557 bits = cs_etm__cache_size(dso,machine);
559 c = auxtrace_cache__new(bits, sizeof(struct cs_etm_cache_entry), 200);
561 dso->auxtrace_cache = c;
566 static int cs_etm__cache_add(struct dso *dso, struct machine *machine,
567 uint64_t offset, uint64_t icount, uint64_t bcount)
569 struct auxtrace_cache *c = cs_etm__cache(dso, machine);
570 struct cs_etm_cache_entry *e;
576 e = auxtrace_cache__alloc_entry(c);
583 err = auxtrace_cache__add(c, offset, &e->entry);
586 auxtrace_cache__free_entry(c, e);
591 static struct cs_etm_cache_entry *cs_etm__cache_lookup(struct dso *dso,
592 struct machine *machine,
595 struct auxtrace_cache *c = cs_etm__cache(dso, machine);
600 return auxtrace_cache__lookup(dso->auxtrace_cache, offset);
604 static int cs_etm__synth_instruction_sample(struct cs_etm_queue *etmq,
605 struct cs_etm_packet *packet)
608 struct cs_etm_auxtrace *etm = etmq->etm;
609 union perf_event *event = etmq->event_buf;
610 struct perf_sample sample = {.ip = 0,};
611 uint64_t start_addr = packet->start_addr;
612 uint64_t end_addr = packet->end_addr;
614 event->sample.header.type = PERF_RECORD_SAMPLE;
615 event->sample.header.misc = PERF_RECORD_MISC_USER;
616 event->sample.header.size = sizeof(struct perf_event_header);
619 sample.ip = start_addr;
620 sample.pid = etmq->pid;
621 sample.tid = etmq->tid;
622 sample.addr = end_addr;
623 sample.id = etmq->etm->instructions_id;
624 sample.stream_id = etmq->etm->instructions_id;
625 sample.period = (end_addr - start_addr) >> 2;
626 sample.cpu = packet->cpu;
627 sample.flags = 0; // etmq->flags;
628 sample.insn_len = 1; // etmq->insn_len;
630 //etmq->last_insn_cnt = etmq->state->tot_insn_cnt;
634 struct addr_location al;
636 struct thread *thread;
637 struct machine *machine = etmq->etm->machine;
639 struct cs_etm_cache_entry *e;
643 thread = etmq->thread;
646 thread = etmq->etm->unknown_thread;
649 if (start_addr > 0xffffffc000000000UL) {
650 cpumode = PERF_RECORD_MISC_KERNEL;
652 cpumode = PERF_RECORD_MISC_USER;
655 thread__find_addr_map(thread, cpumode, MAP__FUNCTION, start_addr,&al);
656 if (!al.map || !al.map->dso) {
659 if (al.map->dso->data.status == DSO_DATA_STATUS_ERROR &&
660 dso__data_status_seen(al.map->dso,DSO_DATA_STATUS_SEEN_ITRACE)) {
664 offset = al.map->map_ip(al.map,start_addr);
667 e = cs_etm__cache_lookup(al.map->dso, machine, offset);
673 map__load(al.map, machine->symbol_filter);
676 len = dso__data_read_offset(al.map->dso, machine,
683 cs_etm__cache_add(al.map->dso, machine, offset, (end_addr - start_addr) >> 2, end_addr - start_addr);
691 ret = perf_session__deliver_synth_event(etm->session,event, &sample);
694 pr_err("CS ETM Trace: failed to deliver instruction event, error %d\n", ret);
700 struct cs_etm_synth {
701 struct perf_tool dummy_tool;
702 struct perf_session *session;
706 static int cs_etm__event_synth(struct perf_tool *tool,
707 union perf_event *event,
708 struct perf_sample *sample,
709 struct machine *machine)
711 struct cs_etm_synth *cs_etm_synth =
712 container_of(tool, struct cs_etm_synth, dummy_tool);
717 return perf_session__deliver_synth_event(cs_etm_synth->session, event, NULL);
722 static int cs_etm__synth_event(struct perf_session *session,
723 struct perf_event_attr *attr, u64 id)
725 struct cs_etm_synth cs_etm_synth;
727 memset(&cs_etm_synth, 0, sizeof(struct cs_etm_synth));
728 cs_etm_synth.session = session;
730 return perf_event__synthesize_attr(&cs_etm_synth.dummy_tool, attr, 1,
731 &id, cs_etm__event_synth);
734 static int cs_etm__synth_events(struct cs_etm_auxtrace *etm,
735 struct perf_session *session)
737 struct perf_evlist *evlist = session->evlist;
738 struct perf_evsel *evsel;
739 struct perf_event_attr attr;
744 evlist__for_each(evlist, evsel) {
746 if (evsel->attr.type == etm->pmu_type) {
753 pr_debug("There are no selected events with Core Sight Trace data\n");
757 memset(&attr, 0, sizeof(struct perf_event_attr));
758 attr.size = sizeof(struct perf_event_attr);
759 attr.type = PERF_TYPE_HARDWARE;
760 attr.sample_type = evsel->attr.sample_type & PERF_SAMPLE_MASK;
761 attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID |
763 if (etm->timeless_decoding)
764 attr.sample_type &= ~(u64)PERF_SAMPLE_TIME;
766 attr.sample_type |= PERF_SAMPLE_TIME;
768 attr.exclude_user = evsel->attr.exclude_user;
769 attr.exclude_kernel = evsel->attr.exclude_kernel;
770 attr.exclude_hv = evsel->attr.exclude_hv;
771 attr.exclude_host = evsel->attr.exclude_host;
772 attr.exclude_guest = evsel->attr.exclude_guest;
773 attr.sample_id_all = evsel->attr.sample_id_all;
774 attr.read_format = evsel->attr.read_format;
776 id = evsel->id[0] + 1000000000;
781 if (etm->synth_opts.instructions) {
782 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
783 attr.sample_period = etm->synth_opts.period;
784 etm->instructions_sample_period = attr.sample_period;
785 err = cs_etm__synth_event(session, &attr, id);
788 pr_err("%s: failed to synthesize 'instructions' event type\n",
792 etm->sample_instructions = true;
793 etm->instructions_sample_type = attr.sample_type;
794 etm->instructions_id = id;
798 etm->synth_needs_swap = evsel->needs_swap;
802 static int cs_etm__sample(struct cs_etm_queue *etmq)
804 //const struct cs_etm_state *state = etmq->state;
805 struct cs_etm_packet packet;
806 //struct cs_etm_auxtrace *etm = etmq->etm;
809 err = cs_etm_decoder__get_packet(etmq->decoder,&packet);
810 // if there is no sample, it returns err = -1, no real error
812 if (!err && packet.sample_type & CS_ETM_RANGE) {
813 err = cs_etm__synth_instruction_sample(etmq,&packet);
820 static int cs_etm__run_decoder(struct cs_etm_queue *etmq, u64 *timestamp)
822 struct cs_etm_buffer buffer;
826 /* Go through each buffer in the queue and decode them one by one */
829 memset(&buffer, 0, sizeof(buffer));
830 err = cs_etm__get_trace(&buffer,etmq);
835 size_t processed = 0;
836 etmq->state = cs_etm_decoder__process_data_block(etmq->decoder,
838 &buffer.buf[buffer_used],
839 buffer.len-buffer_used,
841 err = etmq->state->err;
842 etmq->offset += processed;
843 buffer_used += processed;
845 cs_etm__sample(etmq);
846 } while (!etmq->eot && (buffer.len > buffer_used));
854 static int cs_etm__update_queues(struct cs_etm_auxtrace *etm)
856 if (etm->queues.new_data) {
857 etm->queues.new_data = false;
858 return cs_etm__setup_queues(etm);
863 static int cs_etm__process_queues(struct cs_etm_auxtrace *etm, u64 timestamp)
865 unsigned int queue_nr;
870 struct auxtrace_queue *queue;
871 struct cs_etm_queue *etmq;
873 if (!etm->heap.heap_cnt)
876 if (etm->heap.heap_array[0].ordinal >= timestamp)
879 queue_nr = etm->heap.heap_array[0].queue_nr;
880 queue = &etm->queues.queue_array[queue_nr];
883 //cs_etm__log("queue %u processing 0x%" PRIx64 " to 0x%" PRIx64 "\n",
884 //queue_nr, etm->heap.heap_array[0].ordinal,
887 auxtrace_heap__pop(&etm->heap);
889 if (etm->heap.heap_cnt) {
890 ts = etm->heap.heap_array[0].ordinal + 1;
897 cs_etm__set_pid_tid_cpu(etm, queue);
899 ret = cs_etm__run_decoder(etmq, &ts);
902 auxtrace_heap__add(&etm->heap, queue_nr, ts);
907 ret = auxtrace_heap__add(&etm->heap, queue_nr, ts);
911 etmq->on_heap = false;
917 static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm,
921 struct auxtrace_queues *queues = &etm->queues;
925 for (i = 0; i < queues->nr_queues; ++i) {
926 struct auxtrace_queue *queue = &(etm->queues.queue_array[i]);
927 struct cs_etm_queue *etmq = queue->priv;
929 if (etmq && ((tid == -1) || (etmq->tid == tid))) {
931 cs_etm__set_pid_tid_cpu(etm, queue);
932 cs_etm__run_decoder(etmq,&ts);
939 static struct cs_etm_queue *cs_etm__cpu_to_etmq(struct cs_etm_auxtrace *etm,
944 if (etm->queues.nr_queues == 0)
949 else if ((unsigned) cpu >= etm->queues.nr_queues)
950 q = etm->queues.nr_queues - 1;
954 if (etm->queues.queue_array[q].cpu == cpu)
955 return etm->queues.queue_array[q].priv;
957 for (j = 0; q > 0; j++) {
958 if (etm->queues.queue_array[--q].cpu == cpu)
959 return etm->queues.queue_array[q].priv;
962 for (; j < etm->queues.nr_queues; j++) {
963 if (etm->queues.queue_array[j].cpu == cpu)
964 return etm->queues.queue_array[j].priv;
971 static uint32_t cs_etm__mem_access(struct cs_etm_queue *etmq, uint64_t address, size_t size, uint8_t *buffer)
973 struct addr_location al;
975 struct thread *thread;
976 struct machine *machine;
983 machine = etmq->etm->machine;
984 thread = etmq->thread;
985 if (address > 0xffffffc000000000UL) {
986 cpumode = PERF_RECORD_MISC_KERNEL;
988 cpumode = PERF_RECORD_MISC_USER;
991 thread__find_addr_map(thread, cpumode, MAP__FUNCTION, address,&al);
993 if (!al.map || !al.map->dso) {
997 if (al.map->dso->data.status == DSO_DATA_STATUS_ERROR &&
998 dso__data_status_seen(al.map->dso,DSO_DATA_STATUS_SEEN_ITRACE)) {
1002 offset = al.map->map_ip(al.map,address);
1004 map__load(al.map, machine->symbol_filter);
1006 len = dso__data_read_offset(al.map->dso, machine,
1007 offset, buffer, size);
1016 static bool check_need_swap(int file_endian)
1019 u8 *check = (u8 *)&data;
1023 host_endian = ELFDATA2LSB;
1025 host_endian = ELFDATA2MSB;
1027 return host_endian != file_endian;
1030 static int cs_etm__read_elf_info(const char *fname, uint64_t *foffset, uint64_t *fstart, uint64_t *fsize)
1033 u8 e_ident[EI_NIDENT];
1035 bool need_swap = false;
1040 fp = fopen(fname, "r");
1044 if (fread(e_ident, sizeof(e_ident), 1, fp) != 1)
1047 if (memcmp(e_ident, ELFMAG, SELFMAG) ||
1048 e_ident[EI_VERSION] != EV_CURRENT)
1051 need_swap = check_need_swap(e_ident[EI_DATA]);
1053 /* for simplicity */
1054 fseek(fp, 0, SEEK_SET);
1056 if (e_ident[EI_CLASS] == ELFCLASS32) {
1060 if (fread(&ehdr, sizeof(ehdr), 1, fp) != 1)
1064 ehdr.e_phoff = bswap_32(ehdr.e_phoff);
1065 ehdr.e_phentsize = bswap_16(ehdr.e_phentsize);
1066 ehdr.e_phnum = bswap_16(ehdr.e_phnum);
1069 buf_size = ehdr.e_phentsize * ehdr.e_phnum;
1070 buf = malloc(buf_size);
1074 fseek(fp, ehdr.e_phoff, SEEK_SET);
1075 if (fread(buf, buf_size, 1, fp) != 1)
1078 for (i = 0, phdr = buf; i < ehdr.e_phnum; i++, phdr++) {
1081 phdr->p_type = bswap_32(phdr->p_type);
1082 phdr->p_offset = bswap_32(phdr->p_offset);
1083 phdr->p_filesz = bswap_32(phdr->p_filesz);
1086 if (phdr->p_type != PT_LOAD)
1089 *foffset = phdr->p_offset;
1090 *fstart = phdr->p_vaddr;
1091 *fsize = phdr->p_filesz;
1099 if (fread(&ehdr, sizeof(ehdr), 1, fp) != 1)
1103 ehdr.e_phoff = bswap_64(ehdr.e_phoff);
1104 ehdr.e_phentsize = bswap_16(ehdr.e_phentsize);
1105 ehdr.e_phnum = bswap_16(ehdr.e_phnum);
1108 buf_size = ehdr.e_phentsize * ehdr.e_phnum;
1109 buf = malloc(buf_size);
1113 fseek(fp, ehdr.e_phoff, SEEK_SET);
1114 if (fread(buf, buf_size, 1, fp) != 1)
1117 for (i = 0, phdr = buf; i < ehdr.e_phnum; i++, phdr++) {
1120 phdr->p_type = bswap_32(phdr->p_type);
1121 phdr->p_offset = bswap_64(phdr->p_offset);
1122 phdr->p_filesz = bswap_64(phdr->p_filesz);
1125 if (phdr->p_type != PT_LOAD)
1128 *foffset = phdr->p_offset;
1129 *fstart = phdr->p_vaddr;
1130 *fsize = phdr->p_filesz;
1142 static int cs_etm__process_event(struct perf_session *session,
1143 union perf_event *event,
1144 struct perf_sample *sample,
1145 struct perf_tool *tool)
1147 struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
1148 struct cs_etm_auxtrace,
1157 if (!tool->ordered_events) {
1158 pr_err("CoreSight ETM Trace requires ordered events\n");
1162 if (sample->time && (sample->time != (u64)-1))
1163 timestamp = sample->time;
1167 if (timestamp || etm->timeless_decoding) {
1168 err = cs_etm__update_queues(etm);
1174 if (event->header.type == PERF_RECORD_MMAP2) {
1177 struct cs_etm_queue *etmq;
1181 etmq = cs_etm__cpu_to_etmq(etm,cpu);
1187 dso = dsos__find(&(etm->machine->dsos),event->mmap2.filename,false);
1189 err = cs_etm_decoder__add_mem_access_cb(
1193 cs_etm__mem_access);
1196 if ((symbol_conf.vmlinux_name != NULL) && (!etmq->kernel_mapped)) {
1201 err = cs_etm__read_elf_info(symbol_conf.vmlinux_name,
1202 &foffset,&fstart,&fsize);
1205 cs_etm_decoder__add_bin_file(
1210 symbol_conf.vmlinux_name);
1212 etmq->kernel_mapped = true;
1218 if (etm->timeless_decoding) {
1219 if (event->header.type == PERF_RECORD_EXIT) {
1220 err = cs_etm__process_timeless_queues(etm,
1224 } else if (timestamp) {
1225 err = cs_etm__process_queues(etm, timestamp);
1228 //cs_etm__log("event %s (%u): cpu %d time%"PRIu64" tsc %#"PRIx64"\n",
1229 //perf_event__name(event->header.type), event->header.type,
1230 //sample->cpu, sample->time, timestamp);
1234 static int cs_etm__process_auxtrace_event(struct perf_session *session,
1235 union perf_event *event,
1236 struct perf_tool *tool)
1238 struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
1239 struct cs_etm_auxtrace,
1244 if (!etm->data_queued) {
1245 struct auxtrace_buffer *buffer;
1247 int fd = perf_data_file__fd(session->file);
1248 bool is_pipe = perf_data_file__is_pipe(session->file);
1254 data_offset = lseek(fd, 0, SEEK_CUR);
1255 if (data_offset == -1) {
1260 err = auxtrace_queues__add_event(&etm->queues,
1270 if (auxtrace_buffer__get_data(buffer,fd)) {
1271 cs_etm__dump_event(etm,buffer);
1272 auxtrace_buffer__put_data(buffer);
1281 static const char * const cs_etm_global_header_fmts[] = {
1282 [CS_HEADER_VERSION_0] = " Header version %"PRIx64"\n",
1283 [CS_PMU_TYPE_CPUS] = " PMU type/num cpus %"PRIx64"\n",
1284 [CS_ETM_SNAPSHOT] = " Snapshot %"PRIx64"\n",
1287 static const char * const cs_etm_priv_fmts[] = {
1288 [CS_ETM_MAGIC] = " Magic number %"PRIx64"\n",
1289 [CS_ETM_CPU] = " CPU %"PRIx64"\n",
1290 [CS_ETM_ETMCR] = " ETMCR %"PRIx64"\n",
1291 [CS_ETM_ETMTRACEIDR] = " ETMTRACEIDR %"PRIx64"\n",
1292 [CS_ETM_ETMCCER] = " ETMCCER %"PRIx64"\n",
1293 [CS_ETM_ETMIDR] = " ETMIDR %"PRIx64"\n",
1296 static const char * const cs_etmv4_priv_fmts[] = {
1297 [CS_ETM_MAGIC] = " Magic number %"PRIx64"\n",
1298 [CS_ETM_CPU] = " CPU %"PRIx64"\n",
1299 [CS_ETMV4_TRCCONFIGR] = " TRCCONFIGR %"PRIx64"\n",
1300 [CS_ETMV4_TRCTRACEIDR] = " TRCTRACEIDR %"PRIx64"\n",
1301 [CS_ETMV4_TRCIDR0] = " TRCIDR0 %"PRIx64"\n",
1302 [CS_ETMV4_TRCIDR1] = " TRCIDR1 %"PRIx64"\n",
1303 [CS_ETMV4_TRCIDR2] = " TRCIDR2 %"PRIx64"\n",
1304 [CS_ETMV4_TRCIDR8] = " TRCIDR8 %"PRIx64"\n",
1305 [CS_ETMV4_TRCAUTHSTATUS] = " TRCAUTHSTATUS %"PRIx64"\n",
1308 static void cs_etm__print_auxtrace_info(u64 *val, size_t num)
1312 for (i = 0, cpu = 0; cpu < num; ++cpu) {
1314 if (val[i] == __perf_cs_etmv3_magic) {
1315 for (j = 0; j < CS_ETM_PRIV_MAX; ++j, ++i) {
1316 fprintf(stdout,cs_etm_priv_fmts[j],val[i]);
1318 } else if (val[i] == __perf_cs_etmv4_magic) {
1319 for (j = 0; j < CS_ETMV4_PRIV_MAX; ++j, ++i) {
1320 fprintf(stdout,cs_etmv4_priv_fmts[j],val[i]);
1329 int cs_etm__process_auxtrace_info(union perf_event *event,
1330 struct perf_session *session)
1332 struct auxtrace_info_event *auxtrace_info = &(event->auxtrace_info);
1333 size_t event_header_size = sizeof(struct perf_event_header);
1334 size_t info_header_size = 8;
1335 size_t total_size = auxtrace_info->header.size;
1336 size_t priv_size = 0;
1338 struct cs_etm_auxtrace *etm = 0;
1339 int err = 0, idx = -1;
1342 u64 **metadata = NULL;
1345 struct int_node *inode;
1348 * sizeof(auxtrace_info_event::type) +
1349 * sizeof(auxtrace_info_event::reserved) == 8
1351 info_header_size = 8;
1353 if (total_size < (event_header_size + info_header_size))
1356 priv_size = total_size - event_header_size - info_header_size;
1358 // First the global part
1360 ptr = (u64 *) auxtrace_info->priv;
1362 hdr = zalloc(sizeof(u64 *) * CS_HEADER_VERSION_0_MAX);
1366 for (i = 0; i < CS_HEADER_VERSION_0_MAX; ++i) {
1369 num_cpu = hdr[CS_PMU_TYPE_CPUS] & 0xffffffff;
1370 pmu_type = (unsigned) ((hdr[CS_PMU_TYPE_CPUS] >> 32) & 0xffffffff);
1376 * Create an RB tree for traceID-CPU# tuple. Since the conversion has
1377 * to be made for each packet that gets decoded optimizing access in
1378 * anything other than a sequential array is worth doing.
1380 traceid_list = intlist__new(NULL);
1384 metadata = zalloc(sizeof(u64 *) * num_cpu);
1387 goto err_free_traceid_list;
1390 if (metadata == NULL) {
1394 for (j = 0; j < num_cpu; ++j) {
1395 if (ptr[i] == __perf_cs_etmv3_magic) {
1396 metadata[j] = zalloc(sizeof(u64)*CS_ETM_PRIV_MAX);
1397 if (metadata == NULL)
1399 for (k = 0; k < CS_ETM_PRIV_MAX; k++) {
1400 metadata[j][k] = ptr[i+k];
1403 /* The traceID is our handle */
1404 idx = metadata[j][CS_ETM_ETMIDR];
1405 i += CS_ETM_PRIV_MAX;
1406 } else if (ptr[i] == __perf_cs_etmv4_magic) {
1407 metadata[j] = zalloc(sizeof(u64)*CS_ETMV4_PRIV_MAX);
1408 if (metadata == NULL)
1410 for (k = 0; k < CS_ETMV4_PRIV_MAX; k++) {
1411 metadata[j][k] = ptr[i+k];
1414 /* The traceID is our handle */
1415 idx = metadata[j][CS_ETMV4_TRCTRACEIDR];
1416 i += CS_ETMV4_PRIV_MAX;
1419 /* Get an RB node for this CPU */
1420 inode = intlist__findnew(traceid_list, idx);
1422 /* Something went wrong, no need to continue */
1424 err = PTR_ERR(inode);
1425 goto err_free_metadata;
1429 * The node for that CPU should not have been taken already.
1430 * Backout if that's the case.
1434 goto err_free_metadata;
1437 /* All good, associate the traceID with the CPU# */
1438 inode->priv = &metadata[j][CS_ETM_CPU];
1442 if (i*8 != priv_size)
1446 cs_etm__print_auxtrace_info(auxtrace_info->priv,num_cpu);
1448 etm = zalloc(sizeof(struct cs_etm_auxtrace));
1450 etm->num_cpu = num_cpu;
1451 etm->pmu_type = pmu_type;
1452 etm->snapshot_mode = (hdr[CS_ETM_SNAPSHOT] != 0);
1458 err = auxtrace_queues__init(&etm->queues);
1462 etm->unknown_thread = thread__new(999999999,999999999);
1463 if (etm->unknown_thread == NULL) {
1465 goto err_free_queues;
1467 err = thread__set_comm(etm->unknown_thread, "unknown", 0);
1469 goto err_delete_thread;
1472 if (thread__init_map_groups(etm->unknown_thread,
1475 goto err_delete_thread;
1478 etm->timeless_decoding = true;
1479 etm->sampling_mode = false;
1480 etm->metadata = metadata;
1481 etm->session = session;
1482 etm->machine = &session->machines.host;
1483 etm->auxtrace_type = auxtrace_info->type;
1485 etm->auxtrace.process_event = cs_etm__process_event;
1486 etm->auxtrace.process_auxtrace_event = cs_etm__process_auxtrace_event;
1487 etm->auxtrace.flush_events = cs_etm__flush_events;
1488 etm->auxtrace.free_events = cs_etm__free_events;
1489 etm->auxtrace.free = cs_etm__free;
1490 session->auxtrace = &(etm->auxtrace);
1495 if (session->itrace_synth_opts && session->itrace_synth_opts->set) {
1496 etm->synth_opts = *session->itrace_synth_opts;
1498 itrace_synth_opts__set_default(&etm->synth_opts);
1500 etm->synth_opts.branches = false;
1501 etm->synth_opts.callchain = false;
1502 etm->synth_opts.calls = false;
1503 etm->synth_opts.returns = false;
1505 err = cs_etm__synth_events(etm, session);
1507 goto err_delete_thread;
1509 err = auxtrace_queues__process_index(&etm->queues, session);
1511 goto err_delete_thread;
1513 etm->data_queued = etm->queues.populated;
1518 thread__delete(etm->unknown_thread);
1520 auxtrace_queues__free(&etm->queues);
1521 session->auxtrace = NULL;
1525 /* No need to check @metadata[j], free(NULL) is supported */
1526 for (j = 0; j < num_cpu; ++j)
1529 err_free_traceid_list:
1530 intlist__delete(traceid_list);