tools/perf/builtin-kmem.c

   1 #include "builtin.h"
   2 #include "perf.h"
   3
   4 #include "util/evlist.h"
   5 #include "util/evsel.h"
   6 #include "util/util.h"
   7 #include "util/cache.h"
   8 #include "util/symbol.h"
   9 #include "util/thread.h"
  10 #include "util/header.h"
  11 #include "util/session.h"
  12 #include "util/tool.h"
  13 #include "util/callchain.h"
  14
  15 #include "util/parse-options.h"
  16 #include "util/trace-event.h"
  17 #include "util/data.h"
  18 #include "util/cpumap.h"
  19
  20 #include "util/debug.h"
  21
  22 #include <linux/rbtree.h>
  23 #include <linux/string.h>
  24 #include <locale.h>
  25 #include <regex.h>
  26
  27 static int      kmem_slab;
  28 static int      kmem_page;
  29
  30 static long     kmem_page_size;
  31
  32 struct alloc_stat;
  33 typedef int (*sort_fn_t)(void *, void *);
  34
  35 static int                      alloc_flag;
  36 static int                      caller_flag;
  37
  38 static int                      alloc_lines = -1;
  39 static int                      caller_lines = -1;
  40
  41 static bool                     raw_ip;
  42
  43 struct alloc_stat {
  44         u64     call_site;
  45         u64     ptr;
  46         u64     bytes_req;
  47         u64     bytes_alloc;
  48         u32     hit;
  49         u32     pingpong;
  50
  51         short   alloc_cpu;
  52
  53         struct rb_node node;
  54 };
  55
  56 static struct rb_root root_alloc_stat;
  57 static struct rb_root root_alloc_sorted;
  58 static struct rb_root root_caller_stat;
  59 static struct rb_root root_caller_sorted;
  60
  61 static unsigned long total_requested, total_allocated;
  62 static unsigned long nr_allocs, nr_cross_allocs;
  63
  64 static int insert_alloc_stat(unsigned long call_site, unsigned long ptr,
  65                              int bytes_req, int bytes_alloc, int cpu)
  66 {
  67         struct rb_node **node = &root_alloc_stat.rb_node;
  68         struct rb_node *parent = NULL;
  69         struct alloc_stat *data = NULL;
  70
  71         while (*node) {
  72                 parent = *node;
  73                 data = rb_entry(*node, struct alloc_stat, node);
  74
  75                 if (ptr > data->ptr)
  76                         node = &(*node)->rb_right;
  77                 else if (ptr < data->ptr)
  78                         node = &(*node)->rb_left;
  79                 else
  80                         break;
  81         }
  82
  83         if (data && data->ptr == ptr) {
  84                 data->hit++;
  85                 data->bytes_req += bytes_req;
  86                 data->bytes_alloc += bytes_alloc;
  87         } else {
  88                 data = malloc(sizeof(*data));
  89                 if (!data) {
  90                         pr_err("%s: malloc failed\n", __func__);
  91                         return -1;
  92                 }
  93                 data->ptr = ptr;
  94                 data->pingpong = 0;
  95                 data->hit = 1;
  96                 data->bytes_req = bytes_req;
  97                 data->bytes_alloc = bytes_alloc;
  98
  99                 rb_link_node(&data->node, parent, node);
 100                 rb_insert_color(&data->node, &root_alloc_stat);
 101         }
 102         data->call_site = call_site;
 103         data->alloc_cpu = cpu;
 104         return 0;
 105 }
 106
 107 static int insert_caller_stat(unsigned long call_site,
 108                               int bytes_req, int bytes_alloc)
 109 {
 110         struct rb_node **node = &root_caller_stat.rb_node;
 111         struct rb_node *parent = NULL;
 112         struct alloc_stat *data = NULL;
 113
 114         while (*node) {
 115                 parent = *node;
 116                 data = rb_entry(*node, struct alloc_stat, node);
 117
 118                 if (call_site > data->call_site)
 119                         node = &(*node)->rb_right;
 120                 else if (call_site < data->call_site)
 121                         node = &(*node)->rb_left;
 122                 else
 123                         break;
 124         }
 125
 126         if (data && data->call_site == call_site) {
 127                 data->hit++;
 128                 data->bytes_req += bytes_req;
 129                 data->bytes_alloc += bytes_alloc;
 130         } else {
 131                 data = malloc(sizeof(*data));
 132                 if (!data) {
 133                         pr_err("%s: malloc failed\n", __func__);
 134                         return -1;
 135                 }
 136                 data->call_site = call_site;
 137                 data->pingpong = 0;
 138                 data->hit = 1;
 139                 data->bytes_req = bytes_req;
 140                 data->bytes_alloc = bytes_alloc;
 141
 142                 rb_link_node(&data->node, parent, node);
 143                 rb_insert_color(&data->node, &root_caller_stat);
 144         }
 145
 146         return 0;
 147 }
 148
 149 static int perf_evsel__process_alloc_event(struct perf_evsel *evsel,
 150                                            struct perf_sample *sample)
 151 {
 152         unsigned long ptr = perf_evsel__intval(evsel, sample, "ptr"),
 153                       call_site = perf_evsel__intval(evsel, sample, "call_site");
 154         int bytes_req = perf_evsel__intval(evsel, sample, "bytes_req"),
 155             bytes_alloc = perf_evsel__intval(evsel, sample, "bytes_alloc");
 156
 157         if (insert_alloc_stat(call_site, ptr, bytes_req, bytes_alloc, sample->cpu) ||
 158             insert_caller_stat(call_site, bytes_req, bytes_alloc))
 159                 return -1;
 160
 161         total_requested += bytes_req;
 162         total_allocated += bytes_alloc;
 163
 164         nr_allocs++;
 165         return 0;
 166 }
 167
 168 static int perf_evsel__process_alloc_node_event(struct perf_evsel *evsel,
 169                                                 struct perf_sample *sample)
 170 {
 171         int ret = perf_evsel__process_alloc_event(evsel, sample);
 172
 173         if (!ret) {
 174                 int node1 = cpu__get_node(sample->cpu),
 175                     node2 = perf_evsel__intval(evsel, sample, "node");
 176
 177                 if (node1 != node2)
 178                         nr_cross_allocs++;
 179         }
 180
 181         return ret;
 182 }
 183
 184 static int ptr_cmp(void *, void *);
 185 static int slab_callsite_cmp(void *, void *);
 186
 187 static struct alloc_stat *search_alloc_stat(unsigned long ptr,
 188                                             unsigned long call_site,
 189                                             struct rb_root *root,
 190                                             sort_fn_t sort_fn)
 191 {
 192         struct rb_node *node = root->rb_node;
 193         struct alloc_stat key = { .ptr = ptr, .call_site = call_site };
 194
 195         while (node) {
 196                 struct alloc_stat *data;
 197                 int cmp;
 198
 199                 data = rb_entry(node, struct alloc_stat, node);
 200
 201                 cmp = sort_fn(&key, data);
 202                 if (cmp < 0)
 203                         node = node->rb_left;
 204                 else if (cmp > 0)
 205                         node = node->rb_right;
 206                 else
 207                         return data;
 208         }
 209         return NULL;
 210 }
 211
 212 static int perf_evsel__process_free_event(struct perf_evsel *evsel,
 213                                           struct perf_sample *sample)
 214 {
 215         unsigned long ptr = perf_evsel__intval(evsel, sample, "ptr");
 216         struct alloc_stat *s_alloc, *s_caller;
 217
 218         s_alloc = search_alloc_stat(ptr, 0, &root_alloc_stat, ptr_cmp);
 219         if (!s_alloc)
 220                 return 0;
 221
 222         if ((short)sample->cpu != s_alloc->alloc_cpu) {
 223                 s_alloc->pingpong++;
 224
 225                 s_caller = search_alloc_stat(0, s_alloc->call_site,
 226                                              &root_caller_stat,
 227                                              slab_callsite_cmp);
 228                 if (!s_caller)
 229                         return -1;
 230                 s_caller->pingpong++;
 231         }
 232         s_alloc->alloc_cpu = -1;
 233
 234         return 0;
 235 }
 236
 237 static u64 total_page_alloc_bytes;
 238 static u64 total_page_free_bytes;
 239 static u64 total_page_nomatch_bytes;
 240 static u64 total_page_fail_bytes;
 241 static unsigned long nr_page_allocs;
 242 static unsigned long nr_page_frees;
 243 static unsigned long nr_page_fails;
 244 static unsigned long nr_page_nomatch;
 245
 246 static bool use_pfn;
 247 static struct perf_session *kmem_session;
 248
 249 #define MAX_MIGRATE_TYPES  6
 250 #define MAX_PAGE_ORDER     11
 251
 252 static int order_stats[MAX_PAGE_ORDER][MAX_MIGRATE_TYPES];
 253
 254 struct page_stat {
 255         struct rb_node  node;
 256         u64             page;
 257         u64             callsite;
 258         int             order;
 259         unsigned        gfp_flags;
 260         unsigned        migrate_type;
 261         u64             alloc_bytes;
 262         u64             free_bytes;
 263         int             nr_alloc;
 264         int             nr_free;
 265 };
 266
 267 static struct rb_root page_tree;
 268 static struct rb_root page_alloc_tree;
 269 static struct rb_root page_alloc_sorted;
 270 static struct rb_root page_caller_tree;
 271 static struct rb_root page_caller_sorted;
 272
 273 struct alloc_func {
 274         u64 start;
 275         u64 end;
 276         char *name;
 277 };
 278
 279 static int nr_alloc_funcs;
 280 static struct alloc_func *alloc_func_list;
 281
 282 static int funcmp(const void *a, const void *b)
 283 {
 284         const struct alloc_func *fa = a;
 285         const struct alloc_func *fb = b;
 286
 287         if (fa->start > fb->start)
 288                 return 1;
 289         else
 290                 return -1;
 291 }
 292
 293 static int callcmp(const void *a, const void *b)
 294 {
 295         const struct alloc_func *fa = a;
 296         const struct alloc_func *fb = b;
 297
 298         if (fb->start <= fa->start && fa->end < fb->end)
 299                 return 0;
 300
 301         if (fa->start > fb->start)
 302                 return 1;
 303         else
 304                 return -1;
 305 }
 306
 307 static int build_alloc_func_list(void)
 308 {
 309         int ret;
 310         struct map *kernel_map;
 311         struct symbol *sym;
 312         struct rb_node *node;
 313         struct alloc_func *func;
 314         struct machine *machine = &kmem_session->machines.host;
 315         regex_t alloc_func_regex;
 316         const char pattern[] = "^_?_?(alloc|get_free|get_zeroed)_pages?";
 317
 318         ret = regcomp(&alloc_func_regex, pattern, REG_EXTENDED);
 319         if (ret) {
 320                 char err[BUFSIZ];
 321
 322                 regerror(ret, &alloc_func_regex, err, sizeof(err));
 323                 pr_err("Invalid regex: %s\n%s", pattern, err);
 324                 return -EINVAL;
 325         }
 326
 327         kernel_map = machine->vmlinux_maps[MAP__FUNCTION];
 328         if (map__load(kernel_map, NULL) < 0) {
 329                 pr_err("cannot load kernel map\n");
 330                 return -ENOENT;
 331         }
 332
 333         map__for_each_symbol(kernel_map, sym, node) {
 334                 if (regexec(&alloc_func_regex, sym->name, 0, NULL, 0))
 335                         continue;
 336
 337                 func = realloc(alloc_func_list,
 338                                (nr_alloc_funcs + 1) * sizeof(*func));
 339                 if (func == NULL)
 340                         return -ENOMEM;
 341
 342                 pr_debug("alloc func: %s\n", sym->name);
 343                 func[nr_alloc_funcs].start = sym->start;
 344                 func[nr_alloc_funcs].end   = sym->end;
 345                 func[nr_alloc_funcs].name  = sym->name;
 346
 347                 alloc_func_list = func;
 348                 nr_alloc_funcs++;
 349         }
 350
 351         qsort(alloc_func_list, nr_alloc_funcs, sizeof(*func), funcmp);
 352
 353         regfree(&alloc_func_regex);
 354         return 0;
 355 }
 356
 357 /*
 358  * Find first non-memory allocation function from callchain.
 359  * The allocation functions are in the 'alloc_func_list'.
 360  */
 361 static u64 find_callsite(struct perf_evsel *evsel, struct perf_sample *sample)
 362 {
 363         struct addr_location al;
 364         struct machine *machine = &kmem_session->machines.host;
 365         struct callchain_cursor_node *node;
 366
 367         if (alloc_func_list == NULL) {
 368                 if (build_alloc_func_list() < 0)
 369                         goto out;
 370         }
 371
 372         al.thread = machine__findnew_thread(machine, sample->pid, sample->tid);
 373         sample__resolve_callchain(sample, NULL, evsel, &al, 16);
 374
 375         callchain_cursor_commit(&callchain_cursor);
 376         while (true) {
 377                 struct alloc_func key, *caller;
 378                 u64 addr;
 379
 380                 node = callchain_cursor_current(&callchain_cursor);
 381                 if (node == NULL)
 382                         break;
 383
 384                 key.start = key.end = node->ip;
 385                 caller = bsearch(&key, alloc_func_list, nr_alloc_funcs,
 386                                  sizeof(key), callcmp);
 387                 if (!caller) {
 388                         /* found */
 389                         if (node->map)
 390                                 addr = map__unmap_ip(node->map, node->ip);
 391                         else
 392                                 addr = node->ip;
 393
 394                         return addr;
 395                 } else
 396                         pr_debug3("skipping alloc function: %s\n", caller->name);
 397
 398                 callchain_cursor_advance(&callchain_cursor);
 399         }
 400
 401 out:
 402         pr_debug2("unknown callsite: %"PRIx64 "\n", sample->ip);
 403         return sample->ip;
 404 }
 405
 406 static struct page_stat *
 407 __page_stat__findnew_page(u64 page, bool create)
 408 {
 409         struct rb_node **node = &page_tree.rb_node;
 410         struct rb_node *parent = NULL;
 411         struct page_stat *data;
 412
 413         while (*node) {
 414                 s64 cmp;
 415
 416                 parent = *node;
 417                 data = rb_entry(*node, struct page_stat, node);
 418
 419                 cmp = data->page - page;
 420                 if (cmp < 0)
 421                         node = &parent->rb_left;
 422                 else if (cmp > 0)
 423                         node = &parent->rb_right;
 424                 else
 425                         return data;
 426         }
 427
 428         if (!create)
 429                 return NULL;
 430
 431         data = zalloc(sizeof(*data));
 432         if (data != NULL) {
 433                 data->page = page;
 434
 435                 rb_link_node(&data->node, parent, node);
 436                 rb_insert_color(&data->node, &page_tree);
 437         }
 438
 439         return data;
 440 }
 441
 442 static struct page_stat *page_stat__find_page(u64 page)
 443 {
 444         return __page_stat__findnew_page(page, false);
 445 }
 446
 447 static struct page_stat *page_stat__findnew_page(u64 page)
 448 {
 449         return __page_stat__findnew_page(page, true);
 450 }
 451
 452 struct sort_dimension {
 453         const char              name[20];
 454         sort_fn_t               cmp;
 455         struct list_head        list;
 456 };
 457
 458 static LIST_HEAD(page_alloc_sort_input);
 459 static LIST_HEAD(page_caller_sort_input);
 460
 461 static struct page_stat *
 462 __page_stat__findnew_alloc(struct page_stat *pstat, bool create)
 463 {
 464         struct rb_node **node = &page_alloc_tree.rb_node;
 465         struct rb_node *parent = NULL;
 466         struct page_stat *data;
 467         struct sort_dimension *sort;
 468
 469         while (*node) {
 470                 int cmp = 0;
 471
 472                 parent = *node;
 473                 data = rb_entry(*node, struct page_stat, node);
 474
 475                 list_for_each_entry(sort, &page_alloc_sort_input, list) {
 476                         cmp = sort->cmp(pstat, data);
 477                         if (cmp)
 478                                 break;
 479                 }
 480
 481                 if (cmp < 0)
 482                         node = &parent->rb_left;
 483                 else if (cmp > 0)
 484                         node = &parent->rb_right;
 485                 else
 486                         return data;
 487         }
 488
 489         if (!create)
 490                 return NULL;
 491
 492         data = zalloc(sizeof(*data));
 493         if (data != NULL) {
 494                 data->page = pstat->page;
 495                 data->order = pstat->order;
 496                 data->gfp_flags = pstat->gfp_flags;
 497                 data->migrate_type = pstat->migrate_type;
 498
 499                 rb_link_node(&data->node, parent, node);
 500                 rb_insert_color(&data->node, &page_alloc_tree);
 501         }
 502
 503         return data;
 504 }
 505
 506 static struct page_stat *page_stat__find_alloc(struct page_stat *pstat)
 507 {
 508         return __page_stat__findnew_alloc(pstat, false);
 509 }
 510
 511 static struct page_stat *page_stat__findnew_alloc(struct page_stat *pstat)
 512 {
 513         return __page_stat__findnew_alloc(pstat, true);
 514 }
 515
 516 static struct page_stat *
 517 __page_stat__findnew_caller(struct page_stat *pstat, bool create)
 518 {
 519         struct rb_node **node = &page_caller_tree.rb_node;
 520         struct rb_node *parent = NULL;
 521         struct page_stat *data;
 522         struct sort_dimension *sort;
 523
 524         while (*node) {
 525                 int cmp = 0;
 526
 527                 parent = *node;
 528                 data = rb_entry(*node, struct page_stat, node);
 529
 530                 list_for_each_entry(sort, &page_caller_sort_input, list) {
 531                         cmp = sort->cmp(pstat, data);
 532                         if (cmp)
 533                                 break;
 534                 }
 535
 536                 if (cmp < 0)
 537                         node = &parent->rb_left;
 538                 else if (cmp > 0)
 539                         node = &parent->rb_right;
 540                 else
 541                         return data;
 542         }
 543
 544         if (!create)
 545                 return NULL;
 546
 547         data = zalloc(sizeof(*data));
 548         if (data != NULL) {
 549                 data->callsite = pstat->callsite;
 550                 data->order = pstat->order;
 551                 data->gfp_flags = pstat->gfp_flags;
 552                 data->migrate_type = pstat->migrate_type;
 553
 554                 rb_link_node(&data->node, parent, node);
 555                 rb_insert_color(&data->node, &page_caller_tree);
 556         }
 557
 558         return data;
 559 }
 560
 561 static struct page_stat *page_stat__find_caller(struct page_stat *pstat)
 562 {
 563         return __page_stat__findnew_caller(pstat, false);
 564 }
 565
 566 static struct page_stat *page_stat__findnew_caller(struct page_stat *pstat)
 567 {
 568         return __page_stat__findnew_caller(pstat, true);
 569 }
 570
 571 static bool valid_page(u64 pfn_or_page)
 572 {
 573         if (use_pfn && pfn_or_page == -1UL)
 574                 return false;
 575         if (!use_pfn && pfn_or_page == 0)
 576                 return false;
 577         return true;
 578 }
 579
 580 static int perf_evsel__process_page_alloc_event(struct perf_evsel *evsel,
 581                                                 struct perf_sample *sample)
 582 {
 583         u64 page;
 584         unsigned int order = perf_evsel__intval(evsel, sample, "order");
 585         unsigned int gfp_flags = perf_evsel__intval(evsel, sample, "gfp_flags");
 586         unsigned int migrate_type = perf_evsel__intval(evsel, sample,
 587                                                        "migratetype");
 588         u64 bytes = kmem_page_size << order;
 589         u64 callsite;
 590         struct page_stat *pstat;
 591         struct page_stat this = {
 592                 .order = order,
 593                 .gfp_flags = gfp_flags,
 594                 .migrate_type = migrate_type,
 595         };
 596
 597         if (use_pfn)
 598                 page = perf_evsel__intval(evsel, sample, "pfn");
 599         else
 600                 page = perf_evsel__intval(evsel, sample, "page");
 601
 602         nr_page_allocs++;
 603         total_page_alloc_bytes += bytes;
 604
 605         if (!valid_page(page)) {
 606                 nr_page_fails++;
 607                 total_page_fail_bytes += bytes;
 608
 609                 return 0;
 610         }
 611
 612         callsite = find_callsite(evsel, sample);
 613
 614         /*
 615          * This is to find the current page (with correct gfp flags and
 616          * migrate type) at free event.
 617          */
 618         pstat = page_stat__findnew_page(page);
 619         if (pstat == NULL)
 620                 return -ENOMEM;
 621
 622         pstat->order = order;
 623         pstat->gfp_flags = gfp_flags;
 624         pstat->migrate_type = migrate_type;
 625         pstat->callsite = callsite;
 626
 627         this.page = page;
 628         pstat = page_stat__findnew_alloc(&this);
 629         if (pstat == NULL)
 630                 return -ENOMEM;
 631
 632         pstat->nr_alloc++;
 633         pstat->alloc_bytes += bytes;
 634         pstat->callsite = callsite;
 635
 636         this.callsite = callsite;
 637         pstat = page_stat__findnew_caller(&this);
 638         if (pstat == NULL)
 639                 return -ENOMEM;
 640
 641         pstat->nr_alloc++;
 642         pstat->alloc_bytes += bytes;
 643
 644         order_stats[order][migrate_type]++;
 645
 646         return 0;
 647 }
 648
 649 static int perf_evsel__process_page_free_event(struct perf_evsel *evsel,
 650                                                 struct perf_sample *sample)
 651 {
 652         u64 page;
 653         unsigned int order = perf_evsel__intval(evsel, sample, "order");
 654         u64 bytes = kmem_page_size << order;
 655         struct page_stat *pstat;
 656         struct page_stat this = {
 657                 .order = order,
 658         };
 659
 660         if (use_pfn)
 661                 page = perf_evsel__intval(evsel, sample, "pfn");
 662         else
 663                 page = perf_evsel__intval(evsel, sample, "page");
 664
 665         nr_page_frees++;
 666         total_page_free_bytes += bytes;
 667
 668         pstat = page_stat__find_page(page);
 669         if (pstat == NULL) {
 670                 pr_debug2("missing free at page %"PRIx64" (order: %d)\n",
 671                           page, order);
 672
 673                 nr_page_nomatch++;
 674                 total_page_nomatch_bytes += bytes;
 675
 676                 return 0;
 677         }
 678
 679         this.page = page;
 680         this.gfp_flags = pstat->gfp_flags;
 681         this.migrate_type = pstat->migrate_type;
 682         this.callsite = pstat->callsite;
 683
 684         rb_erase(&pstat->node, &page_tree);
 685         free(pstat);
 686
 687         pstat = page_stat__find_alloc(&this);
 688         if (pstat == NULL)
 689                 return -ENOENT;
 690
 691         pstat->nr_free++;
 692         pstat->free_bytes += bytes;
 693
 694         pstat = page_stat__find_caller(&this);
 695         if (pstat == NULL)
 696                 return -ENOENT;
 697
 698         pstat->nr_free++;
 699         pstat->free_bytes += bytes;
 700
 701         return 0;
 702 }
 703
 704 typedef int (*tracepoint_handler)(struct perf_evsel *evsel,
 705                                   struct perf_sample *sample);
 706
 707 static int process_sample_event(struct perf_tool *tool __maybe_unused,
 708                                 union perf_event *event,
 709                                 struct perf_sample *sample,
 710                                 struct perf_evsel *evsel,
 711                                 struct machine *machine)
 712 {
 713         struct thread *thread = machine__findnew_thread(machine, sample->pid,
 714                                                         sample->tid);
 715
 716         if (thread == NULL) {
 717                 pr_debug("problem processing %d event, skipping it.\n",
 718                          event->header.type);
 719                 return -1;
 720         }
 721
 722         dump_printf(" ... thread: %s:%d\n", thread__comm_str(thread), thread->tid);
 723
 724         if (evsel->handler != NULL) {
 725                 tracepoint_handler f = evsel->handler;
 726                 return f(evsel, sample);
 727         }
 728
 729         return 0;
 730 }
 731
 732 static struct perf_tool perf_kmem = {
 733         .sample          = process_sample_event,
 734         .comm            = perf_event__process_comm,
 735         .mmap            = perf_event__process_mmap,
 736         .mmap2           = perf_event__process_mmap2,
 737         .ordered_events  = true,
 738 };
 739
 740 static double fragmentation(unsigned long n_req, unsigned long n_alloc)
 741 {
 742         if (n_alloc == 0)
 743                 return 0.0;
 744         else
 745                 return 100.0 - (100.0 * n_req / n_alloc);
 746 }
 747
 748 static void __print_slab_result(struct rb_root *root,
 749                                 struct perf_session *session,
 750                                 int n_lines, int is_caller)
 751 {
 752         struct rb_node *next;
 753         struct machine *machine = &session->machines.host;
 754
 755         printf("%.105s\n", graph_dotted_line);
 756         printf(" %-34s |",  is_caller ? "Callsite": "Alloc Ptr");
 757         printf(" Total_alloc/Per | Total_req/Per   | Hit      | Ping-pong | Frag\n");
 758         printf("%.105s\n", graph_dotted_line);
 759
 760         next = rb_first(root);
 761
 762         while (next && n_lines--) {
 763                 struct alloc_stat *data = rb_entry(next, struct alloc_stat,
 764                                                    node);
 765                 struct symbol *sym = NULL;
 766                 struct map *map;
 767                 char buf[BUFSIZ];
 768                 u64 addr;
 769
 770                 if (is_caller) {
 771                         addr = data->call_site;
 772                         if (!raw_ip)
 773                                 sym = machine__find_kernel_function(machine, addr, &map, NULL);
 774                 } else
 775                         addr = data->ptr;
 776
 777                 if (sym != NULL)
 778                         snprintf(buf, sizeof(buf), "%s+%" PRIx64 "", sym->name,
 779                                  addr - map->unmap_ip(map, sym->start));
 780                 else
 781                         snprintf(buf, sizeof(buf), "%#" PRIx64 "", addr);
 782                 printf(" %-34s |", buf);
 783
 784                 printf(" %9llu/%-5lu | %9llu/%-5lu | %8lu | %9lu | %6.3f%%\n",
 785                        (unsigned long long)data->bytes_alloc,
 786                        (unsigned long)data->bytes_alloc / data->hit,
 787                        (unsigned long long)data->bytes_req,
 788                        (unsigned long)data->bytes_req / data->hit,
 789                        (unsigned long)data->hit,
 790                        (unsigned long)data->pingpong,
 791                        fragmentation(data->bytes_req, data->bytes_alloc));
 792
 793                 next = rb_next(next);
 794         }
 795
 796         if (n_lines == -1)
 797                 printf(" ...                                | ...             | ...             | ...      | ...       | ...   \n");
 798
 799         printf("%.105s\n", graph_dotted_line);
 800 }
 801
 802 static const char * const migrate_type_str[] = {
 803         "UNMOVABL",
 804         "RECLAIM",
 805         "MOVABLE",
 806         "RESERVED",
 807         "CMA/ISLT",
 808         "UNKNOWN",
 809 };
 810
 811 static void __print_page_alloc_result(struct perf_session *session, int n_lines)
 812 {
 813         struct rb_node *next = rb_first(&page_alloc_sorted);
 814         struct machine *machine = &session->machines.host;
 815         const char *format;
 816
 817         printf("\n%.105s\n", graph_dotted_line);
 818         printf(" %-16s | Total alloc (KB) | Hits      | Order | Mig.type | GFP flags | Callsite\n",
 819                use_pfn ? "PFN" : "Page");
 820         printf("%.105s\n", graph_dotted_line);
 821
 822         if (use_pfn)
 823                 format = " %16llu | %'16llu | %'9d | %5d | %8s |  %08lx | %s\n";
 824         else
 825                 format = " %016llx | %'16llu | %'9d | %5d | %8s |  %08lx | %s\n";
 826
 827         while (next && n_lines--) {
 828                 struct page_stat *data;
 829                 struct symbol *sym;
 830                 struct map *map;
 831                 char buf[32];
 832                 char *caller = buf;
 833
 834                 data = rb_entry(next, struct page_stat, node);
 835                 sym = machine__find_kernel_function(machine, data->callsite,
 836                                                     &map, NULL);
 837                 if (sym && sym->name)
 838                         caller = sym->name;
 839                 else
 840                         scnprintf(buf, sizeof(buf), "%"PRIx64, data->callsite);
 841
 842                 printf(format, (unsigned long long)data->page,
 843                        (unsigned long long)data->alloc_bytes / 1024,
 844                        data->nr_alloc, data->order,
 845                        migrate_type_str[data->migrate_type],
 846                        (unsigned long)data->gfp_flags, caller);
 847
 848                 next = rb_next(next);
 849         }
 850
 851         if (n_lines == -1)
 852                 printf(" ...              | ...              | ...       | ...   | ...      | ...       | ...\n");
 853
 854         printf("%.105s\n", graph_dotted_line);
 855 }
 856
 857 static void __print_page_caller_result(struct perf_session *session, int n_lines)
 858 {
 859         struct rb_node *next = rb_first(&page_caller_sorted);
 860         struct machine *machine = &session->machines.host;
 861
 862         printf("\n%.105s\n", graph_dotted_line);
 863         printf(" Total alloc (KB) | Hits      | Order | Mig.type | GFP flags | Callsite\n");
 864         printf("%.105s\n", graph_dotted_line);
 865
 866         while (next && n_lines--) {
 867                 struct page_stat *data;
 868                 struct symbol *sym;
 869                 struct map *map;
 870                 char buf[32];
 871                 char *caller = buf;
 872
 873                 data = rb_entry(next, struct page_stat, node);
 874                 sym = machine__find_kernel_function(machine, data->callsite,
 875                                                     &map, NULL);
 876                 if (sym && sym->name)
 877                         caller = sym->name;
 878                 else
 879                         scnprintf(buf, sizeof(buf), "%"PRIx64, data->callsite);
 880
 881                 printf(" %'16llu | %'9d | %5d | %8s |  %08lx | %s\n",
 882                        (unsigned long long)data->alloc_bytes / 1024,
 883                        data->nr_alloc, data->order,
 884                        migrate_type_str[data->migrate_type],
 885                        (unsigned long)data->gfp_flags, caller);
 886
 887                 next = rb_next(next);
 888         }
 889
 890         if (n_lines == -1)
 891                 printf(" ...              | ...       | ...   | ...      | ...       | ...\n");
 892
 893         printf("%.105s\n", graph_dotted_line);
 894 }
 895
 896 static void print_slab_summary(void)
 897 {
 898         printf("\nSUMMARY (SLAB allocator)");
 899         printf("\n========================\n");
 900         printf("Total bytes requested: %'lu\n", total_requested);
 901         printf("Total bytes allocated: %'lu\n", total_allocated);
 902         printf("Total bytes wasted on internal fragmentation: %'lu\n",
 903                total_allocated - total_requested);
 904         printf("Internal fragmentation: %f%%\n",
 905                fragmentation(total_requested, total_allocated));
 906         printf("Cross CPU allocations: %'lu/%'lu\n", nr_cross_allocs, nr_allocs);
 907 }
 908
 909 static void print_page_summary(void)
 910 {
 911         int o, m;
 912         u64 nr_alloc_freed = nr_page_frees - nr_page_nomatch;
 913         u64 total_alloc_freed_bytes = total_page_free_bytes - total_page_nomatch_bytes;
 914
 915         printf("\nSUMMARY (page allocator)");
 916         printf("\n========================\n");
 917         printf("%-30s: %'16lu   [ %'16"PRIu64" KB ]\n", "Total allocation requests",
 918                nr_page_allocs, total_page_alloc_bytes / 1024);
 919         printf("%-30s: %'16lu   [ %'16"PRIu64" KB ]\n", "Total free requests",
 920                nr_page_frees, total_page_free_bytes / 1024);
 921         printf("\n");
 922
 923         printf("%-30s: %'16lu   [ %'16"PRIu64" KB ]\n", "Total alloc+freed requests",
 924                nr_alloc_freed, (total_alloc_freed_bytes) / 1024);
 925         printf("%-30s: %'16lu   [ %'16"PRIu64" KB ]\n", "Total alloc-only requests",
 926                nr_page_allocs - nr_alloc_freed,
 927                (total_page_alloc_bytes - total_alloc_freed_bytes) / 1024);
 928         printf("%-30s: %'16lu   [ %'16"PRIu64" KB ]\n", "Total free-only requests",
 929                nr_page_nomatch, total_page_nomatch_bytes / 1024);
 930         printf("\n");
 931
 932         printf("%-30s: %'16lu   [ %'16"PRIu64" KB ]\n", "Total allocation failures",
 933                nr_page_fails, total_page_fail_bytes / 1024);
 934         printf("\n");
 935
 936         printf("%5s  %12s  %12s  %12s  %12s  %12s\n", "Order",  "Unmovable",
 937                "Reclaimable", "Movable", "Reserved", "CMA/Isolated");
 938         printf("%.5s  %.12s  %.12s  %.12s  %.12s  %.12s\n", graph_dotted_line,
 939                graph_dotted_line, graph_dotted_line, graph_dotted_line,
 940                graph_dotted_line, graph_dotted_line);
 941
 942         for (o = 0; o < MAX_PAGE_ORDER; o++) {
 943                 printf("%5d", o);
 944                 for (m = 0; m < MAX_MIGRATE_TYPES - 1; m++) {
 945                         if (order_stats[o][m])
 946                                 printf("  %'12d", order_stats[o][m]);
 947                         else
 948                                 printf("  %12c", '.');
 949                 }
 950                 printf("\n");
 951         }
 952 }
 953
 954 static void print_slab_result(struct perf_session *session)
 955 {
 956         if (caller_flag)
 957                 __print_slab_result(&root_caller_sorted, session, caller_lines, 1);
 958         if (alloc_flag)
 959                 __print_slab_result(&root_alloc_sorted, session, alloc_lines, 0);
 960         print_slab_summary();
 961 }
 962
 963 static void print_page_result(struct perf_session *session)
 964 {
 965         if (caller_flag)
 966                 __print_page_caller_result(session, caller_lines);
 967         if (alloc_flag)
 968                 __print_page_alloc_result(session, alloc_lines);
 969         print_page_summary();
 970 }
 971
 972 static void print_result(struct perf_session *session)
 973 {
 974         if (kmem_slab)
 975                 print_slab_result(session);
 976         if (kmem_page)
 977                 print_page_result(session);
 978 }
 979
 980 static LIST_HEAD(slab_caller_sort);
 981 static LIST_HEAD(slab_alloc_sort);
 982 static LIST_HEAD(page_caller_sort);
 983 static LIST_HEAD(page_alloc_sort);
 984
 985 static void sort_slab_insert(struct rb_root *root, struct alloc_stat *data,
 986                              struct list_head *sort_list)
 987 {
 988         struct rb_node **new = &(root->rb_node);
 989         struct rb_node *parent = NULL;
 990         struct sort_dimension *sort;
 991
 992         while (*new) {
 993                 struct alloc_stat *this;
 994                 int cmp = 0;
 995
 996                 this = rb_entry(*new, struct alloc_stat, node);
 997                 parent = *new;
 998
 999                 list_for_each_entry(sort, sort_list, list) {
1000                         cmp = sort->cmp(data, this);
1001                         if (cmp)
1002                                 break;
1003                 }
1004
1005                 if (cmp > 0)
1006                         new = &((*new)->rb_left);
1007                 else
1008                         new = &((*new)->rb_right);
1009         }
1010
1011         rb_link_node(&data->node, parent, new);
1012         rb_insert_color(&data->node, root);
1013 }
1014
1015 static void __sort_slab_result(struct rb_root *root, struct rb_root *root_sorted,
1016                                struct list_head *sort_list)
1017 {
1018         struct rb_node *node;
1019         struct alloc_stat *data;
1020
1021         for (;;) {
1022                 node = rb_first(root);
1023                 if (!node)
1024                         break;
1025
1026                 rb_erase(node, root);
1027                 data = rb_entry(node, struct alloc_stat, node);
1028                 sort_slab_insert(root_sorted, data, sort_list);
1029         }
1030 }
1031
1032 static void sort_page_insert(struct rb_root *root, struct page_stat *data,
1033                              struct list_head *sort_list)
1034 {
1035         struct rb_node **new = &root->rb_node;
1036         struct rb_node *parent = NULL;
1037         struct sort_dimension *sort;
1038
1039         while (*new) {
1040                 struct page_stat *this;
1041                 int cmp = 0;
1042
1043                 this = rb_entry(*new, struct page_stat, node);
1044                 parent = *new;
1045
1046                 list_for_each_entry(sort, sort_list, list) {
1047                         cmp = sort->cmp(data, this);
1048                         if (cmp)
1049                                 break;
1050                 }
1051
1052                 if (cmp > 0)
1053                         new = &parent->rb_left;
1054                 else
1055                         new = &parent->rb_right;
1056         }
1057
1058         rb_link_node(&data->node, parent, new);
1059         rb_insert_color(&data->node, root);
1060 }
1061
1062 static void __sort_page_result(struct rb_root *root, struct rb_root *root_sorted,
1063                                struct list_head *sort_list)
1064 {
1065         struct rb_node *node;
1066         struct page_stat *data;
1067
1068         for (;;) {
1069                 node = rb_first(root);
1070                 if (!node)
1071                         break;
1072
1073                 rb_erase(node, root);
1074                 data = rb_entry(node, struct page_stat, node);
1075                 sort_page_insert(root_sorted, data, sort_list);
1076         }
1077 }
1078
1079 static void sort_result(void)
1080 {
1081         if (kmem_slab) {
1082                 __sort_slab_result(&root_alloc_stat, &root_alloc_sorted,
1083                                    &slab_alloc_sort);
1084                 __sort_slab_result(&root_caller_stat, &root_caller_sorted,
1085                                    &slab_caller_sort);
1086         }
1087         if (kmem_page) {
1088                 __sort_page_result(&page_alloc_tree, &page_alloc_sorted,
1089                                    &page_alloc_sort);
1090                 __sort_page_result(&page_caller_tree, &page_caller_sorted,
1091                                    &page_caller_sort);
1092         }
1093 }
1094
1095 static int __cmd_kmem(struct perf_session *session)
1096 {
1097         int err = -EINVAL;
1098         struct perf_evsel *evsel;
1099         const struct perf_evsel_str_handler kmem_tracepoints[] = {
1100                 /* slab allocator */
1101                 { "kmem:kmalloc",               perf_evsel__process_alloc_event, },
1102                 { "kmem:kmem_cache_alloc",      perf_evsel__process_alloc_event, },
1103                 { "kmem:kmalloc_node",          perf_evsel__process_alloc_node_event, },
1104                 { "kmem:kmem_cache_alloc_node", perf_evsel__process_alloc_node_event, },
1105                 { "kmem:kfree",                 perf_evsel__process_free_event, },
1106                 { "kmem:kmem_cache_free",       perf_evsel__process_free_event, },
1107                 /* page allocator */
1108                 { "kmem:mm_page_alloc",         perf_evsel__process_page_alloc_event, },
1109                 { "kmem:mm_page_free",          perf_evsel__process_page_free_event, },
1110         };
1111
1112         if (!perf_session__has_traces(session, "kmem record"))
1113                 goto out;
1114
1115         if (perf_session__set_tracepoints_handlers(session, kmem_tracepoints)) {
1116                 pr_err("Initializing perf session tracepoint handlers failed\n");
1117                 goto out;
1118         }
1119
1120         evlist__for_each(session->evlist, evsel) {
1121                 if (!strcmp(perf_evsel__name(evsel), "kmem:mm_page_alloc") &&
1122                     perf_evsel__field(evsel, "pfn")) {
1123                         use_pfn = true;
1124                         break;
1125                 }
1126         }
1127
1128         setup_pager();
1129         err = perf_session__process_events(session);
1130         if (err != 0) {
1131                 pr_err("error during process events: %d\n", err);
1132                 goto out;
1133         }
1134         sort_result();
1135         print_result(session);
1136 out:
1137         return err;
1138 }
1139
1140 /* slab sort keys */
1141 static int ptr_cmp(void *a, void *b)
1142 {
1143         struct alloc_stat *l = a;
1144         struct alloc_stat *r = b;
1145
1146         if (l->ptr < r->ptr)
1147                 return -1;
1148         else if (l->ptr > r->ptr)
1149                 return 1;
1150         return 0;
1151 }
1152
1153 static struct sort_dimension ptr_sort_dimension = {
1154         .name   = "ptr",
1155         .cmp    = ptr_cmp,
1156 };
1157
1158 static int slab_callsite_cmp(void *a, void *b)
1159 {
1160         struct alloc_stat *l = a;
1161         struct alloc_stat *r = b;
1162
1163         if (l->call_site < r->call_site)
1164                 return -1;
1165         else if (l->call_site > r->call_site)
1166                 return 1;
1167         return 0;
1168 }
1169
1170 static struct sort_dimension callsite_sort_dimension = {
1171         .name   = "callsite",
1172         .cmp    = slab_callsite_cmp,
1173 };
1174
1175 static int hit_cmp(void *a, void *b)
1176 {
1177         struct alloc_stat *l = a;
1178         struct alloc_stat *r = b;
1179
1180         if (l->hit < r->hit)
1181                 return -1;
1182         else if (l->hit > r->hit)
1183                 return 1;
1184         return 0;
1185 }
1186
1187 static struct sort_dimension hit_sort_dimension = {
1188         .name   = "hit",
1189         .cmp    = hit_cmp,
1190 };
1191
1192 static int bytes_cmp(void *a, void *b)
1193 {
1194         struct alloc_stat *l = a;
1195         struct alloc_stat *r = b;
1196
1197         if (l->bytes_alloc < r->bytes_alloc)
1198                 return -1;
1199         else if (l->bytes_alloc > r->bytes_alloc)
1200                 return 1;
1201         return 0;
1202 }
1203
1204 static struct sort_dimension bytes_sort_dimension = {
1205         .name   = "bytes",
1206         .cmp    = bytes_cmp,
1207 };
1208
1209 static int frag_cmp(void *a, void *b)
1210 {
1211         double x, y;
1212         struct alloc_stat *l = a;
1213         struct alloc_stat *r = b;
1214
1215         x = fragmentation(l->bytes_req, l->bytes_alloc);
1216         y = fragmentation(r->bytes_req, r->bytes_alloc);
1217
1218         if (x < y)
1219                 return -1;
1220         else if (x > y)
1221                 return 1;
1222         return 0;
1223 }
1224
1225 static struct sort_dimension frag_sort_dimension = {
1226         .name   = "frag",
1227         .cmp    = frag_cmp,
1228 };
1229
1230 static int pingpong_cmp(void *a, void *b)
1231 {
1232         struct alloc_stat *l = a;
1233         struct alloc_stat *r = b;
1234
1235         if (l->pingpong < r->pingpong)
1236                 return -1;
1237         else if (l->pingpong > r->pingpong)
1238                 return 1;
1239         return 0;
1240 }
1241
1242 static struct sort_dimension pingpong_sort_dimension = {
1243         .name   = "pingpong",
1244         .cmp    = pingpong_cmp,
1245 };
1246
1247 /* page sort keys */
1248 static int page_cmp(void *a, void *b)
1249 {
1250         struct page_stat *l = a;
1251         struct page_stat *r = b;
1252
1253         if (l->page < r->page)
1254                 return -1;
1255         else if (l->page > r->page)
1256                 return 1;
1257         return 0;
1258 }
1259
1260 static struct sort_dimension page_sort_dimension = {
1261         .name   = "page",
1262         .cmp    = page_cmp,
1263 };
1264
1265 static int page_callsite_cmp(void *a, void *b)
1266 {
1267         struct page_stat *l = a;
1268         struct page_stat *r = b;
1269
1270         if (l->callsite < r->callsite)
1271                 return -1;
1272         else if (l->callsite > r->callsite)
1273                 return 1;
1274         return 0;
1275 }
1276
1277 static struct sort_dimension page_callsite_sort_dimension = {
1278         .name   = "callsite",
1279         .cmp    = page_callsite_cmp,
1280 };
1281
1282 static int page_hit_cmp(void *a, void *b)
1283 {
1284         struct page_stat *l = a;
1285         struct page_stat *r = b;
1286
1287         if (l->nr_alloc < r->nr_alloc)
1288                 return -1;
1289         else if (l->nr_alloc > r->nr_alloc)
1290                 return 1;
1291         return 0;
1292 }
1293
1294 static struct sort_dimension page_hit_sort_dimension = {
1295         .name   = "hit",
1296         .cmp    = page_hit_cmp,
1297 };
1298
1299 static int page_bytes_cmp(void *a, void *b)
1300 {
1301         struct page_stat *l = a;
1302         struct page_stat *r = b;
1303
1304         if (l->alloc_bytes < r->alloc_bytes)
1305                 return -1;
1306         else if (l->alloc_bytes > r->alloc_bytes)
1307                 return 1;
1308         return 0;
1309 }
1310
1311 static struct sort_dimension page_bytes_sort_dimension = {
1312         .name   = "bytes",
1313         .cmp    = page_bytes_cmp,
1314 };
1315
1316 static int page_order_cmp(void *a, void *b)
1317 {
1318         struct page_stat *l = a;
1319         struct page_stat *r = b;
1320
1321         if (l->order < r->order)
1322                 return -1;
1323         else if (l->order > r->order)
1324                 return 1;
1325         return 0;
1326 }
1327
1328 static struct sort_dimension page_order_sort_dimension = {
1329         .name   = "order",
1330         .cmp    = page_order_cmp,
1331 };
1332
1333 static int migrate_type_cmp(void *a, void *b)
1334 {
1335         struct page_stat *l = a;
1336         struct page_stat *r = b;
1337
1338         /* for internal use to find free'd page */
1339         if (l->migrate_type == -1U)
1340                 return 0;
1341
1342         if (l->migrate_type < r->migrate_type)
1343                 return -1;
1344         else if (l->migrate_type > r->migrate_type)
1345                 return 1;
1346         return 0;
1347 }
1348
1349 static struct sort_dimension migrate_type_sort_dimension = {
1350         .name   = "migtype",
1351         .cmp    = migrate_type_cmp,
1352 };
1353
1354 static int gfp_flags_cmp(void *a, void *b)
1355 {
1356         struct page_stat *l = a;
1357         struct page_stat *r = b;
1358
1359         /* for internal use to find free'd page */
1360         if (l->gfp_flags == -1U)
1361                 return 0;
1362
1363         if (l->gfp_flags < r->gfp_flags)
1364                 return -1;
1365         else if (l->gfp_flags > r->gfp_flags)
1366                 return 1;
1367         return 0;
1368 }
1369
1370 static struct sort_dimension gfp_flags_sort_dimension = {
1371         .name   = "gfp",
1372         .cmp    = gfp_flags_cmp,
1373 };
1374
1375 static struct sort_dimension *slab_sorts[] = {
1376         &ptr_sort_dimension,
1377         &callsite_sort_dimension,
1378         &hit_sort_dimension,
1379         &bytes_sort_dimension,
1380         &frag_sort_dimension,
1381         &pingpong_sort_dimension,
1382 };
1383
1384 static struct sort_dimension *page_sorts[] = {
1385         &page_sort_dimension,
1386         &page_callsite_sort_dimension,
1387         &page_hit_sort_dimension,
1388         &page_bytes_sort_dimension,
1389         &page_order_sort_dimension,
1390         &migrate_type_sort_dimension,
1391         &gfp_flags_sort_dimension,
1392 };
1393
1394 static int slab_sort_dimension__add(const char *tok, struct list_head *list)
1395 {
1396         struct sort_dimension *sort;
1397         int i;
1398
1399         for (i = 0; i < (int)ARRAY_SIZE(slab_sorts); i++) {
1400                 if (!strcmp(slab_sorts[i]->name, tok)) {
1401                         sort = memdup(slab_sorts[i], sizeof(*slab_sorts[i]));
1402                         if (!sort) {
1403                                 pr_err("%s: memdup failed\n", __func__);
1404                                 return -1;
1405                         }
1406                         list_add_tail(&sort->list, list);
1407                         return 0;
1408                 }
1409         }
1410
1411         return -1;
1412 }
1413
1414 static int page_sort_dimension__add(const char *tok, struct list_head *list)
1415 {
1416         struct sort_dimension *sort;
1417         int i;
1418
1419         for (i = 0; i < (int)ARRAY_SIZE(page_sorts); i++) {
1420                 if (!strcmp(page_sorts[i]->name, tok)) {
1421                         sort = memdup(page_sorts[i], sizeof(*page_sorts[i]));
1422                         if (!sort) {
1423                                 pr_err("%s: memdup failed\n", __func__);
1424                                 return -1;
1425                         }
1426                         list_add_tail(&sort->list, list);
1427                         return 0;
1428                 }
1429         }
1430
1431         return -1;
1432 }
1433
1434 static int setup_slab_sorting(struct list_head *sort_list, const char *arg)
1435 {
1436         char *tok;
1437         char *str = strdup(arg);
1438         char *pos = str;
1439
1440         if (!str) {
1441                 pr_err("%s: strdup failed\n", __func__);
1442                 return -1;
1443         }
1444
1445         while (true) {
1446                 tok = strsep(&pos, ",");
1447                 if (!tok)
1448                         break;
1449                 if (slab_sort_dimension__add(tok, sort_list) < 0) {
1450                         error("Unknown slab --sort key: '%s'", tok);
1451                         free(str);
1452                         return -1;
1453                 }
1454         }
1455
1456         free(str);
1457         return 0;
1458 }
1459
1460 static int setup_page_sorting(struct list_head *sort_list, const char *arg)
1461 {
1462         char *tok;
1463         char *str = strdup(arg);
1464         char *pos = str;
1465
1466         if (!str) {
1467                 pr_err("%s: strdup failed\n", __func__);
1468                 return -1;
1469         }
1470
1471         while (true) {
1472                 tok = strsep(&pos, ",");
1473                 if (!tok)
1474                         break;
1475                 if (page_sort_dimension__add(tok, sort_list) < 0) {
1476                         error("Unknown page --sort key: '%s'", tok);
1477                         free(str);
1478                         return -1;
1479                 }
1480         }
1481
1482         free(str);
1483         return 0;
1484 }
1485
1486 static int parse_sort_opt(const struct option *opt __maybe_unused,
1487                           const char *arg, int unset __maybe_unused)
1488 {
1489         if (!arg)
1490                 return -1;
1491
1492         if (kmem_page > kmem_slab) {
1493                 if (caller_flag > alloc_flag)
1494                         return setup_page_sorting(&page_caller_sort, arg);
1495                 else
1496                         return setup_page_sorting(&page_alloc_sort, arg);
1497         } else {
1498                 if (caller_flag > alloc_flag)
1499                         return setup_slab_sorting(&slab_caller_sort, arg);
1500                 else
1501                         return setup_slab_sorting(&slab_alloc_sort, arg);
1502         }
1503
1504         return 0;
1505 }
1506
1507 static int parse_caller_opt(const struct option *opt __maybe_unused,
1508                             const char *arg __maybe_unused,
1509                             int unset __maybe_unused)
1510 {
1511         caller_flag = (alloc_flag + 1);
1512         return 0;
1513 }
1514
1515 static int parse_alloc_opt(const struct option *opt __maybe_unused,
1516                            const char *arg __maybe_unused,
1517                            int unset __maybe_unused)
1518 {
1519         alloc_flag = (caller_flag + 1);
1520         return 0;
1521 }
1522
1523 static int parse_slab_opt(const struct option *opt __maybe_unused,
1524                           const char *arg __maybe_unused,
1525                           int unset __maybe_unused)
1526 {
1527         kmem_slab = (kmem_page + 1);
1528         return 0;
1529 }
1530
1531 static int parse_page_opt(const struct option *opt __maybe_unused,
1532                           const char *arg __maybe_unused,
1533                           int unset __maybe_unused)
1534 {
1535         kmem_page = (kmem_slab + 1);
1536         return 0;
1537 }
1538
1539 static int parse_line_opt(const struct option *opt __maybe_unused,
1540                           const char *arg, int unset __maybe_unused)
1541 {
1542         int lines;
1543
1544         if (!arg)
1545                 return -1;
1546
1547         lines = strtoul(arg, NULL, 10);
1548
1549         if (caller_flag > alloc_flag)
1550                 caller_lines = lines;
1551         else
1552                 alloc_lines = lines;
1553
1554         return 0;
1555 }
1556
1557 static int __cmd_record(int argc, const char **argv)
1558 {
1559         const char * const record_args[] = {
1560         "record", "-a", "-R", "-c", "1",
1561         };
1562         const char * const slab_events[] = {
1563         "-e", "kmem:kmalloc",
1564         "-e", "kmem:kmalloc_node",
1565         "-e", "kmem:kfree",
1566         "-e", "kmem:kmem_cache_alloc",
1567         "-e", "kmem:kmem_cache_alloc_node",
1568         "-e", "kmem:kmem_cache_free",
1569         };
1570         const char * const page_events[] = {
1571         "-e", "kmem:mm_page_alloc",
1572         "-e", "kmem:mm_page_free",
1573         };
1574         unsigned int rec_argc, i, j;
1575         const char **rec_argv;
1576
1577         rec_argc = ARRAY_SIZE(record_args) + argc - 1;
1578         if (kmem_slab)
1579                 rec_argc += ARRAY_SIZE(slab_events);
1580         if (kmem_page)
1581                 rec_argc += ARRAY_SIZE(page_events) + 1; /* for -g */
1582
1583         rec_argv = calloc(rec_argc + 1, sizeof(char *));
1584
1585         if (rec_argv == NULL)
1586                 return -ENOMEM;
1587
1588         for (i = 0; i < ARRAY_SIZE(record_args); i++)
1589                 rec_argv[i] = strdup(record_args[i]);
1590
1591         if (kmem_slab) {
1592                 for (j = 0; j < ARRAY_SIZE(slab_events); j++, i++)
1593                         rec_argv[i] = strdup(slab_events[j]);
1594         }
1595         if (kmem_page) {
1596                 rec_argv[i++] = strdup("-g");
1597
1598                 for (j = 0; j < ARRAY_SIZE(page_events); j++, i++)
1599                         rec_argv[i] = strdup(page_events[j]);
1600         }
1601
1602         for (j = 1; j < (unsigned int)argc; j++, i++)
1603                 rec_argv[i] = argv[j];
1604
1605         return cmd_record(i, rec_argv, NULL);
1606 }
1607
1608 int cmd_kmem(int argc, const char **argv, const char *prefix __maybe_unused)
1609 {
1610         const char * const default_slab_sort = "frag,hit,bytes";
1611         const char * const default_page_sort = "bytes,hit";
1612         struct perf_data_file file = {
1613                 .mode = PERF_DATA_MODE_READ,
1614         };
1615         const struct option kmem_options[] = {
1616         OPT_STRING('i', "input", &input_name, "file", "input file name"),
1617         OPT_INCR('v', "verbose", &verbose,
1618                     "be more verbose (show symbol address, etc)"),
1619         OPT_CALLBACK_NOOPT(0, "caller", NULL, NULL,
1620                            "show per-callsite statistics", parse_caller_opt),
1621         OPT_CALLBACK_NOOPT(0, "alloc", NULL, NULL,
1622                            "show per-allocation statistics", parse_alloc_opt),
1623         OPT_CALLBACK('s', "sort", NULL, "key[,key2...]",
1624                      "sort by keys: ptr, callsite, bytes, hit, pingpong, frag, "
1625                      "page, order, migtype, gfp", parse_sort_opt),
1626         OPT_CALLBACK('l', "line", NULL, "num", "show n lines", parse_line_opt),
1627         OPT_BOOLEAN(0, "raw-ip", &raw_ip, "show raw ip instead of symbol"),
1628         OPT_BOOLEAN('f', "force", &file.force, "don't complain, do it"),
1629         OPT_CALLBACK_NOOPT(0, "slab", NULL, NULL, "Analyze slab allocator",
1630                            parse_slab_opt),
1631         OPT_CALLBACK_NOOPT(0, "page", NULL, NULL, "Analyze page allocator",
1632                            parse_page_opt),
1633         OPT_END()
1634         };
1635         const char *const kmem_subcommands[] = { "record", "stat", NULL };
1636         const char *kmem_usage[] = {
1637                 NULL,
1638                 NULL
1639         };
1640         struct perf_session *session;
1641         int ret = -1;
1642
1643         argc = parse_options_subcommand(argc, argv, kmem_options,
1644                                         kmem_subcommands, kmem_usage, 0);
1645
1646         if (!argc)
1647                 usage_with_options(kmem_usage, kmem_options);
1648
1649         if (kmem_slab == 0 && kmem_page == 0)
1650                 kmem_slab = 1;  /* for backward compatibility */
1651
1652         if (!strncmp(argv[0], "rec", 3)) {
1653                 symbol__init(NULL);
1654                 return __cmd_record(argc, argv);
1655         }
1656
1657         file.path = input_name;
1658
1659         kmem_session = session = perf_session__new(&file, false, &perf_kmem);
1660         if (session == NULL)
1661                 return -1;
1662
1663         if (kmem_page) {
1664                 struct perf_evsel *evsel = perf_evlist__first(session->evlist);
1665
1666                 if (evsel == NULL || evsel->tp_format == NULL) {
1667                         pr_err("invalid event found.. aborting\n");
1668                         return -1;
1669                 }
1670
1671                 kmem_page_size = pevent_get_page_size(evsel->tp_format->pevent);
1672                 symbol_conf.use_callchain = true;
1673         }
1674
1675         symbol__init(&session->header.env);
1676
1677         if (!strcmp(argv[0], "stat")) {
1678                 setlocale(LC_ALL, "");
1679
1680                 if (cpu__setup_cpunode_map())
1681                         goto out_delete;
1682
1683                 if (list_empty(&slab_caller_sort))
1684                         setup_slab_sorting(&slab_caller_sort, default_slab_sort);
1685                 if (list_empty(&slab_alloc_sort))
1686                         setup_slab_sorting(&slab_alloc_sort, default_slab_sort);
1687                 if (list_empty(&page_caller_sort))
1688                         setup_page_sorting(&page_caller_sort, default_page_sort);
1689                 if (list_empty(&page_alloc_sort))
1690                         setup_page_sorting(&page_alloc_sort, default_page_sort);
1691
1692                 if (kmem_page) {
1693                         setup_page_sorting(&page_alloc_sort_input,
1694                                            "page,order,migtype,gfp");
1695                         setup_page_sorting(&page_caller_sort_input,
1696                                            "callsite,order,migtype,gfp");
1697                 }
1698                 ret = __cmd_kmem(session);
1699         } else
1700                 usage_with_options(kmem_usage, kmem_options);
1701
1702 out_delete:
1703         perf_session__delete(session);
1704
1705         return ret;
1706 }
1707