slub: get_map() function to establish map of free objects in a slab

[firefly-linux-kernel-4.4.55.git] / mm / slub.c

diff --git a/mm/slub.c b/mm/slub.c
index 7e4f835e32ab21d19378cf27492e415bcbf26a6e..837f932671a1d3b7b1e0988e1e78001aeadfc009 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -217,7 +217,7 @@ static inline void sysfs_slab_remove(struct kmem_cache *s)
 
 #endif
 
-static inline void stat(struct kmem_cache *s, enum stat_item si)
+static inline void stat(const struct kmem_cache *s, enum stat_item si)
 {
 #ifdef CONFIG_SLUB_STATS
        __this_cpu_inc(s->cpu_slab->stat[si]);
@@ -271,10 +271,6 @@ static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
        for (__p = (__addr); __p < (__addr) + (__objects) * (__s)->size;\
                        __p += (__s)->size)
 
-/* Scan freelist */
-#define for_each_free_object(__p, __s, __free) \
-       for (__p = (__free); __p; __p = get_freepointer((__s), __p))
-
 /* Determine object index from a given position */
 static inline int slab_index(void *p, struct kmem_cache *s, void *addr)
 {
@@ -330,6 +326,21 @@ static inline int oo_objects(struct kmem_cache_order_objects x)
        return x.x & OO_MASK;
 }
 
+/*
+ * Determine a map of object in use on a page.
+ *
+ * Slab lock or node listlock must be held to guarantee that the page does
+ * not vanish from under us.
+ */
+static void get_map(struct kmem_cache *s, struct page *page, unsigned long *map)
+{
+       void *p;
+       void *addr = page_address(page);
+
+       for (p = page->freelist; p; p = get_freepointer(s, p))
+               set_bit(slab_index(p, s, addr), map);
+}
+
 #ifdef CONFIG_SLUB_DEBUG
 /*
  * Debug settings:
@@ -849,11 +860,11 @@ static inline void slab_free_hook(struct kmem_cache *s, void *x)
                local_irq_save(flags);
                kmemcheck_slab_free(s, x, s->objsize);
                debug_check_no_locks_freed(x, s->objsize);
-               if (!(s->flags & SLAB_DEBUG_OBJECTS))
-                       debug_check_no_obj_freed(x, s->objsize);
                local_irq_restore(flags);
        }
 #endif
+       if (!(s->flags & SLAB_DEBUG_OBJECTS))
+               debug_check_no_obj_freed(x, s->objsize);
 }
 
 /*
@@ -1487,7 +1498,7 @@ static struct page *get_partial(struct kmem_cache *s, gfp_t flags, int node)
        int searchnode = (node == NUMA_NO_NODE) ? numa_node_id() : node;
 
        page = get_partial_node(get_node(s, searchnode));
-       if (page || node != -1)
+       if (page || node != NUMA_NO_NODE)
                return page;
 
        return get_any_partial(s, flags);
@@ -1597,13 +1608,14 @@ static inline void note_cmpxchg_failure(const char *n,
                printk("for unknown reason: actual=%lx was=%lx target=%lx\n",
                        actual_tid, tid, next_tid(tid));
 #endif
+       stat(s, CMPXCHG_DOUBLE_CPU_FAIL);
 }
 
 #endif
 
 void init_kmem_cache_cpus(struct kmem_cache *s)
 {
-#if defined(CONFIG_CMPXCHG_LOCAL) && defined(CONFIG_PREEMPT)
+#ifdef CONFIG_CMPXCHG_LOCAL
        int cpu;
 
        for_each_possible_cpu(cpu)
@@ -1857,6 +1869,9 @@ new_slab:
        }
        if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit())
                slab_out_of_memory(s, gfpflags, node);
+#ifdef CONFIG_CMPXCHG_LOCAL
+       local_irq_restore(flags);
+#endif
        return NULL;
 debug:
        if (!alloc_debug_processing(s, c->page, object, addr))
@@ -2669,9 +2684,8 @@ static void list_slab_objects(struct kmem_cache *s, struct page *page,
                return;
        slab_err(s, page, "%s", text);
        slab_lock(page);
-       for_each_free_object(p, s, page->freelist)
-               set_bit(slab_index(p, s, addr), map);
 
+       get_map(s, page, map);
        for_each_object(p, s, addr, page->objects) {
 
                if (!test_bit(slab_index(p, s, addr), map)) {
@@ -3199,7 +3213,7 @@ static void __init kmem_cache_bootstrap_fixup(struct kmem_cache *s)
                        list_for_each_entry(p, &n->partial, lru)
                                p->slab = s;
 
-#ifdef CONFIG_SLAB_DEBUG
+#ifdef CONFIG_SLUB_DEBUG
                        list_for_each_entry(p, &n->full, lru)
                                p->slab = s;
 #endif
@@ -3606,10 +3620,11 @@ static int validate_slab(struct kmem_cache *s, struct page *page,
        /* Now we know that a valid freelist exists */
        bitmap_zero(map, page->objects);
 
-       for_each_free_object(p, s, page->freelist) {
-               set_bit(slab_index(p, s, addr), map);
-               if (!check_object(s, page, p, SLUB_RED_INACTIVE))
-                       return 0;
+       get_map(s, page, map);
+       for_each_object(p, s, addr, page->objects) {
+               if (test_bit(slab_index(p, s, addr), map))
+                       if (!check_object(s, page, p, SLUB_RED_INACTIVE))
+                               return 0;
        }
 
        for_each_object(p, s, addr, page->objects)
@@ -3817,8 +3832,7 @@ static void process_slab(struct loc_track *t, struct kmem_cache *s,
        void *p;
 
        bitmap_zero(map, page->objects);
-       for_each_free_object(p, s, page->freelist)
-               set_bit(slab_index(p, s, addr), map);
+       get_map(s, page, map);
 
        for_each_object(p, s, addr, page->objects)
                if (!test_bit(slab_index(p, s, addr), map))