slub: optimize bulk slowpath free by detached freelist

[firefly-linux-kernel-4.4.55.git] / mm / slub.c

diff --git a/mm/slub.c b/mm/slub.c
index d52f0d0ab71231787d445c983ddde19f99f9a09e..c17c5202864d9f091b210eab4dd2eba2e426fcd4 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -2807,44 +2807,93 @@ void kmem_cache_free(struct kmem_cache *s, void *x)
 }
 EXPORT_SYMBOL(kmem_cache_free);
 
-/* Note that interrupts must be enabled when calling this function. */
-void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
-{
-       struct kmem_cache_cpu *c;
+struct detached_freelist {
        struct page *page;
-       int i;
+       void *tail;
+       void *freelist;
+       int cnt;
+};
 
-       local_irq_disable();
-       c = this_cpu_ptr(s->cpu_slab);
+/*
+ * This function progressively scans the array with free objects (with
+ * a limited look ahead) and extract objects belonging to the same
+ * page.  It builds a detached freelist directly within the given
+ * page/objects.  This can happen without any need for
+ * synchronization, because the objects are owned by running process.
+ * The freelist is build up as a single linked list in the objects.
+ * The idea is, that this detached freelist can then be bulk
+ * transferred to the real freelist(s), but only requiring a single
+ * synchronization primitive.  Look ahead in the array is limited due
+ * to performance reasons.
+ */
+static int build_detached_freelist(struct kmem_cache *s, size_t size,
+                                  void **p, struct detached_freelist *df)
+{
+       size_t first_skipped_index = 0;
+       int lookahead = 3;
+       void *object;
 
-       for (i = 0; i < size; i++) {
-               void *object = p[i];
+       /* Always re-init detached_freelist */
+       df->page = NULL;
 
-               BUG_ON(!object);
-               /* kmem cache debug support */
-               s = cache_from_obj(s, object);
-               if (unlikely(!s))
-                       goto exit;
-               slab_free_hook(s, object);
+       do {
+               object = p[--size];
+       } while (!object && size);
 
-               page = virt_to_head_page(object);
+       if (!object)
+               return 0;
 
-               if (c->page == page) {
-                       /* Fastpath: local CPU free */
-                       set_freepointer(s, object, c->freelist);
-                       c->freelist = object;
-               } else {
-                       c->tid = next_tid(c->tid);
-                       local_irq_enable();
-                       /* Slowpath: overhead locked cmpxchg_double_slab */
-                       __slab_free(s, page, object, object, 1, _RET_IP_);
-                       local_irq_disable();
-                       c = this_cpu_ptr(s->cpu_slab);
+       /* Start new detached freelist */
+       set_freepointer(s, object, NULL);
+       df->page = virt_to_head_page(object);
+       df->tail = object;
+       df->freelist = object;
+       p[size] = NULL; /* mark object processed */
+       df->cnt = 1;
+
+       while (size) {
+               object = p[--size];
+               if (!object)
+                       continue; /* Skip processed objects */
+
+               /* df->page is always set at this point */
+               if (df->page == virt_to_head_page(object)) {
+                       /* Opportunity build freelist */
+                       set_freepointer(s, object, df->freelist);
+                       df->freelist = object;
+                       df->cnt++;
+                       p[size] = NULL; /* mark object processed */
+
+                       continue;
                }
+
+               /* Limit look ahead search */
+               if (!--lookahead)
+                       break;
+
+               if (!first_skipped_index)
+                       first_skipped_index = size + 1;
        }
-exit:
-       c->tid = next_tid(c->tid);
-       local_irq_enable();
+
+       return first_skipped_index;
+}
+
+
+/* Note that interrupts must be enabled when calling this function. */
+void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
+{
+       if (WARN_ON(!size))
+               return;
+
+       do {
+               struct detached_freelist df;
+
+               size = build_detached_freelist(s, size, p, &df);
+               if (unlikely(!df.page))
+                       continue;
+
+               slab_free(s, df.page, df.freelist, df.tail, df.cnt, _RET_IP_);
+       } while (likely(size));
 }
 EXPORT_SYMBOL(kmem_cache_free_bulk);