2 * In-kernel transcendent memory (generic implementation)
4 * Copyright (c) 2009-2011, Dan Magenheimer, Oracle Corp.
6 * The primary purpose of Transcedent Memory ("tmem") is to map object-oriented
7 * "handles" (triples containing a pool id, and object id, and an index), to
8 * pages in a page-accessible memory (PAM). Tmem references the PAM pages via
9 * an abstract "pampd" (PAM page-descriptor), which can be operated on by a
10 * set of functions (pamops). Each pampd contains some representation of
11 * PAGE_SIZE bytes worth of data. Tmem must support potentially millions of
12 * pages and must be able to insert, find, and delete these pages at a
13 * potential frequency of thousands per second concurrently across many CPUs,
14 * (and, if used with KVM, across many vcpus across many guests).
15 * Tmem is tracked with a hierarchy of data structures, organized by
16 * the elements in a handle-tuple: pool_id, object_id, and page index.
17 * One or more "clients" (e.g. guests) each provide one or more tmem_pools.
18 * Each pool, contains a hash table of rb_trees of tmem_objs. Each
19 * tmem_obj contains a radix-tree-like tree of pointers, with intermediate
20 * nodes called tmem_objnodes. Each leaf pointer in this tree points to
21 * a pampd, which is accessible only through a small set of callbacks
22 * registered by the PAM implementation (see tmem_register_pamops). Tmem
23 * does all memory allocation via a set of callbacks registered by the tmem
24 * host implementation (e.g. see tmem_register_hostops).
27 #include <linux/list.h>
28 #include <linux/spinlock.h>
29 #include <linux/atomic.h>
30 #include <linux/delay.h>
34 /* data structure sentinels used for debugging... see tmem.h */
35 #define POOL_SENTINEL 0x87658765
36 #define OBJ_SENTINEL 0x12345678
37 #define OBJNODE_SENTINEL 0xfedcba09
40 * A tmem host implementation must use this function to register callbacks
41 * for memory allocation.
43 static struct tmem_hostops tmem_hostops;
45 static void tmem_objnode_tree_init(void);
47 void tmem_register_hostops(struct tmem_hostops *m)
49 tmem_objnode_tree_init();
54 * A tmem host implementation must use this function to register
55 * callbacks for a page-accessible memory (PAM) implementation
57 static struct tmem_pamops tmem_pamops;
59 void tmem_register_pamops(struct tmem_pamops *m)
65 * Oid's are potentially very sparse and tmem_objs may have an indeterminately
66 * short life, being added and deleted at a relatively high frequency.
67 * So an rb_tree is an ideal data structure to manage tmem_objs. But because
68 * of the potentially huge number of tmem_objs, each pool manages a hashtable
69 * of rb_trees to reduce search, insert, delete, and rebalancing time.
70 * Each hashbucket also has a lock to manage concurrent access.
72 * The following routines manage tmem_objs. When any tmem_obj is accessed,
73 * the hashbucket lock must be held.
76 /* searches for object==oid in pool, returns locked object if found */
77 static struct tmem_obj *tmem_obj_find(struct tmem_hashbucket *hb,
78 struct tmem_oid *oidp)
80 struct rb_node *rbnode;
83 rbnode = hb->obj_rb_root.rb_node;
85 BUG_ON(RB_EMPTY_NODE(rbnode));
86 obj = rb_entry(rbnode, struct tmem_obj, rb_tree_node);
87 switch (tmem_oid_compare(oidp, &obj->oid)) {
91 rbnode = rbnode->rb_left;
94 rbnode = rbnode->rb_right;
103 static void tmem_pampd_destroy_all_in_obj(struct tmem_obj *);
105 /* free an object that has no more pampds in it */
106 static void tmem_obj_free(struct tmem_obj *obj, struct tmem_hashbucket *hb)
108 struct tmem_pool *pool;
111 ASSERT_SENTINEL(obj, OBJ);
112 BUG_ON(obj->pampd_count > 0);
114 BUG_ON(pool == NULL);
115 if (obj->objnode_tree_root != NULL) /* may be "stump" with no leaves */
116 tmem_pampd_destroy_all_in_obj(obj);
117 BUG_ON(obj->objnode_tree_root != NULL);
118 BUG_ON((long)obj->objnode_count != 0);
119 atomic_dec(&pool->obj_count);
120 BUG_ON(atomic_read(&pool->obj_count) < 0);
121 INVERT_SENTINEL(obj, OBJ);
123 tmem_oid_set_invalid(&obj->oid);
124 rb_erase(&obj->rb_tree_node, &hb->obj_rb_root);
128 * initialize, and insert an tmem_object_root (called only if find failed)
130 static void tmem_obj_init(struct tmem_obj *obj, struct tmem_hashbucket *hb,
131 struct tmem_pool *pool,
132 struct tmem_oid *oidp)
134 struct rb_root *root = &hb->obj_rb_root;
135 struct rb_node **new = &(root->rb_node), *parent = NULL;
136 struct tmem_obj *this;
138 BUG_ON(pool == NULL);
139 atomic_inc(&pool->obj_count);
140 obj->objnode_tree_height = 0;
141 obj->objnode_tree_root = NULL;
144 obj->objnode_count = 0;
145 obj->pampd_count = 0;
146 (*tmem_pamops.new_obj)(obj);
147 SET_SENTINEL(obj, OBJ);
149 BUG_ON(RB_EMPTY_NODE(*new));
150 this = rb_entry(*new, struct tmem_obj, rb_tree_node);
152 switch (tmem_oid_compare(oidp, &this->oid)) {
154 BUG(); /* already present; should never happen! */
157 new = &(*new)->rb_left;
160 new = &(*new)->rb_right;
164 rb_link_node(&obj->rb_tree_node, parent, new);
165 rb_insert_color(&obj->rb_tree_node, root);
169 * Tmem is managed as a set of tmem_pools with certain attributes, such as
170 * "ephemeral" vs "persistent". These attributes apply to all tmem_objs
171 * and all pampds that belong to a tmem_pool. A tmem_pool is created
172 * or deleted relatively rarely (for example, when a filesystem is
173 * mounted or unmounted.
176 /* flush all data from a pool and, optionally, free it */
177 static void tmem_pool_flush(struct tmem_pool *pool, bool destroy)
179 struct rb_node *rbnode;
180 struct tmem_obj *obj;
181 struct tmem_hashbucket *hb = &pool->hashbucket[0];
184 BUG_ON(pool == NULL);
185 for (i = 0; i < TMEM_HASH_BUCKETS; i++, hb++) {
186 spin_lock(&hb->lock);
187 rbnode = rb_first(&hb->obj_rb_root);
188 while (rbnode != NULL) {
189 obj = rb_entry(rbnode, struct tmem_obj, rb_tree_node);
190 rbnode = rb_next(rbnode);
191 tmem_pampd_destroy_all_in_obj(obj);
192 tmem_obj_free(obj, hb);
193 (*tmem_hostops.obj_free)(obj, pool);
195 spin_unlock(&hb->lock);
198 list_del(&pool->pool_list);
202 * A tmem_obj contains a radix-tree-like tree in which the intermediate
203 * nodes are called tmem_objnodes. (The kernel lib/radix-tree.c implementation
204 * is very specialized and tuned for specific uses and is not particularly
205 * suited for use from this code, though some code from the core algorithms has
206 * been reused, thus the copyright notices below). Each tmem_objnode contains
207 * a set of pointers which point to either a set of intermediate tmem_objnodes
208 * or a set of of pampds.
210 * Portions Copyright (C) 2001 Momchil Velikov
211 * Portions Copyright (C) 2001 Christoph Hellwig
212 * Portions Copyright (C) 2005 SGI, Christoph Lameter <clameter@sgi.com>
215 struct tmem_objnode_tree_path {
216 struct tmem_objnode *objnode;
220 /* objnode height_to_maxindex translation */
221 static unsigned long tmem_objnode_tree_h2max[OBJNODE_TREE_MAX_PATH + 1];
223 static void tmem_objnode_tree_init(void)
225 unsigned int ht, tmp;
227 for (ht = 0; ht < ARRAY_SIZE(tmem_objnode_tree_h2max); ht++) {
228 tmp = ht * OBJNODE_TREE_MAP_SHIFT;
229 if (tmp >= OBJNODE_TREE_INDEX_BITS)
230 tmem_objnode_tree_h2max[ht] = ~0UL;
232 tmem_objnode_tree_h2max[ht] =
233 (~0UL >> (OBJNODE_TREE_INDEX_BITS - tmp - 1)) >> 1;
237 static struct tmem_objnode *tmem_objnode_alloc(struct tmem_obj *obj)
239 struct tmem_objnode *objnode;
241 ASSERT_SENTINEL(obj, OBJ);
242 BUG_ON(obj->pool == NULL);
243 ASSERT_SENTINEL(obj->pool, POOL);
244 objnode = (*tmem_hostops.objnode_alloc)(obj->pool);
245 if (unlikely(objnode == NULL))
248 SET_SENTINEL(objnode, OBJNODE);
249 memset(&objnode->slots, 0, sizeof(objnode->slots));
250 objnode->slots_in_use = 0;
251 obj->objnode_count++;
256 static void tmem_objnode_free(struct tmem_objnode *objnode)
258 struct tmem_pool *pool;
261 BUG_ON(objnode == NULL);
262 for (i = 0; i < OBJNODE_TREE_MAP_SIZE; i++)
263 BUG_ON(objnode->slots[i] != NULL);
264 ASSERT_SENTINEL(objnode, OBJNODE);
265 INVERT_SENTINEL(objnode, OBJNODE);
266 BUG_ON(objnode->obj == NULL);
267 ASSERT_SENTINEL(objnode->obj, OBJ);
268 pool = objnode->obj->pool;
269 BUG_ON(pool == NULL);
270 ASSERT_SENTINEL(pool, POOL);
271 objnode->obj->objnode_count--;
273 (*tmem_hostops.objnode_free)(objnode, pool);
277 * lookup index in object and return associated pampd (or NULL if not found)
279 static void **__tmem_pampd_lookup_in_obj(struct tmem_obj *obj, uint32_t index)
281 unsigned int height, shift;
282 struct tmem_objnode **slot = NULL;
285 ASSERT_SENTINEL(obj, OBJ);
286 BUG_ON(obj->pool == NULL);
287 ASSERT_SENTINEL(obj->pool, POOL);
289 height = obj->objnode_tree_height;
290 if (index > tmem_objnode_tree_h2max[obj->objnode_tree_height])
292 if (height == 0 && obj->objnode_tree_root) {
293 slot = &obj->objnode_tree_root;
296 shift = (height-1) * OBJNODE_TREE_MAP_SHIFT;
297 slot = &obj->objnode_tree_root;
301 slot = (struct tmem_objnode **)
303 ((index >> shift) & OBJNODE_TREE_MAP_MASK));
304 shift -= OBJNODE_TREE_MAP_SHIFT;
308 return slot != NULL ? (void **)slot : NULL;
311 static void *tmem_pampd_lookup_in_obj(struct tmem_obj *obj, uint32_t index)
313 struct tmem_objnode **slot;
315 slot = (struct tmem_objnode **)__tmem_pampd_lookup_in_obj(obj, index);
316 return slot != NULL ? *slot : NULL;
319 static void *tmem_pampd_replace_in_obj(struct tmem_obj *obj, uint32_t index,
320 void *new_pampd, bool no_free)
322 struct tmem_objnode **slot;
325 slot = (struct tmem_objnode **)__tmem_pampd_lookup_in_obj(obj, index);
326 if ((slot != NULL) && (*slot != NULL)) {
327 void *old_pampd = *(void **)slot;
328 *(void **)slot = new_pampd;
330 (*tmem_pamops.free)(old_pampd, obj->pool,
337 static int tmem_pampd_add_to_obj(struct tmem_obj *obj, uint32_t index,
341 struct tmem_objnode *objnode = NULL, *newnode, *slot;
342 unsigned int height, shift;
345 /* if necessary, extend the tree to be higher */
346 if (index > tmem_objnode_tree_h2max[obj->objnode_tree_height]) {
347 height = obj->objnode_tree_height + 1;
348 if (index > tmem_objnode_tree_h2max[height])
349 while (index > tmem_objnode_tree_h2max[height])
351 if (obj->objnode_tree_root == NULL) {
352 obj->objnode_tree_height = height;
356 newnode = tmem_objnode_alloc(obj);
361 newnode->slots[0] = obj->objnode_tree_root;
362 newnode->slots_in_use = 1;
363 obj->objnode_tree_root = newnode;
364 obj->objnode_tree_height++;
365 } while (height > obj->objnode_tree_height);
368 slot = obj->objnode_tree_root;
369 height = obj->objnode_tree_height;
370 shift = (height-1) * OBJNODE_TREE_MAP_SHIFT;
373 /* add a child objnode. */
374 slot = tmem_objnode_alloc(obj);
381 objnode->slots[offset] = slot;
382 objnode->slots_in_use++;
384 obj->objnode_tree_root = slot;
386 /* go down a level */
387 offset = (index >> shift) & OBJNODE_TREE_MAP_MASK;
389 slot = objnode->slots[offset];
390 shift -= OBJNODE_TREE_MAP_SHIFT;
393 BUG_ON(slot != NULL);
395 objnode->slots_in_use++;
396 objnode->slots[offset] = pampd;
398 obj->objnode_tree_root = pampd;
404 static void *tmem_pampd_delete_from_obj(struct tmem_obj *obj, uint32_t index)
406 struct tmem_objnode_tree_path path[OBJNODE_TREE_MAX_PATH + 1];
407 struct tmem_objnode_tree_path *pathp = path;
408 struct tmem_objnode *slot = NULL;
409 unsigned int height, shift;
413 ASSERT_SENTINEL(obj, OBJ);
414 BUG_ON(obj->pool == NULL);
415 ASSERT_SENTINEL(obj->pool, POOL);
416 height = obj->objnode_tree_height;
417 if (index > tmem_objnode_tree_h2max[height])
419 slot = obj->objnode_tree_root;
420 if (height == 0 && obj->objnode_tree_root) {
421 obj->objnode_tree_root = NULL;
424 shift = (height - 1) * OBJNODE_TREE_MAP_SHIFT;
425 pathp->objnode = NULL;
430 offset = (index >> shift) & OBJNODE_TREE_MAP_MASK;
431 pathp->offset = offset;
432 pathp->objnode = slot;
433 slot = slot->slots[offset];
434 shift -= OBJNODE_TREE_MAP_SHIFT;
436 } while (height > 0);
439 while (pathp->objnode) {
440 pathp->objnode->slots[pathp->offset] = NULL;
441 pathp->objnode->slots_in_use--;
442 if (pathp->objnode->slots_in_use) {
443 if (pathp->objnode == obj->objnode_tree_root) {
444 while (obj->objnode_tree_height > 0 &&
445 obj->objnode_tree_root->slots_in_use == 1 &&
446 obj->objnode_tree_root->slots[0]) {
447 struct tmem_objnode *to_free =
448 obj->objnode_tree_root;
450 obj->objnode_tree_root =
452 obj->objnode_tree_height--;
453 to_free->slots[0] = NULL;
454 to_free->slots_in_use = 0;
455 tmem_objnode_free(to_free);
460 tmem_objnode_free(pathp->objnode); /* 0 slots used, free it */
463 obj->objnode_tree_height = 0;
464 obj->objnode_tree_root = NULL;
469 BUG_ON(obj->pampd_count < 0);
473 /* recursively walk the objnode_tree destroying pampds and objnodes */
474 static void tmem_objnode_node_destroy(struct tmem_obj *obj,
475 struct tmem_objnode *objnode,
482 for (i = 0; i < OBJNODE_TREE_MAP_SIZE; i++) {
483 if (objnode->slots[i]) {
486 (*tmem_pamops.free)(objnode->slots[i],
487 obj->pool, NULL, 0, true);
488 objnode->slots[i] = NULL;
491 tmem_objnode_node_destroy(obj, objnode->slots[i], ht-1);
492 tmem_objnode_free(objnode->slots[i]);
493 objnode->slots[i] = NULL;
498 static void tmem_pampd_destroy_all_in_obj(struct tmem_obj *obj)
500 if (obj->objnode_tree_root == NULL)
502 if (obj->objnode_tree_height == 0) {
504 (*tmem_pamops.free)(obj->objnode_tree_root,
505 obj->pool, NULL, 0, true);
507 tmem_objnode_node_destroy(obj, obj->objnode_tree_root,
508 obj->objnode_tree_height);
509 tmem_objnode_free(obj->objnode_tree_root);
510 obj->objnode_tree_height = 0;
512 obj->objnode_tree_root = NULL;
513 (*tmem_pamops.free_obj)(obj->pool, obj);
517 * Tmem is operated on by a set of well-defined actions:
518 * "put", "get", "flush", "flush_object", "new pool" and "destroy pool".
519 * (The tmem ABI allows for subpages and exchanges but these operations
520 * are not included in this implementation.)
522 * These "tmem core" operations are implemented in the following functions.
526 * "Put" a page, e.g. copy a page from the kernel into newly allocated
527 * PAM space (if such space is available). Tmem_put is complicated by
528 * a corner case: What if a page with matching handle already exists in
529 * tmem? To guarantee coherency, one of two actions is necessary: Either
530 * the data for the page must be overwritten, or the page must be
531 * "flushed" so that the data is not accessible to a subsequent "get".
532 * Since these "duplicate puts" are relatively rare, this implementation
533 * always flushes for simplicity.
535 int tmem_put(struct tmem_pool *pool, struct tmem_oid *oidp, uint32_t index,
536 char *data, size_t size, bool raw, int ephemeral)
538 struct tmem_obj *obj = NULL, *objfound = NULL, *objnew = NULL;
539 void *pampd = NULL, *pampd_del = NULL;
541 struct tmem_hashbucket *hb;
543 hb = &pool->hashbucket[tmem_oid_hash(oidp)];
544 spin_lock(&hb->lock);
545 obj = objfound = tmem_obj_find(hb, oidp);
547 pampd = tmem_pampd_lookup_in_obj(objfound, index);
549 /* if found, is a dup put, flush the old one */
550 pampd_del = tmem_pampd_delete_from_obj(obj, index);
551 BUG_ON(pampd_del != pampd);
552 (*tmem_pamops.free)(pampd, pool, oidp, index, true);
553 if (obj->pampd_count == 0) {
560 obj = objnew = (*tmem_hostops.obj_alloc)(pool);
561 if (unlikely(obj == NULL)) {
565 tmem_obj_init(obj, hb, pool, oidp);
568 BUG_ON(((objnew != obj) && (objfound != obj)) || (objnew == objfound));
569 pampd = (*tmem_pamops.create)(data, size, raw, ephemeral,
570 obj->pool, &obj->oid, index);
571 if (unlikely(pampd == NULL))
573 ret = tmem_pampd_add_to_obj(obj, index, pampd);
574 if (unlikely(ret == -ENOMEM))
575 /* may have partially built objnode tree ("stump") */
576 goto delete_and_free;
580 (void)tmem_pampd_delete_from_obj(obj, index);
583 (*tmem_pamops.free)(pampd, pool, NULL, 0, true);
585 tmem_obj_free(objnew, hb);
586 (*tmem_hostops.obj_free)(objnew, pool);
589 spin_unlock(&hb->lock);
593 void *tmem_localify_get_pampd(struct tmem_pool *pool, struct tmem_oid *oidp,
594 uint32_t index, struct tmem_obj **ret_obj,
597 struct tmem_hashbucket *hb;
598 struct tmem_obj *obj = NULL;
601 hb = &pool->hashbucket[tmem_oid_hash(oidp)];
602 spin_lock(&hb->lock);
603 obj = tmem_obj_find(hb, oidp);
604 if (likely(obj != NULL))
605 pampd = tmem_pampd_lookup_in_obj(obj, index);
607 *saved_hb = (void *)hb;
608 /* note, hashbucket remains locked */
612 void tmem_localify_finish(struct tmem_obj *obj, uint32_t index,
613 void *pampd, void *saved_hb, bool delete)
615 struct tmem_hashbucket *hb = (struct tmem_hashbucket *)saved_hb;
617 BUG_ON(!spin_is_locked(&hb->lock));
620 (void)tmem_pampd_replace_in_obj(obj, index, pampd, 1);
623 (void)tmem_pampd_delete_from_obj(obj, index);
625 spin_unlock(&hb->lock);
628 static int tmem_repatriate(void **ppampd, struct tmem_hashbucket *hb,
629 struct tmem_pool *pool, struct tmem_oid *oidp,
630 uint32_t index, bool free, char *data)
632 void *old_pampd = *ppampd, *new_pampd = NULL;
633 bool intransit = false;
637 if (!is_ephemeral(pool))
638 new_pampd = (*tmem_pamops.repatriate_preload)(
639 old_pampd, pool, oidp, index, &intransit);
642 else if (new_pampd != NULL)
644 /* must release the hb->lock else repatriate can't sleep */
645 spin_unlock(&hb->lock);
647 ret = (*tmem_pamops.repatriate)(old_pampd, new_pampd, pool,
648 oidp, index, free, data);
653 * "Get" a page, e.g. if one can be found, copy the tmem page with the
654 * matching handle from PAM space to the kernel. By tmem definition,
655 * when a "get" is successful on an ephemeral page, the page is "flushed",
656 * and when a "get" is successful on a persistent page, the page is retained
657 * in tmem. Note that to preserve
658 * coherency, "get" can never be skipped if tmem contains the data.
659 * That is, if a get is done with a certain handle and fails, any
660 * subsequent "get" must also fail (unless of course there is a
661 * "put" done with the same handle).
664 int tmem_get(struct tmem_pool *pool, struct tmem_oid *oidp, uint32_t index,
665 char *data, size_t *size, bool raw, int get_and_free)
667 struct tmem_obj *obj;
669 bool ephemeral = is_ephemeral(pool);
671 struct tmem_hashbucket *hb;
672 bool free = (get_and_free == 1) || ((get_and_free == 0) && ephemeral);
677 hb = &pool->hashbucket[tmem_oid_hash(oidp)];
678 spin_lock(&hb->lock);
680 obj = tmem_obj_find(hb, oidp);
683 ppampd = __tmem_pampd_lookup_in_obj(obj, index);
686 if (tmem_pamops.is_remote(*ppampd)) {
687 ret = tmem_repatriate(ppampd, hb, pool, oidp,
689 lock_held = 0; /* note hb->lock has been unlocked */
690 if (ret == -EAGAIN) {
691 /* rare I think, but should cond_resched()??? */
692 usleep_range(10, 1000);
694 } else if (ret != 0) {
696 pr_err("UNTESTED case in tmem_get, ret=%d\n",
704 pampd = tmem_pampd_delete_from_obj(obj, index);
706 pampd = tmem_pampd_lookup_in_obj(obj, index);
710 if (obj->pampd_count == 0) {
711 tmem_obj_free(obj, hb);
712 (*tmem_hostops.obj_free)(obj, pool);
717 ret = (*tmem_pamops.get_data_and_free)(
718 data, size, raw, pampd, pool, oidp, index);
720 ret = (*tmem_pamops.get_data)(
721 data, size, raw, pampd, pool, oidp, index);
727 spin_unlock(&hb->lock);
732 * If a page in tmem matches the handle, "flush" this page from tmem such
733 * that any subsequent "get" does not succeed (unless, of course, there
734 * was another "put" with the same handle).
736 int tmem_flush_page(struct tmem_pool *pool,
737 struct tmem_oid *oidp, uint32_t index)
739 struct tmem_obj *obj;
742 struct tmem_hashbucket *hb;
744 hb = &pool->hashbucket[tmem_oid_hash(oidp)];
745 spin_lock(&hb->lock);
746 obj = tmem_obj_find(hb, oidp);
749 pampd = tmem_pampd_delete_from_obj(obj, index);
752 (*tmem_pamops.free)(pampd, pool, oidp, index, true);
753 if (obj->pampd_count == 0) {
754 tmem_obj_free(obj, hb);
755 (*tmem_hostops.obj_free)(obj, pool);
760 spin_unlock(&hb->lock);
765 * If a page in tmem matches the handle, replace the page so that any
766 * subsequent "get" gets the new page. Returns the new page if
767 * there was a page to replace, else returns NULL.
769 int tmem_replace(struct tmem_pool *pool, struct tmem_oid *oidp,
770 uint32_t index, void *new_pampd)
772 struct tmem_obj *obj;
774 struct tmem_hashbucket *hb;
776 hb = &pool->hashbucket[tmem_oid_hash(oidp)];
777 spin_lock(&hb->lock);
778 obj = tmem_obj_find(hb, oidp);
781 new_pampd = tmem_pampd_replace_in_obj(obj, index, new_pampd, 0);
782 ret = (*tmem_pamops.replace_in_obj)(new_pampd, obj);
784 spin_unlock(&hb->lock);
789 * "Flush" all pages in tmem matching this oid.
791 int tmem_flush_object(struct tmem_pool *pool, struct tmem_oid *oidp)
793 struct tmem_obj *obj;
794 struct tmem_hashbucket *hb;
797 hb = &pool->hashbucket[tmem_oid_hash(oidp)];
798 spin_lock(&hb->lock);
799 obj = tmem_obj_find(hb, oidp);
802 tmem_pampd_destroy_all_in_obj(obj);
803 tmem_obj_free(obj, hb);
804 (*tmem_hostops.obj_free)(obj, pool);
808 spin_unlock(&hb->lock);
813 * "Flush" all pages (and tmem_objs) from this tmem_pool and disable
814 * all subsequent access to this tmem_pool.
816 int tmem_destroy_pool(struct tmem_pool *pool)
822 tmem_pool_flush(pool, 1);
828 static LIST_HEAD(tmem_global_pool_list);
831 * Create a new tmem_pool with the provided flag and return
832 * a pool id provided by the tmem host implementation.
834 void tmem_new_pool(struct tmem_pool *pool, uint32_t flags)
836 int persistent = flags & TMEM_POOL_PERSIST;
837 int shared = flags & TMEM_POOL_SHARED;
838 struct tmem_hashbucket *hb = &pool->hashbucket[0];
841 for (i = 0; i < TMEM_HASH_BUCKETS; i++, hb++) {
842 hb->obj_rb_root = RB_ROOT;
843 spin_lock_init(&hb->lock);
845 INIT_LIST_HEAD(&pool->pool_list);
846 atomic_set(&pool->obj_count, 0);
847 SET_SENTINEL(pool, POOL);
848 list_add_tail(&pool->pool_list, &tmem_global_pool_list);
849 pool->persistent = persistent;
850 pool->shared = shared;