1 /******************************************************************************
4 * Granting foreign access to our memory reservation.
6 * Copyright (c) 2005-2006, Christopher Clark
7 * Copyright (c) 2004-2005, K A Fraser
9 * This program is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU General Public License version 2
11 * as published by the Free Software Foundation; or, when distributed
12 * separately from the Linux kernel or incorporated into other
13 * software packages, subject to the following license:
15 * Permission is hereby granted, free of charge, to any person obtaining a copy
16 * of this source file (the "Software"), to deal in the Software without
17 * restriction, including without limitation the rights to use, copy, modify,
18 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
19 * and to permit persons to whom the Software is furnished to do so, subject to
20 * the following conditions:
22 * The above copyright notice and this permission notice shall be included in
23 * all copies or substantial portions of the Software.
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
26 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
27 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
28 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
29 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
30 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
34 #include <linux/module.h>
35 #include <linux/sched.h>
37 #include <linux/slab.h>
38 #include <linux/vmalloc.h>
39 #include <linux/uaccess.h>
41 #include <linux/delay.h>
42 #include <linux/hardirq.h>
45 #include <xen/interface/xen.h>
47 #include <xen/grant_table.h>
48 #include <xen/interface/memory.h>
49 #include <xen/hvc-console.h>
50 #include <asm/xen/hypercall.h>
51 #include <asm/xen/interface.h>
53 #include <asm/pgtable.h>
54 #include <asm/sync_bitops.h>
56 /* External tools reserve first few grant table entries. */
57 #define NR_RESERVED_ENTRIES 8
58 #define GNTTAB_LIST_END 0xffffffff
60 static grant_ref_t **gnttab_list;
61 static unsigned int nr_grant_frames;
62 static unsigned int boot_max_nr_grant_frames;
63 static int gnttab_free_count;
64 static grant_ref_t gnttab_free_head;
65 static DEFINE_SPINLOCK(gnttab_list_lock);
66 unsigned long xen_hvm_resume_frames;
67 EXPORT_SYMBOL_GPL(xen_hvm_resume_frames);
70 struct grant_entry_v1 *v1;
71 union grant_entry_v2 *v2;
75 /*This is a structure of function pointers for grant table*/
78 * Mapping a list of frames for storing grant entries. Frames parameter
79 * is used to store grant table address when grant table being setup,
80 * nr_gframes is the number of frames to map grant table. Returning
81 * GNTST_okay means success and negative value means failure.
83 int (*map_frames)(xen_pfn_t *frames, unsigned int nr_gframes);
85 * Release a list of frames which are mapped in map_frames for grant
88 void (*unmap_frames)(void);
90 * Introducing a valid entry into the grant table, granting the frame of
91 * this grant entry to domain for accessing or transfering. Ref
92 * parameter is reference of this introduced grant entry, domid is id of
93 * granted domain, frame is the page frame to be granted, and flags is
94 * status of the grant entry to be updated.
96 void (*update_entry)(grant_ref_t ref, domid_t domid,
97 unsigned long frame, unsigned flags);
99 * Stop granting a grant entry to domain for accessing. Ref parameter is
100 * reference of a grant entry whose grant access will be stopped,
101 * readonly is not in use in this function. If the grant entry is
102 * currently mapped for reading or writing, just return failure(==0)
103 * directly and don't tear down the grant access. Otherwise, stop grant
104 * access for this entry and return success(==1).
106 int (*end_foreign_access_ref)(grant_ref_t ref, int readonly);
108 * Stop granting a grant entry to domain for transfer. Ref parameter is
109 * reference of a grant entry whose grant transfer will be stopped. If
110 * tranfer has not started, just reclaim the grant entry and return
111 * failure(==0). Otherwise, wait for the transfer to complete and then
114 unsigned long (*end_foreign_transfer_ref)(grant_ref_t ref);
116 * Query the status of a grant entry. Ref parameter is reference of
117 * queried grant entry, return value is the status of queried entry.
118 * Detailed status(writing/reading) can be gotten from the return value
121 int (*query_foreign_access)(grant_ref_t ref);
123 * Grant a domain to access a range of bytes within the page referred by
124 * an available grant entry. Ref parameter is reference of a grant entry
125 * which will be sub-page accessed, domid is id of grantee domain, frame
126 * is frame address of subpage grant, flags is grant type and flag
127 * information, page_off is offset of the range of bytes, and length is
128 * length of bytes to be accessed.
130 void (*update_subpage_entry)(grant_ref_t ref, domid_t domid,
131 unsigned long frame, int flags,
132 unsigned page_off, unsigned length);
134 * Redirect an available grant entry on domain A to another grant
135 * reference of domain B, then allow domain C to use grant reference
136 * of domain B transitively. Ref parameter is an available grant entry
137 * reference on domain A, domid is id of domain C which accesses grant
138 * entry transitively, flags is grant type and flag information,
139 * trans_domid is id of domain B whose grant entry is finally accessed
140 * transitively, trans_gref is grant entry transitive reference of
143 void (*update_trans_entry)(grant_ref_t ref, domid_t domid, int flags,
144 domid_t trans_domid, grant_ref_t trans_gref);
147 static struct gnttab_ops *gnttab_interface;
149 /*This reflects status of grant entries, so act as a global value*/
150 static grant_status_t *grstatus;
152 static int grant_table_version;
153 static int grefs_per_grant_frame;
155 static struct gnttab_free_callback *gnttab_free_callback_list;
157 static int gnttab_expand(unsigned int req_entries);
159 #define RPP (PAGE_SIZE / sizeof(grant_ref_t))
160 #define SPP (PAGE_SIZE / sizeof(grant_status_t))
162 static inline grant_ref_t *__gnttab_entry(grant_ref_t entry)
164 return &gnttab_list[(entry) / RPP][(entry) % RPP];
166 /* This can be used as an l-value */
167 #define gnttab_entry(entry) (*__gnttab_entry(entry))
169 static int get_free_entries(unsigned count)
175 spin_lock_irqsave(&gnttab_list_lock, flags);
177 if ((gnttab_free_count < count) &&
178 ((rc = gnttab_expand(count - gnttab_free_count)) < 0)) {
179 spin_unlock_irqrestore(&gnttab_list_lock, flags);
183 ref = head = gnttab_free_head;
184 gnttab_free_count -= count;
186 head = gnttab_entry(head);
187 gnttab_free_head = gnttab_entry(head);
188 gnttab_entry(head) = GNTTAB_LIST_END;
190 spin_unlock_irqrestore(&gnttab_list_lock, flags);
195 static void do_free_callbacks(void)
197 struct gnttab_free_callback *callback, *next;
199 callback = gnttab_free_callback_list;
200 gnttab_free_callback_list = NULL;
202 while (callback != NULL) {
203 next = callback->next;
204 if (gnttab_free_count >= callback->count) {
205 callback->next = NULL;
206 callback->fn(callback->arg);
208 callback->next = gnttab_free_callback_list;
209 gnttab_free_callback_list = callback;
215 static inline void check_free_callbacks(void)
217 if (unlikely(gnttab_free_callback_list))
221 static void put_free_entry(grant_ref_t ref)
224 spin_lock_irqsave(&gnttab_list_lock, flags);
225 gnttab_entry(ref) = gnttab_free_head;
226 gnttab_free_head = ref;
228 check_free_callbacks();
229 spin_unlock_irqrestore(&gnttab_list_lock, flags);
233 * Following applies to gnttab_update_entry_v1 and gnttab_update_entry_v2.
234 * Introducing a valid entry into the grant table:
235 * 1. Write ent->domid.
236 * 2. Write ent->frame:
237 * GTF_permit_access: Frame to which access is permitted.
238 * GTF_accept_transfer: Pseudo-phys frame slot being filled by new
239 * frame, or zero if none.
240 * 3. Write memory barrier (WMB).
241 * 4. Write ent->flags, inc. valid type.
243 static void gnttab_update_entry_v1(grant_ref_t ref, domid_t domid,
244 unsigned long frame, unsigned flags)
246 gnttab_shared.v1[ref].domid = domid;
247 gnttab_shared.v1[ref].frame = frame;
249 gnttab_shared.v1[ref].flags = flags;
252 static void gnttab_update_entry_v2(grant_ref_t ref, domid_t domid,
253 unsigned long frame, unsigned flags)
255 gnttab_shared.v2[ref].hdr.domid = domid;
256 gnttab_shared.v2[ref].full_page.frame = frame;
258 gnttab_shared.v2[ref].hdr.flags = GTF_permit_access | flags;
262 * Public grant-issuing interface functions
264 void gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid,
265 unsigned long frame, int readonly)
267 gnttab_interface->update_entry(ref, domid, frame,
268 GTF_permit_access | (readonly ? GTF_readonly : 0));
270 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_ref);
272 int gnttab_grant_foreign_access(domid_t domid, unsigned long frame,
277 ref = get_free_entries(1);
278 if (unlikely(ref < 0))
281 gnttab_grant_foreign_access_ref(ref, domid, frame, readonly);
285 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access);
287 static void gnttab_update_subpage_entry_v2(grant_ref_t ref, domid_t domid,
288 unsigned long frame, int flags,
289 unsigned page_off, unsigned length)
291 gnttab_shared.v2[ref].sub_page.frame = frame;
292 gnttab_shared.v2[ref].sub_page.page_off = page_off;
293 gnttab_shared.v2[ref].sub_page.length = length;
294 gnttab_shared.v2[ref].hdr.domid = domid;
296 gnttab_shared.v2[ref].hdr.flags =
297 GTF_permit_access | GTF_sub_page | flags;
300 int gnttab_grant_foreign_access_subpage_ref(grant_ref_t ref, domid_t domid,
301 unsigned long frame, int flags,
305 if (flags & (GTF_accept_transfer | GTF_reading |
306 GTF_writing | GTF_transitive))
309 if (gnttab_interface->update_subpage_entry == NULL)
312 gnttab_interface->update_subpage_entry(ref, domid, frame, flags,
317 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_subpage_ref);
319 int gnttab_grant_foreign_access_subpage(domid_t domid, unsigned long frame,
320 int flags, unsigned page_off,
325 ref = get_free_entries(1);
326 if (unlikely(ref < 0))
329 rc = gnttab_grant_foreign_access_subpage_ref(ref, domid, frame, flags,
338 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_subpage);
340 bool gnttab_subpage_grants_available(void)
342 return gnttab_interface->update_subpage_entry != NULL;
344 EXPORT_SYMBOL_GPL(gnttab_subpage_grants_available);
346 static void gnttab_update_trans_entry_v2(grant_ref_t ref, domid_t domid,
347 int flags, domid_t trans_domid,
348 grant_ref_t trans_gref)
350 gnttab_shared.v2[ref].transitive.trans_domid = trans_domid;
351 gnttab_shared.v2[ref].transitive.gref = trans_gref;
352 gnttab_shared.v2[ref].hdr.domid = domid;
354 gnttab_shared.v2[ref].hdr.flags =
355 GTF_permit_access | GTF_transitive | flags;
358 int gnttab_grant_foreign_access_trans_ref(grant_ref_t ref, domid_t domid,
359 int flags, domid_t trans_domid,
360 grant_ref_t trans_gref)
362 if (flags & (GTF_accept_transfer | GTF_reading |
363 GTF_writing | GTF_sub_page))
366 if (gnttab_interface->update_trans_entry == NULL)
369 gnttab_interface->update_trans_entry(ref, domid, flags, trans_domid,
374 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_trans_ref);
376 int gnttab_grant_foreign_access_trans(domid_t domid, int flags,
378 grant_ref_t trans_gref)
382 ref = get_free_entries(1);
383 if (unlikely(ref < 0))
386 rc = gnttab_grant_foreign_access_trans_ref(ref, domid, flags,
387 trans_domid, trans_gref);
395 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_trans);
397 bool gnttab_trans_grants_available(void)
399 return gnttab_interface->update_trans_entry != NULL;
401 EXPORT_SYMBOL_GPL(gnttab_trans_grants_available);
403 static int gnttab_query_foreign_access_v1(grant_ref_t ref)
405 return gnttab_shared.v1[ref].flags & (GTF_reading|GTF_writing);
408 static int gnttab_query_foreign_access_v2(grant_ref_t ref)
410 return grstatus[ref] & (GTF_reading|GTF_writing);
413 int gnttab_query_foreign_access(grant_ref_t ref)
415 return gnttab_interface->query_foreign_access(ref);
417 EXPORT_SYMBOL_GPL(gnttab_query_foreign_access);
419 static int gnttab_end_foreign_access_ref_v1(grant_ref_t ref, int readonly)
424 pflags = &gnttab_shared.v1[ref].flags;
428 if (flags & (GTF_reading|GTF_writing))
430 } while ((nflags = sync_cmpxchg(pflags, flags, 0)) != flags);
435 static int gnttab_end_foreign_access_ref_v2(grant_ref_t ref, int readonly)
437 gnttab_shared.v2[ref].hdr.flags = 0;
439 if (grstatus[ref] & (GTF_reading|GTF_writing)) {
442 /* The read of grstatus needs to have acquire
443 semantics. On x86, reads already have
444 that, and we just need to protect against
445 compiler reorderings. On other
446 architectures we may need a full
458 static inline int _gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly)
460 return gnttab_interface->end_foreign_access_ref(ref, readonly);
463 int gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly)
465 if (_gnttab_end_foreign_access_ref(ref, readonly))
467 pr_warn("WARNING: g.e. %#x still in use!\n", ref);
470 EXPORT_SYMBOL_GPL(gnttab_end_foreign_access_ref);
472 struct deferred_entry {
473 struct list_head list;
479 static LIST_HEAD(deferred_list);
480 static void gnttab_handle_deferred(unsigned long);
481 static DEFINE_TIMER(deferred_timer, gnttab_handle_deferred, 0, 0);
483 static void gnttab_handle_deferred(unsigned long unused)
485 unsigned int nr = 10;
486 struct deferred_entry *first = NULL;
489 spin_lock_irqsave(&gnttab_list_lock, flags);
491 struct deferred_entry *entry
492 = list_first_entry(&deferred_list,
493 struct deferred_entry, list);
497 list_del(&entry->list);
498 spin_unlock_irqrestore(&gnttab_list_lock, flags);
499 if (_gnttab_end_foreign_access_ref(entry->ref, entry->ro)) {
500 put_free_entry(entry->ref);
502 pr_debug("freeing g.e. %#x (pfn %#lx)\n",
503 entry->ref, page_to_pfn(entry->page));
504 __free_page(entry->page);
506 pr_info("freeing g.e. %#x\n", entry->ref);
510 if (!--entry->warn_delay)
511 pr_info("g.e. %#x still pending\n",
516 spin_lock_irqsave(&gnttab_list_lock, flags);
518 list_add_tail(&entry->list, &deferred_list);
519 else if (list_empty(&deferred_list))
522 if (!list_empty(&deferred_list) && !timer_pending(&deferred_timer)) {
523 deferred_timer.expires = jiffies + HZ;
524 add_timer(&deferred_timer);
526 spin_unlock_irqrestore(&gnttab_list_lock, flags);
529 static void gnttab_add_deferred(grant_ref_t ref, bool readonly,
532 struct deferred_entry *entry = kmalloc(sizeof(*entry), GFP_ATOMIC);
533 const char *what = KERN_WARNING "leaking";
539 entry->ro = readonly;
541 entry->warn_delay = 60;
542 spin_lock_irqsave(&gnttab_list_lock, flags);
543 list_add_tail(&entry->list, &deferred_list);
544 if (!timer_pending(&deferred_timer)) {
545 deferred_timer.expires = jiffies + HZ;
546 add_timer(&deferred_timer);
548 spin_unlock_irqrestore(&gnttab_list_lock, flags);
549 what = KERN_DEBUG "deferring";
551 printk("%s g.e. %#x (pfn %#lx)\n",
552 what, ref, page ? page_to_pfn(page) : -1);
555 void gnttab_end_foreign_access(grant_ref_t ref, int readonly,
558 if (gnttab_end_foreign_access_ref(ref, readonly)) {
563 gnttab_add_deferred(ref, readonly,
564 page ? virt_to_page(page) : NULL);
566 EXPORT_SYMBOL_GPL(gnttab_end_foreign_access);
568 int gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn)
572 ref = get_free_entries(1);
573 if (unlikely(ref < 0))
575 gnttab_grant_foreign_transfer_ref(ref, domid, pfn);
579 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer);
581 void gnttab_grant_foreign_transfer_ref(grant_ref_t ref, domid_t domid,
584 gnttab_interface->update_entry(ref, domid, pfn, GTF_accept_transfer);
586 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer_ref);
588 static unsigned long gnttab_end_foreign_transfer_ref_v1(grant_ref_t ref)
594 pflags = &gnttab_shared.v1[ref].flags;
597 * If a transfer is not even yet started, try to reclaim the grant
598 * reference and return failure (== 0).
600 while (!((flags = *pflags) & GTF_transfer_committed)) {
601 if (sync_cmpxchg(pflags, flags, 0) == flags)
606 /* If a transfer is in progress then wait until it is completed. */
607 while (!(flags & GTF_transfer_completed)) {
612 rmb(); /* Read the frame number /after/ reading completion status. */
613 frame = gnttab_shared.v1[ref].frame;
619 static unsigned long gnttab_end_foreign_transfer_ref_v2(grant_ref_t ref)
625 pflags = &gnttab_shared.v2[ref].hdr.flags;
628 * If a transfer is not even yet started, try to reclaim the grant
629 * reference and return failure (== 0).
631 while (!((flags = *pflags) & GTF_transfer_committed)) {
632 if (sync_cmpxchg(pflags, flags, 0) == flags)
637 /* If a transfer is in progress then wait until it is completed. */
638 while (!(flags & GTF_transfer_completed)) {
643 rmb(); /* Read the frame number /after/ reading completion status. */
644 frame = gnttab_shared.v2[ref].full_page.frame;
650 unsigned long gnttab_end_foreign_transfer_ref(grant_ref_t ref)
652 return gnttab_interface->end_foreign_transfer_ref(ref);
654 EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer_ref);
656 unsigned long gnttab_end_foreign_transfer(grant_ref_t ref)
658 unsigned long frame = gnttab_end_foreign_transfer_ref(ref);
662 EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer);
664 void gnttab_free_grant_reference(grant_ref_t ref)
668 EXPORT_SYMBOL_GPL(gnttab_free_grant_reference);
670 void gnttab_free_grant_references(grant_ref_t head)
675 if (head == GNTTAB_LIST_END)
677 spin_lock_irqsave(&gnttab_list_lock, flags);
679 while (gnttab_entry(ref) != GNTTAB_LIST_END) {
680 ref = gnttab_entry(ref);
683 gnttab_entry(ref) = gnttab_free_head;
684 gnttab_free_head = head;
685 gnttab_free_count += count;
686 check_free_callbacks();
687 spin_unlock_irqrestore(&gnttab_list_lock, flags);
689 EXPORT_SYMBOL_GPL(gnttab_free_grant_references);
691 int gnttab_alloc_grant_references(u16 count, grant_ref_t *head)
693 int h = get_free_entries(count);
702 EXPORT_SYMBOL_GPL(gnttab_alloc_grant_references);
704 int gnttab_empty_grant_references(const grant_ref_t *private_head)
706 return (*private_head == GNTTAB_LIST_END);
708 EXPORT_SYMBOL_GPL(gnttab_empty_grant_references);
710 int gnttab_claim_grant_reference(grant_ref_t *private_head)
712 grant_ref_t g = *private_head;
713 if (unlikely(g == GNTTAB_LIST_END))
715 *private_head = gnttab_entry(g);
718 EXPORT_SYMBOL_GPL(gnttab_claim_grant_reference);
720 void gnttab_release_grant_reference(grant_ref_t *private_head,
723 gnttab_entry(release) = *private_head;
724 *private_head = release;
726 EXPORT_SYMBOL_GPL(gnttab_release_grant_reference);
728 void gnttab_request_free_callback(struct gnttab_free_callback *callback,
729 void (*fn)(void *), void *arg, u16 count)
732 struct gnttab_free_callback *cb;
734 spin_lock_irqsave(&gnttab_list_lock, flags);
736 /* Check if the callback is already on the list */
737 cb = gnttab_free_callback_list;
746 callback->count = count;
747 callback->next = gnttab_free_callback_list;
748 gnttab_free_callback_list = callback;
749 check_free_callbacks();
751 spin_unlock_irqrestore(&gnttab_list_lock, flags);
753 EXPORT_SYMBOL_GPL(gnttab_request_free_callback);
755 void gnttab_cancel_free_callback(struct gnttab_free_callback *callback)
757 struct gnttab_free_callback **pcb;
760 spin_lock_irqsave(&gnttab_list_lock, flags);
761 for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) {
762 if (*pcb == callback) {
763 *pcb = callback->next;
767 spin_unlock_irqrestore(&gnttab_list_lock, flags);
769 EXPORT_SYMBOL_GPL(gnttab_cancel_free_callback);
771 static int grow_gnttab_list(unsigned int more_frames)
773 unsigned int new_nr_grant_frames, extra_entries, i;
774 unsigned int nr_glist_frames, new_nr_glist_frames;
776 BUG_ON(grefs_per_grant_frame == 0);
778 new_nr_grant_frames = nr_grant_frames + more_frames;
779 extra_entries = more_frames * grefs_per_grant_frame;
781 nr_glist_frames = (nr_grant_frames * grefs_per_grant_frame + RPP - 1) / RPP;
782 new_nr_glist_frames =
783 (new_nr_grant_frames * grefs_per_grant_frame + RPP - 1) / RPP;
784 for (i = nr_glist_frames; i < new_nr_glist_frames; i++) {
785 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_ATOMIC);
791 for (i = grefs_per_grant_frame * nr_grant_frames;
792 i < grefs_per_grant_frame * new_nr_grant_frames - 1; i++)
793 gnttab_entry(i) = i + 1;
795 gnttab_entry(i) = gnttab_free_head;
796 gnttab_free_head = grefs_per_grant_frame * nr_grant_frames;
797 gnttab_free_count += extra_entries;
799 nr_grant_frames = new_nr_grant_frames;
801 check_free_callbacks();
806 for ( ; i >= nr_glist_frames; i--)
807 free_page((unsigned long) gnttab_list[i]);
811 static unsigned int __max_nr_grant_frames(void)
813 struct gnttab_query_size query;
816 query.dom = DOMID_SELF;
818 rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1);
819 if ((rc < 0) || (query.status != GNTST_okay))
820 return 4; /* Legacy max supported number of frames */
822 return query.max_nr_frames;
825 unsigned int gnttab_max_grant_frames(void)
827 unsigned int xen_max = __max_nr_grant_frames();
829 if (xen_max > boot_max_nr_grant_frames)
830 return boot_max_nr_grant_frames;
833 EXPORT_SYMBOL_GPL(gnttab_max_grant_frames);
835 /* Handling of paged out grant targets (GNTST_eagain) */
836 #define MAX_DELAY 256
838 gnttab_retry_eagain_gop(unsigned int cmd, void *gop, int16_t *status,
844 BUG_ON(HYPERVISOR_grant_table_op(cmd, gop, 1));
845 if (*status == GNTST_eagain)
847 } while ((*status == GNTST_eagain) && (delay < MAX_DELAY));
849 if (delay >= MAX_DELAY) {
850 printk(KERN_ERR "%s: %s eagain grant\n", func, current->comm);
851 *status = GNTST_bad_page;
855 void gnttab_batch_map(struct gnttab_map_grant_ref *batch, unsigned count)
857 struct gnttab_map_grant_ref *op;
859 if (HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, batch, count))
861 for (op = batch; op < batch + count; op++)
862 if (op->status == GNTST_eagain)
863 gnttab_retry_eagain_gop(GNTTABOP_map_grant_ref, op,
864 &op->status, __func__);
866 EXPORT_SYMBOL_GPL(gnttab_batch_map);
868 void gnttab_batch_copy(struct gnttab_copy *batch, unsigned count)
870 struct gnttab_copy *op;
872 if (HYPERVISOR_grant_table_op(GNTTABOP_copy, batch, count))
874 for (op = batch; op < batch + count; op++)
875 if (op->status == GNTST_eagain)
876 gnttab_retry_eagain_gop(GNTTABOP_copy, op,
877 &op->status, __func__);
879 EXPORT_SYMBOL_GPL(gnttab_batch_copy);
881 int gnttab_map_refs(struct gnttab_map_grant_ref *map_ops,
882 struct gnttab_map_grant_ref *kmap_ops,
883 struct page **pages, unsigned int count)
890 ret = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, map_ops, count);
894 /* Retry eagain maps */
895 for (i = 0; i < count; i++)
896 if (map_ops[i].status == GNTST_eagain)
897 gnttab_retry_eagain_gop(GNTTABOP_map_grant_ref, map_ops + i,
898 &map_ops[i].status, __func__);
900 if (xen_feature(XENFEAT_auto_translated_physmap))
903 if (!in_interrupt() && paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) {
904 arch_enter_lazy_mmu_mode();
908 for (i = 0; i < count; i++) {
909 /* Do not add to override if the map failed. */
910 if (map_ops[i].status)
913 if (map_ops[i].flags & GNTMAP_contains_pte) {
914 pte = (pte_t *) (mfn_to_virt(PFN_DOWN(map_ops[i].host_addr)) +
915 (map_ops[i].host_addr & ~PAGE_MASK));
918 mfn = PFN_DOWN(map_ops[i].dev_bus_addr);
920 ret = m2p_add_override(mfn, pages[i], kmap_ops ?
921 &kmap_ops[i] : NULL);
928 arch_leave_lazy_mmu_mode();
932 EXPORT_SYMBOL_GPL(gnttab_map_refs);
934 int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops,
935 struct gnttab_map_grant_ref *kmap_ops,
936 struct page **pages, unsigned int count)
941 ret = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, unmap_ops, count);
945 if (xen_feature(XENFEAT_auto_translated_physmap))
948 if (!in_interrupt() && paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) {
949 arch_enter_lazy_mmu_mode();
953 for (i = 0; i < count; i++) {
954 ret = m2p_remove_override(pages[i], kmap_ops ?
955 &kmap_ops[i] : NULL);
962 arch_leave_lazy_mmu_mode();
966 EXPORT_SYMBOL_GPL(gnttab_unmap_refs);
968 static unsigned nr_status_frames(unsigned nr_grant_frames)
970 BUG_ON(grefs_per_grant_frame == 0);
971 return (nr_grant_frames * grefs_per_grant_frame + SPP - 1) / SPP;
974 static int gnttab_map_frames_v1(xen_pfn_t *frames, unsigned int nr_gframes)
978 rc = arch_gnttab_map_shared(frames, nr_gframes,
979 gnttab_max_grant_frames(),
980 &gnttab_shared.addr);
986 static void gnttab_unmap_frames_v1(void)
988 arch_gnttab_unmap(gnttab_shared.addr, nr_grant_frames);
991 static int gnttab_map_frames_v2(xen_pfn_t *frames, unsigned int nr_gframes)
994 unsigned int nr_sframes;
995 struct gnttab_get_status_frames getframes;
998 nr_sframes = nr_status_frames(nr_gframes);
1000 /* No need for kzalloc as it is initialized in following hypercall
1001 * GNTTABOP_get_status_frames.
1003 sframes = kmalloc(nr_sframes * sizeof(uint64_t), GFP_ATOMIC);
1007 getframes.dom = DOMID_SELF;
1008 getframes.nr_frames = nr_sframes;
1009 set_xen_guest_handle(getframes.frame_list, sframes);
1011 rc = HYPERVISOR_grant_table_op(GNTTABOP_get_status_frames,
1013 if (rc == -ENOSYS) {
1018 BUG_ON(rc || getframes.status);
1020 rc = arch_gnttab_map_status(sframes, nr_sframes,
1021 nr_status_frames(gnttab_max_grant_frames()),
1026 rc = arch_gnttab_map_shared(frames, nr_gframes,
1027 gnttab_max_grant_frames(),
1028 &gnttab_shared.addr);
1034 static void gnttab_unmap_frames_v2(void)
1036 arch_gnttab_unmap(gnttab_shared.addr, nr_grant_frames);
1037 arch_gnttab_unmap(grstatus, nr_status_frames(nr_grant_frames));
1040 static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
1042 struct gnttab_setup_table setup;
1044 unsigned int nr_gframes = end_idx + 1;
1047 if (xen_hvm_domain()) {
1048 struct xen_add_to_physmap xatp;
1049 unsigned int i = end_idx;
1052 * Loop backwards, so that the first hypercall has the largest
1053 * index, ensuring that the table will grow only once.
1056 xatp.domid = DOMID_SELF;
1058 xatp.space = XENMAPSPACE_grant_table;
1059 xatp.gpfn = (xen_hvm_resume_frames >> PAGE_SHIFT) + i;
1060 rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp);
1063 "grant table add_to_physmap failed, err=%d\n", rc);
1066 } while (i-- > start_idx);
1071 /* No need for kzalloc as it is initialized in following hypercall
1072 * GNTTABOP_setup_table.
1074 frames = kmalloc(nr_gframes * sizeof(unsigned long), GFP_ATOMIC);
1078 setup.dom = DOMID_SELF;
1079 setup.nr_frames = nr_gframes;
1080 set_xen_guest_handle(setup.frame_list, frames);
1082 rc = HYPERVISOR_grant_table_op(GNTTABOP_setup_table, &setup, 1);
1083 if (rc == -ENOSYS) {
1088 BUG_ON(rc || setup.status);
1090 rc = gnttab_interface->map_frames(frames, nr_gframes);
1097 static struct gnttab_ops gnttab_v1_ops = {
1098 .map_frames = gnttab_map_frames_v1,
1099 .unmap_frames = gnttab_unmap_frames_v1,
1100 .update_entry = gnttab_update_entry_v1,
1101 .end_foreign_access_ref = gnttab_end_foreign_access_ref_v1,
1102 .end_foreign_transfer_ref = gnttab_end_foreign_transfer_ref_v1,
1103 .query_foreign_access = gnttab_query_foreign_access_v1,
1106 static struct gnttab_ops gnttab_v2_ops = {
1107 .map_frames = gnttab_map_frames_v2,
1108 .unmap_frames = gnttab_unmap_frames_v2,
1109 .update_entry = gnttab_update_entry_v2,
1110 .end_foreign_access_ref = gnttab_end_foreign_access_ref_v2,
1111 .end_foreign_transfer_ref = gnttab_end_foreign_transfer_ref_v2,
1112 .query_foreign_access = gnttab_query_foreign_access_v2,
1113 .update_subpage_entry = gnttab_update_subpage_entry_v2,
1114 .update_trans_entry = gnttab_update_trans_entry_v2,
1117 static void gnttab_request_version(void)
1120 struct gnttab_set_version gsv;
1122 if (xen_hvm_domain())
1126 rc = HYPERVISOR_grant_table_op(GNTTABOP_set_version, &gsv, 1);
1127 if (rc == 0 && gsv.version == 2) {
1128 grant_table_version = 2;
1129 grefs_per_grant_frame = PAGE_SIZE / sizeof(union grant_entry_v2);
1130 gnttab_interface = &gnttab_v2_ops;
1131 } else if (grant_table_version == 2) {
1133 * If we've already used version 2 features,
1134 * but then suddenly discover that they're not
1135 * available (e.g. migrating to an older
1136 * version of Xen), almost unbounded badness
1139 panic("we need grant tables version 2, but only version 1 is available");
1141 grant_table_version = 1;
1142 grefs_per_grant_frame = PAGE_SIZE / sizeof(struct grant_entry_v1);
1143 gnttab_interface = &gnttab_v1_ops;
1145 printk(KERN_INFO "Grant tables using version %d layout.\n",
1146 grant_table_version);
1149 static int gnttab_setup(void)
1151 unsigned int max_nr_gframes;
1153 max_nr_gframes = gnttab_max_grant_frames();
1154 if (max_nr_gframes < nr_grant_frames)
1157 if (xen_pv_domain())
1158 return gnttab_map(0, nr_grant_frames - 1);
1160 if (gnttab_shared.addr == NULL) {
1161 gnttab_shared.addr = xen_remap(xen_hvm_resume_frames,
1162 PAGE_SIZE * max_nr_gframes);
1163 if (gnttab_shared.addr == NULL) {
1165 "Failed to ioremap gnttab share frames!");
1170 gnttab_map(0, nr_grant_frames - 1);
1175 int gnttab_resume(void)
1177 gnttab_request_version();
1178 return gnttab_setup();
1181 int gnttab_suspend(void)
1183 gnttab_interface->unmap_frames();
1187 static int gnttab_expand(unsigned int req_entries)
1190 unsigned int cur, extra;
1192 BUG_ON(grefs_per_grant_frame == 0);
1193 cur = nr_grant_frames;
1194 extra = ((req_entries + (grefs_per_grant_frame-1)) /
1195 grefs_per_grant_frame);
1196 if (cur + extra > gnttab_max_grant_frames())
1199 rc = gnttab_map(cur, cur + extra - 1);
1201 rc = grow_gnttab_list(extra);
1206 int gnttab_init(void)
1209 unsigned int max_nr_glist_frames, nr_glist_frames;
1210 unsigned int nr_init_grefs;
1213 gnttab_request_version();
1214 nr_grant_frames = 1;
1215 boot_max_nr_grant_frames = __max_nr_grant_frames();
1217 /* Determine the maximum number of frames required for the
1218 * grant reference free list on the current hypervisor.
1220 BUG_ON(grefs_per_grant_frame == 0);
1221 max_nr_glist_frames = (boot_max_nr_grant_frames *
1222 grefs_per_grant_frame / RPP);
1224 gnttab_list = kmalloc(max_nr_glist_frames * sizeof(grant_ref_t *),
1226 if (gnttab_list == NULL)
1229 nr_glist_frames = (nr_grant_frames * grefs_per_grant_frame + RPP - 1) / RPP;
1230 for (i = 0; i < nr_glist_frames; i++) {
1231 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_KERNEL);
1232 if (gnttab_list[i] == NULL) {
1238 if (gnttab_setup() < 0) {
1243 nr_init_grefs = nr_grant_frames * grefs_per_grant_frame;
1245 for (i = NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++)
1246 gnttab_entry(i) = i + 1;
1248 gnttab_entry(nr_init_grefs - 1) = GNTTAB_LIST_END;
1249 gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES;
1250 gnttab_free_head = NR_RESERVED_ENTRIES;
1252 printk("Grant table initialized\n");
1256 for (i--; i >= 0; i--)
1257 free_page((unsigned long)gnttab_list[i]);
1261 EXPORT_SYMBOL_GPL(gnttab_init);
1263 static int __gnttab_init(void)
1265 /* Delay grant-table initialization in the PV on HVM case */
1266 if (xen_hvm_domain())
1269 if (!xen_pv_domain())
1272 return gnttab_init();
1275 core_initcall(__gnttab_init);