#include <linux/delay.h>
#include <linux/freezer.h>
-#include <xen/balloon.h>
#include <xen/events.h>
#include <xen/page.h>
#include <asm/xen/hypervisor.h>
#include <asm/xen/hypercall.h>
#include "common.h"
+#define WRITE_BARRIER (REQ_WRITE | REQ_FLUSH | REQ_FUA)
+
/*
* These are rather arbitrary. They are fairly large because adjacent requests
* pulled from a communication ring are quite likely to end up being part of
struct list_head free_list;
} pending_req_t;
-static pending_req_t *pending_reqs;
-static struct list_head pending_free;
-static DEFINE_SPINLOCK(pending_free_lock);
-static DECLARE_WAIT_QUEUE_HEAD(pending_free_wq);
-
#define BLKBACK_INVALID_HANDLE (~0)
-static struct page **pending_pages;
-static grant_handle_t *pending_grant_handles;
+struct xen_blkbk {
+ pending_req_t *pending_reqs;
+ struct list_head pending_free;
+ spinlock_t pending_free_lock;
+ wait_queue_head_t pending_free_wq;
+ struct page **pending_pages;
+ grant_handle_t *pending_grant_handles;
+};
+
+static struct xen_blkbk *blkbk;
static inline int vaddr_pagenr(pending_req_t *req, int seg)
{
- return (req - pending_reqs) * BLKIF_MAX_SEGMENTS_PER_REQUEST + seg;
+ return (req - blkbk->pending_reqs) * BLKIF_MAX_SEGMENTS_PER_REQUEST + seg;
}
+#define pending_page(req, seg) pending_pages[vaddr_pagenr(req, seg)]
+
static inline unsigned long vaddr(pending_req_t *req, int seg)
{
- unsigned long pfn = page_to_pfn(pending_pages[vaddr_pagenr(req, seg)]);
+ unsigned long pfn = page_to_pfn(blkbk->pending_page(req, seg));
return (unsigned long)pfn_to_kaddr(pfn);
}
#define pending_handle(_req, _seg) \
- (pending_grant_handles[vaddr_pagenr(_req, _seg)])
+ (blkbk->pending_grant_handles[vaddr_pagenr(_req, _seg)])
static int do_block_io_op(blkif_t *blkif);
pending_req_t *req = NULL;
unsigned long flags;
- spin_lock_irqsave(&pending_free_lock, flags);
- if (!list_empty(&pending_free)) {
- req = list_entry(pending_free.next, pending_req_t, free_list);
+ spin_lock_irqsave(&blkbk->pending_free_lock, flags);
+ if (!list_empty(&blkbk->pending_free)) {
+ req = list_entry(blkbk->pending_free.next, pending_req_t, free_list);
list_del(&req->free_list);
}
- spin_unlock_irqrestore(&pending_free_lock, flags);
+ spin_unlock_irqrestore(&blkbk->pending_free_lock, flags);
return req;
}
unsigned long flags;
int was_empty;
- spin_lock_irqsave(&pending_free_lock, flags);
- was_empty = list_empty(&pending_free);
- list_add(&req->free_list, &pending_free);
- spin_unlock_irqrestore(&pending_free_lock, flags);
+ spin_lock_irqsave(&blkbk->pending_free_lock, flags);
+ was_empty = list_empty(&blkbk->pending_free);
+ list_add(&req->free_list, &blkbk->pending_free);
+ spin_unlock_irqrestore(&blkbk->pending_free_lock, flags);
if (was_empty)
- wake_up(&pending_free_wq);
+ wake_up(&blkbk->pending_free_wq);
}
static void unplug_queue(blkif_t *blkif)
ret = HYPERVISOR_grant_table_op(
GNTTABOP_unmap_grant_ref, unmap, invcount);
BUG_ON(ret);
+ /* Note, we use invcount, so nr->pages, so we can't index
+ * using vaddr(req, i). */
+ for (i = 0; i < invcount; i++) {
+ ret = m2p_remove_override(
+ virt_to_page(unmap[i].host_addr), false);
+ if (ret) {
+ printk(KERN_ALERT "Failed to remove M2P override for " \
+ "%lx\n", (unsigned long)unmap[i].host_addr);
+ continue;
+ }
+ }
}
/******************************************************************
blkif->wq,
blkif->waiting_reqs || kthread_should_stop());
wait_event_interruptible(
- pending_free_wq,
- !list_empty(&pending_free) || kthread_should_stop());
+ blkbk->pending_free_wq,
+ !list_empty(&blkbk->pending_free) || kthread_should_stop());
blkif->waiting_reqs = 0;
smp_mb(); /* clear flag *before* checking for work */
}
preq.dev = req->handle;
- preq.sector_number = req->sector_number;
+ preq.sector_number = req->u.rw.sector_number;
preq.nr_sects = 0;
pending_req->blkif = blkif;
for (i = 0; i < nseg; i++) {
uint32_t flags;
- seg[i].nsec = req->seg[i].last_sect -
- req->seg[i].first_sect + 1;
+ seg[i].nsec = req->u.rw.seg[i].last_sect -
+ req->u.rw.seg[i].first_sect + 1;
- if ((req->seg[i].last_sect >= (PAGE_SIZE >> 9)) ||
- (req->seg[i].last_sect < req->seg[i].first_sect))
+ if ((req->u.rw.seg[i].last_sect >= (PAGE_SIZE >> 9)) ||
+ (req->u.rw.seg[i].last_sect < req->u.rw.seg[i].first_sect))
goto fail_response;
preq.nr_sects += seg[i].nsec;
if (operation != READ)
flags |= GNTMAP_readonly;
gnttab_set_map_op(&map[i], vaddr(pending_req, i), flags,
- req->seg[i].gref, blkif->domid);
+ req->u.rw.seg[i].gref, blkif->domid);
}
ret = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, map, nseg);
if (ret)
continue;
+
+ ret = m2p_add_override(PFN_DOWN(map[i].dev_bus_addr),
+ blkbk->pending_page(pending_req, i), false);
+ if (ret) {
+ printk(KERN_ALERT "Failed to install M2P override for"\
+ " %lx (ret: %d)\n", (unsigned long)map[i].dev_bus_addr, ret);
+ continue;
+ }
- set_phys_to_machine(__pa(vaddr(
- pending_req, i)) >> PAGE_SHIFT,
- FOREIGN_FRAME(map[i].dev_bus_addr >> PAGE_SHIFT));
seg[i].buf = map[i].dev_bus_addr |
- (req->seg[i].first_sect << 9);
+ (req->u.rw.seg[i].first_sect << 9);
}
if (ret)
while ((bio == NULL) ||
(bio_add_page(bio,
- virt_to_page(vaddr(pending_req, i)),
+ blkbk->pending_page(pending_req, i),
seg[i].nsec << 9,
seg[i].buf & ~PAGE_MASK) == 0)) {
if (bio) {
if (!xen_pv_domain())
return -ENODEV;
+ blkbk = (struct xen_blkbk *)kzalloc(sizeof(struct xen_blkbk), GFP_KERNEL);
+ if (!blkbk) {
+ printk(KERN_ALERT "%s: out of memory!\n", __func__);
+ return -ENOMEM;
+ }
+
mmap_pages = blkif_reqs * BLKIF_MAX_SEGMENTS_PER_REQUEST;
- pending_reqs = kmalloc(sizeof(pending_reqs[0]) *
+ blkbk->pending_reqs = kmalloc(sizeof(blkbk->pending_reqs[0]) *
blkif_reqs, GFP_KERNEL);
- pending_grant_handles = kmalloc(sizeof(pending_grant_handles[0]) *
+ blkbk->pending_grant_handles = kzalloc(sizeof(blkbk->pending_grant_handles[0]) *
+ mmap_pages, GFP_KERNEL);
+ blkbk->pending_pages = kzalloc(sizeof(blkbk->pending_pages[0]) *
mmap_pages, GFP_KERNEL);
- pending_pages = alloc_empty_pages_and_pagevec(mmap_pages);
- if (!pending_reqs || !pending_grant_handles || !pending_pages) {
+ if (!blkbk->pending_reqs || !blkbk->pending_grant_handles || !blkbk->pending_pages) {
rc = -ENOMEM;
goto out_of_memory;
}
- for (i = 0; i < mmap_pages; i++)
- pending_grant_handles[i] = BLKBACK_INVALID_HANDLE;
-
+ for (i = 0; i < mmap_pages; i++) {
+ blkbk->pending_grant_handles[i] = BLKBACK_INVALID_HANDLE;
+ blkbk->pending_pages[i] = alloc_page(GFP_KERNEL);
+ if (blkbk->pending_pages[i] == NULL) {
+ rc = -ENOMEM;
+ goto out_of_memory;
+ }
+ }
rc = blkif_interface_init();
if (rc)
goto failed_init;
- memset(pending_reqs, 0, sizeof(pending_reqs));
- INIT_LIST_HEAD(&pending_free);
+ memset(blkbk->pending_reqs, 0, sizeof(blkbk->pending_reqs));
+
+ INIT_LIST_HEAD(&blkbk->pending_free);
+ spin_lock_init(&blkbk->pending_free_lock);
+ init_waitqueue_head(&blkbk->pending_free_wq);
for (i = 0; i < blkif_reqs; i++)
- list_add_tail(&pending_reqs[i].free_list, &pending_free);
+ list_add_tail(&blkbk->pending_reqs[i].free_list, &blkbk->pending_free);
rc = blkif_xenbus_init();
if (rc)
out_of_memory:
printk(KERN_ERR "%s: out of memory\n", __func__);
failed_init:
- kfree(pending_reqs);
- kfree(pending_grant_handles);
- free_empty_pages_and_pagevec(pending_pages, mmap_pages);
+ kfree(blkbk->pending_reqs);
+ kfree(blkbk->pending_grant_handles);
+ for (i = 0; i < mmap_pages; i++) {
+ if (blkbk->pending_pages[i])
+ __free_page(blkbk->pending_pages[i]);
+ }
+ kfree(blkbk->pending_pages);
+ kfree(blkbk);
+ blkbk = NULL;
return rc;
}
projects / firefly-linux-kernel-4.4.55.git / blobdiff