2 * IBM eServer eHCA Infiniband device driver for Linux on POWER
4 * Completion queue handling
6 * Authors: Waleri Fomin <fomin@de.ibm.com>
7 * Khadija Souissi <souissi@de.ibm.com>
8 * Reinhard Ernst <rernst@de.ibm.com>
9 * Heiko J Schick <schickhj@de.ibm.com>
10 * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
13 * Copyright (c) 2005 IBM Corporation
15 * All rights reserved.
17 * This source code is distributed under a dual license of GPL v2.0 and OpenIB
22 * Redistribution and use in source and binary forms, with or without
23 * modification, are permitted provided that the following conditions are met:
25 * Redistributions of source code must retain the above copyright notice, this
26 * list of conditions and the following disclaimer.
28 * Redistributions in binary form must reproduce the above copyright notice,
29 * this list of conditions and the following disclaimer in the documentation
30 * and/or other materials
31 * provided with the distribution.
33 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
34 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
35 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
36 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
37 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
38 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
39 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
40 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
41 * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
42 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
43 * POSSIBILITY OF SUCH DAMAGE.
46 #include <linux/slab.h>
48 #include "ehca_iverbs.h"
49 #include "ehca_classes.h"
53 static struct kmem_cache *cq_cache;
55 int ehca_cq_assign_qp(struct ehca_cq *cq, struct ehca_qp *qp)
57 unsigned int qp_num = qp->real_qp_num;
58 unsigned int key = qp_num & (QP_HASHTAB_LEN-1);
61 spin_lock_irqsave(&cq->spinlock, flags);
62 hlist_add_head(&qp->list_entries, &cq->qp_hashtab[key]);
63 spin_unlock_irqrestore(&cq->spinlock, flags);
65 ehca_dbg(cq->ib_cq.device, "cq_num=%x real_qp_num=%x",
66 cq->cq_number, qp_num);
71 int ehca_cq_unassign_qp(struct ehca_cq *cq, unsigned int real_qp_num)
74 unsigned int key = real_qp_num & (QP_HASHTAB_LEN-1);
75 struct hlist_node *iter;
79 spin_lock_irqsave(&cq->spinlock, flags);
80 hlist_for_each(iter, &cq->qp_hashtab[key]) {
81 qp = hlist_entry(iter, struct ehca_qp, list_entries);
82 if (qp->real_qp_num == real_qp_num) {
84 ehca_dbg(cq->ib_cq.device,
85 "removed qp from cq .cq_num=%x real_qp_num=%x",
86 cq->cq_number, real_qp_num);
91 spin_unlock_irqrestore(&cq->spinlock, flags);
93 ehca_err(cq->ib_cq.device,
94 "qp not found cq_num=%x real_qp_num=%x",
95 cq->cq_number, real_qp_num);
100 struct ehca_qp *ehca_cq_get_qp(struct ehca_cq *cq, int real_qp_num)
102 struct ehca_qp *ret = NULL;
103 unsigned int key = real_qp_num & (QP_HASHTAB_LEN-1);
104 struct hlist_node *iter;
106 hlist_for_each(iter, &cq->qp_hashtab[key]) {
107 qp = hlist_entry(iter, struct ehca_qp, list_entries);
108 if (qp->real_qp_num == real_qp_num) {
116 struct ib_cq *ehca_create_cq(struct ib_device *device,
117 const struct ib_cq_init_attr *attr,
118 struct ib_ucontext *context,
119 struct ib_udata *udata)
122 static const u32 additional_cqe = 20;
124 struct ehca_cq *my_cq;
125 struct ehca_shca *shca =
126 container_of(device, struct ehca_shca, ib_device);
127 struct ipz_adapter_handle adapter_handle;
128 struct ehca_alloc_cq_parms param; /* h_call's out parameters */
132 u64 rpage, cqx_fec, h_ret;
137 return ERR_PTR(-EINVAL);
139 if (cqe >= 0xFFFFFFFF - 64 - additional_cqe)
140 return ERR_PTR(-EINVAL);
142 if (!atomic_add_unless(&shca->num_cqs, 1, shca->max_num_cqs)) {
143 ehca_err(device, "Unable to create CQ, max number of %i "
144 "CQs reached.", shca->max_num_cqs);
145 ehca_err(device, "To increase the maximum number of CQs "
146 "use the number_of_cqs module parameter.\n");
147 return ERR_PTR(-ENOSPC);
150 my_cq = kmem_cache_zalloc(cq_cache, GFP_KERNEL);
152 ehca_err(device, "Out of memory for ehca_cq struct device=%p",
154 atomic_dec(&shca->num_cqs);
155 return ERR_PTR(-ENOMEM);
158 memset(¶m, 0, sizeof(struct ehca_alloc_cq_parms));
160 spin_lock_init(&my_cq->spinlock);
161 spin_lock_init(&my_cq->cb_lock);
162 spin_lock_init(&my_cq->task_lock);
163 atomic_set(&my_cq->nr_events, 0);
164 init_waitqueue_head(&my_cq->wait_completion);
168 adapter_handle = shca->ipz_hca_handle;
169 param.eq_handle = shca->eq.ipz_eq_handle;
171 idr_preload(GFP_KERNEL);
172 write_lock_irqsave(&ehca_cq_idr_lock, flags);
173 my_cq->token = idr_alloc(&ehca_cq_idr, my_cq, 0, 0x2000000, GFP_NOWAIT);
174 write_unlock_irqrestore(&ehca_cq_idr_lock, flags);
177 if (my_cq->token < 0) {
178 cq = ERR_PTR(-ENOMEM);
179 ehca_err(device, "Can't allocate new idr entry. device=%p",
181 goto create_cq_exit1;
185 * CQs maximum depth is 4GB-64, but we need additional 20 as buffer
186 * for receiving errors CQEs.
188 param.nr_cqe = cqe + additional_cqe;
189 h_ret = hipz_h_alloc_resource_cq(adapter_handle, my_cq, ¶m);
191 if (h_ret != H_SUCCESS) {
192 ehca_err(device, "hipz_h_alloc_resource_cq() failed "
193 "h_ret=%lli device=%p", h_ret, device);
194 cq = ERR_PTR(ehca2ib_return_code(h_ret));
195 goto create_cq_exit2;
198 ipz_rc = ipz_queue_ctor(NULL, &my_cq->ipz_queue, param.act_pages,
199 EHCA_PAGESIZE, sizeof(struct ehca_cqe), 0, 0);
201 ehca_err(device, "ipz_queue_ctor() failed ipz_rc=%i device=%p",
203 cq = ERR_PTR(-EINVAL);
204 goto create_cq_exit3;
207 for (counter = 0; counter < param.act_pages; counter++) {
208 vpage = ipz_qpageit_get_inc(&my_cq->ipz_queue);
210 ehca_err(device, "ipz_qpageit_get_inc() "
211 "returns NULL device=%p", device);
212 cq = ERR_PTR(-EAGAIN);
213 goto create_cq_exit4;
217 h_ret = hipz_h_register_rpage_cq(adapter_handle,
218 my_cq->ipz_cq_handle,
227 if (h_ret < H_SUCCESS) {
228 ehca_err(device, "hipz_h_register_rpage_cq() failed "
229 "ehca_cq=%p cq_num=%x h_ret=%lli counter=%i "
230 "act_pages=%i", my_cq, my_cq->cq_number,
231 h_ret, counter, param.act_pages);
232 cq = ERR_PTR(-EINVAL);
233 goto create_cq_exit4;
236 if (counter == (param.act_pages - 1)) {
237 vpage = ipz_qpageit_get_inc(&my_cq->ipz_queue);
238 if ((h_ret != H_SUCCESS) || vpage) {
239 ehca_err(device, "Registration of pages not "
240 "complete ehca_cq=%p cq_num=%x "
241 "h_ret=%lli", my_cq, my_cq->cq_number,
243 cq = ERR_PTR(-EAGAIN);
244 goto create_cq_exit4;
247 if (h_ret != H_PAGE_REGISTERED) {
248 ehca_err(device, "Registration of page failed "
249 "ehca_cq=%p cq_num=%x h_ret=%lli "
250 "counter=%i act_pages=%i",
251 my_cq, my_cq->cq_number,
252 h_ret, counter, param.act_pages);
253 cq = ERR_PTR(-ENOMEM);
254 goto create_cq_exit4;
259 ipz_qeit_reset(&my_cq->ipz_queue);
261 gal = my_cq->galpas.kernel;
262 cqx_fec = hipz_galpa_load(gal, CQTEMM_OFFSET(cqx_fec));
263 ehca_dbg(device, "ehca_cq=%p cq_num=%x CQX_FEC=%llx",
264 my_cq, my_cq->cq_number, cqx_fec);
266 my_cq->ib_cq.cqe = my_cq->nr_of_entries =
267 param.act_nr_of_entries - additional_cqe;
268 my_cq->cq_number = (my_cq->ipz_cq_handle.handle) & 0xffff;
270 for (i = 0; i < QP_HASHTAB_LEN; i++)
271 INIT_HLIST_HEAD(&my_cq->qp_hashtab[i]);
273 INIT_LIST_HEAD(&my_cq->sqp_err_list);
274 INIT_LIST_HEAD(&my_cq->rqp_err_list);
277 struct ipz_queue *ipz_queue = &my_cq->ipz_queue;
278 struct ehca_create_cq_resp resp;
279 memset(&resp, 0, sizeof(resp));
280 resp.cq_number = my_cq->cq_number;
281 resp.token = my_cq->token;
282 resp.ipz_queue.qe_size = ipz_queue->qe_size;
283 resp.ipz_queue.act_nr_of_sg = ipz_queue->act_nr_of_sg;
284 resp.ipz_queue.queue_length = ipz_queue->queue_length;
285 resp.ipz_queue.pagesize = ipz_queue->pagesize;
286 resp.ipz_queue.toggle_state = ipz_queue->toggle_state;
287 resp.fw_handle_ofs = (u32)
288 (my_cq->galpas.user.fw_handle & (PAGE_SIZE - 1));
289 if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
290 ehca_err(device, "Copy to udata failed.");
291 cq = ERR_PTR(-EFAULT);
292 goto create_cq_exit4;
299 ipz_queue_dtor(NULL, &my_cq->ipz_queue);
302 h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 1);
303 if (h_ret != H_SUCCESS)
304 ehca_err(device, "hipz_h_destroy_cq() failed ehca_cq=%p "
305 "cq_num=%x h_ret=%lli", my_cq, my_cq->cq_number, h_ret);
308 write_lock_irqsave(&ehca_cq_idr_lock, flags);
309 idr_remove(&ehca_cq_idr, my_cq->token);
310 write_unlock_irqrestore(&ehca_cq_idr_lock, flags);
313 kmem_cache_free(cq_cache, my_cq);
315 atomic_dec(&shca->num_cqs);
319 int ehca_destroy_cq(struct ib_cq *cq)
322 struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
323 int cq_num = my_cq->cq_number;
324 struct ib_device *device = cq->device;
325 struct ehca_shca *shca = container_of(device, struct ehca_shca,
327 struct ipz_adapter_handle adapter_handle = shca->ipz_hca_handle;
331 if (my_cq->mm_count_galpa || my_cq->mm_count_queue) {
332 ehca_err(device, "Resources still referenced in "
333 "user space cq_num=%x", my_cq->cq_number);
339 * remove the CQ from the idr first to make sure
340 * no more interrupt tasklets will touch this CQ
342 write_lock_irqsave(&ehca_cq_idr_lock, flags);
343 idr_remove(&ehca_cq_idr, my_cq->token);
344 write_unlock_irqrestore(&ehca_cq_idr_lock, flags);
346 /* now wait until all pending events have completed */
347 wait_event(my_cq->wait_completion, !atomic_read(&my_cq->nr_events));
349 /* nobody's using our CQ any longer -- we can destroy it */
350 h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 0);
351 if (h_ret == H_R_STATE) {
352 /* cq in err: read err data and destroy it forcibly */
353 ehca_dbg(device, "ehca_cq=%p cq_num=%x resource=%llx in err "
354 "state. Try to delete it forcibly.",
355 my_cq, cq_num, my_cq->ipz_cq_handle.handle);
356 ehca_error_data(shca, my_cq, my_cq->ipz_cq_handle.handle);
357 h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 1);
358 if (h_ret == H_SUCCESS)
359 ehca_dbg(device, "cq_num=%x deleted successfully.",
362 if (h_ret != H_SUCCESS) {
363 ehca_err(device, "hipz_h_destroy_cq() failed h_ret=%lli "
364 "ehca_cq=%p cq_num=%x", h_ret, my_cq, cq_num);
365 return ehca2ib_return_code(h_ret);
367 ipz_queue_dtor(NULL, &my_cq->ipz_queue);
368 kmem_cache_free(cq_cache, my_cq);
370 atomic_dec(&shca->num_cqs);
374 int ehca_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata)
376 /* TODO: proper resize needs to be done */
377 ehca_err(cq->device, "not implemented yet");
382 int ehca_init_cq_cache(void)
384 cq_cache = kmem_cache_create("ehca_cache_cq",
385 sizeof(struct ehca_cq), 0,
393 void ehca_cleanup_cq_cache(void)
396 kmem_cache_destroy(cq_cache);