1 /*
2 * Broadcom NetXtreme-E RoCE driver.
3 *
4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5 * Broadcom refers to Broadcom Limited and/or its subsidiaries.
6 *
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * BSD license below:
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 *
17 * 1. Redistributions of source code must retain the above copyright
18 * notice, this list of conditions and the following disclaimer.
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in
21 * the documentation and/or other materials provided with the
22 * distribution.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 *
36 * Description: Fast Path Operators
37 */
39 #include <linux/interrupt.h>
40 #include <linux/spinlock.h>
41 #include <linux/sched.h>
42 #include <linux/slab.h>
43 #include <linux/pci.h>
44 #include <linux/prefetch.h>
46 #include "roce_hsi.h"
48 #include "qplib_res.h"
49 #include "qplib_rcfw.h"
50 #include "qplib_sp.h"
51 #include "qplib_fp.h"
53 static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq);
54 static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp);
55 static void bnxt_qplib_arm_srq(struct bnxt_qplib_srq *srq, u32 arm_type);
57 static void bnxt_qplib_cancel_phantom_processing(struct bnxt_qplib_qp *qp)
58 {
59 qp->sq.condition = false;
60 qp->sq.send_phantom = false;
61 qp->sq.single = false;
62 }
64 /* Flush list */
65 static void __bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp)
66 {
67 struct bnxt_qplib_cq *scq, *rcq;
69 scq = qp->scq;
70 rcq = qp->rcq;
72 if (!qp->sq.flushed) {
73 dev_dbg(&scq->hwq.pdev->dev,
74 "QPLIB: FP: Adding to SQ Flush list = %p",
75 qp);
76 bnxt_qplib_cancel_phantom_processing(qp);
77 list_add_tail(&qp->sq_flush, &scq->sqf_head);
78 qp->sq.flushed = true;
79 }
80 if (!qp->srq) {
81 if (!qp->rq.flushed) {
82 dev_dbg(&rcq->hwq.pdev->dev,
83 "QPLIB: FP: Adding to RQ Flush list = %p",
84 qp);
85 list_add_tail(&qp->rq_flush, &rcq->rqf_head);
86 qp->rq.flushed = true;
87 }
88 }
89 }
91 static void bnxt_qplib_acquire_cq_flush_locks(struct bnxt_qplib_qp *qp,
92 unsigned long *flags)
93 __acquires(&qp->scq->flush_lock) __acquires(&qp->rcq->flush_lock)
94 {
95 spin_lock_irqsave(&qp->scq->flush_lock, *flags);
96 if (qp->scq == qp->rcq)
97 __acquire(&qp->rcq->flush_lock);
98 else
99 spin_lock(&qp->rcq->flush_lock);
100 }
102 static void bnxt_qplib_release_cq_flush_locks(struct bnxt_qplib_qp *qp,
103 unsigned long *flags)
104 __releases(&qp->scq->flush_lock) __releases(&qp->rcq->flush_lock)
105 {
106 if (qp->scq == qp->rcq)
107 __release(&qp->rcq->flush_lock);
108 else
109 spin_unlock(&qp->rcq->flush_lock);
110 spin_unlock_irqrestore(&qp->scq->flush_lock, *flags);
111 }
113 void bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp)
114 {
115 unsigned long flags;
117 bnxt_qplib_acquire_cq_flush_locks(qp, &flags);
118 __bnxt_qplib_add_flush_qp(qp);
119 bnxt_qplib_release_cq_flush_locks(qp, &flags);
120 }
122 static void __bnxt_qplib_del_flush_qp(struct bnxt_qplib_qp *qp)
123 {
124 if (qp->sq.flushed) {
125 qp->sq.flushed = false;
126 list_del(&qp->sq_flush);
127 }
128 if (!qp->srq) {
129 if (qp->rq.flushed) {
130 qp->rq.flushed = false;
131 list_del(&qp->rq_flush);
132 }
133 }
134 }
136 void bnxt_qplib_clean_qp(struct bnxt_qplib_qp *qp)
137 {
138 unsigned long flags;
140 bnxt_qplib_acquire_cq_flush_locks(qp, &flags);
141 __clean_cq(qp->scq, (u64)(unsigned long)qp);
142 qp->sq.hwq.prod = 0;
143 qp->sq.hwq.cons = 0;
144 __clean_cq(qp->rcq, (u64)(unsigned long)qp);
145 qp->rq.hwq.prod = 0;
146 qp->rq.hwq.cons = 0;
148 __bnxt_qplib_del_flush_qp(qp);
149 bnxt_qplib_release_cq_flush_locks(qp, &flags);
150 }
152 static void bnxt_qpn_cqn_sched_task(struct work_struct *work)
153 {
154 struct bnxt_qplib_nq_work *nq_work =
155 container_of(work, struct bnxt_qplib_nq_work, work);
157 struct bnxt_qplib_cq *cq = nq_work->cq;
158 struct bnxt_qplib_nq *nq = nq_work->nq;
160 if (cq && nq) {
161 spin_lock_bh(&cq->compl_lock);
162 if (atomic_read(&cq->arm_state) && nq->cqn_handler) {
163 dev_dbg(&nq->pdev->dev,
164 "%s:Trigger cq = %p event nq = %p\n",
165 __func__, cq, nq);
166 nq->cqn_handler(nq, cq);
167 }
168 spin_unlock_bh(&cq->compl_lock);
169 }
170 kfree(nq_work);
171 }
173 static void bnxt_qplib_free_qp_hdr_buf(struct bnxt_qplib_res *res,
174 struct bnxt_qplib_qp *qp)
175 {
176 struct bnxt_qplib_q *rq = &qp->rq;
177 struct bnxt_qplib_q *sq = &qp->sq;
179 if (qp->rq_hdr_buf)
180 dma_free_coherent(&res->pdev->dev,
181 rq->hwq.max_elements * qp->rq_hdr_buf_size,
182 qp->rq_hdr_buf, qp->rq_hdr_buf_map);
183 if (qp->sq_hdr_buf)
184 dma_free_coherent(&res->pdev->dev,
185 sq->hwq.max_elements * qp->sq_hdr_buf_size,
186 qp->sq_hdr_buf, qp->sq_hdr_buf_map);
187 qp->rq_hdr_buf = NULL;
188 qp->sq_hdr_buf = NULL;
189 qp->rq_hdr_buf_map = 0;
190 qp->sq_hdr_buf_map = 0;
191 qp->sq_hdr_buf_size = 0;
192 qp->rq_hdr_buf_size = 0;
193 }
195 static int bnxt_qplib_alloc_qp_hdr_buf(struct bnxt_qplib_res *res,
196 struct bnxt_qplib_qp *qp)
197 {
198 struct bnxt_qplib_q *rq = &qp->rq;
199 struct bnxt_qplib_q *sq = &qp->sq;
200 int rc = 0;
202 if (qp->sq_hdr_buf_size && sq->hwq.max_elements) {
203 qp->sq_hdr_buf = dma_alloc_coherent(&res->pdev->dev,
204 sq->hwq.max_elements *
205 qp->sq_hdr_buf_size,
206 &qp->sq_hdr_buf_map, GFP_KERNEL);
207 if (!qp->sq_hdr_buf) {
208 rc = -ENOMEM;
209 dev_err(&res->pdev->dev,
210 "QPLIB: Failed to create sq_hdr_buf");
211 goto fail;
212 }
213 }
215 if (qp->rq_hdr_buf_size && rq->hwq.max_elements) {
216 qp->rq_hdr_buf = dma_alloc_coherent(&res->pdev->dev,
217 rq->hwq.max_elements *
218 qp->rq_hdr_buf_size,
219 &qp->rq_hdr_buf_map,
220 GFP_KERNEL);
221 if (!qp->rq_hdr_buf) {
222 rc = -ENOMEM;
223 dev_err(&res->pdev->dev,
224 "QPLIB: Failed to create rq_hdr_buf");
225 goto fail;
226 }
227 }
228 return 0;
230 fail:
231 bnxt_qplib_free_qp_hdr_buf(res, qp);
232 return rc;
233 }
235 static void bnxt_qplib_service_nq(unsigned long data)
236 {
237 struct bnxt_qplib_nq *nq = (struct bnxt_qplib_nq *)data;
238 struct bnxt_qplib_hwq *hwq = &nq->hwq;
239 struct nq_base *nqe, **nq_ptr;
240 struct bnxt_qplib_cq *cq;
241 int num_cqne_processed = 0;
242 int num_srqne_processed = 0;
243 u32 sw_cons, raw_cons;
244 u16 type;
245 int budget = nq->budget;
246 uintptr_t q_handle;
248 /* Service the NQ until empty */
249 raw_cons = hwq->cons;
250 while (budget--) {
251 sw_cons = HWQ_CMP(raw_cons, hwq);
252 nq_ptr = (struct nq_base **)hwq->pbl_ptr;
253 nqe = &nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)];
254 if (!NQE_CMP_VALID(nqe, raw_cons, hwq->max_elements))
255 break;
257 /*
258 * The valid test of the entry must be done first before
259 * reading any further.
260 */
261 dma_rmb();
263 type = le16_to_cpu(nqe->info10_type) & NQ_BASE_TYPE_MASK;
264 switch (type) {
265 case NQ_BASE_TYPE_CQ_NOTIFICATION:
266 {
267 struct nq_cn *nqcne = (struct nq_cn *)nqe;
269 q_handle = le32_to_cpu(nqcne->cq_handle_low);
270 q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high)
271 << 32;
272 cq = (struct bnxt_qplib_cq *)(unsigned long)q_handle;
273 bnxt_qplib_arm_cq_enable(cq);
274 spin_lock_bh(&cq->compl_lock);
275 atomic_set(&cq->arm_state, 0);
276 if (!nq->cqn_handler(nq, (cq)))
277 num_cqne_processed++;
278 else
279 dev_warn(&nq->pdev->dev,
280 "QPLIB: cqn - type 0x%x not handled",
281 type);
282 spin_unlock_bh(&cq->compl_lock);
283 break;
284 }
285 case NQ_BASE_TYPE_SRQ_EVENT:
286 {
287 struct nq_srq_event *nqsrqe =
288 (struct nq_srq_event *)nqe;
290 q_handle = le32_to_cpu(nqsrqe->srq_handle_low);
291 q_handle |= (u64)le32_to_cpu(nqsrqe->srq_handle_high)
292 << 32;
293 bnxt_qplib_arm_srq((struct bnxt_qplib_srq *)q_handle,
294 DBR_DBR_TYPE_SRQ_ARMENA);
295 if (!nq->srqn_handler(nq,
296 (struct bnxt_qplib_srq *)q_handle,
297 nqsrqe->event))
298 num_srqne_processed++;
299 else
300 dev_warn(&nq->pdev->dev,
301 "QPLIB: SRQ event 0x%x not handled",
302 nqsrqe->event);
303 break;
304 }
305 case NQ_BASE_TYPE_DBQ_EVENT:
306 break;
307 default:
308 dev_warn(&nq->pdev->dev,
309 "QPLIB: nqe with type = 0x%x not handled",
310 type);
311 break;
312 }
313 raw_cons++;
314 }
315 if (hwq->cons != raw_cons) {
316 hwq->cons = raw_cons;
317 NQ_DB_REARM(nq->bar_reg_iomem, hwq->cons, hwq->max_elements);
318 }
319 }
321 static irqreturn_t bnxt_qplib_nq_irq(int irq, void *dev_instance)
322 {
323 struct bnxt_qplib_nq *nq = dev_instance;
324 struct bnxt_qplib_hwq *hwq = &nq->hwq;
325 struct nq_base **nq_ptr;
326 u32 sw_cons;
328 /* Prefetch the NQ element */
329 sw_cons = HWQ_CMP(hwq->cons, hwq);
330 nq_ptr = (struct nq_base **)nq->hwq.pbl_ptr;
331 prefetch(&nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)]);
333 /* Fan out to CPU affinitized kthreads? */
334 tasklet_schedule(&nq->worker);
336 return IRQ_HANDLED;
337 }
339 void bnxt_qplib_nq_stop_irq(struct bnxt_qplib_nq *nq, bool kill)
340 {
341 tasklet_disable(&nq->worker);
342 /* Mask h/w interrupt */
343 NQ_DB(nq->bar_reg_iomem, nq->hwq.cons, nq->hwq.max_elements);
344 /* Sync with last running IRQ handler */
345 synchronize_irq(nq->vector);
346 if (kill)
347 tasklet_kill(&nq->worker);
348 if (nq->requested) {
349 irq_set_affinity_hint(nq->vector, NULL);
350 free_irq(nq->vector, nq);
351 nq->requested = false;
352 }
353 }
355 void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq)
356 {
357 if (nq->cqn_wq) {
358 destroy_workqueue(nq->cqn_wq);
359 nq->cqn_wq = NULL;
360 }
362 /* Make sure the HW is stopped! */
363 bnxt_qplib_nq_stop_irq(nq, true);
365 if (nq->bar_reg_iomem)
366 iounmap(nq->bar_reg_iomem);
367 nq->bar_reg_iomem = NULL;
369 nq->cqn_handler = NULL;
370 nq->srqn_handler = NULL;
371 nq->vector = 0;
372 }
374 int bnxt_qplib_nq_start_irq(struct bnxt_qplib_nq *nq, int nq_indx,
375 int msix_vector, bool need_init)
376 {
377 int rc;
379 if (nq->requested)
380 return -EFAULT;
382 nq->vector = msix_vector;
383 if (need_init)
384 tasklet_init(&nq->worker, bnxt_qplib_service_nq,
385 (unsigned long)nq);
386 else
387 tasklet_enable(&nq->worker);
389 snprintf(nq->name, sizeof(nq->name), "bnxt_qplib_nq-%d", nq_indx);
390 rc = request_irq(nq->vector, bnxt_qplib_nq_irq, 0, nq->name, nq);
391 if (rc)
392 return rc;
394 cpumask_clear(&nq->mask);
395 cpumask_set_cpu(nq_indx, &nq->mask);
396 rc = irq_set_affinity_hint(nq->vector, &nq->mask);
397 if (rc) {
398 dev_warn(&nq->pdev->dev,
399 "QPLIB: set affinity failed; vector: %d nq_idx: %d\n",
400 nq->vector, nq_indx);
401 }
402 nq->requested = true;
403 NQ_DB_REARM(nq->bar_reg_iomem, nq->hwq.cons, nq->hwq.max_elements);
405 return rc;
406 }
408 int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq,
409 int nq_idx, int msix_vector, int bar_reg_offset,
410 int (*cqn_handler)(struct bnxt_qplib_nq *nq,
411 struct bnxt_qplib_cq *),
412 int (*srqn_handler)(struct bnxt_qplib_nq *nq,
413 struct bnxt_qplib_srq *,
414 u8 event))
415 {
416 resource_size_t nq_base;
417 int rc = -1;
419 if (cqn_handler)
420 nq->cqn_handler = cqn_handler;
422 if (srqn_handler)
423 nq->srqn_handler = srqn_handler;
425 /* Have a task to schedule CQ notifiers in post send case */
426 nq->cqn_wq = create_singlethread_workqueue("bnxt_qplib_nq");
427 if (!nq->cqn_wq)
428 return -ENOMEM;
430 nq->bar_reg = NQ_CONS_PCI_BAR_REGION;
431 nq->bar_reg_off = bar_reg_offset;
432 nq_base = pci_resource_start(pdev, nq->bar_reg);
433 if (!nq_base) {
434 rc = -ENOMEM;
435 goto fail;
436 }
437 nq->bar_reg_iomem = ioremap_nocache(nq_base + nq->bar_reg_off, 4);
438 if (!nq->bar_reg_iomem) {
439 rc = -ENOMEM;
440 goto fail;
441 }
443 rc = bnxt_qplib_nq_start_irq(nq, nq_idx, msix_vector, true);
444 if (rc) {
445 dev_err(&nq->pdev->dev,
446 "QPLIB: Failed to request irq for nq-idx %d", nq_idx);
447 goto fail;
448 }
450 return 0;
451 fail:
452 bnxt_qplib_disable_nq(nq);
453 return rc;
454 }
456 void bnxt_qplib_free_nq(struct bnxt_qplib_nq *nq)
457 {
458 if (nq->hwq.max_elements) {
459 bnxt_qplib_free_hwq(nq->pdev, &nq->hwq);
460 nq->hwq.max_elements = 0;
461 }
462 }
464 int bnxt_qplib_alloc_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq)
465 {
466 nq->pdev = pdev;
467 if (!nq->hwq.max_elements ||
468 nq->hwq.max_elements > BNXT_QPLIB_NQE_MAX_CNT)
469 nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT;
471 if (bnxt_qplib_alloc_init_hwq(nq->pdev, &nq->hwq, NULL, 0,
472 &nq->hwq.max_elements,
473 BNXT_QPLIB_MAX_NQE_ENTRY_SIZE, 0,
474 PAGE_SIZE, HWQ_TYPE_L2_CMPL))
475 return -ENOMEM;
477 nq->budget = 8;
478 return 0;
479 }
481 /* SRQ */
482 static void bnxt_qplib_arm_srq(struct bnxt_qplib_srq *srq, u32 arm_type)
483 {
484 struct bnxt_qplib_hwq *srq_hwq = &srq->hwq;
485 struct dbr_dbr db_msg = { 0 };
486 void __iomem *db;
487 u32 sw_prod = 0;
489 /* Ring DB */
490 sw_prod = (arm_type == DBR_DBR_TYPE_SRQ_ARM) ? srq->threshold :
491 HWQ_CMP(srq_hwq->prod, srq_hwq);
492 db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) &
493 DBR_DBR_INDEX_MASK);
494 db_msg.type_xid = cpu_to_le32(((srq->id << DBR_DBR_XID_SFT) &
495 DBR_DBR_XID_MASK) | arm_type);
496 db = (arm_type == DBR_DBR_TYPE_SRQ_ARMENA) ?
497 srq->dbr_base : srq->dpi->dbr;
498 wmb(); /* barrier before db ring */
499 __iowrite64_copy(db, &db_msg, sizeof(db_msg) / sizeof(u64));
500 }
502 int bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res,
503 struct bnxt_qplib_srq *srq)
504 {
505 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
506 struct cmdq_destroy_srq req;
507 struct creq_destroy_srq_resp resp;
508 u16 cmd_flags = 0;
509 int rc;
511 RCFW_CMD_PREP(req, DESTROY_SRQ, cmd_flags);
513 /* Configure the request */
514 req.srq_cid = cpu_to_le32(srq->id);
516 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
517 (void *)&resp, NULL, 0);
518 if (rc)
519 return rc;
521 bnxt_qplib_free_hwq(res->pdev, &srq->hwq);
522 kfree(srq->swq);
523 return 0;
524 }
526 int bnxt_qplib_create_srq(struct bnxt_qplib_res *res,
527 struct bnxt_qplib_srq *srq)
528 {
529 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
530 struct cmdq_create_srq req;
531 struct creq_create_srq_resp resp;
532 struct bnxt_qplib_pbl *pbl;
533 u16 cmd_flags = 0;
534 int rc, idx;
536 srq->hwq.max_elements = srq->max_wqe;
537 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &srq->hwq, srq->sglist,
538 srq->nmap, &srq->hwq.max_elements,
539 BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0,
540 PAGE_SIZE, HWQ_TYPE_QUEUE);
541 if (rc)
542 goto exit;
544 srq->swq = kcalloc(srq->hwq.max_elements, sizeof(*srq->swq),
545 GFP_KERNEL);
546 if (!srq->swq) {
547 rc = -ENOMEM;
548 goto fail;
549 }
551 RCFW_CMD_PREP(req, CREATE_SRQ, cmd_flags);
553 /* Configure the request */
554 req.dpi = cpu_to_le32(srq->dpi->dpi);
555 req.srq_handle = cpu_to_le64((uintptr_t)srq);
557 req.srq_size = cpu_to_le16((u16)srq->hwq.max_elements);
558 pbl = &srq->hwq.pbl[PBL_LVL_0];
559 req.pg_size_lvl = cpu_to_le16((((u16)srq->hwq.level &
560 CMDQ_CREATE_SRQ_LVL_MASK) <<
561 CMDQ_CREATE_SRQ_LVL_SFT) |
562 (pbl->pg_size == ROCE_PG_SIZE_4K ?
563 CMDQ_CREATE_SRQ_PG_SIZE_PG_4K :
564 pbl->pg_size == ROCE_PG_SIZE_8K ?
565 CMDQ_CREATE_SRQ_PG_SIZE_PG_8K :
566 pbl->pg_size == ROCE_PG_SIZE_64K ?
567 CMDQ_CREATE_SRQ_PG_SIZE_PG_64K :
568 pbl->pg_size == ROCE_PG_SIZE_2M ?
569 CMDQ_CREATE_SRQ_PG_SIZE_PG_2M :
570 pbl->pg_size == ROCE_PG_SIZE_8M ?
571 CMDQ_CREATE_SRQ_PG_SIZE_PG_8M :
572 pbl->pg_size == ROCE_PG_SIZE_1G ?
573 CMDQ_CREATE_SRQ_PG_SIZE_PG_1G :
574 CMDQ_CREATE_SRQ_PG_SIZE_PG_4K));
575 req.pbl = cpu_to_le64(pbl->pg_map_arr[0]);
576 req.pd_id = cpu_to_le32(srq->pd->id);
577 req.eventq_id = cpu_to_le16(srq->eventq_hw_ring_id);
579 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
580 (void *)&resp, NULL, 0);
581 if (rc)
582 goto fail;
584 spin_lock_init(&srq->lock);
585 srq->start_idx = 0;
586 srq->last_idx = srq->hwq.max_elements - 1;
587 for (idx = 0; idx < srq->hwq.max_elements; idx++)
588 srq->swq[idx].next_idx = idx + 1;
589 srq->swq[srq->last_idx].next_idx = -1;
591 srq->id = le32_to_cpu(resp.xid);
592 srq->dbr_base = res->dpi_tbl.dbr_bar_reg_iomem;
593 if (srq->threshold)
594 bnxt_qplib_arm_srq(srq, DBR_DBR_TYPE_SRQ_ARMENA);
595 srq->arm_req = false;
597 return 0;
598 fail:
599 bnxt_qplib_free_hwq(res->pdev, &srq->hwq);
600 kfree(srq->swq);
601 exit:
602 return rc;
603 }
605 int bnxt_qplib_modify_srq(struct bnxt_qplib_res *res,
606 struct bnxt_qplib_srq *srq)
607 {
608 struct bnxt_qplib_hwq *srq_hwq = &srq->hwq;
609 u32 sw_prod, sw_cons, count = 0;
611 sw_prod = HWQ_CMP(srq_hwq->prod, srq_hwq);
612 sw_cons = HWQ_CMP(srq_hwq->cons, srq_hwq);
614 count = sw_prod > sw_cons ? sw_prod - sw_cons :
615 srq_hwq->max_elements - sw_cons + sw_prod;
616 if (count > srq->threshold) {
617 srq->arm_req = false;
618 bnxt_qplib_arm_srq(srq, DBR_DBR_TYPE_SRQ_ARM);
619 } else {
620 /* Deferred arming */
621 srq->arm_req = true;
622 }
624 return 0;
625 }
627 int bnxt_qplib_query_srq(struct bnxt_qplib_res *res,
628 struct bnxt_qplib_srq *srq)
629 {
630 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
631 struct cmdq_query_srq req;
632 struct creq_query_srq_resp resp;
633 struct bnxt_qplib_rcfw_sbuf *sbuf;
634 struct creq_query_srq_resp_sb *sb;
635 u16 cmd_flags = 0;
636 int rc = 0;
638 RCFW_CMD_PREP(req, QUERY_SRQ, cmd_flags);
639 req.srq_cid = cpu_to_le32(srq->id);
641 /* Configure the request */
642 sbuf = bnxt_qplib_rcfw_alloc_sbuf(rcfw, sizeof(*sb));
643 if (!sbuf)
644 return -ENOMEM;
645 sb = sbuf->sb;
646 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp,
647 (void *)sbuf, 0);
648 srq->threshold = le16_to_cpu(sb->srq_limit);
649 bnxt_qplib_rcfw_free_sbuf(rcfw, sbuf);
651 return rc;
652 }
654 int bnxt_qplib_post_srq_recv(struct bnxt_qplib_srq *srq,
655 struct bnxt_qplib_swqe *wqe)
656 {
657 struct bnxt_qplib_hwq *srq_hwq = &srq->hwq;
658 struct rq_wqe *srqe, **srqe_ptr;
659 struct sq_sge *hw_sge;
660 u32 sw_prod, sw_cons, count = 0;
661 int i, rc = 0, next;
663 spin_lock(&srq_hwq->lock);
664 if (srq->start_idx == srq->last_idx) {
665 dev_err(&srq_hwq->pdev->dev, "QPLIB: FP: SRQ (0x%x) is full!",
666 srq->id);
667 rc = -EINVAL;
668 spin_unlock(&srq_hwq->lock);
669 goto done;
670 }
671 next = srq->start_idx;
672 srq->start_idx = srq->swq[next].next_idx;
673 spin_unlock(&srq_hwq->lock);
675 sw_prod = HWQ_CMP(srq_hwq->prod, srq_hwq);
676 srqe_ptr = (struct rq_wqe **)srq_hwq->pbl_ptr;
677 srqe = &srqe_ptr[RQE_PG(sw_prod)][RQE_IDX(sw_prod)];
678 memset(srqe, 0, BNXT_QPLIB_MAX_RQE_ENTRY_SIZE);
679 /* Calculate wqe_size16 and data_len */
680 for (i = 0, hw_sge = (struct sq_sge *)srqe->data;
681 i < wqe->num_sge; i++, hw_sge++) {
682 hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr);
683 hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey);
684 hw_sge->size = cpu_to_le32(wqe->sg_list[i].size);
685 }
686 srqe->wqe_type = wqe->type;
687 srqe->flags = wqe->flags;
688 srqe->wqe_size = wqe->num_sge +
689 ((offsetof(typeof(*srqe), data) + 15) >> 4);
690 srqe->wr_id[0] = cpu_to_le32((u32)next);
691 srq->swq[next].wr_id = wqe->wr_id;
693 srq_hwq->prod++;
695 spin_lock(&srq_hwq->lock);
696 sw_prod = HWQ_CMP(srq_hwq->prod, srq_hwq);
697 /* retaining srq_hwq->cons for this logic
698 * actually the lock is only required to
699 * read srq_hwq->cons.
700 */
701 sw_cons = HWQ_CMP(srq_hwq->cons, srq_hwq);
702 count = sw_prod > sw_cons ? sw_prod - sw_cons :
703 srq_hwq->max_elements - sw_cons + sw_prod;
704 spin_unlock(&srq_hwq->lock);
705 /* Ring DB */
706 bnxt_qplib_arm_srq(srq, DBR_DBR_TYPE_SRQ);
707 if (srq->arm_req == true && count > srq->threshold) {
708 srq->arm_req = false;
709 bnxt_qplib_arm_srq(srq, DBR_DBR_TYPE_SRQ_ARM);
710 }
711 done:
712 return rc;
713 }
715 /* QP */
716 int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
717 {
718 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
719 struct cmdq_create_qp1 req;
720 struct creq_create_qp1_resp resp;
721 struct bnxt_qplib_pbl *pbl;
722 struct bnxt_qplib_q *sq = &qp->sq;
723 struct bnxt_qplib_q *rq = &qp->rq;
724 int rc;
725 u16 cmd_flags = 0;
726 u32 qp_flags = 0;
728 RCFW_CMD_PREP(req, CREATE_QP1, cmd_flags);
730 /* General */
731 req.type = qp->type;
732 req.dpi = cpu_to_le32(qp->dpi->dpi);
733 req.qp_handle = cpu_to_le64(qp->qp_handle);
735 /* SQ */
736 sq->hwq.max_elements = sq->max_wqe;
737 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, NULL, 0,
738 &sq->hwq.max_elements,
739 BNXT_QPLIB_MAX_SQE_ENTRY_SIZE, 0,
740 PAGE_SIZE, HWQ_TYPE_QUEUE);
741 if (rc)
742 goto exit;
744 sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL);
745 if (!sq->swq) {
746 rc = -ENOMEM;
747 goto fail_sq;
748 }
749 pbl = &sq->hwq.pbl[PBL_LVL_0];
750 req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
751 req.sq_pg_size_sq_lvl =
752 ((sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK)
753 << CMDQ_CREATE_QP1_SQ_LVL_SFT) |
754 (pbl->pg_size == ROCE_PG_SIZE_4K ?
755 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K :
756 pbl->pg_size == ROCE_PG_SIZE_8K ?
757 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8K :
758 pbl->pg_size == ROCE_PG_SIZE_64K ?
759 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_64K :
760 pbl->pg_size == ROCE_PG_SIZE_2M ?
761 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_2M :
762 pbl->pg_size == ROCE_PG_SIZE_8M ?
763 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8M :
764 pbl->pg_size == ROCE_PG_SIZE_1G ?
765 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_1G :
766 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K);
768 if (qp->scq)
769 req.scq_cid = cpu_to_le32(qp->scq->id);
771 qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE;
773 /* RQ */
774 if (rq->max_wqe) {
775 rq->hwq.max_elements = qp->rq.max_wqe;
776 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, NULL, 0,
777 &rq->hwq.max_elements,
778 BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0,
779 PAGE_SIZE, HWQ_TYPE_QUEUE);
780 if (rc)
781 goto fail_sq;
783 rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq),
784 GFP_KERNEL);
785 if (!rq->swq) {
786 rc = -ENOMEM;
787 goto fail_rq;
788 }
789 pbl = &rq->hwq.pbl[PBL_LVL_0];
790 req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
791 req.rq_pg_size_rq_lvl =
792 ((rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK) <<
793 CMDQ_CREATE_QP1_RQ_LVL_SFT) |
794 (pbl->pg_size == ROCE_PG_SIZE_4K ?
795 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K :
796 pbl->pg_size == ROCE_PG_SIZE_8K ?
797 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8K :
798 pbl->pg_size == ROCE_PG_SIZE_64K ?
799 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_64K :
800 pbl->pg_size == ROCE_PG_SIZE_2M ?
801 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_2M :
802 pbl->pg_size == ROCE_PG_SIZE_8M ?
803 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8M :
804 pbl->pg_size == ROCE_PG_SIZE_1G ?
805 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_1G :
806 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K);
807 if (qp->rcq)
808 req.rcq_cid = cpu_to_le32(qp->rcq->id);
809 }
811 /* Header buffer - allow hdr_buf pass in */
812 rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp);
813 if (rc) {
814 rc = -ENOMEM;
815 goto fail;
816 }
817 req.qp_flags = cpu_to_le32(qp_flags);
818 req.sq_size = cpu_to_le32(sq->hwq.max_elements);
819 req.rq_size = cpu_to_le32(rq->hwq.max_elements);
821 req.sq_fwo_sq_sge =
822 cpu_to_le16((sq->max_sge & CMDQ_CREATE_QP1_SQ_SGE_MASK) <<
823 CMDQ_CREATE_QP1_SQ_SGE_SFT);
824 req.rq_fwo_rq_sge =
825 cpu_to_le16((rq->max_sge & CMDQ_CREATE_QP1_RQ_SGE_MASK) <<
826 CMDQ_CREATE_QP1_RQ_SGE_SFT);
828 req.pd_id = cpu_to_le32(qp->pd->id);
830 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
831 (void *)&resp, NULL, 0);
832 if (rc)
833 goto fail;
835 qp->id = le32_to_cpu(resp.xid);
836 qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
837 rcfw->qp_tbl[qp->id].qp_id = qp->id;
838 rcfw->qp_tbl[qp->id].qp_handle = (void *)qp;
840 return 0;
842 fail:
843 bnxt_qplib_free_qp_hdr_buf(res, qp);
844 fail_rq:
845 bnxt_qplib_free_hwq(res->pdev, &rq->hwq);
846 kfree(rq->swq);
847 fail_sq:
848 bnxt_qplib_free_hwq(res->pdev, &sq->hwq);
849 kfree(sq->swq);
850 exit:
851 return rc;
852 }
854 int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
855 {
856 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
857 struct sq_send *hw_sq_send_hdr, **hw_sq_send_ptr;
858 struct cmdq_create_qp req;
859 struct creq_create_qp_resp resp;
860 struct bnxt_qplib_pbl *pbl;
861 struct sq_psn_search **psn_search_ptr;
862 unsigned long int psn_search, poff = 0;
863 struct bnxt_qplib_q *sq = &qp->sq;
864 struct bnxt_qplib_q *rq = &qp->rq;
865 struct bnxt_qplib_hwq *xrrq;
866 int i, rc, req_size, psn_sz;
867 u16 cmd_flags = 0, max_ssge;
868 u32 sw_prod, qp_flags = 0;
870 RCFW_CMD_PREP(req, CREATE_QP, cmd_flags);
872 /* General */
873 req.type = qp->type;
874 req.dpi = cpu_to_le32(qp->dpi->dpi);
875 req.qp_handle = cpu_to_le64(qp->qp_handle);
877 /* SQ */
878 psn_sz = (qp->type == CMDQ_CREATE_QP_TYPE_RC) ?
879 sizeof(struct sq_psn_search) : 0;
880 sq->hwq.max_elements = sq->max_wqe;
881 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, sq->sglist,
882 sq->nmap, &sq->hwq.max_elements,
883 BNXT_QPLIB_MAX_SQE_ENTRY_SIZE,
884 psn_sz,
885 PAGE_SIZE, HWQ_TYPE_QUEUE);
886 if (rc)
887 goto exit;
889 sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL);
890 if (!sq->swq) {
891 rc = -ENOMEM;
892 goto fail_sq;
893 }
894 hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;
895 if (psn_sz) {
896 psn_search_ptr = (struct sq_psn_search **)
897 &hw_sq_send_ptr[get_sqe_pg
898 (sq->hwq.max_elements)];
899 psn_search = (unsigned long int)
900 &hw_sq_send_ptr[get_sqe_pg(sq->hwq.max_elements)]
901 [get_sqe_idx(sq->hwq.max_elements)];
902 if (psn_search & ~PAGE_MASK) {
903 /* If the psn_search does not start on a page boundary,
904 * then calculate the offset
905 */
906 poff = (psn_search & ~PAGE_MASK) /
907 BNXT_QPLIB_MAX_PSNE_ENTRY_SIZE;
908 }
909 for (i = 0; i < sq->hwq.max_elements; i++)
910 sq->swq[i].psn_search =
911 &psn_search_ptr[get_psne_pg(i + poff)]
912 [get_psne_idx(i + poff)];
913 }
914 pbl = &sq->hwq.pbl[PBL_LVL_0];
915 req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
916 req.sq_pg_size_sq_lvl =
917 ((sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK)
918 << CMDQ_CREATE_QP_SQ_LVL_SFT) |
919 (pbl->pg_size == ROCE_PG_SIZE_4K ?
920 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K :
921 pbl->pg_size == ROCE_PG_SIZE_8K ?
922 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8K :
923 pbl->pg_size == ROCE_PG_SIZE_64K ?
924 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_64K :
925 pbl->pg_size == ROCE_PG_SIZE_2M ?
926 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_2M :
927 pbl->pg_size == ROCE_PG_SIZE_8M ?
928 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8M :
929 pbl->pg_size == ROCE_PG_SIZE_1G ?
930 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_1G :
931 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K);
933 /* initialize all SQ WQEs to LOCAL_INVALID (sq prep for hw fetch) */
934 hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;
935 for (sw_prod = 0; sw_prod < sq->hwq.max_elements; sw_prod++) {
936 hw_sq_send_hdr = &hw_sq_send_ptr[get_sqe_pg(sw_prod)]
937 [get_sqe_idx(sw_prod)];
938 hw_sq_send_hdr->wqe_type = SQ_BASE_WQE_TYPE_LOCAL_INVALID;
939 }
941 if (qp->scq)
942 req.scq_cid = cpu_to_le32(qp->scq->id);
944 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE;
945 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED;
946 if (qp->sig_type)
947 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION;
949 /* RQ */
950 if (rq->max_wqe) {
951 rq->hwq.max_elements = rq->max_wqe;
952 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, rq->sglist,
953 rq->nmap, &rq->hwq.max_elements,
954 BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0,
955 PAGE_SIZE, HWQ_TYPE_QUEUE);
956 if (rc)
957 goto fail_sq;
959 rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq),
960 GFP_KERNEL);
961 if (!rq->swq) {
962 rc = -ENOMEM;
963 goto fail_rq;
964 }
965 pbl = &rq->hwq.pbl[PBL_LVL_0];
966 req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
967 req.rq_pg_size_rq_lvl =
968 ((rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK) <<
969 CMDQ_CREATE_QP_RQ_LVL_SFT) |
970 (pbl->pg_size == ROCE_PG_SIZE_4K ?
971 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K :
972 pbl->pg_size == ROCE_PG_SIZE_8K ?
973 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8K :
974 pbl->pg_size == ROCE_PG_SIZE_64K ?
975 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_64K :
976 pbl->pg_size == ROCE_PG_SIZE_2M ?
977 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_2M :
978 pbl->pg_size == ROCE_PG_SIZE_8M ?
979 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8M :
980 pbl->pg_size == ROCE_PG_SIZE_1G ?
981 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_1G :
982 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K);
983 } else {
984 /* SRQ */
985 if (qp->srq) {
986 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED;
987 req.srq_cid = cpu_to_le32(qp->srq->id);
988 }
989 }
991 if (qp->rcq)
992 req.rcq_cid = cpu_to_le32(qp->rcq->id);
993 req.qp_flags = cpu_to_le32(qp_flags);
994 req.sq_size = cpu_to_le32(sq->hwq.max_elements);
995 req.rq_size = cpu_to_le32(rq->hwq.max_elements);
996 qp->sq_hdr_buf = NULL;
997 qp->rq_hdr_buf = NULL;
999 rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp);
1000 if (rc)
1001 goto fail_rq;
1003 /* CTRL-22434: Irrespective of the requested SGE count on the SQ
1004 * always create the QP with max send sges possible if the requested
1005 * inline size is greater than 0.
1006 */
1007 max_ssge = qp->max_inline_data ? 6 : sq->max_sge;
1008 req.sq_fwo_sq_sge = cpu_to_le16(
1009 ((max_ssge & CMDQ_CREATE_QP_SQ_SGE_MASK)
1010 << CMDQ_CREATE_QP_SQ_SGE_SFT) | 0);
1011 req.rq_fwo_rq_sge = cpu_to_le16(
1012 ((rq->max_sge & CMDQ_CREATE_QP_RQ_SGE_MASK)
1013 << CMDQ_CREATE_QP_RQ_SGE_SFT) | 0);
1014 /* ORRQ and IRRQ */
1015 if (psn_sz) {
1016 xrrq = &qp->orrq;
1017 xrrq->max_elements =
1018 ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic);
1019 req_size = xrrq->max_elements *
1020 BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE + PAGE_SIZE - 1;
1021 req_size &= ~(PAGE_SIZE - 1);
1022 rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL, 0,
1023 &xrrq->max_elements,
1024 BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE,
1025 0, req_size, HWQ_TYPE_CTX);
1026 if (rc)
1027 goto fail_buf_free;
1028 pbl = &xrrq->pbl[PBL_LVL_0];
1029 req.orrq_addr = cpu_to_le64(pbl->pg_map_arr[0]);
1031 xrrq = &qp->irrq;
1032 xrrq->max_elements = IRD_LIMIT_TO_IRRQ_SLOTS(
1033 qp->max_dest_rd_atomic);
1034 req_size = xrrq->max_elements *
1035 BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE + PAGE_SIZE - 1;
1036 req_size &= ~(PAGE_SIZE - 1);
1038 rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL, 0,
1039 &xrrq->max_elements,
1040 BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE,
1041 0, req_size, HWQ_TYPE_CTX);
1042 if (rc)
1043 goto fail_orrq;
1045 pbl = &xrrq->pbl[PBL_LVL_0];
1046 req.irrq_addr = cpu_to_le64(pbl->pg_map_arr[0]);
1047 }
1048 req.pd_id = cpu_to_le32(qp->pd->id);
1050 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
1051 (void *)&resp, NULL, 0);
1052 if (rc)
1053 goto fail;
1055 qp->id = le32_to_cpu(resp.xid);
1056 qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
1057 INIT_LIST_HEAD(&qp->sq_flush);
1058 INIT_LIST_HEAD(&qp->rq_flush);
1059 rcfw->qp_tbl[qp->id].qp_id = qp->id;
1060 rcfw->qp_tbl[qp->id].qp_handle = (void *)qp;
1062 return 0;
1064 fail:
1065 if (qp->irrq.max_elements)
1066 bnxt_qplib_free_hwq(res->pdev, &qp->irrq);
1067 fail_orrq:
1068 if (qp->orrq.max_elements)
1069 bnxt_qplib_free_hwq(res->pdev, &qp->orrq);
1070 fail_buf_free:
1071 bnxt_qplib_free_qp_hdr_buf(res, qp);
1072 fail_rq:
1073 bnxt_qplib_free_hwq(res->pdev, &rq->hwq);
1074 kfree(rq->swq);
1075 fail_sq:
1076 bnxt_qplib_free_hwq(res->pdev, &sq->hwq);
1077 kfree(sq->swq);
1078 exit:
1079 return rc;
1080 }
1082 static void __modify_flags_from_init_state(struct bnxt_qplib_qp *qp)
1083 {
1084 switch (qp->state) {
1085 case CMDQ_MODIFY_QP_NEW_STATE_RTR:
1086 /* INIT->RTR, configure the path_mtu to the default
1087 * 2048 if not being requested
1088 */
1089 if (!(qp->modify_flags &
1090 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU)) {
1091 qp->modify_flags |=
1092 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
1093 qp->path_mtu =
1094 CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
1095 }
1096 qp->modify_flags &=
1097 ~CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID;
1098 /* Bono FW require the max_dest_rd_atomic to be >= 1 */
1099 if (qp->max_dest_rd_atomic < 1)
1100 qp->max_dest_rd_atomic = 1;
1101 qp->modify_flags &= ~CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC;
1102 /* Bono FW 20.6.5 requires SGID_INDEX configuration */
1103 if (!(qp->modify_flags &
1104 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX)) {
1105 qp->modify_flags |=
1106 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX;
1107 qp->ah.sgid_index = 0;
1108 }
1109 break;
1110 default:
1111 break;
1112 }
1113 }
1115 static void __modify_flags_from_rtr_state(struct bnxt_qplib_qp *qp)
1116 {
1117 switch (qp->state) {
1118 case CMDQ_MODIFY_QP_NEW_STATE_RTS:
1119 /* Bono FW requires the max_rd_atomic to be >= 1 */
1120 if (qp->max_rd_atomic < 1)
1121 qp->max_rd_atomic = 1;
1122 /* Bono FW does not allow PKEY_INDEX,
1123 * DGID, FLOW_LABEL, SGID_INDEX, HOP_LIMIT,
1124 * TRAFFIC_CLASS, DEST_MAC, PATH_MTU, RQ_PSN,
1125 * MIN_RNR_TIMER, MAX_DEST_RD_ATOMIC, DEST_QP_ID
1126 * modification
1127 */
1128 qp->modify_flags &=
1129 ~(CMDQ_MODIFY_QP_MODIFY_MASK_PKEY |
1130 CMDQ_MODIFY_QP_MODIFY_MASK_DGID |
1131 CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL |
1132 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX |
1133 CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT |
1134 CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS |
1135 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC |
1136 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU |
1137 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN |
1138 CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER |
1139 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC |
1140 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID);
1141 break;
1142 default:
1143 break;
1144 }
1145 }
1147 static void __filter_modify_flags(struct bnxt_qplib_qp *qp)
1148 {
1149 switch (qp->cur_qp_state) {
1150 case CMDQ_MODIFY_QP_NEW_STATE_RESET:
1151 break;
1152 case CMDQ_MODIFY_QP_NEW_STATE_INIT:
1153 __modify_flags_from_init_state(qp);
1154 break;
1155 case CMDQ_MODIFY_QP_NEW_STATE_RTR:
1156 __modify_flags_from_rtr_state(qp);
1157 break;
1158 case CMDQ_MODIFY_QP_NEW_STATE_RTS:
1159 break;
1160 case CMDQ_MODIFY_QP_NEW_STATE_SQD:
1161 break;
1162 case CMDQ_MODIFY_QP_NEW_STATE_SQE:
1163 break;
1164 case CMDQ_MODIFY_QP_NEW_STATE_ERR:
1165 break;
1166 default:
1167 break;
1168 }
1169 }
1171 int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
1172 {
1173 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1174 struct cmdq_modify_qp req;
1175 struct creq_modify_qp_resp resp;
1176 u16 cmd_flags = 0, pkey;
1177 u32 temp32[4];
1178 u32 bmask;
1179 int rc;
1181 RCFW_CMD_PREP(req, MODIFY_QP, cmd_flags);
1183 /* Filter out the qp_attr_mask based on the state->new transition */
1184 __filter_modify_flags(qp);
1185 bmask = qp->modify_flags;
1186 req.modify_mask = cpu_to_le32(qp->modify_flags);
1187 req.qp_cid = cpu_to_le32(qp->id);
1188 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_STATE) {
1189 req.network_type_en_sqd_async_notify_new_state =
1190 (qp->state & CMDQ_MODIFY_QP_NEW_STATE_MASK) |
1191 (qp->en_sqd_async_notify ?
1192 CMDQ_MODIFY_QP_EN_SQD_ASYNC_NOTIFY : 0);
1193 }
1194 req.network_type_en_sqd_async_notify_new_state |= qp->nw_type;
1196 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS)
1197 req.access = qp->access;
1199 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PKEY) {
1200 if (!bnxt_qplib_get_pkey(res, &res->pkey_tbl,
1201 qp->pkey_index, &pkey))
1202 req.pkey = cpu_to_le16(pkey);
1203 }
1204 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_QKEY)
1205 req.qkey = cpu_to_le32(qp->qkey);
1207 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DGID) {
1208 memcpy(temp32, qp->ah.dgid.data, sizeof(struct bnxt_qplib_gid));
1209 req.dgid[0] = cpu_to_le32(temp32[0]);
1210 req.dgid[1] = cpu_to_le32(temp32[1]);
1211 req.dgid[2] = cpu_to_le32(temp32[2]);
1212 req.dgid[3] = cpu_to_le32(temp32[3]);
1213 }
1214 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL)
1215 req.flow_label = cpu_to_le32(qp->ah.flow_label);
1217 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX)
1218 req.sgid_index = cpu_to_le16(res->sgid_tbl.hw_id
1219 [qp->ah.sgid_index]);
1221 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT)
1222 req.hop_limit = qp->ah.hop_limit;
1224 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS)
1225 req.traffic_class = qp->ah.traffic_class;
1227 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC)
1228 memcpy(req.dest_mac, qp->ah.dmac, 6);
1230 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU)
1231 req.path_mtu = qp->path_mtu;
1233 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT)
1234 req.timeout = qp->timeout;
1236 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT)
1237 req.retry_cnt = qp->retry_cnt;
1239 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY)
1240 req.rnr_retry = qp->rnr_retry;
1242 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER)
1243 req.min_rnr_timer = qp->min_rnr_timer;
1245 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN)
1246 req.rq_psn = cpu_to_le32(qp->rq.psn);
1248 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN)
1249 req.sq_psn = cpu_to_le32(qp->sq.psn);
1251 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC)
1252 req.max_rd_atomic =
1253 ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic);
1255 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC)
1256 req.max_dest_rd_atomic =
1257 IRD_LIMIT_TO_IRRQ_SLOTS(qp->max_dest_rd_atomic);
1259 req.sq_size = cpu_to_le32(qp->sq.hwq.max_elements);
1260 req.rq_size = cpu_to_le32(qp->rq.hwq.max_elements);
1261 req.sq_sge = cpu_to_le16(qp->sq.max_sge);
1262 req.rq_sge = cpu_to_le16(qp->rq.max_sge);
1263 req.max_inline_data = cpu_to_le32(qp->max_inline_data);
1264 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID)
1265 req.dest_qp_id = cpu_to_le32(qp->dest_qpn);
1267 req.vlan_pcp_vlan_dei_vlan_id = cpu_to_le16(qp->vlan_id);
1269 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
1270 (void *)&resp, NULL, 0);
1271 if (rc)
1272 return rc;
1273 qp->cur_qp_state = qp->state;
1274 return 0;
1275 }
1277 int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
1278 {
1279 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1280 struct cmdq_query_qp req;
1281 struct creq_query_qp_resp resp;
1282 struct bnxt_qplib_rcfw_sbuf *sbuf;
1283 struct creq_query_qp_resp_sb *sb;
1284 u16 cmd_flags = 0;
1285 u32 temp32[4];
1286 int i, rc = 0;
1288 RCFW_CMD_PREP(req, QUERY_QP, cmd_flags);
1290 sbuf = bnxt_qplib_rcfw_alloc_sbuf(rcfw, sizeof(*sb));
1291 if (!sbuf)
1292 return -ENOMEM;
1293 sb = sbuf->sb;
1295 req.qp_cid = cpu_to_le32(qp->id);
1296 req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS;
1297 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp,
1298 (void *)sbuf, 0);
1299 if (rc)
1300 goto bail;
1301 /* Extract the context from the side buffer */
1302 qp->state = sb->en_sqd_async_notify_state &
1303 CREQ_QUERY_QP_RESP_SB_STATE_MASK;
1304 qp->en_sqd_async_notify = sb->en_sqd_async_notify_state &
1305 CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY ?
1306 true : false;
1307 qp->access = sb->access;
1308 qp->pkey_index = le16_to_cpu(sb->pkey);
1309 qp->qkey = le32_to_cpu(sb->qkey);
1311 temp32[0] = le32_to_cpu(sb->dgid[0]);
1312 temp32[1] = le32_to_cpu(sb->dgid[1]);
1313 temp32[2] = le32_to_cpu(sb->dgid[2]);
1314 temp32[3] = le32_to_cpu(sb->dgid[3]);
1315 memcpy(qp->ah.dgid.data, temp32, sizeof(qp->ah.dgid.data));
1317 qp->ah.flow_label = le32_to_cpu(sb->flow_label);
1319 qp->ah.sgid_index = 0;
1320 for (i = 0; i < res->sgid_tbl.max; i++) {
1321 if (res->sgid_tbl.hw_id[i] == le16_to_cpu(sb->sgid_index)) {
1322 qp->ah.sgid_index = i;
1323 break;
1324 }
1325 }
1326 if (i == res->sgid_tbl.max)
1327 dev_warn(&res->pdev->dev, "QPLIB: SGID not found??");
1329 qp->ah.hop_limit = sb->hop_limit;
1330 qp->ah.traffic_class = sb->traffic_class;
1331 memcpy(qp->ah.dmac, sb->dest_mac, 6);
1332 qp->ah.vlan_id = (le16_to_cpu(sb->path_mtu_dest_vlan_id) &
1333 CREQ_QUERY_QP_RESP_SB_VLAN_ID_MASK) >>
1334 CREQ_QUERY_QP_RESP_SB_VLAN_ID_SFT;
1335 qp->path_mtu = (le16_to_cpu(sb->path_mtu_dest_vlan_id) &
1336 CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) >>
1337 CREQ_QUERY_QP_RESP_SB_PATH_MTU_SFT;
1338 qp->timeout = sb->timeout;
1339 qp->retry_cnt = sb->retry_cnt;
1340 qp->rnr_retry = sb->rnr_retry;
1341 qp->min_rnr_timer = sb->min_rnr_timer;
1342 qp->rq.psn = le32_to_cpu(sb->rq_psn);
1343 qp->max_rd_atomic = ORRQ_SLOTS_TO_ORD_LIMIT(sb->max_rd_atomic);
1344 qp->sq.psn = le32_to_cpu(sb->sq_psn);
1345 qp->max_dest_rd_atomic =
1346 IRRQ_SLOTS_TO_IRD_LIMIT(sb->max_dest_rd_atomic);
1347 qp->sq.max_wqe = qp->sq.hwq.max_elements;
1348 qp->rq.max_wqe = qp->rq.hwq.max_elements;
1349 qp->sq.max_sge = le16_to_cpu(sb->sq_sge);
1350 qp->rq.max_sge = le16_to_cpu(sb->rq_sge);
1351 qp->max_inline_data = le32_to_cpu(sb->max_inline_data);
1352 qp->dest_qpn = le32_to_cpu(sb->dest_qp_id);
1353 memcpy(qp->smac, sb->src_mac, 6);
1354 qp->vlan_id = le16_to_cpu(sb->vlan_pcp_vlan_dei_vlan_id);
1355 bail:
1356 bnxt_qplib_rcfw_free_sbuf(rcfw, sbuf);
1357 return rc;
1358 }
1360 static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp)
1361 {
1362 struct bnxt_qplib_hwq *cq_hwq = &cq->hwq;
1363 struct cq_base *hw_cqe, **hw_cqe_ptr;
1364 int i;
1366 for (i = 0; i < cq_hwq->max_elements; i++) {
1367 hw_cqe_ptr = (struct cq_base **)cq_hwq->pbl_ptr;
1368 hw_cqe = &hw_cqe_ptr[CQE_PG(i)][CQE_IDX(i)];
1369 if (!CQE_CMP_VALID(hw_cqe, i, cq_hwq->max_elements))
1370 continue;
1371 /*
1372 * The valid test of the entry must be done first before
1373 * reading any further.
1374 */
1375 dma_rmb();
1376 switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) {
1377 case CQ_BASE_CQE_TYPE_REQ:
1378 case CQ_BASE_CQE_TYPE_TERMINAL:
1379 {
1380 struct cq_req *cqe = (struct cq_req *)hw_cqe;
1382 if (qp == le64_to_cpu(cqe->qp_handle))
1383 cqe->qp_handle = 0;
1384 break;
1385 }
1386 case CQ_BASE_CQE_TYPE_RES_RC:
1387 case CQ_BASE_CQE_TYPE_RES_UD:
1388 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
1389 {
1390 struct cq_res_rc *cqe = (struct cq_res_rc *)hw_cqe;
1392 if (qp == le64_to_cpu(cqe->qp_handle))
1393 cqe->qp_handle = 0;
1394 break;
1395 }
1396 default:
1397 break;
1398 }
1399 }
1400 }
1402 int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res,
1403 struct bnxt_qplib_qp *qp)
1404 {
1405 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1406 struct cmdq_destroy_qp req;
1407 struct creq_destroy_qp_resp resp;
1408 u16 cmd_flags = 0;
1409 int rc;
1411 rcfw->qp_tbl[qp->id].qp_id = BNXT_QPLIB_QP_ID_INVALID;
1412 rcfw->qp_tbl[qp->id].qp_handle = NULL;
1414 RCFW_CMD_PREP(req, DESTROY_QP, cmd_flags);
1416 req.qp_cid = cpu_to_le32(qp->id);
1417 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
1418 (void *)&resp, NULL, 0);
1419 if (rc) {
1420 rcfw->qp_tbl[qp->id].qp_id = qp->id;
1421 rcfw->qp_tbl[qp->id].qp_handle = qp;
1422 return rc;
1423 }
1425 return 0;
1426 }
1428 void bnxt_qplib_free_qp_res(struct bnxt_qplib_res *res,
1429 struct bnxt_qplib_qp *qp)
1430 {
1431 bnxt_qplib_free_qp_hdr_buf(res, qp);
1432 bnxt_qplib_free_hwq(res->pdev, &qp->sq.hwq);
1433 kfree(qp->sq.swq);
1435 bnxt_qplib_free_hwq(res->pdev, &qp->rq.hwq);
1436 kfree(qp->rq.swq);
1438 if (qp->irrq.max_elements)
1439 bnxt_qplib_free_hwq(res->pdev, &qp->irrq);
1440 if (qp->orrq.max_elements)
1441 bnxt_qplib_free_hwq(res->pdev, &qp->orrq);
1443 }
1445 void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp,
1446 struct bnxt_qplib_sge *sge)
1447 {
1448 struct bnxt_qplib_q *sq = &qp->sq;
1449 u32 sw_prod;
1451 memset(sge, 0, sizeof(*sge));
1453 if (qp->sq_hdr_buf) {
1454 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1455 sge->addr = (dma_addr_t)(qp->sq_hdr_buf_map +
1456 sw_prod * qp->sq_hdr_buf_size);
1457 sge->lkey = 0xFFFFFFFF;
1458 sge->size = qp->sq_hdr_buf_size;
1459 return qp->sq_hdr_buf + sw_prod * sge->size;
1460 }
1461 return NULL;
1462 }
1464 u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp)
1465 {
1466 struct bnxt_qplib_q *rq = &qp->rq;
1468 return HWQ_CMP(rq->hwq.prod, &rq->hwq);
1469 }
1471 dma_addr_t bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp, u32 index)
1472 {
1473 return (qp->rq_hdr_buf_map + index * qp->rq_hdr_buf_size);
1474 }
1476 void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp,
1477 struct bnxt_qplib_sge *sge)
1478 {
1479 struct bnxt_qplib_q *rq = &qp->rq;
1480 u32 sw_prod;
1482 memset(sge, 0, sizeof(*sge));
1484 if (qp->rq_hdr_buf) {
1485 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1486 sge->addr = (dma_addr_t)(qp->rq_hdr_buf_map +
1487 sw_prod * qp->rq_hdr_buf_size);
1488 sge->lkey = 0xFFFFFFFF;
1489 sge->size = qp->rq_hdr_buf_size;
1490 return qp->rq_hdr_buf + sw_prod * sge->size;
1491 }
1492 return NULL;
1493 }
1495 void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp)
1496 {
1497 struct bnxt_qplib_q *sq = &qp->sq;
1498 struct dbr_dbr db_msg = { 0 };
1499 u32 sw_prod;
1501 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1503 db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) &
1504 DBR_DBR_INDEX_MASK);
1505 db_msg.type_xid =
1506 cpu_to_le32(((qp->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) |
1507 DBR_DBR_TYPE_SQ);
1508 /* Flush all the WQE writes to HW */
1509 wmb();
1510 __iowrite64_copy(qp->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64));
1511 }
1513 int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,
1514 struct bnxt_qplib_swqe *wqe)
1515 {
1516 struct bnxt_qplib_q *sq = &qp->sq;
1517 struct bnxt_qplib_swq *swq;
1518 struct sq_send *hw_sq_send_hdr, **hw_sq_send_ptr;
1519 struct sq_sge *hw_sge;
1520 struct bnxt_qplib_nq_work *nq_work = NULL;
1521 bool sch_handler = false;
1522 u32 sw_prod;
1523 u8 wqe_size16;
1524 int i, rc = 0, data_len = 0, pkt_num = 0;
1525 __le32 temp32;
1527 if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS) {
1528 if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
1529 sch_handler = true;
1530 dev_dbg(&sq->hwq.pdev->dev,
1531 "%s Error QP. Scheduling for poll_cq\n",
1532 __func__);
1533 goto queue_err;
1534 }
1535 }
1537 if (bnxt_qplib_queue_full(sq)) {
1538 dev_err(&sq->hwq.pdev->dev,
1539 "QPLIB: prod = %#x cons = %#x qdepth = %#x delta = %#x",
1540 sq->hwq.prod, sq->hwq.cons, sq->hwq.max_elements,
1541 sq->q_full_delta);
1542 rc = -ENOMEM;
1543 goto done;
1544 }
1545 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1546 swq = &sq->swq[sw_prod];
1547 swq->wr_id = wqe->wr_id;
1548 swq->type = wqe->type;
1549 swq->flags = wqe->flags;
1550 if (qp->sig_type)
1551 swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP;
1552 swq->start_psn = sq->psn & BTH_PSN_MASK;
1554 hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;
1555 hw_sq_send_hdr = &hw_sq_send_ptr[get_sqe_pg(sw_prod)]
1556 [get_sqe_idx(sw_prod)];
1558 memset(hw_sq_send_hdr, 0, BNXT_QPLIB_MAX_SQE_ENTRY_SIZE);
1560 if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) {
1561 /* Copy the inline data */
1562 if (wqe->inline_len > BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) {
1563 dev_warn(&sq->hwq.pdev->dev,
1564 "QPLIB: Inline data length > 96 detected");
1565 data_len = BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH;
1566 } else {
1567 data_len = wqe->inline_len;
1568 }
1569 memcpy(hw_sq_send_hdr->data, wqe->inline_data, data_len);
1570 wqe_size16 = (data_len + 15) >> 4;
1571 } else {
1572 for (i = 0, hw_sge = (struct sq_sge *)hw_sq_send_hdr->data;
1573 i < wqe->num_sge; i++, hw_sge++) {
1574 hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr);
1575 hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey);
1576 hw_sge->size = cpu_to_le32(wqe->sg_list[i].size);
1577 data_len += wqe->sg_list[i].size;
1578 }
1579 /* Each SGE entry = 1 WQE size16 */
1580 wqe_size16 = wqe->num_sge;
1581 /* HW requires wqe size has room for atleast one SGE even if
1582 * none was supplied by ULP
1583 */
1584 if (!wqe->num_sge)
1585 wqe_size16++;
1586 }
1588 /* Specifics */
1589 switch (wqe->type) {
1590 case BNXT_QPLIB_SWQE_TYPE_SEND:
1591 if (qp->type == CMDQ_CREATE_QP1_TYPE_GSI) {
1592 /* Assemble info for Raw Ethertype QPs */
1593 struct sq_send_raweth_qp1 *sqe =
1594 (struct sq_send_raweth_qp1 *)hw_sq_send_hdr;
1596 sqe->wqe_type = wqe->type;
1597 sqe->flags = wqe->flags;
1598 sqe->wqe_size = wqe_size16 +
1599 ((offsetof(typeof(*sqe), data) + 15) >> 4);
1600 sqe->cfa_action = cpu_to_le16(wqe->rawqp1.cfa_action);
1601 sqe->lflags = cpu_to_le16(wqe->rawqp1.lflags);
1602 sqe->length = cpu_to_le32(data_len);
1603 sqe->cfa_meta = cpu_to_le32((wqe->rawqp1.cfa_meta &
1604 SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK) <<
1605 SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT);
1607 break;
1608 }
1609 /* fall thru */
1610 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM:
1611 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV:
1612 {
1613 struct sq_send *sqe = (struct sq_send *)hw_sq_send_hdr;
1615 sqe->wqe_type = wqe->type;
1616 sqe->flags = wqe->flags;
1617 sqe->wqe_size = wqe_size16 +
1618 ((offsetof(typeof(*sqe), data) + 15) >> 4);
1619 sqe->inv_key_or_imm_data = cpu_to_le32(
1620 wqe->send.inv_key);
1621 if (qp->type == CMDQ_CREATE_QP_TYPE_UD) {
1622 sqe->q_key = cpu_to_le32(wqe->send.q_key);
1623 sqe->dst_qp = cpu_to_le32(
1624 wqe->send.dst_qp & SQ_SEND_DST_QP_MASK);
1625 sqe->length = cpu_to_le32(data_len);
1626 sqe->avid = cpu_to_le32(wqe->send.avid &
1627 SQ_SEND_AVID_MASK);
1628 sq->psn = (sq->psn + 1) & BTH_PSN_MASK;
1629 } else {
1630 sqe->length = cpu_to_le32(data_len);
1631 sqe->dst_qp = 0;
1632 sqe->avid = 0;
1633 if (qp->mtu)
1634 pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1635 if (!pkt_num)
1636 pkt_num = 1;
1637 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
1638 }
1639 break;
1640 }
1641 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE:
1642 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM:
1643 case BNXT_QPLIB_SWQE_TYPE_RDMA_READ:
1644 {
1645 struct sq_rdma *sqe = (struct sq_rdma *)hw_sq_send_hdr;
1647 sqe->wqe_type = wqe->type;
1648 sqe->flags = wqe->flags;
1649 sqe->wqe_size = wqe_size16 +
1650 ((offsetof(typeof(*sqe), data) + 15) >> 4);
1651 sqe->imm_data = cpu_to_le32(wqe->rdma.inv_key);
1652 sqe->length = cpu_to_le32((u32)data_len);
1653 sqe->remote_va = cpu_to_le64(wqe->rdma.remote_va);
1654 sqe->remote_key = cpu_to_le32(wqe->rdma.r_key);
1655 if (qp->mtu)
1656 pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1657 if (!pkt_num)
1658 pkt_num = 1;
1659 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
1660 break;
1661 }
1662 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP:
1663 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD:
1664 {
1665 struct sq_atomic *sqe = (struct sq_atomic *)hw_sq_send_hdr;
1667 sqe->wqe_type = wqe->type;
1668 sqe->flags = wqe->flags;
1669 sqe->remote_key = cpu_to_le32(wqe->atomic.r_key);
1670 sqe->remote_va = cpu_to_le64(wqe->atomic.remote_va);
1671 sqe->swap_data = cpu_to_le64(wqe->atomic.swap_data);
1672 sqe->cmp_data = cpu_to_le64(wqe->atomic.cmp_data);
1673 if (qp->mtu)
1674 pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1675 if (!pkt_num)
1676 pkt_num = 1;
1677 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
1678 break;
1679 }
1680 case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV:
1681 {
1682 struct sq_localinvalidate *sqe =
1683 (struct sq_localinvalidate *)hw_sq_send_hdr;
1685 sqe->wqe_type = wqe->type;
1686 sqe->flags = wqe->flags;
1687 sqe->inv_l_key = cpu_to_le32(wqe->local_inv.inv_l_key);
1689 break;
1690 }
1691 case BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR:
1692 {
1693 struct sq_fr_pmr *sqe = (struct sq_fr_pmr *)hw_sq_send_hdr;
1695 sqe->wqe_type = wqe->type;
1696 sqe->flags = wqe->flags;
1697 sqe->access_cntl = wqe->frmr.access_cntl |
1698 SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE;
1699 sqe->zero_based_page_size_log =
1700 (wqe->frmr.pg_sz_log & SQ_FR_PMR_PAGE_SIZE_LOG_MASK) <<
1701 SQ_FR_PMR_PAGE_SIZE_LOG_SFT |
1702 (wqe->frmr.zero_based ? SQ_FR_PMR_ZERO_BASED : 0);
1703 sqe->l_key = cpu_to_le32(wqe->frmr.l_key);
1704 temp32 = cpu_to_le32(wqe->frmr.length);
1705 memcpy(sqe->length, &temp32, sizeof(wqe->frmr.length));
1706 sqe->numlevels_pbl_page_size_log =
1707 ((wqe->frmr.pbl_pg_sz_log <<
1708 SQ_FR_PMR_PBL_PAGE_SIZE_LOG_SFT) &
1709 SQ_FR_PMR_PBL_PAGE_SIZE_LOG_MASK) |
1710 ((wqe->frmr.levels << SQ_FR_PMR_NUMLEVELS_SFT) &
1711 SQ_FR_PMR_NUMLEVELS_MASK);
1713 for (i = 0; i < wqe->frmr.page_list_len; i++)
1714 wqe->frmr.pbl_ptr[i] = cpu_to_le64(
1715 wqe->frmr.page_list[i] |
1716 PTU_PTE_VALID);
1717 sqe->pblptr = cpu_to_le64(wqe->frmr.pbl_dma_ptr);
1718 sqe->va = cpu_to_le64(wqe->frmr.va);
1720 break;
1721 }
1722 case BNXT_QPLIB_SWQE_TYPE_BIND_MW:
1723 {
1724 struct sq_bind *sqe = (struct sq_bind *)hw_sq_send_hdr;
1726 sqe->wqe_type = wqe->type;
1727 sqe->flags = wqe->flags;
1728 sqe->access_cntl = wqe->bind.access_cntl;
1729 sqe->mw_type_zero_based = wqe->bind.mw_type |
1730 (wqe->bind.zero_based ? SQ_BIND_ZERO_BASED : 0);
1731 sqe->parent_l_key = cpu_to_le32(wqe->bind.parent_l_key);
1732 sqe->l_key = cpu_to_le32(wqe->bind.r_key);
1733 sqe->va = cpu_to_le64(wqe->bind.va);
1734 temp32 = cpu_to_le32(wqe->bind.length);
1735 memcpy(&sqe->length, &temp32, sizeof(wqe->bind.length));
1736 break;
1737 }
1738 default:
1739 /* Bad wqe, return error */
1740 rc = -EINVAL;
1741 goto done;
1742 }
1743 swq->next_psn = sq->psn & BTH_PSN_MASK;
1744 if (swq->psn_search) {
1745 swq->psn_search->opcode_start_psn = cpu_to_le32(
1746 ((swq->start_psn << SQ_PSN_SEARCH_START_PSN_SFT) &
1747 SQ_PSN_SEARCH_START_PSN_MASK) |
1748 ((wqe->type << SQ_PSN_SEARCH_OPCODE_SFT) &
1749 SQ_PSN_SEARCH_OPCODE_MASK));
1750 swq->psn_search->flags_next_psn = cpu_to_le32(
1751 ((swq->next_psn << SQ_PSN_SEARCH_NEXT_PSN_SFT) &
1752 SQ_PSN_SEARCH_NEXT_PSN_MASK));
1753 }
1754 queue_err:
1755 if (sch_handler) {
1756 /* Store the ULP info in the software structures */
1757 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1758 swq = &sq->swq[sw_prod];
1759 swq->wr_id = wqe->wr_id;
1760 swq->type = wqe->type;
1761 swq->flags = wqe->flags;
1762 if (qp->sig_type)
1763 swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP;
1764 swq->start_psn = sq->psn & BTH_PSN_MASK;
1765 }
1766 sq->hwq.prod++;
1767 qp->wqe_cnt++;
1769 done:
1770 if (sch_handler) {
1771 nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC);
1772 if (nq_work) {
1773 nq_work->cq = qp->scq;
1774 nq_work->nq = qp->scq->nq;
1775 INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task);
1776 queue_work(qp->scq->nq->cqn_wq, &nq_work->work);
1777 } else {
1778 dev_err(&sq->hwq.pdev->dev,
1779 "QPLIB: FP: Failed to allocate SQ nq_work!");
1780 rc = -ENOMEM;
1781 }
1782 }
1783 return rc;
1784 }
1786 void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp)
1787 {
1788 struct bnxt_qplib_q *rq = &qp->rq;
1789 struct dbr_dbr db_msg = { 0 };
1790 u32 sw_prod;
1792 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1793 db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) &
1794 DBR_DBR_INDEX_MASK);
1795 db_msg.type_xid =
1796 cpu_to_le32(((qp->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) |
1797 DBR_DBR_TYPE_RQ);
1799 /* Flush the writes to HW Rx WQE before the ringing Rx DB */
1800 wmb();
1801 __iowrite64_copy(qp->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64));
1802 }
1804 int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp,
1805 struct bnxt_qplib_swqe *wqe)
1806 {
1807 struct bnxt_qplib_q *rq = &qp->rq;
1808 struct rq_wqe *rqe, **rqe_ptr;
1809 struct sq_sge *hw_sge;
1810 struct bnxt_qplib_nq_work *nq_work = NULL;
1811 bool sch_handler = false;
1812 u32 sw_prod;
1813 int i, rc = 0;
1815 if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
1816 sch_handler = true;
1817 dev_dbg(&rq->hwq.pdev->dev,
1818 "%s Error QP. Scheduling for poll_cq\n",
1819 __func__);
1820 goto queue_err;
1821 }
1822 if (bnxt_qplib_queue_full(rq)) {
1823 dev_err(&rq->hwq.pdev->dev,
1824 "QPLIB: FP: QP (0x%x) RQ is full!", qp->id);
1825 rc = -EINVAL;
1826 goto done;
1827 }
1828 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1829 rq->swq[sw_prod].wr_id = wqe->wr_id;
1831 rqe_ptr = (struct rq_wqe **)rq->hwq.pbl_ptr;
1832 rqe = &rqe_ptr[RQE_PG(sw_prod)][RQE_IDX(sw_prod)];
1834 memset(rqe, 0, BNXT_QPLIB_MAX_RQE_ENTRY_SIZE);
1836 /* Calculate wqe_size16 and data_len */
1837 for (i = 0, hw_sge = (struct sq_sge *)rqe->data;
1838 i < wqe->num_sge; i++, hw_sge++) {
1839 hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr);
1840 hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey);
1841 hw_sge->size = cpu_to_le32(wqe->sg_list[i].size);
1842 }
1843 rqe->wqe_type = wqe->type;
1844 rqe->flags = wqe->flags;
1845 rqe->wqe_size = wqe->num_sge +
1846 ((offsetof(typeof(*rqe), data) + 15) >> 4);
1847 /* HW requires wqe size has room for atleast one SGE even if none
1848 * was supplied by ULP
1849 */
1850 if (!wqe->num_sge)
1851 rqe->wqe_size++;
1853 /* Supply the rqe->wr_id index to the wr_id_tbl for now */
1854 rqe->wr_id[0] = cpu_to_le32(sw_prod);
1856 queue_err:
1857 if (sch_handler) {
1858 /* Store the ULP info in the software structures */
1859 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1860 rq->swq[sw_prod].wr_id = wqe->wr_id;
1861 }
1863 rq->hwq.prod++;
1864 if (sch_handler) {
1865 nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC);
1866 if (nq_work) {
1867 nq_work->cq = qp->rcq;
1868 nq_work->nq = qp->rcq->nq;
1869 INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task);
1870 queue_work(qp->rcq->nq->cqn_wq, &nq_work->work);
1871 } else {
1872 dev_err(&rq->hwq.pdev->dev,
1873 "QPLIB: FP: Failed to allocate RQ nq_work!");
1874 rc = -ENOMEM;
1875 }
1876 }
1877 done:
1878 return rc;
1879 }
1881 /* CQ */
1883 /* Spinlock must be held */
1884 static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq)
1885 {
1886 struct dbr_dbr db_msg = { 0 };
1888 db_msg.type_xid =
1889 cpu_to_le32(((cq->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) |
1890 DBR_DBR_TYPE_CQ_ARMENA);
1891 /* Flush memory writes before enabling the CQ */
1892 wmb();
1893 __iowrite64_copy(cq->dbr_base, &db_msg, sizeof(db_msg) / sizeof(u64));
1894 }
1896 static void bnxt_qplib_arm_cq(struct bnxt_qplib_cq *cq, u32 arm_type)
1897 {
1898 struct bnxt_qplib_hwq *cq_hwq = &cq->hwq;
1899 struct dbr_dbr db_msg = { 0 };
1900 u32 sw_cons;
1902 /* Ring DB */
1903 sw_cons = HWQ_CMP(cq_hwq->cons, cq_hwq);
1904 db_msg.index = cpu_to_le32((sw_cons << DBR_DBR_INDEX_SFT) &
1905 DBR_DBR_INDEX_MASK);
1906 db_msg.type_xid =
1907 cpu_to_le32(((cq->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) |
1908 arm_type);
1909 /* flush memory writes before arming the CQ */
1910 wmb();
1911 __iowrite64_copy(cq->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64));
1912 }
1914 int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
1915 {
1916 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1917 struct cmdq_create_cq req;
1918 struct creq_create_cq_resp resp;
1919 struct bnxt_qplib_pbl *pbl;
1920 u16 cmd_flags = 0;
1921 int rc;
1923 cq->hwq.max_elements = cq->max_wqe;
1924 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &cq->hwq, cq->sghead,
1925 cq->nmap, &cq->hwq.max_elements,
1926 BNXT_QPLIB_MAX_CQE_ENTRY_SIZE, 0,
1927 PAGE_SIZE, HWQ_TYPE_QUEUE);
1928 if (rc)
1929 goto exit;
1931 RCFW_CMD_PREP(req, CREATE_CQ, cmd_flags);
1933 if (!cq->dpi) {
1934 dev_err(&rcfw->pdev->dev,
1935 "QPLIB: FP: CREATE_CQ failed due to NULL DPI");
1936 return -EINVAL;
1937 }
1938 req.dpi = cpu_to_le32(cq->dpi->dpi);
1939 req.cq_handle = cpu_to_le64(cq->cq_handle);
1941 req.cq_size = cpu_to_le32(cq->hwq.max_elements);
1942 pbl = &cq->hwq.pbl[PBL_LVL_0];
1943 req.pg_size_lvl = cpu_to_le32(
1944 ((cq->hwq.level & CMDQ_CREATE_CQ_LVL_MASK) <<
1945 CMDQ_CREATE_CQ_LVL_SFT) |
1946 (pbl->pg_size == ROCE_PG_SIZE_4K ? CMDQ_CREATE_CQ_PG_SIZE_PG_4K :
1947 pbl->pg_size == ROCE_PG_SIZE_8K ? CMDQ_CREATE_CQ_PG_SIZE_PG_8K :
1948 pbl->pg_size == ROCE_PG_SIZE_64K ? CMDQ_CREATE_CQ_PG_SIZE_PG_64K :
1949 pbl->pg_size == ROCE_PG_SIZE_2M ? CMDQ_CREATE_CQ_PG_SIZE_PG_2M :
1950 pbl->pg_size == ROCE_PG_SIZE_8M ? CMDQ_CREATE_CQ_PG_SIZE_PG_8M :
1951 pbl->pg_size == ROCE_PG_SIZE_1G ? CMDQ_CREATE_CQ_PG_SIZE_PG_1G :
1952 CMDQ_CREATE_CQ_PG_SIZE_PG_4K));
1954 req.pbl = cpu_to_le64(pbl->pg_map_arr[0]);
1956 req.cq_fco_cnq_id = cpu_to_le32(
1957 (cq->cnq_hw_ring_id & CMDQ_CREATE_CQ_CNQ_ID_MASK) <<
1958 CMDQ_CREATE_CQ_CNQ_ID_SFT);
1960 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
1961 (void *)&resp, NULL, 0);
1962 if (rc)
1963 goto fail;
1965 cq->id = le32_to_cpu(resp.xid);
1966 cq->dbr_base = res->dpi_tbl.dbr_bar_reg_iomem;
1967 cq->period = BNXT_QPLIB_QUEUE_START_PERIOD;
1968 init_waitqueue_head(&cq->waitq);
1969 INIT_LIST_HEAD(&cq->sqf_head);
1970 INIT_LIST_HEAD(&cq->rqf_head);
1971 spin_lock_init(&cq->compl_lock);
1973 bnxt_qplib_arm_cq_enable(cq);
1974 return 0;
1976 fail:
1977 bnxt_qplib_free_hwq(res->pdev, &cq->hwq);
1978 exit:
1979 return rc;
1980 }
1982 int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
1983 {
1984 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1985 struct cmdq_destroy_cq req;
1986 struct creq_destroy_cq_resp resp;
1987 u16 cmd_flags = 0;
1988 int rc;
1990 RCFW_CMD_PREP(req, DESTROY_CQ, cmd_flags);
1992 req.cq_cid = cpu_to_le32(cq->id);
1993 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
1994 (void *)&resp, NULL, 0);
1995 if (rc)
1996 return rc;
1997 bnxt_qplib_free_hwq(res->pdev, &cq->hwq);
1998 return 0;
1999 }
2001 static int __flush_sq(struct bnxt_qplib_q *sq, struct bnxt_qplib_qp *qp,
2002 struct bnxt_qplib_cqe **pcqe, int *budget)
2003 {
2004 u32 sw_prod, sw_cons;
2005 struct bnxt_qplib_cqe *cqe;
2006 int rc = 0;
2008 /* Now complete all outstanding SQEs with FLUSHED_ERR */
2009 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
2010 cqe = *pcqe;
2011 while (*budget) {
2012 sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
2013 if (sw_cons == sw_prod) {
2014 break;
2015 }
2016 /* Skip the FENCE WQE completions */
2017 if (sq->swq[sw_cons].wr_id == BNXT_QPLIB_FENCE_WRID) {
2018 bnxt_qplib_cancel_phantom_processing(qp);
2019 goto skip_compl;
2020 }
2021 memset(cqe, 0, sizeof(*cqe));
2022 cqe->status = CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR;
2023 cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
2024 cqe->qp_handle = (u64)(unsigned long)qp;
2025 cqe->wr_id = sq->swq[sw_cons].wr_id;
2026 cqe->src_qp = qp->id;
2027 cqe->type = sq->swq[sw_cons].type;
2028 cqe++;
2029 (*budget)--;
2030 skip_compl:
2031 sq->hwq.cons++;
2032 }
2033 *pcqe = cqe;
2034 if (!(*budget) && HWQ_CMP(sq->hwq.cons, &sq->hwq) != sw_prod)
2035 /* Out of budget */
2036 rc = -EAGAIN;
2038 return rc;
2039 }
2041 static int __flush_rq(struct bnxt_qplib_q *rq, struct bnxt_qplib_qp *qp,
2042 struct bnxt_qplib_cqe **pcqe, int *budget)
2043 {
2044 struct bnxt_qplib_cqe *cqe;
2045 u32 sw_prod, sw_cons;
2046 int rc = 0;
2047 int opcode = 0;
2049 switch (qp->type) {
2050 case CMDQ_CREATE_QP1_TYPE_GSI:
2051 opcode = CQ_BASE_CQE_TYPE_RES_RAWETH_QP1;
2052 break;
2053 case CMDQ_CREATE_QP_TYPE_RC:
2054 opcode = CQ_BASE_CQE_TYPE_RES_RC;
2055 break;
2056 case CMDQ_CREATE_QP_TYPE_UD:
2057 opcode = CQ_BASE_CQE_TYPE_RES_UD;
2058 break;
2059 }
2061 /* Flush the rest of the RQ */
2062 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
2063 cqe = *pcqe;
2064 while (*budget) {
2065 sw_cons = HWQ_CMP(rq->hwq.cons, &rq->hwq);
2066 if (sw_cons == sw_prod)
2067 break;
2068 memset(cqe, 0, sizeof(*cqe));
2069 cqe->status =
2070 CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR;
2071 cqe->opcode = opcode;
2072 cqe->qp_handle = (unsigned long)qp;
2073 cqe->wr_id = rq->swq[sw_cons].wr_id;
2074 cqe++;
2075 (*budget)--;
2076 rq->hwq.cons++;
2077 }
2078 *pcqe = cqe;
2079 if (!*budget && HWQ_CMP(rq->hwq.cons, &rq->hwq) != sw_prod)
2080 /* Out of budget */
2081 rc = -EAGAIN;
2083 return rc;
2084 }
2086 void bnxt_qplib_mark_qp_error(void *qp_handle)
2087 {
2088 struct bnxt_qplib_qp *qp = qp_handle;
2090 if (!qp)
2091 return;
2093 /* Must block new posting of SQ and RQ */
2094 qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
2095 bnxt_qplib_cancel_phantom_processing(qp);
2096 }
2098 /* Note: SQE is valid from sw_sq_cons up to cqe_sq_cons (exclusive)
2099 * CQE is track from sw_cq_cons to max_element but valid only if VALID=1
2100 */
2101 static int do_wa9060(struct bnxt_qplib_qp *qp, struct bnxt_qplib_cq *cq,
2102 u32 cq_cons, u32 sw_sq_cons, u32 cqe_sq_cons)
2103 {
2104 struct bnxt_qplib_q *sq = &qp->sq;
2105 struct bnxt_qplib_swq *swq;
2106 u32 peek_sw_cq_cons, peek_raw_cq_cons, peek_sq_cons_idx;
2107 struct cq_base *peek_hwcqe, **peek_hw_cqe_ptr;
2108 struct cq_req *peek_req_hwcqe;
2109 struct bnxt_qplib_qp *peek_qp;
2110 struct bnxt_qplib_q *peek_sq;
2111 int i, rc = 0;
2113 /* Normal mode */
2114 /* Check for the psn_search marking before completing */
2115 swq = &sq->swq[sw_sq_cons];
2116 if (swq->psn_search &&
2117 le32_to_cpu(swq->psn_search->flags_next_psn) & 0x80000000) {
2118 /* Unmark */
2119 swq->psn_search->flags_next_psn = cpu_to_le32
2120 (le32_to_cpu(swq->psn_search->flags_next_psn)
2121 & ~0x80000000);
2122 dev_dbg(&cq->hwq.pdev->dev,
2123 "FP: Process Req cq_cons=0x%x qp=0x%x sq cons sw=0x%x cqe=0x%x marked!\n",
2124 cq_cons, qp->id, sw_sq_cons, cqe_sq_cons);
2125 sq->condition = true;
2126 sq->send_phantom = true;
2128 /* TODO: Only ARM if the previous SQE is ARMALL */
2129 bnxt_qplib_arm_cq(cq, DBR_DBR_TYPE_CQ_ARMALL);
2131 rc = -EAGAIN;
2132 goto out;
2133 }
2134 if (sq->condition) {
2135 /* Peek at the completions */
2136 peek_raw_cq_cons = cq->hwq.cons;
2137 peek_sw_cq_cons = cq_cons;
2138 i = cq->hwq.max_elements;
2139 while (i--) {
2140 peek_sw_cq_cons = HWQ_CMP((peek_sw_cq_cons), &cq->hwq);
2141 peek_hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr;
2142 peek_hwcqe = &peek_hw_cqe_ptr[CQE_PG(peek_sw_cq_cons)]
2143 [CQE_IDX(peek_sw_cq_cons)];
2144 /* If the next hwcqe is VALID */
2145 if (CQE_CMP_VALID(peek_hwcqe, peek_raw_cq_cons,
2146 cq->hwq.max_elements)) {
2147 /*
2148 * The valid test of the entry must be done first before
2149 * reading any further.
2150 */
2151 dma_rmb();
2152 /* If the next hwcqe is a REQ */
2153 if ((peek_hwcqe->cqe_type_toggle &
2154 CQ_BASE_CQE_TYPE_MASK) ==
2155 CQ_BASE_CQE_TYPE_REQ) {
2156 peek_req_hwcqe = (struct cq_req *)
2157 peek_hwcqe;
2158 peek_qp = (struct bnxt_qplib_qp *)
2159 ((unsigned long)
2160 le64_to_cpu
2161 (peek_req_hwcqe->qp_handle));
2162 peek_sq = &peek_qp->sq;
2163 peek_sq_cons_idx = HWQ_CMP(le16_to_cpu(
2164 peek_req_hwcqe->sq_cons_idx) - 1
2165 , &sq->hwq);
2166 /* If the hwcqe's sq's wr_id matches */
2167 if (peek_sq == sq &&
2168 sq->swq[peek_sq_cons_idx].wr_id ==
2169 BNXT_QPLIB_FENCE_WRID) {
2170 /*
2171 * Unbreak only if the phantom
2172 * comes back
2173 */
2174 dev_dbg(&cq->hwq.pdev->dev,
2175 "FP:Got Phantom CQE");
2176 sq->condition = false;
2177 sq->single = true;
2178 rc = 0;
2179 goto out;
2180 }
2181 }
2182 /* Valid but not the phantom, so keep looping */
2183 } else {
2184 /* Not valid yet, just exit and wait */
2185 rc = -EINVAL;
2186 goto out;
2187 }
2188 peek_sw_cq_cons++;
2189 peek_raw_cq_cons++;
2190 }
2191 dev_err(&cq->hwq.pdev->dev,
2192 "Should not have come here! cq_cons=0x%x qp=0x%x sq cons sw=0x%x hw=0x%x",
2193 cq_cons, qp->id, sw_sq_cons, cqe_sq_cons);
2194 rc = -EINVAL;
2195 }
2196 out:
2197 return rc;
2198 }
2200 static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq,
2201 struct cq_req *hwcqe,
2202 struct bnxt_qplib_cqe **pcqe, int *budget,
2203 u32 cq_cons, struct bnxt_qplib_qp **lib_qp)
2204 {
2205 struct bnxt_qplib_qp *qp;
2206 struct bnxt_qplib_q *sq;
2207 struct bnxt_qplib_cqe *cqe;
2208 u32 sw_sq_cons, cqe_sq_cons;
2209 struct bnxt_qplib_swq *swq;
2210 int rc = 0;
2212 qp = (struct bnxt_qplib_qp *)((unsigned long)
2213 le64_to_cpu(hwcqe->qp_handle));
2214 if (!qp) {
2215 dev_err(&cq->hwq.pdev->dev,
2216 "QPLIB: FP: Process Req qp is NULL");
2217 return -EINVAL;
2218 }
2219 sq = &qp->sq;
2221 cqe_sq_cons = HWQ_CMP(le16_to_cpu(hwcqe->sq_cons_idx), &sq->hwq);
2222 if (cqe_sq_cons > sq->hwq.max_elements) {
2223 dev_err(&cq->hwq.pdev->dev,
2224 "QPLIB: FP: CQ Process req reported ");
2225 dev_err(&cq->hwq.pdev->dev,
2226 "QPLIB: sq_cons_idx 0x%x which exceeded max 0x%x",
2227 cqe_sq_cons, sq->hwq.max_elements);
2228 return -EINVAL;
2229 }
2231 if (qp->sq.flushed) {
2232 dev_dbg(&cq->hwq.pdev->dev,
2233 "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
2234 goto done;
2235 }
2236 /* Require to walk the sq's swq to fabricate CQEs for all previously
2237 * signaled SWQEs due to CQE aggregation from the current sq cons
2238 * to the cqe_sq_cons
2239 */
2240 cqe = *pcqe;
2241 while (*budget) {
2242 sw_sq_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
2243 if (sw_sq_cons == cqe_sq_cons)
2244 /* Done */
2245 break;
2247 swq = &sq->swq[sw_sq_cons];
2248 memset(cqe, 0, sizeof(*cqe));
2249 cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
2250 cqe->qp_handle = (u64)(unsigned long)qp;
2251 cqe->src_qp = qp->id;
2252 cqe->wr_id = swq->wr_id;
2253 if (cqe->wr_id == BNXT_QPLIB_FENCE_WRID)
2254 goto skip;
2255 cqe->type = swq->type;
2257 /* For the last CQE, check for status. For errors, regardless
2258 * of the request being signaled or not, it must complete with
2259 * the hwcqe error status
2260 */
2261 if (HWQ_CMP((sw_sq_cons + 1), &sq->hwq) == cqe_sq_cons &&
2262 hwcqe->status != CQ_REQ_STATUS_OK) {
2263 cqe->status = hwcqe->status;
2264 dev_err(&cq->hwq.pdev->dev,
2265 "QPLIB: FP: CQ Processed Req ");
2266 dev_err(&cq->hwq.pdev->dev,
2267 "QPLIB: wr_id[%d] = 0x%llx with status 0x%x",
2268 sw_sq_cons, cqe->wr_id, cqe->status);
2269 cqe++;
2270 (*budget)--;
2271 bnxt_qplib_mark_qp_error(qp);
2272 /* Add qp to flush list of the CQ */
2273 bnxt_qplib_add_flush_qp(qp);
2274 } else {
2275 if (swq->flags & SQ_SEND_FLAGS_SIGNAL_COMP) {
2276 /* Before we complete, do WA 9060 */
2277 if (do_wa9060(qp, cq, cq_cons, sw_sq_cons,
2278 cqe_sq_cons)) {
2279 *lib_qp = qp;
2280 goto out;
2281 }
2282 cqe->status = CQ_REQ_STATUS_OK;
2283 cqe++;
2284 (*budget)--;
2285 }
2286 }
2287 skip:
2288 sq->hwq.cons++;
2289 if (sq->single)
2290 break;
2291 }
2292 out:
2293 *pcqe = cqe;
2294 if (HWQ_CMP(sq->hwq.cons, &sq->hwq) != cqe_sq_cons) {
2295 /* Out of budget */
2296 rc = -EAGAIN;
2297 goto done;
2298 }
2299 /*
2300 * Back to normal completion mode only after it has completed all of
2301 * the WC for this CQE
2302 */
2303 sq->single = false;
2304 done:
2305 return rc;
2306 }
2308 static void bnxt_qplib_release_srqe(struct bnxt_qplib_srq *srq, u32 tag)
2309 {
2310 spin_lock(&srq->hwq.lock);
2311 srq->swq[srq->last_idx].next_idx = (int)tag;
2312 srq->last_idx = (int)tag;
2313 srq->swq[srq->last_idx].next_idx = -1;
2314 srq->hwq.cons++; /* Support for SRQE counter */
2315 spin_unlock(&srq->hwq.lock);
2316 }
2318 static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq *cq,
2319 struct cq_res_rc *hwcqe,
2320 struct bnxt_qplib_cqe **pcqe,
2321 int *budget)
2322 {
2323 struct bnxt_qplib_qp *qp;
2324 struct bnxt_qplib_q *rq;
2325 struct bnxt_qplib_srq *srq;
2326 struct bnxt_qplib_cqe *cqe;
2327 u32 wr_id_idx;
2328 int rc = 0;
2330 qp = (struct bnxt_qplib_qp *)((unsigned long)
2331 le64_to_cpu(hwcqe->qp_handle));
2332 if (!qp) {
2333 dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq RC qp is NULL");
2334 return -EINVAL;
2335 }
2336 if (qp->rq.flushed) {
2337 dev_dbg(&cq->hwq.pdev->dev,
2338 "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
2339 goto done;
2340 }
2342 cqe = *pcqe;
2343 cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
2344 cqe->length = le32_to_cpu(hwcqe->length);
2345 cqe->invrkey = le32_to_cpu(hwcqe->imm_data_or_inv_r_key);
2346 cqe->mr_handle = le64_to_cpu(hwcqe->mr_handle);
2347 cqe->flags = le16_to_cpu(hwcqe->flags);
2348 cqe->status = hwcqe->status;
2349 cqe->qp_handle = (u64)(unsigned long)qp;
2351 wr_id_idx = le32_to_cpu(hwcqe->srq_or_rq_wr_id) &
2352 CQ_RES_RC_SRQ_OR_RQ_WR_ID_MASK;
2353 if (cqe->flags & CQ_RES_RC_FLAGS_SRQ_SRQ) {
2354 srq = qp->srq;
2355 if (!srq)
2356 return -EINVAL;
2357 if (wr_id_idx >= srq->hwq.max_elements) {
2358 dev_err(&cq->hwq.pdev->dev,
2359 "QPLIB: FP: CQ Process RC ");
2360 dev_err(&cq->hwq.pdev->dev,
2361 "QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x",
2362 wr_id_idx, srq->hwq.max_elements);
2363 return -EINVAL;
2364 }
2365 cqe->wr_id = srq->swq[wr_id_idx].wr_id;
2366 bnxt_qplib_release_srqe(srq, wr_id_idx);
2367 cqe++;
2368 (*budget)--;
2369 *pcqe = cqe;
2370 } else {
2371 rq = &qp->rq;
2372 if (wr_id_idx >= rq->hwq.max_elements) {
2373 dev_err(&cq->hwq.pdev->dev,
2374 "QPLIB: FP: CQ Process RC ");
2375 dev_err(&cq->hwq.pdev->dev,
2376 "QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x",
2377 wr_id_idx, rq->hwq.max_elements);
2378 return -EINVAL;
2379 }
2380 cqe->wr_id = rq->swq[wr_id_idx].wr_id;
2381 cqe++;
2382 (*budget)--;
2383 rq->hwq.cons++;
2384 *pcqe = cqe;
2386 if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
2387 qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
2388 /* Add qp to flush list of the CQ */
2389 bnxt_qplib_add_flush_qp(qp);
2390 }
2391 }
2393 done:
2394 return rc;
2395 }
2397 static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq,
2398 struct cq_res_ud *hwcqe,
2399 struct bnxt_qplib_cqe **pcqe,
2400 int *budget)
2401 {
2402 struct bnxt_qplib_qp *qp;
2403 struct bnxt_qplib_q *rq;
2404 struct bnxt_qplib_srq *srq;
2405 struct bnxt_qplib_cqe *cqe;
2406 u32 wr_id_idx;
2407 int rc = 0;
2409 qp = (struct bnxt_qplib_qp *)((unsigned long)
2410 le64_to_cpu(hwcqe->qp_handle));
2411 if (!qp) {
2412 dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq UD qp is NULL");
2413 return -EINVAL;
2414 }
2415 if (qp->rq.flushed) {
2416 dev_dbg(&cq->hwq.pdev->dev,
2417 "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
2418 goto done;
2419 }
2420 cqe = *pcqe;
2421 cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
2422 cqe->length = le32_to_cpu(hwcqe->length);
2423 cqe->invrkey = le32_to_cpu(hwcqe->imm_data);
2424 cqe->flags = le16_to_cpu(hwcqe->flags);
2425 cqe->status = hwcqe->status;
2426 cqe->qp_handle = (u64)(unsigned long)qp;
2427 memcpy(cqe->smac, hwcqe->src_mac, 6);
2428 wr_id_idx = le32_to_cpu(hwcqe->src_qp_high_srq_or_rq_wr_id)
2429 & CQ_RES_UD_SRQ_OR_RQ_WR_ID_MASK;
2430 cqe->src_qp = le16_to_cpu(hwcqe->src_qp_low) |
2431 ((le32_to_cpu(
2432 hwcqe->src_qp_high_srq_or_rq_wr_id) &
2433 CQ_RES_UD_SRC_QP_HIGH_MASK) >> 8);
2435 if (cqe->flags & CQ_RES_RC_FLAGS_SRQ_SRQ) {
2436 srq = qp->srq;
2437 if (!srq)
2438 return -EINVAL;
2440 if (wr_id_idx >= srq->hwq.max_elements) {
2441 dev_err(&cq->hwq.pdev->dev,
2442 "QPLIB: FP: CQ Process UD ");
2443 dev_err(&cq->hwq.pdev->dev,
2444 "QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x",
2445 wr_id_idx, srq->hwq.max_elements);
2446 return -EINVAL;
2447 }
2448 cqe->wr_id = srq->swq[wr_id_idx].wr_id;
2449 bnxt_qplib_release_srqe(srq, wr_id_idx);
2450 cqe++;
2451 (*budget)--;
2452 *pcqe = cqe;
2453 } else {
2454 rq = &qp->rq;
2455 if (wr_id_idx >= rq->hwq.max_elements) {
2456 dev_err(&cq->hwq.pdev->dev,
2457 "QPLIB: FP: CQ Process UD ");
2458 dev_err(&cq->hwq.pdev->dev,
2459 "QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x",
2460 wr_id_idx, rq->hwq.max_elements);
2461 return -EINVAL;
2462 }
2464 cqe->wr_id = rq->swq[wr_id_idx].wr_id;
2465 cqe++;
2466 (*budget)--;
2467 rq->hwq.cons++;
2468 *pcqe = cqe;
2470 if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
2471 qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
2472 /* Add qp to flush list of the CQ */
2473 bnxt_qplib_add_flush_qp(qp);
2474 }
2475 }
2476 done:
2477 return rc;
2478 }
2480 bool bnxt_qplib_is_cq_empty(struct bnxt_qplib_cq *cq)
2481 {
2482 struct cq_base *hw_cqe, **hw_cqe_ptr;
2483 u32 sw_cons, raw_cons;
2484 bool rc = true;
2486 raw_cons = cq->hwq.cons;
2487 sw_cons = HWQ_CMP(raw_cons, &cq->hwq);
2488 hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr;
2489 hw_cqe = &hw_cqe_ptr[CQE_PG(sw_cons)][CQE_IDX(sw_cons)];
2491 /* Check for Valid bit. If the CQE is valid, return false */
2492 rc = !CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements);
2493 return rc;
2494 }
2496 static int bnxt_qplib_cq_process_res_raweth_qp1(struct bnxt_qplib_cq *cq,
2497 struct cq_res_raweth_qp1 *hwcqe,
2498 struct bnxt_qplib_cqe **pcqe,
2499 int *budget)
2500 {
2501 struct bnxt_qplib_qp *qp;
2502 struct bnxt_qplib_q *rq;
2503 struct bnxt_qplib_srq *srq;
2504 struct bnxt_qplib_cqe *cqe;
2505 u32 wr_id_idx;
2506 int rc = 0;
2508 qp = (struct bnxt_qplib_qp *)((unsigned long)
2509 le64_to_cpu(hwcqe->qp_handle));
2510 if (!qp) {
2511 dev_err(&cq->hwq.pdev->dev,
2512 "QPLIB: process_cq Raw/QP1 qp is NULL");
2513 return -EINVAL;
2514 }
2515 if (qp->rq.flushed) {
2516 dev_dbg(&cq->hwq.pdev->dev,
2517 "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
2518 goto done;
2519 }
2520 cqe = *pcqe;
2521 cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
2522 cqe->flags = le16_to_cpu(hwcqe->flags);
2523 cqe->qp_handle = (u64)(unsigned long)qp;
2525 wr_id_idx =
2526 le32_to_cpu(hwcqe->raweth_qp1_payload_offset_srq_or_rq_wr_id)
2527 & CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_MASK;
2528 cqe->src_qp = qp->id;
2529 if (qp->id == 1 && !cqe->length) {
2530 /* Add workaround for the length misdetection */
2531 cqe->length = 296;
2532 } else {
2533 cqe->length = le16_to_cpu(hwcqe->length);
2534 }
2535 cqe->pkey_index = qp->pkey_index;
2536 memcpy(cqe->smac, qp->smac, 6);
2538 cqe->raweth_qp1_flags = le16_to_cpu(hwcqe->raweth_qp1_flags);
2539 cqe->raweth_qp1_flags2 = le32_to_cpu(hwcqe->raweth_qp1_flags2);
2540 cqe->raweth_qp1_metadata = le32_to_cpu(hwcqe->raweth_qp1_metadata);
2542 if (cqe->flags & CQ_RES_RAWETH_QP1_FLAGS_SRQ_SRQ) {
2543 srq = qp->srq;
2544 if (!srq) {
2545 dev_err(&cq->hwq.pdev->dev,
2546 "QPLIB: FP: SRQ used but not defined??");
2547 return -EINVAL;
2548 }
2549 if (wr_id_idx >= srq->hwq.max_elements) {
2550 dev_err(&cq->hwq.pdev->dev,
2551 "QPLIB: FP: CQ Process Raw/QP1 ");
2552 dev_err(&cq->hwq.pdev->dev,
2553 "QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x",
2554 wr_id_idx, srq->hwq.max_elements);
2555 return -EINVAL;
2556 }
2557 cqe->wr_id = srq->swq[wr_id_idx].wr_id;
2558 bnxt_qplib_release_srqe(srq, wr_id_idx);
2559 cqe++;
2560 (*budget)--;
2561 *pcqe = cqe;
2562 } else {
2563 rq = &qp->rq;
2564 if (wr_id_idx >= rq->hwq.max_elements) {
2565 dev_err(&cq->hwq.pdev->dev,
2566 "QPLIB: FP: CQ Process Raw/QP1 RQ wr_id ");
2567 dev_err(&cq->hwq.pdev->dev,
2568 "QPLIB: ix 0x%x exceeded RQ max 0x%x",
2569 wr_id_idx, rq->hwq.max_elements);
2570 return -EINVAL;
2571 }
2572 cqe->wr_id = rq->swq[wr_id_idx].wr_id;
2573 cqe++;
2574 (*budget)--;
2575 rq->hwq.cons++;
2576 *pcqe = cqe;
2578 if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
2579 qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
2580 /* Add qp to flush list of the CQ */
2581 bnxt_qplib_add_flush_qp(qp);
2582 }
2583 }
2585 done:
2586 return rc;
2587 }
2589 static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq,
2590 struct cq_terminal *hwcqe,
2591 struct bnxt_qplib_cqe **pcqe,
2592 int *budget)
2593 {
2594 struct bnxt_qplib_qp *qp;
2595 struct bnxt_qplib_q *sq, *rq;
2596 struct bnxt_qplib_cqe *cqe;
2597 u32 sw_cons = 0, cqe_cons;
2598 int rc = 0;
2600 /* Check the Status */
2601 if (hwcqe->status != CQ_TERMINAL_STATUS_OK)
2602 dev_warn(&cq->hwq.pdev->dev,
2603 "QPLIB: FP: CQ Process Terminal Error status = 0x%x",
2604 hwcqe->status);
2606 qp = (struct bnxt_qplib_qp *)((unsigned long)
2607 le64_to_cpu(hwcqe->qp_handle));
2608 if (!qp) {
2609 dev_err(&cq->hwq.pdev->dev,
2610 "QPLIB: FP: CQ Process terminal qp is NULL");
2611 return -EINVAL;
2612 }
2614 /* Must block new posting of SQ and RQ */
2615 qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
2617 sq = &qp->sq;
2618 rq = &qp->rq;
2620 cqe_cons = le16_to_cpu(hwcqe->sq_cons_idx);
2621 if (cqe_cons == 0xFFFF)
2622 goto do_rq;
2624 if (cqe_cons > sq->hwq.max_elements) {
2625 dev_err(&cq->hwq.pdev->dev,
2626 "QPLIB: FP: CQ Process terminal reported ");
2627 dev_err(&cq->hwq.pdev->dev,
2628 "QPLIB: sq_cons_idx 0x%x which exceeded max 0x%x",
2629 cqe_cons, sq->hwq.max_elements);
2630 goto do_rq;
2631 }
2633 if (qp->sq.flushed) {
2634 dev_dbg(&cq->hwq.pdev->dev,
2635 "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
2636 goto sq_done;
2637 }
2639 /* Terminal CQE can also include aggregated successful CQEs prior.
2640 * So we must complete all CQEs from the current sq's cons to the
2641 * cq_cons with status OK
2642 */
2643 cqe = *pcqe;
2644 while (*budget) {
2645 sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
2646 if (sw_cons == cqe_cons)
2647 break;
2648 if (sq->swq[sw_cons].flags & SQ_SEND_FLAGS_SIGNAL_COMP) {
2649 memset(cqe, 0, sizeof(*cqe));
2650 cqe->status = CQ_REQ_STATUS_OK;
2651 cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
2652 cqe->qp_handle = (u64)(unsigned long)qp;
2653 cqe->src_qp = qp->id;
2654 cqe->wr_id = sq->swq[sw_cons].wr_id;
2655 cqe->type = sq->swq[sw_cons].type;
2656 cqe++;
2657 (*budget)--;
2658 }
2659 sq->hwq.cons++;
2660 }
2661 *pcqe = cqe;
2662 if (!(*budget) && sw_cons != cqe_cons) {
2663 /* Out of budget */
2664 rc = -EAGAIN;
2665 goto sq_done;
2666 }
2667 sq_done:
2668 if (rc)
2669 return rc;
2670 do_rq:
2671 cqe_cons = le16_to_cpu(hwcqe->rq_cons_idx);
2672 if (cqe_cons == 0xFFFF) {
2673 goto done;
2674 } else if (cqe_cons > rq->hwq.max_elements) {
2675 dev_err(&cq->hwq.pdev->dev,
2676 "QPLIB: FP: CQ Processed terminal ");
2677 dev_err(&cq->hwq.pdev->dev,
2678 "QPLIB: reported rq_cons_idx 0x%x exceeds max 0x%x",
2679 cqe_cons, rq->hwq.max_elements);
2680 goto done;
2681 }
2683 if (qp->rq.flushed) {
2684 dev_dbg(&cq->hwq.pdev->dev,
2685 "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
2686 rc = 0;
2687 goto done;
2688 }
2690 /* Terminal CQE requires all posted RQEs to complete with FLUSHED_ERR
2691 * from the current rq->cons to the rq->prod regardless what the
2692 * rq->cons the terminal CQE indicates
2693 */
2695 /* Add qp to flush list of the CQ */
2696 bnxt_qplib_add_flush_qp(qp);
2697 done:
2698 return rc;
2699 }
2701 static int bnxt_qplib_cq_process_cutoff(struct bnxt_qplib_cq *cq,
2702 struct cq_cutoff *hwcqe)
2703 {
2704 /* Check the Status */
2705 if (hwcqe->status != CQ_CUTOFF_STATUS_OK) {
2706 dev_err(&cq->hwq.pdev->dev,
2707 "QPLIB: FP: CQ Process Cutoff Error status = 0x%x",
2708 hwcqe->status);
2709 return -EINVAL;
2710 }
2711 clear_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags);
2712 wake_up_interruptible(&cq->waitq);
2714 return 0;
2715 }
2717 int bnxt_qplib_process_flush_list(struct bnxt_qplib_cq *cq,
2718 struct bnxt_qplib_cqe *cqe,
2719 int num_cqes)
2720 {
2721 struct bnxt_qplib_qp *qp = NULL;
2722 u32 budget = num_cqes;
2723 unsigned long flags;
2725 spin_lock_irqsave(&cq->flush_lock, flags);
2726 list_for_each_entry(qp, &cq->sqf_head, sq_flush) {
2727 dev_dbg(&cq->hwq.pdev->dev,
2728 "QPLIB: FP: Flushing SQ QP= %p",
2729 qp);
2730 __flush_sq(&qp->sq, qp, &cqe, &budget);
2731 }
2733 list_for_each_entry(qp, &cq->rqf_head, rq_flush) {
2734 dev_dbg(&cq->hwq.pdev->dev,
2735 "QPLIB: FP: Flushing RQ QP= %p",
2736 qp);
2737 __flush_rq(&qp->rq, qp, &cqe, &budget);
2738 }
2739 spin_unlock_irqrestore(&cq->flush_lock, flags);
2741 return num_cqes - budget;
2742 }
2744 int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe,
2745 int num_cqes, struct bnxt_qplib_qp **lib_qp)
2746 {
2747 struct cq_base *hw_cqe, **hw_cqe_ptr;
2748 u32 sw_cons, raw_cons;
2749 int budget, rc = 0;
2751 raw_cons = cq->hwq.cons;
2752 budget = num_cqes;
2754 while (budget) {
2755 sw_cons = HWQ_CMP(raw_cons, &cq->hwq);
2756 hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr;
2757 hw_cqe = &hw_cqe_ptr[CQE_PG(sw_cons)][CQE_IDX(sw_cons)];
2759 /* Check for Valid bit */
2760 if (!CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements))
2761 break;
2763 /*
2764 * The valid test of the entry must be done first before
2765 * reading any further.
2766 */
2767 dma_rmb();
2768 /* From the device's respective CQE format to qplib_wc*/
2769 switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) {
2770 case CQ_BASE_CQE_TYPE_REQ:
2771 rc = bnxt_qplib_cq_process_req(cq,
2772 (struct cq_req *)hw_cqe,
2773 &cqe, &budget,
2774 sw_cons, lib_qp);
2775 break;
2776 case CQ_BASE_CQE_TYPE_RES_RC:
2777 rc = bnxt_qplib_cq_process_res_rc(cq,
2778 (struct cq_res_rc *)
2779 hw_cqe, &cqe,
2780 &budget);
2781 break;
2782 case CQ_BASE_CQE_TYPE_RES_UD:
2783 rc = bnxt_qplib_cq_process_res_ud
2784 (cq, (struct cq_res_ud *)hw_cqe, &cqe,
2785 &budget);
2786 break;
2787 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
2788 rc = bnxt_qplib_cq_process_res_raweth_qp1
2789 (cq, (struct cq_res_raweth_qp1 *)
2790 hw_cqe, &cqe, &budget);
2791 break;
2792 case CQ_BASE_CQE_TYPE_TERMINAL:
2793 rc = bnxt_qplib_cq_process_terminal
2794 (cq, (struct cq_terminal *)hw_cqe,
2795 &cqe, &budget);
2796 break;
2797 case CQ_BASE_CQE_TYPE_CUT_OFF:
2798 bnxt_qplib_cq_process_cutoff
2799 (cq, (struct cq_cutoff *)hw_cqe);
2800 /* Done processing this CQ */
2801 goto exit;
2802 default:
2803 dev_err(&cq->hwq.pdev->dev,
2804 "QPLIB: process_cq unknown type 0x%lx",
2805 hw_cqe->cqe_type_toggle &
2806 CQ_BASE_CQE_TYPE_MASK);
2807 rc = -EINVAL;
2808 break;
2809 }
2810 if (rc < 0) {
2811 if (rc == -EAGAIN)
2812 break;
2813 /* Error while processing the CQE, just skip to the
2814 * next one
2815 */
2816 dev_err(&cq->hwq.pdev->dev,
2817 "QPLIB: process_cqe error rc = 0x%x", rc);
2818 }
2819 raw_cons++;
2820 }
2821 if (cq->hwq.cons != raw_cons) {
2822 cq->hwq.cons = raw_cons;
2823 bnxt_qplib_arm_cq(cq, DBR_DBR_TYPE_CQ);
2824 }
2825 exit:
2826 return num_cqes - budget;
2827 }
2829 void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type)
2830 {
2831 if (arm_type)
2832 bnxt_qplib_arm_cq(cq, arm_type);
2833 /* Using cq->arm_state variable to track whether to issue cq handler */
2834 atomic_set(&cq->arm_state, 1);
2835 }
2837 void bnxt_qplib_flush_cqn_wq(struct bnxt_qplib_qp *qp)
2838 {
2839 flush_workqueue(qp->scq->nq->cqn_wq);
2840 if (qp->scq != qp->rcq)
2841 flush_workqueue(qp->rcq->nq->cqn_wq);
2842 }