1 /*
2 * Copyright (C) 2005-2006 by Texas Instruments
3 * Copyright (c) 2008, MontaVista Software, Inc. <source@mvista.com>
4 *
5 * This file implements a DMA interface using TI's CPPI 4.1 DMA.
6 *
7 * This program is free software; you can distribute it and/or modify it
8 * under the terms of the GNU General Public License (Version 2) as
9 * published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * for more details.
15 *
16 * You should have received a copy of the GNU General Public License along
17 * with this program; if not, write to the Free Software Foundation, Inc.,
18 * 59 Temple Place - Suite 330, Boston MA 02111-1307, USA.
19 *
20 */
22 #include <linux/errno.h>
23 #include <linux/dma-mapping.h>
24 #include <linux/module.h>
26 #include "cppi41.h"
28 #include "musb_core.h"
29 #include "musb_dma.h"
30 #include "cppi41_dma.h"
32 /* Configuration */
33 #define USB_CPPI41_DESC_SIZE_SHIFT 6
34 #define USB_CPPI41_DESC_ALIGN (1 << USB_CPPI41_DESC_SIZE_SHIFT)
35 #define USB_CPPI41_CH_NUM_PD 128 /* 4K bulk data at full speed */
36 #define USB_CPPI41_MAX_PD (USB_CPPI41_CH_NUM_PD * (USB_CPPI41_NUM_CH+1))
38 #undef DEBUG_CPPI_TD
39 #undef USBDRV_DEBUG
41 #ifdef USBDRV_DEBUG
42 #define dprintk(x, ...) printk(x, ## __VA_ARGS__)
43 #else
44 #define dprintk(x, ...)
45 #endif
47 /*
48 * Data structure definitions
49 */
51 /*
52 * USB Packet Descriptor
53 */
54 struct usb_pkt_desc;
56 struct usb_pkt_desc {
57 /* Hardware descriptor fields from this point */
58 struct cppi41_host_pkt_desc hw_desc; /* 40 bytes */
59 /* Protocol specific data */
60 dma_addr_t dma_addr; /* offs:44 byte */
61 struct usb_pkt_desc *next_pd_ptr; /* offs:48 byte*/
62 u8 ch_num;
63 u8 ep_num;
64 u8 eop;
65 u8 res1; /* offs:52 */
66 u8 res2[12]; /* offs:64 */
67 };
69 /**
70 * struct cppi41_channel - DMA Channel Control Structure
71 *
72 * Using the same for Tx/Rx.
73 */
74 struct cppi41_channel {
75 struct dma_channel channel;
77 struct cppi41_dma_ch_obj dma_ch_obj; /* DMA channel object */
78 struct cppi41_queue src_queue; /* Tx queue or Rx free descriptor/ */
79 /* buffer queue */
80 struct cppi41_queue_obj queue_obj; /* Tx queue object or Rx free */
81 /* descriptor/buffer queue object */
83 u32 tag_info; /* Tx PD Tag Information field */
85 /* Which direction of which endpoint? */
86 struct musb_hw_ep *end_pt;
87 u8 transmit;
88 u8 ch_num; /* Channel number of Tx/Rx 0..3 */
90 /* DMA mode: "transparent", RNDIS, CDC, or Generic RNDIS */
91 u8 dma_mode;
92 u8 autoreq;
94 /* Book keeping for the current transfer request */
95 dma_addr_t start_addr;
96 u32 length;
97 u32 curr_offset;
98 u16 pkt_size;
99 u8 transfer_mode;
100 u8 zlp_queued;
101 u8 inf_mode;
102 u8 tx_complete;
103 };
105 /**
106 * struct cppi41 - CPPI 4.1 DMA Controller Object
107 *
108 * Encapsulates all book keeping and data structures pertaining to
109 * the CPPI 1.4 DMA controller.
110 */
111 struct cppi41 {
112 struct dma_controller controller;
113 struct musb *musb;
115 struct cppi41_channel tx_cppi_ch[USB_CPPI41_NUM_CH];
116 struct cppi41_channel rx_cppi_ch[USB_CPPI41_NUM_CH];
117 struct work_struct txdma_work;
119 struct usb_pkt_desc *pd_pool_head; /* Free PD pool head */
120 dma_addr_t pd_mem_phys; /* PD memory physical address */
121 void *pd_mem; /* PD memory pointer */
122 u8 pd_mem_rgn; /* PD memory region number */
124 u16 teardownQNum; /* Teardown completion queue number */
125 struct cppi41_queue_obj queue_obj; /* Teardown completion queue */
126 /* object */
127 u32 pkt_info; /* Tx PD Packet Information field */
128 struct usb_cppi41_info *cppi_info; /* cppi channel information */
129 u8 en_bd_intr; /* enable bd interrupt */
130 u32 automode_reg_offs; /* USB_AUTOREQ_REG offset */
131 u32 teardown_reg_offs; /* USB_TEARDOWN_REG offset */
132 u32 bd_size;
133 u8 inf_mode;
134 };
136 struct usb_cppi41_info usb_cppi41_info[2];
137 EXPORT_SYMBOL(usb_cppi41_info);
139 #ifdef DEBUG_CPPI_TD
140 static void print_pd_list(struct usb_pkt_desc *pd_pool_head)
141 {
142 struct usb_pkt_desc *curr_pd = pd_pool_head;
143 int cnt = 0;
145 while (curr_pd != NULL) {
146 if (cnt % 8 == 0)
147 dprintk("\n%02x ", cnt);
148 cnt++;
149 dprintk(" %p", curr_pd);
150 curr_pd = curr_pd->next_pd_ptr;
151 }
152 dprintk("\n");
153 }
154 #endif
156 static struct usb_pkt_desc *usb_get_free_pd(struct cppi41 *cppi)
157 {
158 struct usb_pkt_desc *free_pd = cppi->pd_pool_head;
160 if (free_pd != NULL) {
161 cppi->pd_pool_head = free_pd->next_pd_ptr;
162 free_pd->next_pd_ptr = NULL;
163 }
164 return free_pd;
165 }
167 static void usb_put_free_pd(struct cppi41 *cppi, struct usb_pkt_desc *free_pd)
168 {
169 free_pd->next_pd_ptr = cppi->pd_pool_head;
170 cppi->pd_pool_head = free_pd;
171 }
173 /**
174 * cppi41_controller_start - start DMA controller
175 * @controller: the controller
176 *
177 * This function initializes the CPPI 4.1 Tx/Rx channels.
178 */
179 static int __devinit cppi41_controller_start(struct dma_controller *controller)
180 {
181 struct cppi41 *cppi;
182 struct cppi41_channel *cppi_ch;
183 void __iomem *reg_base;
184 struct usb_pkt_desc *curr_pd;
185 unsigned long pd_addr;
186 int i;
187 struct usb_cppi41_info *cppi_info;
188 struct musb *musb;
190 cppi = container_of(controller, struct cppi41, controller);
191 cppi_info = cppi->cppi_info;
192 musb = cppi->musb;
194 if (cpu_is_ti816x() || cpu_is_am33xx()) {
195 cppi->automode_reg_offs = TI81XX_USB_AUTOREQ_REG;
196 cppi->teardown_reg_offs = TI81XX_USB_TEARDOWN_REG;
197 } else {
198 cppi->automode_reg_offs = USB_AUTOREQ_REG;
199 cppi->teardown_reg_offs = USB_TEARDOWN_REG;
200 }
202 /*
203 * TODO: We may need to check USB_CPPI41_MAX_PD here since CPPI 4.1
204 * requires the descriptor count to be a multiple of 2 ^ 5 (i.e. 32).
205 * Similarly, the descriptor size should also be a multiple of 32.
206 */
208 /*
209 * Allocate free packet descriptor pool for all Tx/Rx endpoints --
210 * dma_alloc_coherent() will return a page aligned address, so our
211 * alignment requirement will be honored.
212 */
213 cppi->bd_size = USB_CPPI41_MAX_PD * sizeof(struct usb_pkt_desc);
214 cppi->pd_mem = dma_alloc_coherent(cppi->musb->controller,
215 cppi->bd_size,
216 &cppi->pd_mem_phys,
217 GFP_KERNEL | GFP_DMA);
218 if (cppi->pd_mem == NULL) {
219 dev_dbg(musb->controller, "ERROR: packet descriptor memory allocation failed\n");
220 return 0;
221 }
223 if (cppi41_mem_rgn_alloc(cppi_info->q_mgr, cppi->pd_mem_phys,
224 USB_CPPI41_DESC_SIZE_SHIFT,
225 get_count_order(USB_CPPI41_MAX_PD),
226 &cppi->pd_mem_rgn)) {
227 dev_dbg(musb->controller, "ERROR: queue manager memory region allocation "
228 "failed\n");
229 goto free_pds;
230 }
232 /* Allocate the teardown completion queue */
233 if (cppi41_queue_alloc(CPPI41_UNASSIGNED_QUEUE,
234 0, &cppi->teardownQNum)) {
235 dev_dbg(musb->controller, "ERROR: teardown completion queue allocation failed\n");
236 goto free_mem_rgn;
237 }
238 dev_dbg(musb->controller, "Allocated teardown completion queue %d in queue manager 0\n",
239 cppi->teardownQNum);
241 if (cppi41_queue_init(&cppi->queue_obj, 0, cppi->teardownQNum)) {
242 dev_dbg(musb->controller, "ERROR: teardown completion queue initialization "
243 "failed\n");
244 goto free_queue;
245 }
247 /*
248 * "Slice" PDs one-by-one from the big chunk and
249 * add them to the free pool.
250 */
251 curr_pd = (struct usb_pkt_desc *)cppi->pd_mem;
252 pd_addr = cppi->pd_mem_phys;
253 for (i = 0; i < USB_CPPI41_MAX_PD; i++) {
254 curr_pd->dma_addr = pd_addr;
256 usb_put_free_pd(cppi, curr_pd);
257 curr_pd = (struct usb_pkt_desc *)((char *)curr_pd +
258 USB_CPPI41_DESC_ALIGN);
259 pd_addr += USB_CPPI41_DESC_ALIGN;
260 }
262 /* Configure the Tx channels */
263 for (i = 0, cppi_ch = cppi->tx_cppi_ch;
264 i < ARRAY_SIZE(cppi->tx_cppi_ch); ++i, ++cppi_ch) {
265 const struct cppi41_tx_ch *tx_info;
267 memset(cppi_ch, 0, sizeof(struct cppi41_channel));
268 cppi_ch->transmit = 1;
269 cppi_ch->ch_num = i;
270 cppi_ch->channel.private_data = cppi;
272 /*
273 * Extract the CPPI 4.1 DMA Tx channel configuration and
274 * construct/store the Tx PD tag info field for later use...
275 */
276 tx_info = cppi41_dma_block[cppi_info->dma_block].tx_ch_info
277 + cppi_info->ep_dma_ch[i];
278 cppi_ch->src_queue = tx_info->tx_queue[0];
279 cppi_ch->tag_info = (tx_info->port_num <<
280 CPPI41_SRC_TAG_PORT_NUM_SHIFT) |
281 (tx_info->ch_num <<
282 CPPI41_SRC_TAG_CH_NUM_SHIFT) |
283 (tx_info->sub_ch_num <<
284 CPPI41_SRC_TAG_SUB_CH_NUM_SHIFT);
285 }
287 /* Configure the Rx channels */
288 for (i = 0, cppi_ch = cppi->rx_cppi_ch;
289 i < ARRAY_SIZE(cppi->rx_cppi_ch); ++i, ++cppi_ch) {
290 memset(cppi_ch, 0, sizeof(struct cppi41_channel));
291 cppi_ch->ch_num = i;
292 cppi_ch->channel.private_data = cppi;
293 }
295 /* Construct/store Tx PD packet info field for later use */
296 cppi->pkt_info = (CPPI41_PKT_TYPE_USB << CPPI41_PKT_TYPE_SHIFT) |
297 (CPPI41_RETURN_LINKED << CPPI41_RETURN_POLICY_SHIFT);
299 /* Do necessary configuartion in hardware to get started */
300 reg_base = cppi->musb->ctrl_base;
302 /* Disable auto request mode */
303 musb_writel(reg_base, cppi->automode_reg_offs, 0);
305 /* Disable the CDC/RNDIS modes */
306 musb_writel(reg_base, USB_TX_MODE_REG, 0);
307 musb_writel(reg_base, USB_RX_MODE_REG, 0);
309 return 1;
311 free_queue:
312 if (cppi41_queue_free(0, cppi->teardownQNum))
313 dev_dbg(musb->controller, "ERROR: failed to free teardown completion queue\n");
315 free_mem_rgn:
316 if (cppi41_mem_rgn_free(cppi_info->q_mgr, cppi->pd_mem_rgn))
317 dev_dbg(musb->controller, "ERROR: failed to free queue manager memory region\n");
319 free_pds:
320 dma_free_coherent(cppi->musb->controller,
321 cppi->bd_size,
322 cppi->pd_mem, cppi->pd_mem_phys);
324 return 0;
325 }
327 /**
328 * cppi41_controller_stop - stop DMA controller
329 * @controller: the controller
330 *
331 * De-initialize the DMA Controller as necessary.
332 */
333 static int cppi41_controller_stop(struct dma_controller *controller)
334 {
335 struct cppi41 *cppi;
336 void __iomem *reg_base;
337 struct usb_cppi41_info *cppi_info;
338 struct musb *musb;
340 cppi = container_of(controller, struct cppi41, controller);
341 cppi_info = cppi->cppi_info;
342 musb = cppi->musb;
344 /* Free the teardown completion queue */
345 if (cppi41_queue_free(cppi_info->q_mgr, cppi->teardownQNum))
346 dev_dbg(musb->controller, "ERROR: failed to free teardown completion queue\n");
348 /*
349 * Free the packet descriptor region allocated
350 * for all Tx/Rx channels.
351 */
352 if (cppi41_mem_rgn_free(cppi_info->q_mgr, cppi->pd_mem_rgn))
353 dev_dbg(musb->controller, "ERROR: failed to free queue manager memory region\n");
355 dma_free_coherent(cppi->musb->controller, cppi->bd_size,
356 cppi->pd_mem, cppi->pd_mem_phys);
358 cppi->pd_mem = 0;
359 cppi->pd_mem_phys = 0;
360 cppi->pd_pool_head = 0;
361 cppi->bd_size = 0;
363 reg_base = cppi->musb->ctrl_base;
365 /* Disable auto request mode */
366 musb_writel(reg_base, cppi->automode_reg_offs, 0);
368 /* Disable the CDC/RNDIS modes */
369 musb_writel(reg_base, USB_TX_MODE_REG, 0);
370 musb_writel(reg_base, USB_RX_MODE_REG, 0);
372 return 1;
373 }
375 /**
376 * cppi41_channel_alloc - allocate a CPPI channel for DMA.
377 * @controller: the controller
378 * @ep: the endpoint
379 * @is_tx: 1 for Tx channel, 0 for Rx channel
380 *
381 * With CPPI, channels are bound to each transfer direction of a non-control
382 * endpoint, so allocating (and deallocating) is mostly a way to notice bad
383 * housekeeping on the software side. We assume the IRQs are always active.
384 */
385 static struct dma_channel *cppi41_channel_alloc(struct dma_controller
386 *controller,
387 struct musb_hw_ep *ep, u8 is_tx)
388 {
389 struct cppi41 *cppi;
390 struct cppi41_channel *cppi_ch;
391 u32 ch_num, ep_num = ep->epnum;
392 struct usb_cppi41_info *cppi_info;
393 struct musb *musb;
395 cppi = container_of(controller, struct cppi41, controller);
396 cppi_info = cppi->cppi_info;
397 musb = cppi->musb;
399 /* Remember, ep_num: 1 .. Max_EP, and CPPI ch_num: 0 .. Max_EP - 1 */
400 ch_num = ep_num - 1;
402 if (ep_num > USB_CPPI41_NUM_CH) {
403 dev_dbg(musb->controller, "No %cx DMA channel for EP%d\n",
404 is_tx ? 'T' : 'R', ep_num);
405 return NULL;
406 }
408 cppi_ch = (is_tx ? cppi->tx_cppi_ch : cppi->rx_cppi_ch) + ch_num;
410 /* As of now, just return the corresponding CPPI 4.1 channel handle */
411 if (is_tx) {
412 /* Initialize the CPPI 4.1 Tx DMA channel */
413 if (cppi41_tx_ch_init(&cppi_ch->dma_ch_obj,
414 cppi_info->dma_block,
415 cppi_info->ep_dma_ch[ch_num])) {
416 dev_dbg(musb->controller, "ERROR: cppi41_tx_ch_init failed for "
417 "channel %d\n", ch_num);
418 return NULL;
419 }
420 /*
421 * Teardown descriptors will be pushed to the dedicated
422 * completion queue.
423 */
424 cppi41_dma_ch_default_queue(&cppi_ch->dma_ch_obj,
425 0, cppi->teardownQNum);
426 } else {
427 struct cppi41_rx_ch_cfg rx_cfg;
428 u8 q_mgr = cppi_info->q_mgr;
429 int i;
431 /* Initialize the CPPI 4.1 Rx DMA channel */
432 if (cppi41_rx_ch_init(&cppi_ch->dma_ch_obj,
433 cppi_info->dma_block,
434 cppi_info->ep_dma_ch[ch_num])) {
435 dev_dbg(musb->controller, "ERROR: cppi41_rx_ch_init failed\n");
436 return NULL;
437 }
439 if (cppi41_queue_alloc(CPPI41_FREE_DESC_BUF_QUEUE |
440 CPPI41_UNASSIGNED_QUEUE,
441 q_mgr, &cppi_ch->src_queue.q_num)) {
442 dev_dbg(musb->controller, "ERROR: cppi41_queue_alloc failed for "
443 "free descriptor/buffer queue\n");
444 return NULL;
445 }
446 dev_dbg(musb->controller, "Allocated free descriptor/buffer queue %d in "
447 "queue manager %d\n", cppi_ch->src_queue.q_num, q_mgr);
449 rx_cfg.default_desc_type = cppi41_rx_host_desc;
450 rx_cfg.sop_offset = 0;
451 rx_cfg.retry_starved = 1;
452 rx_cfg.rx_max_buf_cnt = 0;
453 rx_cfg.rx_queue.q_mgr = cppi_ch->src_queue.q_mgr = q_mgr;
454 rx_cfg.rx_queue.q_num = cppi_info->rx_comp_q[ch_num];
455 for (i = 0; i < 4; i++)
456 rx_cfg.cfg.host_pkt.fdb_queue[i] = cppi_ch->src_queue;
457 cppi41_rx_ch_configure(&cppi_ch->dma_ch_obj, &rx_cfg);
458 }
460 /* Initialize the CPPI 4.1 DMA source queue */
461 if (cppi41_queue_init(&cppi_ch->queue_obj, cppi_ch->src_queue.q_mgr,
462 cppi_ch->src_queue.q_num)) {
463 dev_dbg(musb->controller, "ERROR: cppi41_queue_init failed for %s queue",
464 is_tx ? "Tx" : "Rx free descriptor/buffer");
465 if (is_tx == 0 &&
466 cppi41_queue_free(cppi_ch->src_queue.q_mgr,
467 cppi_ch->src_queue.q_num))
468 dev_dbg(musb->controller, "ERROR: failed to free Rx descriptor/buffer "
469 "queue\n");
470 return NULL;
471 }
473 /* Enable the DMA channel */
474 cppi41_dma_ch_enable(&cppi_ch->dma_ch_obj);
476 if (cppi_ch->end_pt)
477 dev_dbg(musb->controller, "Re-allocating DMA %cx channel %d (%p)\n",
478 is_tx ? 'T' : 'R', ch_num, cppi_ch);
480 cppi_ch->end_pt = ep;
481 cppi_ch->ch_num = ch_num;
482 cppi_ch->channel.status = MUSB_DMA_STATUS_FREE;
483 cppi_ch->channel.max_len = is_tx ?
484 CPPI41_TXDMA_MAXLEN : CPPI41_RXDMA_MAXLEN;
486 dev_dbg(musb->controller, "Allocated DMA %cx channel %d for EP%d\n", is_tx ? 'T' : 'R',
487 ch_num, ep_num);
489 return &cppi_ch->channel;
490 }
492 /**
493 * cppi41_channel_release - release a CPPI DMA channel
494 * @channel: the channel
495 */
496 static void cppi41_channel_release(struct dma_channel *channel)
497 {
498 struct cppi41_channel *cppi_ch;
500 /* REVISIT: for paranoia, check state and abort if needed... */
501 cppi_ch = container_of(channel, struct cppi41_channel, channel);
503 if (cppi_ch->end_pt == NULL)
504 printk(KERN_INFO "Releasing idle DMA channel %p\n", cppi_ch);
506 /* But for now, not its IRQ */
507 cppi_ch->end_pt = NULL;
508 channel->status = MUSB_DMA_STATUS_UNKNOWN;
510 cppi41_dma_ch_disable(&cppi_ch->dma_ch_obj);
512 /* De-allocate Rx free descriptior/buffer queue */
513 if (cppi_ch->transmit == 0 &&
514 cppi41_queue_free(cppi_ch->src_queue.q_mgr,
515 cppi_ch->src_queue.q_num))
516 printk(KERN_ERR "ERROR: failed to free Rx descriptor/buffer queue\n");
517 }
519 static void cppi41_mode_update(struct cppi41_channel *cppi_ch, u8 mode)
520 {
521 if (mode != cppi_ch->dma_mode) {
522 struct cppi41 *cppi = cppi_ch->channel.private_data;
523 void *__iomem reg_base = cppi->musb->ctrl_base;
524 u32 reg_val;
525 u8 ep_num = cppi_ch->ch_num + 1;
527 if (cppi_ch->transmit) {
528 reg_val = musb_readl(reg_base, USB_TX_MODE_REG);
529 reg_val &= ~USB_TX_MODE_MASK(ep_num);
530 reg_val |= mode << USB_TX_MODE_SHIFT(ep_num);
531 musb_writel(reg_base, USB_TX_MODE_REG, reg_val);
532 } else {
533 reg_val = musb_readl(reg_base, USB_RX_MODE_REG);
534 reg_val &= ~USB_RX_MODE_MASK(ep_num);
535 reg_val |= mode << USB_RX_MODE_SHIFT(ep_num);
536 musb_writel(reg_base, USB_RX_MODE_REG, reg_val);
537 }
538 cppi_ch->dma_mode = mode;
539 }
540 }
542 /*
543 * CPPI 4.1 Tx:
544 * ============
545 * Tx is a lot more reasonable than Rx: RNDIS mode seems to behave well except
546 * how it handles the exactly-N-packets case. It appears that there's a hiccup
547 * in that case (maybe the DMA completes before a ZLP gets written?) boiling
548 * down to not being able to rely on the XFER DMA writing any terminating zero
549 * length packet before the next transfer is started...
550 *
551 * The generic RNDIS mode does not have this misfeature, so we prefer using it
552 * instead. We then send the terminating ZLP *explictly* using DMA instead of
553 * doing it by PIO after an IRQ.
554 *
555 */
557 /**
558 * cppi41_next_tx_segment - DMA write for the next chunk of a buffer
559 * @tx_ch: Tx channel
560 *
561 * Context: controller IRQ-locked
562 */
563 static unsigned cppi41_next_tx_segment(struct cppi41_channel *tx_ch)
564 {
565 struct cppi41 *cppi = tx_ch->channel.private_data;
566 struct musb *musb = cppi->musb;
567 struct usb_pkt_desc *curr_pd;
568 u32 length = tx_ch->length - tx_ch->curr_offset;
569 u32 pkt_size = tx_ch->pkt_size;
570 unsigned num_pds, n;
571 struct usb_cppi41_info *cppi_info = cppi->cppi_info;
572 u16 q_mgr = cppi_info->q_mgr;
573 u16 tx_comp_q = cppi_info->tx_comp_q[tx_ch->ch_num];
574 u8 en_bd_intr = cppi->en_bd_intr;
576 /*
577 * Tx can use the generic RNDIS mode where we can probably fit this
578 * transfer in one PD and one IRQ. The only time we would NOT want
579 * to use it is when the hardware constraints prevent it...
580 */
581 if ((pkt_size & 0x3f) == 0) {
582 num_pds = length ? 1 : 0;
583 cppi41_mode_update(tx_ch, USB_GENERIC_RNDIS_MODE);
584 } else {
585 num_pds = (length + pkt_size - 1) / pkt_size;
586 cppi41_mode_update(tx_ch, USB_TRANSPARENT_MODE);
587 }
589 pkt_size = length;
590 /*
591 * If length of transmit buffer is 0 or a multiple of the endpoint size,
592 * then send the zero length packet.
593 */
594 if (!length || (tx_ch->transfer_mode && length % pkt_size == 0))
595 num_pds++;
597 dev_dbg(musb->controller, "TX DMA%u, %s, maxpkt %u, %u PDs, addr %#x, len %u\n",
598 tx_ch->ch_num, tx_ch->dma_mode ? "accelerated" : "transparent",
599 pkt_size, num_pds, tx_ch->start_addr + tx_ch->curr_offset, length);
601 for (n = 0; n < num_pds; n++) {
602 struct cppi41_host_pkt_desc *hw_desc;
604 /* Get Tx host packet descriptor from the free pool */
605 curr_pd = usb_get_free_pd(cppi);
606 if (curr_pd == NULL) {
607 dev_dbg(musb->controller, "No Tx PDs\n");
608 break;
609 }
611 if (length < pkt_size)
612 pkt_size = length;
614 hw_desc = &curr_pd->hw_desc;
615 hw_desc->desc_info = (CPPI41_DESC_TYPE_HOST <<
616 CPPI41_DESC_TYPE_SHIFT) | pkt_size;
617 hw_desc->tag_info = tx_ch->tag_info;
618 hw_desc->pkt_info = cppi->pkt_info;
619 hw_desc->pkt_info |= ((q_mgr << CPPI41_RETURN_QMGR_SHIFT) |
620 (tx_comp_q << CPPI41_RETURN_QNUM_SHIFT));
622 hw_desc->buf_ptr = tx_ch->start_addr + tx_ch->curr_offset;
623 hw_desc->buf_len = pkt_size;
624 hw_desc->next_desc_ptr = 0;
625 hw_desc->orig_buf_len = pkt_size;
627 curr_pd->ch_num = tx_ch->ch_num;
628 curr_pd->ep_num = tx_ch->end_pt->epnum;
630 tx_ch->curr_offset += pkt_size;
631 length -= pkt_size;
633 if (pkt_size == 0)
634 tx_ch->zlp_queued = 1;
636 if (en_bd_intr)
637 hw_desc->orig_buf_len |= CPPI41_PKT_INTR_FLAG;
639 dev_dbg(musb->controller, "TX PD %p: buf %08x, len %08x, pkt info %08x\n", curr_pd,
640 hw_desc->buf_ptr, hw_desc->buf_len, hw_desc->pkt_info);
642 cppi41_queue_push(&tx_ch->queue_obj, curr_pd->dma_addr,
643 USB_CPPI41_DESC_ALIGN, pkt_size);
644 }
646 return n;
647 }
649 static void cppi41_autoreq_update(struct cppi41_channel *rx_ch, u8 autoreq)
650 {
651 struct cppi41 *cppi = rx_ch->channel.private_data;
653 if (is_host_active(cppi->musb) &&
654 autoreq != rx_ch->autoreq) {
655 void *__iomem reg_base = cppi->musb->ctrl_base;
656 u32 reg_val = musb_readl(reg_base, cppi->automode_reg_offs);
657 u8 ep_num = rx_ch->ch_num + 1;
659 reg_val &= ~USB_RX_AUTOREQ_MASK(ep_num);
660 reg_val |= autoreq << USB_RX_AUTOREQ_SHIFT(ep_num);
662 musb_writel(reg_base, cppi->automode_reg_offs, reg_val);
663 rx_ch->autoreq = autoreq;
664 }
665 }
667 static void cppi41_set_ep_size(struct cppi41_channel *rx_ch, u32 pkt_size)
668 {
669 struct cppi41 *cppi = rx_ch->channel.private_data;
670 void *__iomem reg_base = cppi->musb->ctrl_base;
671 u8 ep_num = rx_ch->ch_num + 1;
672 u32 res = pkt_size % 64;
674 /* epsize register must be multiple of 64 */
675 pkt_size += res ? (64 - res) : res;
677 musb_writel(reg_base, USB_GENERIC_RNDIS_EP_SIZE_REG(ep_num), pkt_size);
678 }
680 /*
681 * CPPI 4.1 Rx:
682 * ============
683 * Consider a 1KB bulk Rx buffer in two scenarios: (a) it's fed two 300 byte
684 * packets back-to-back, and (b) it's fed two 512 byte packets back-to-back.
685 * (Full speed transfers have similar scenarios.)
686 *
687 * The correct behavior for Linux is that (a) fills the buffer with 300 bytes,
688 * and the next packet goes into a buffer that's queued later; while (b) fills
689 * the buffer with 1024 bytes. How to do that with accelerated DMA modes?
690 *
691 * Rx queues in RNDIS mode (one single BD) handle (a) correctly but (b) loses
692 * BADLY because nothing (!) happens when that second packet fills the buffer,
693 * much less when a third one arrives -- which makes it not a "true" RNDIS mode.
694 * In the RNDIS protocol short-packet termination is optional, and it's fine if
695 * the peripherals (not hosts!) pad the messages out to end of buffer. Standard
696 * PCI host controller DMA descriptors implement that mode by default... which
697 * is no accident.
698 *
699 * Generic RNDIS mode is the only way to reliably make both cases work. This
700 * mode is identical to the "normal" RNDIS mode except for the case where the
701 * last packet of the segment matches the max USB packet size -- in this case,
702 * the packet will be closed when a value (0x10000 max) in the Generic RNDIS
703 * EP Size register is reached. This mode will work for the network drivers
704 * (CDC/RNDIS) as well as for the mass storage drivers where there is no short
705 * packet.
706 *
707 * BUT we can only use non-transparent modes when USB packet size is a multiple
708 * of 64 bytes. Let's see what happens when this is not the case...
709 *
710 * Rx queues (2 BDs with 512 bytes each) have converse problems to RNDIS mode:
711 * (b) is handled right but (a) loses badly. DMA doesn't stop after receiving
712 * a short packet and processes both of those PDs; so both packets are loaded
713 * into the buffer (with 212 byte gap between them), and the next buffer queued
714 * will NOT get its 300 bytes of data. Even in the case when there should be
715 * no short packets (URB_SHORT_NOT_OK is set), queueing several packets in the
716 * host mode doesn't win us anything since we have to manually "prod" the Rx
717 * process after each packet is received by setting ReqPkt bit in endpoint's
718 * RXCSR; in the peripheral mode without short packets, queueing could be used
719 * BUT we'll have to *teardown* the channel if a short packet still arrives in
720 * the peripheral mode, and to "collect" the left-over packet descriptors from
721 * the free descriptor/buffer queue in both cases...
722 *
723 * One BD at a time is the only way to make make both cases work reliably, with
724 * software handling both cases correctly, at the significant penalty of needing
725 * an IRQ per packet. (The lack of I/O overlap can be slightly ameliorated by
726 * enabling double buffering.)
727 *
728 * There seems to be no way to identify for sure the cases where the CDC mode
729 * is appropriate...
730 *
731 */
733 /**
734 * cppi41_next_rx_segment - DMA read for the next chunk of a buffer
735 * @rx_ch: Rx channel
736 *
737 * Context: controller IRQ-locked
738 *
739 * NOTE: In the transparent mode, we have to queue one packet at a time since:
740 * - we must avoid starting reception of another packet after receiving
741 * a short packet;
742 * - in host mode we have to set ReqPkt bit in the endpoint's RXCSR after
743 * receiving each packet but the last one... ugly!
744 */
745 static unsigned cppi41_next_rx_segment(struct cppi41_channel *rx_ch)
746 {
747 struct cppi41 *cppi = rx_ch->channel.private_data;
748 struct musb *musb = cppi->musb;
749 struct usb_pkt_desc *curr_pd;
750 struct cppi41_host_pkt_desc *hw_desc;
751 u32 length = rx_ch->length - rx_ch->curr_offset;
752 u32 pkt_size = rx_ch->pkt_size;
753 u32 max_rx_transfer_size = 64 * 1024;
754 u32 i, n_bd , pkt_len;
755 struct usb_gadget_driver *gadget_driver;
756 u8 en_bd_intr = cppi->en_bd_intr, mode;
758 if (is_peripheral_active(cppi->musb)) {
759 /* TODO: temporary fix for CDC/RNDIS which needs to be in
760 * GENERIC_RNDIS mode. Without this RNDIS gadget taking
761 * more then 2K ms for a 64 byte pings.
762 */
763 gadget_driver = cppi->musb->gadget_driver;
765 pkt_len = rx_ch->pkt_size;
766 mode = USB_GENERIC_RNDIS_MODE;
767 if (!strcmp(gadget_driver->driver.name, "g_file_storage")) {
768 if (cppi->inf_mode && length > pkt_len) {
769 pkt_len = 0;
770 length = length - rx_ch->pkt_size;
771 cppi41_rx_ch_set_maxbufcnt(&rx_ch->dma_ch_obj,
772 DMA_CH_RX_MAX_BUF_CNT_1);
773 rx_ch->inf_mode = 1;
774 } else {
775 max_rx_transfer_size = rx_ch->pkt_size;
776 mode = USB_TRANSPARENT_MODE;
777 }
778 } else
779 if (rx_ch->length < max_rx_transfer_size)
780 pkt_len = rx_ch->length;
782 if (mode != USB_TRANSPARENT_MODE)
783 cppi41_set_ep_size(rx_ch, pkt_len);
784 cppi41_mode_update(rx_ch, mode);
785 } else {
786 /*
787 * Rx can use the generic RNDIS mode where we can
788 * probably fit this transfer in one PD and one IRQ
789 * (or two with a short packet).
790 */
791 if ((pkt_size & 0x3f) == 0) {
792 cppi41_mode_update(rx_ch, USB_GENERIC_RNDIS_MODE);
793 cppi41_autoreq_update(rx_ch, USB_AUTOREQ_ALL_BUT_EOP);
795 pkt_size = (length > 0x10000) ? 0x10000 : length;
796 cppi41_set_ep_size(rx_ch, pkt_size);
797 } else {
798 cppi41_mode_update(rx_ch, USB_TRANSPARENT_MODE);
799 cppi41_autoreq_update(rx_ch, USB_NO_AUTOREQ);
800 max_rx_transfer_size = rx_ch->pkt_size;
801 }
802 }
804 dev_dbg(musb->controller, "RX DMA%u, %s, maxpkt %u, addr %#x, rec'd %u/%u\n",
805 rx_ch->ch_num, rx_ch->dma_mode ? "accelerated" : "transparent",
806 pkt_size, rx_ch->start_addr + rx_ch->curr_offset,
807 rx_ch->curr_offset, rx_ch->length);
809 /* calculate number of bd required */
810 n_bd = (length + max_rx_transfer_size - 1)/max_rx_transfer_size;
812 for (i = 0; i < n_bd ; ++i) {
813 /* Get Rx packet descriptor from the free pool */
814 curr_pd = usb_get_free_pd(cppi);
815 if (curr_pd == NULL) {
816 /* Shouldn't ever happen! */
817 dev_dbg(musb->controller, "No Rx PDs\n");
818 goto sched;
819 }
821 pkt_len =
822 (length > max_rx_transfer_size) ? max_rx_transfer_size : length;
824 hw_desc = &curr_pd->hw_desc;
825 hw_desc->desc_info = (CPPI41_DESC_TYPE_HOST <<
826 CPPI41_DESC_TYPE_SHIFT);
827 hw_desc->orig_buf_ptr = rx_ch->start_addr + rx_ch->curr_offset;
828 hw_desc->orig_buf_len = pkt_len;
830 /* buf_len field of buffer descriptor updated by dma
831 * after reception of data is completed
832 */
833 hw_desc->buf_len = 0;
835 curr_pd->ch_num = rx_ch->ch_num;
836 curr_pd->ep_num = rx_ch->end_pt->epnum;
838 curr_pd->eop = (length -= pkt_len) ? 0 : 1;
839 rx_ch->curr_offset += pkt_len;
841 if (en_bd_intr)
842 hw_desc->orig_buf_len |= CPPI41_PKT_INTR_FLAG;
843 /*
844 * Push the free Rx packet descriptor
845 * to the free descriptor/buffer queue.
846 */
847 cppi41_queue_push(&rx_ch->queue_obj, curr_pd->dma_addr,
848 USB_CPPI41_DESC_ALIGN, 0);
849 }
851 sched:
852 /*
853 * HCD arranged ReqPkt for the first packet.
854 * We arrange it for all but the last one.
855 */
856 if (is_host_active(cppi->musb) && rx_ch->channel.actual_len) {
857 void __iomem *epio = rx_ch->end_pt->regs;
858 u16 csr = musb_readw(epio, MUSB_RXCSR);
860 csr |= MUSB_RXCSR_H_REQPKT | MUSB_RXCSR_H_WZC_BITS;
861 musb_writew(epio, MUSB_RXCSR, csr);
862 }
864 /* enable schedular if not enabled */
865 if (is_peripheral_active(cppi->musb) && (n_bd > 0))
866 cppi41_schedtbl_add_dma_ch(0, 0, rx_ch->ch_num, 0);
867 return 1;
868 }
870 /**
871 * cppi41_channel_program - program channel for data transfer
872 * @channel: the channel
873 * @maxpacket: max packet size
874 * @mode: for Rx, 1 unless the USB protocol driver promised to treat
875 * all short reads as errors and kick in high level fault recovery;
876 * for Tx, 0 unless the protocol driver _requires_ short-packet
877 * termination mode
878 * @dma_addr: DMA address of buffer
879 * @length: length of buffer
880 *
881 * Context: controller IRQ-locked
882 */
883 static int cppi41_channel_program(struct dma_channel *channel, u16 maxpacket,
884 u8 mode, dma_addr_t dma_addr, u32 length)
885 {
886 struct cppi41_channel *cppi_ch;
887 unsigned queued;
889 cppi_ch = container_of(channel, struct cppi41_channel, channel);
891 switch (channel->status) {
892 case MUSB_DMA_STATUS_BUS_ABORT:
893 case MUSB_DMA_STATUS_CORE_ABORT:
894 /* Fault IRQ handler should have handled cleanup */
895 WARNING("%cx DMA%d not cleaned up after abort!\n",
896 cppi_ch->transmit ? 'T' : 'R', cppi_ch->ch_num);
897 break;
898 case MUSB_DMA_STATUS_BUSY:
899 WARNING("Program active channel? %cx DMA%d\n",
900 cppi_ch->transmit ? 'T' : 'R', cppi_ch->ch_num);
901 break;
902 case MUSB_DMA_STATUS_UNKNOWN:
903 WARNING("%cx DMA%d not allocated!\n",
904 cppi_ch->transmit ? 'T' : 'R', cppi_ch->ch_num);
905 return 0;
906 case MUSB_DMA_STATUS_FREE:
907 break;
908 }
910 channel->status = MUSB_DMA_STATUS_BUSY;
912 /* Set the transfer parameters, then queue up the first segment */
913 cppi_ch->start_addr = dma_addr;
914 cppi_ch->curr_offset = 0;
915 cppi_ch->pkt_size = maxpacket;
916 cppi_ch->length = length;
917 cppi_ch->transfer_mode = mode;
918 cppi_ch->zlp_queued = 0;
919 cppi_ch->channel.actual_len = 0;
921 /* Tx or Rx channel? */
922 if (cppi_ch->transmit)
923 queued = cppi41_next_tx_segment(cppi_ch);
924 else
925 queued = cppi41_next_rx_segment(cppi_ch);
927 return queued > 0;
928 }
930 static struct usb_pkt_desc *usb_get_pd_ptr(struct cppi41 *cppi,
931 unsigned long pd_addr)
932 {
933 if (pd_addr >= cppi->pd_mem_phys && pd_addr < cppi->pd_mem_phys +
934 USB_CPPI41_MAX_PD * USB_CPPI41_DESC_ALIGN)
935 return pd_addr - cppi->pd_mem_phys + cppi->pd_mem;
936 else
937 return NULL;
938 }
940 static int usb_check_teardown(struct cppi41_channel *cppi_ch,
941 unsigned long pd_addr)
942 {
943 u32 info;
944 struct cppi41 *cppi = cppi_ch->channel.private_data;
945 struct usb_cppi41_info *cppi_info = cppi->cppi_info;
946 struct musb *musb = cppi->musb;
948 if (cppi41_get_teardown_info(pd_addr, &info)) {
949 dev_dbg(musb->controller, "ERROR: not a teardown descriptor\n");
950 return 0;
951 }
953 if ((info & CPPI41_TEARDOWN_TX_RX_MASK) ==
954 (!cppi_ch->transmit << CPPI41_TEARDOWN_TX_RX_SHIFT) &&
955 (info & CPPI41_TEARDOWN_DMA_NUM_MASK) ==
956 (cppi_info->dma_block << CPPI41_TEARDOWN_DMA_NUM_SHIFT) &&
957 (info & CPPI41_TEARDOWN_CHAN_NUM_MASK) ==
958 (cppi_info->ep_dma_ch[cppi_ch->ch_num] <<
959 CPPI41_TEARDOWN_CHAN_NUM_SHIFT))
960 return 1;
962 dev_dbg(musb->controller, "ERROR: unexpected values in teardown descriptor\n");
963 return 0;
964 }
966 /*
967 * We can't handle the channel teardown via the default completion queue in
968 * context of the controller IRQ-locked, so we use the dedicated teardown
969 * completion queue which we can just poll for a teardown descriptor, not
970 * interfering with the Tx completion queue processing.
971 */
972 static void usb_tx_ch_teardown(struct cppi41_channel *tx_ch)
973 {
974 struct cppi41 *cppi = tx_ch->channel.private_data;
975 struct musb *musb = cppi->musb;
976 void __iomem *reg_base = musb->ctrl_base;
977 u32 td_reg, timeout = 0xfffff;
978 u8 ep_num = tx_ch->ch_num + 1;
979 unsigned long pd_addr;
980 struct cppi41_queue_obj tx_queue_obj;
981 struct usb_cppi41_info *cppi_info;
983 /* Initiate teardown for Tx DMA channel */
984 cppi41_dma_ch_teardown(&tx_ch->dma_ch_obj);
986 /* Wait for a descriptor to be queued and pop it... */
987 do {
988 td_reg = musb_readl(reg_base, cppi->teardown_reg_offs);
989 td_reg |= USB_TX_TDOWN_MASK(ep_num);
990 musb_writel(reg_base, cppi->teardown_reg_offs, td_reg);
992 pd_addr = cppi41_queue_pop(&cppi->queue_obj);
993 } while (!pd_addr && timeout--);
995 if (pd_addr) {
997 dev_dbg(musb->controller, "Descriptor (%08lx) popped from teardown completion "
998 "queue\n", pd_addr);
1000 if (usb_check_teardown(tx_ch, pd_addr)) {
1001 dev_dbg(musb->controller, "Teardown Desc (%lx) rcvd\n", pd_addr);
1002 } else
1003 ERR("Invalid PD(%08lx)popped from TearDn completion"
1004 "queue\n", pd_addr);
1005 } else {
1006 if (timeout <= 0)
1007 ERR("Teardown Desc not rcvd\n");
1008 }
1010 /* read the tx completion queue and remove
1011 * completion bd if any
1012 */
1013 cppi_info = cppi->cppi_info;
1014 if (cppi41_queue_init(&tx_queue_obj, cppi_info->q_mgr,
1015 cppi_info->tx_comp_q[tx_ch->ch_num])) {
1016 ERR("ERROR: cppi41_queue_init failed for "
1017 "Tx completion queue");
1018 return;
1019 }
1021 while ((pd_addr = cppi41_queue_pop(&tx_queue_obj)) != 0) {
1022 struct usb_pkt_desc *curr_pd;
1024 curr_pd = usb_get_pd_ptr(cppi, pd_addr);
1025 if (curr_pd == NULL) {
1026 ERR("Invalid PD popped from Tx completion queue\n");
1027 continue;
1028 }
1030 dev_dbg(musb->controller, "Tx-PD(%p) popped from completion queue\n", curr_pd);
1031 dev_dbg(musb->controller, "ch(%d)epnum(%d)len(%d)\n", curr_pd->ch_num,
1032 curr_pd->ep_num, curr_pd->hw_desc.buf_len);
1034 usb_put_free_pd(cppi, curr_pd);
1035 }
1036 }
1038 /*
1039 * For Rx DMA channels, the situation is more complex: there's only a single
1040 * completion queue for all our needs, so we have to temporarily redirect the
1041 * completed descriptors to our teardown completion queue, with a possibility
1042 * of a completed packet landing there as well...
1043 */
1044 static void usb_rx_ch_teardown(struct cppi41_channel *rx_ch)
1045 {
1046 struct cppi41 *cppi = rx_ch->channel.private_data;
1047 struct musb *musb = cppi->musb;
1048 struct usb_cppi41_info *cppi_info = cppi->cppi_info;
1049 u32 timeout = 0xfffff, pd_addr;
1050 struct cppi41_queue_obj rx_queue_obj;
1052 cppi41_dma_ch_default_queue(&rx_ch->dma_ch_obj, 0, cppi->teardownQNum);
1054 /* Initiate teardown for Rx DMA channel */
1055 cppi41_dma_ch_teardown(&rx_ch->dma_ch_obj);
1057 do {
1058 struct usb_pkt_desc *curr_pd;
1059 unsigned long pd_addr;
1061 /* Wait for a descriptor to be queued and pop it... */
1062 do {
1063 pd_addr = cppi41_queue_pop(&cppi->queue_obj);
1064 } while (!pd_addr && timeout--);
1066 if (timeout <= 0 || !pd_addr) {
1067 ERR("teardown Desc not found\n");
1068 break;
1069 }
1071 dev_dbg(musb->controller, "Descriptor (%08lx) popped from teardown completion "
1072 "queue\n", pd_addr);
1074 /*
1075 * We might have popped a completed Rx PD, so check if the
1076 * physical address is within the PD region first. If it's
1077 * not the case, it must be a teardown descriptor...
1078 * */
1079 curr_pd = usb_get_pd_ptr(cppi, pd_addr);
1080 if (curr_pd == NULL) {
1081 if (usb_check_teardown(rx_ch, pd_addr))
1082 break;
1083 continue;
1084 }
1086 /* Paranoia: check if PD is from the right channel... */
1087 if (curr_pd->ch_num != rx_ch->ch_num) {
1088 ERR("Unexpected channel %d in Rx PD\n",
1089 curr_pd->ch_num);
1090 continue;
1091 }
1093 /* Extract the buffer length from the completed PD */
1094 rx_ch->channel.actual_len += curr_pd->hw_desc.buf_len;
1096 /*
1097 * Return Rx PDs to the software list --
1098 * this is protected by critical section.
1099 */
1100 usb_put_free_pd(cppi, curr_pd);
1101 } while (0);
1103 /* read the rx completion queue and remove
1104 * completion bd if any
1105 */
1106 if (cppi41_queue_init(&rx_queue_obj, cppi_info->q_mgr,
1107 cppi_info->rx_comp_q[rx_ch->ch_num])) {
1108 ERR("ERROR: cppi41_queue_init failed for "
1109 "Rx completion queue");
1110 return;
1111 }
1113 while ((pd_addr = cppi41_queue_pop(&rx_queue_obj)) != 0) {
1114 struct usb_pkt_desc *curr_pd;
1116 curr_pd = usb_get_pd_ptr(cppi, pd_addr);
1117 if (curr_pd == NULL) {
1118 ERR("Invalid PD popped from Rx completion queue\n");
1119 continue;
1120 }
1122 dev_dbg(musb->controller, "Rx-PD(%p) popped from completion queue\n", curr_pd);
1123 dev_dbg(musb->controller, "ch(%d)epnum(%d)len(%d)\n", curr_pd->ch_num,
1124 curr_pd->ep_num, curr_pd->hw_desc.buf_len);
1126 usb_put_free_pd(cppi, curr_pd);
1127 }
1129 /* Now restore the default Rx completion queue... */
1130 cppi41_dma_ch_default_queue(&rx_ch->dma_ch_obj, cppi_info->q_mgr,
1131 cppi_info->rx_comp_q[rx_ch->ch_num]);
1132 }
1134 /*
1135 * cppi41_channel_abort
1136 *
1137 * Context: controller IRQ-locked, endpoint selected.
1138 */
1139 static int cppi41_channel_abort(struct dma_channel *channel)
1140 {
1141 struct cppi41 *cppi;
1142 struct cppi41_channel *cppi_ch;
1143 struct musb *musb;
1144 void __iomem *reg_base, *epio;
1145 unsigned long pd_addr;
1146 u32 csr, td_reg;
1147 u8 ch_num, ep_num;
1149 cppi_ch = container_of(channel, struct cppi41_channel, channel);
1150 ch_num = cppi_ch->ch_num;
1151 cppi = cppi_ch->channel.private_data;
1152 musb = cppi->musb;
1154 switch (channel->status) {
1155 case MUSB_DMA_STATUS_BUS_ABORT:
1156 case MUSB_DMA_STATUS_CORE_ABORT:
1157 /* From Rx or Tx fault IRQ handler */
1158 case MUSB_DMA_STATUS_BUSY:
1159 /* The hardware needs shutting down... */
1160 dprintk("%s: DMA busy, status = %x\n",
1161 __func__, channel->status);
1162 break;
1163 case MUSB_DMA_STATUS_UNKNOWN:
1164 dev_dbg(musb->controller, "%cx DMA%d not allocated\n",
1165 cppi_ch->transmit ? 'T' : 'R', ch_num);
1166 /* FALLTHROUGH */
1167 case MUSB_DMA_STATUS_FREE:
1168 return 0;
1169 }
1171 reg_base = musb->ctrl_base;
1172 epio = cppi_ch->end_pt->regs;
1173 ep_num = ch_num + 1;
1175 #ifdef DEBUG_CPPI_TD
1176 printk("Before teardown:");
1177 print_pd_list(cppi->pd_pool_head);
1178 #endif
1180 if (cppi_ch->transmit) {
1181 dprintk("Tx channel teardown, cppi_ch = %p\n", cppi_ch);
1183 /* Tear down Tx DMA channel */
1184 usb_tx_ch_teardown(cppi_ch);
1186 /* Issue CPPI FIFO teardown for Tx channel */
1187 td_reg = musb_readl(reg_base, cppi->teardown_reg_offs);
1188 td_reg |= USB_TX_TDOWN_MASK(ep_num);
1189 musb_writel(reg_base, cppi->teardown_reg_offs, td_reg);
1191 /* Flush FIFO of the endpoint */
1192 csr = musb_readw(epio, MUSB_TXCSR);
1193 csr |= MUSB_TXCSR_FLUSHFIFO | MUSB_TXCSR_H_WZC_BITS;
1194 musb_writew(epio, MUSB_TXCSR, csr);
1195 musb_writew(epio, MUSB_TXCSR, csr);
1196 cppi_ch->tx_complete = 0;
1197 } else { /* Rx */
1198 dprintk("Rx channel teardown, cppi_ch = %p\n", cppi_ch);
1200 /* Flush FIFO of the endpoint */
1201 csr = musb_readw(epio, MUSB_RXCSR);
1202 csr |= MUSB_RXCSR_FLUSHFIFO | MUSB_RXCSR_H_WZC_BITS;
1203 musb_writew(epio, MUSB_RXCSR, csr);
1204 musb_writew(epio, MUSB_RXCSR, csr);
1206 /* Issue CPPI FIFO teardown for Rx channel */
1207 td_reg = musb_readl(reg_base, cppi->teardown_reg_offs);
1208 td_reg |= USB_RX_TDOWN_MASK(ep_num);
1209 musb_writel(reg_base, cppi->teardown_reg_offs, td_reg);
1211 /* Tear down Rx DMA channel */
1212 usb_rx_ch_teardown(cppi_ch);
1214 /*
1215 * NOTE: docs don't guarantee any of this works... we expect
1216 * that if the USB core stops telling the CPPI core to pull
1217 * more data from it, then it'll be safe to flush current Rx
1218 * DMA state iff any pending FIFO transfer is done.
1219 */
1221 /* For host, ensure ReqPkt is never set again */
1222 cppi41_autoreq_update(cppi_ch, USB_NO_AUTOREQ);
1224 /* For host, clear (just) ReqPkt at end of current packet(s) */
1225 if (is_host_active(cppi->musb))
1226 csr &= ~MUSB_RXCSR_H_REQPKT;
1227 csr |= MUSB_RXCSR_H_WZC_BITS;
1229 /* Clear DMA enable */
1230 csr &= ~MUSB_RXCSR_DMAENAB;
1231 musb_writew(epio, MUSB_RXCSR, csr);
1233 /* Flush the FIFO of endpoint once again */
1234 csr = musb_readw(epio, MUSB_RXCSR);
1235 csr |= MUSB_RXCSR_FLUSHFIFO | MUSB_RXCSR_H_WZC_BITS;
1236 musb_writew(epio, MUSB_RXCSR, csr);
1238 udelay(50);
1239 }
1241 /*
1242 * There might be PDs in the Rx/Tx source queue that were not consumed
1243 * by the DMA controller -- they need to be recycled properly.
1244 */
1245 while ((pd_addr = cppi41_queue_pop(&cppi_ch->queue_obj)) != 0) {
1246 struct usb_pkt_desc *curr_pd;
1248 curr_pd = usb_get_pd_ptr(cppi, pd_addr);
1249 if (curr_pd == NULL) {
1250 ERR("Invalid PD popped from source queue\n");
1251 continue;
1252 }
1254 /*
1255 * Return Rx/Tx PDs to the software list --
1256 * this is protected by critical section.
1257 */
1258 dprintk("Returning PD %p to the free PD list\n", curr_pd);
1259 usb_put_free_pd(cppi, curr_pd);
1260 }
1262 #ifdef DEBUG_CPPI_TD
1263 printk("After teardown:");
1264 print_pd_list(cppi->pd_pool_head);
1265 #endif
1267 /* Re-enable the DMA channel */
1268 cppi41_dma_ch_enable(&cppi_ch->dma_ch_obj);
1270 channel->status = MUSB_DMA_STATUS_FREE;
1272 return 0;
1273 }
1275 void txdma_completion_work(struct work_struct *data)
1276 {
1277 struct cppi41 *cppi = container_of(data, struct cppi41, txdma_work);
1278 struct cppi41_channel *tx_ch;
1279 struct musb *musb = cppi->musb;
1280 unsigned index;
1281 u8 resched = 0;
1282 unsigned long flags;
1284 while (1) {
1285 for (index = 0; index < USB_CPPI41_NUM_CH; index++) {
1286 void __iomem *epio;
1287 u16 csr;
1289 tx_ch = &cppi->tx_cppi_ch[index];
1290 spin_lock_irqsave(&musb->lock, flags);
1291 if (tx_ch->tx_complete) {
1292 /* Sometimes a EP can unregister from a DMA
1293 * channel while the data is still in the FIFO.
1294 * Probable reason a proper abort was not
1295 * called before taking such a step.
1296 * Protect against such cases.
1297 */
1298 if (!tx_ch->end_pt) {
1299 tx_ch->tx_complete = 0;
1300 continue;
1301 }
1303 epio = tx_ch->end_pt->regs;
1304 csr = musb_readw(epio, MUSB_TXCSR);
1306 if (csr & (MUSB_TXCSR_TXPKTRDY |
1307 MUSB_TXCSR_FIFONOTEMPTY)) {
1308 resched = 1;
1309 } else {
1310 tx_ch->channel.status =
1311 MUSB_DMA_STATUS_FREE;
1312 tx_ch->tx_complete = 0;
1313 musb_dma_completion(musb, index+1, 1);
1314 }
1315 }
1316 spin_unlock_irqrestore(&musb->lock, flags);
1318 if (!resched)
1319 cond_resched();
1320 }
1322 if (resched) {
1323 resched = 0;
1324 cond_resched();
1325 } else {
1326 return ;
1327 }
1328 }
1330 }
1332 /**
1333 * cppi41_dma_controller_create -
1334 * instantiate an object representing DMA controller.
1335 */
1336 struct dma_controller * __devinit
1337 cppi41_dma_controller_create(struct musb *musb, void __iomem *mregs)
1338 {
1339 struct cppi41 *cppi;
1341 cppi = kzalloc(sizeof *cppi, GFP_KERNEL);
1342 if (!cppi)
1343 return NULL;
1345 /* Initialize the CPPI 4.1 DMA controller structure */
1346 cppi->musb = musb;
1347 cppi->controller.start = cppi41_controller_start;
1348 cppi->controller.stop = cppi41_controller_stop;
1349 cppi->controller.channel_alloc = cppi41_channel_alloc;
1350 cppi->controller.channel_release = cppi41_channel_release;
1351 cppi->controller.channel_program = cppi41_channel_program;
1352 cppi->controller.channel_abort = cppi41_channel_abort;
1353 cppi->cppi_info = (struct usb_cppi41_info *)&usb_cppi41_info[musb->id];;
1354 cppi->en_bd_intr = cppi->cppi_info->bd_intr_ctrl;
1355 INIT_WORK(&cppi->txdma_work, txdma_completion_work);
1357 /* enable infinite mode only for ti81xx silicon rev2 */
1358 if (cpu_is_am33xx() || cpu_is_ti816x()) {
1359 dev_dbg(musb->controller, "cppi41dma supports infinite mode\n");
1360 cppi->inf_mode = 1;
1361 }
1363 return &cppi->controller;
1364 }
1365 EXPORT_SYMBOL(cppi41_dma_controller_create);
1367 /**
1368 * cppi41_dma_controller_destroy -
1369 * destroy a previously instantiated DMA controller
1370 * @controller: the controller
1371 */
1372 void cppi41_dma_controller_destroy(struct dma_controller *controller)
1373 {
1374 struct cppi41 *cppi;
1376 cppi = container_of(controller, struct cppi41, controller);
1378 /* Free the CPPI object */
1379 kfree(cppi);
1380 }
1381 EXPORT_SYMBOL(cppi41_dma_controller_destroy);
1383 static void usb_process_tx_queue(struct cppi41 *cppi, unsigned index)
1384 {
1385 struct cppi41_queue_obj tx_queue_obj;
1386 unsigned long pd_addr;
1387 struct usb_cppi41_info *cppi_info = cppi->cppi_info;
1388 struct musb *musb = cppi->musb;
1390 if (cppi41_queue_init(&tx_queue_obj, cppi_info->q_mgr,
1391 cppi_info->tx_comp_q[index])) {
1392 dev_dbg(musb->controller, "ERROR: cppi41_queue_init failed for "
1393 "Tx completion queue");
1394 return;
1395 }
1397 while ((pd_addr = cppi41_queue_pop(&tx_queue_obj)) != 0) {
1398 struct usb_pkt_desc *curr_pd;
1399 struct cppi41_channel *tx_ch;
1400 u8 ch_num, ep_num;
1401 u32 length;
1403 curr_pd = usb_get_pd_ptr(cppi, pd_addr);
1404 if (curr_pd == NULL) {
1405 ERR("Invalid PD popped from Tx completion queue\n");
1406 continue;
1407 }
1409 /* Extract the data from received packet descriptor */
1410 ch_num = curr_pd->ch_num;
1411 ep_num = curr_pd->ep_num;
1412 length = curr_pd->hw_desc.buf_len;
1414 tx_ch = &cppi->tx_cppi_ch[ch_num];
1415 tx_ch->channel.actual_len += length;
1417 /*
1418 * Return Tx PD to the software list --
1419 * this is protected by critical section
1420 */
1421 usb_put_free_pd(cppi, curr_pd);
1423 if ((tx_ch->curr_offset < tx_ch->length) ||
1424 (tx_ch->transfer_mode && !tx_ch->zlp_queued))
1425 cppi41_next_tx_segment(tx_ch);
1426 else if (tx_ch->channel.actual_len >= tx_ch->length) {
1427 /*
1428 * We get Tx DMA completion interrupt even when
1429 * data is still in FIFO and not moved out to
1430 * USB bus. As we program the next request we
1431 * flush out and old data in FIFO which affects
1432 * USB functionality. So far, we have obsered
1433 * failure with iperf.
1434 */
1435 tx_ch->tx_complete = 1;
1436 schedule_work(&cppi->txdma_work);
1437 }
1438 }
1439 }
1441 static void usb_process_rx_queue(struct cppi41 *cppi, unsigned index)
1442 {
1443 struct cppi41_queue_obj rx_queue_obj;
1444 unsigned long pd_addr;
1445 struct usb_cppi41_info *cppi_info = cppi->cppi_info;
1446 struct musb *musb = cppi->musb;
1447 u8 en_bd_intr = cppi->en_bd_intr;
1449 if (cppi41_queue_init(&rx_queue_obj, cppi_info->q_mgr,
1450 cppi_info->rx_comp_q[index])) {
1451 dev_dbg(musb->controller, "ERROR: cppi41_queue_init failed for Rx queue\n");
1452 return;
1453 }
1455 while ((pd_addr = cppi41_queue_pop(&rx_queue_obj)) != 0) {
1456 struct usb_pkt_desc *curr_pd;
1457 struct cppi41_channel *rx_ch;
1458 u8 ch_num, ep_num;
1459 u32 length = 0, orig_buf_len, timeout = 50;
1461 curr_pd = usb_get_pd_ptr(cppi, pd_addr);
1462 if (curr_pd == NULL) {
1463 ERR("Invalid PD popped from Rx completion queue\n");
1464 continue;
1465 }
1467 /* This delay is required to overcome the dma race condition
1468 * where software reads buffer descriptor before being updated
1469 * by dma as buffer descriptor's writes by dma still pending in
1470 * interconnect bridge.
1471 */
1472 while (timeout--) {
1473 length = curr_pd->hw_desc.desc_info &
1474 CPPI41_PKT_LEN_MASK;
1475 if (length != 0)
1476 break;
1477 udelay(1);
1478 }
1480 if (length == 0)
1481 ERR("!Race condtion: rxBD read before updated by dma");
1483 /* Extract the data from received packet descriptor */
1484 ch_num = curr_pd->ch_num;
1485 ep_num = curr_pd->ep_num;
1487 dev_dbg(musb->controller, "Rx complete: dma channel(%d) ep%d len %d timeout %d\n",
1488 ch_num, ep_num, length, (50-timeout));
1490 rx_ch = &cppi->rx_cppi_ch[ch_num];
1491 rx_ch->channel.actual_len += length;
1493 if (curr_pd->eop) {
1494 curr_pd->eop = 0;
1495 /* disable the rx dma schedular */
1496 if (is_peripheral_active(cppi->musb) && !cppi->inf_mode)
1497 cppi41_schedtbl_remove_dma_ch(0, 0, ch_num, 0);
1498 }
1500 /*
1501 * Return Rx PD to the software list --
1502 * this is protected by critical section
1503 */
1504 usb_put_free_pd(cppi, curr_pd);
1506 orig_buf_len = curr_pd->hw_desc.orig_buf_len;
1507 if (en_bd_intr)
1508 orig_buf_len &= ~CPPI41_PKT_INTR_FLAG;
1510 if (unlikely(rx_ch->channel.actual_len >= rx_ch->length ||
1511 length < orig_buf_len)) {
1513 #if defined(CONFIG_SOC_OMAPTI81XX) || defined(CONFIG_SOC_OMAPAM33XX)
1514 struct musb_hw_ep *ep;
1515 u8 isoc, next_seg = 0;
1517 /* Workaround for early rx completion of
1518 * cppi41 dma in Generic RNDIS mode for ti81xx
1519 */
1520 if (is_host_enabled(cppi->musb)) {
1521 u32 pkt_size = rx_ch->pkt_size;
1522 ep = cppi->musb->endpoints + ep_num;
1523 isoc = musb_readb(ep->regs, MUSB_RXTYPE);
1524 isoc = (isoc >> 4) & 0x1;
1526 if (!isoc
1527 && (rx_ch->dma_mode == USB_GENERIC_RNDIS_MODE)
1528 && (rx_ch->channel.actual_len < rx_ch->length)
1529 && !(rx_ch->channel.actual_len % pkt_size))
1530 next_seg = 1;
1531 }
1532 if (next_seg) {
1533 rx_ch->curr_offset = rx_ch->channel.actual_len;
1534 cppi41_next_rx_segment(rx_ch);
1535 } else
1536 #endif
1537 {
1538 rx_ch->channel.status = MUSB_DMA_STATUS_FREE;
1540 if (rx_ch->inf_mode) {
1541 cppi41_rx_ch_set_maxbufcnt(
1542 &rx_ch->dma_ch_obj, 0);
1543 rx_ch->inf_mode = 0;
1544 }
1545 /* Rx completion routine callback */
1546 musb_dma_completion(cppi->musb, ep_num, 0);
1547 }
1548 } else {
1549 if (is_peripheral_active(cppi->musb) &&
1550 ((rx_ch->length - rx_ch->curr_offset) > 0))
1551 cppi41_next_rx_segment(rx_ch);
1552 }
1553 }
1554 }
1556 /*
1557 * cppi41_completion - handle interrupts from the Tx/Rx completion queues
1558 *
1559 * NOTE: since we have to manually prod the Rx process in the transparent mode,
1560 * we certainly want to handle the Rx queues first.
1561 */
1562 void cppi41_completion(struct musb *musb, u32 rx, u32 tx)
1563 {
1564 struct cppi41 *cppi;
1565 unsigned index;
1567 cppi = container_of(musb->dma_controller, struct cppi41, controller);
1569 /* Process packet descriptors from the Rx queues */
1570 for (index = 0; rx != 0; rx >>= 1, index++)
1571 if (rx & 1)
1572 usb_process_rx_queue(cppi, index);
1574 /* Process packet descriptors from the Tx completion queues */
1575 for (index = 0; tx != 0; tx >>= 1, index++)
1576 if (tx & 1)
1577 usb_process_tx_queue(cppi, index);
1578 }
1579 EXPORT_SYMBOL(cppi41_completion);
1581 MODULE_DESCRIPTION("CPPI4.1 dma controller driver for musb");
1582 MODULE_LICENSE("GPL v2");
1584 static int __init cppi41_dma_init(void)
1585 {
1586 return 0;
1587 }
1588 module_init(cppi41_dma_init);
1590 static void __exit cppi41_dma__exit(void)
1591 {
1592 }
1593 module_exit(cppi41_dma__exit);