1 /*
2 * Copyright (C) 2005-2006 by Texas Instruments
3 * Copyright (c) 2008, MontaVista Software, Inc. <source@mvista.com>
4 *
5 * This file implements a DMA interface using TI's CPPI 4.1 DMA.
6 *
7 * This program is free software; you can distribute it and/or modify it
8 * under the terms of the GNU General Public License (Version 2) as
9 * published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * for more details.
15 *
16 * You should have received a copy of the GNU General Public License along
17 * with this program; if not, write to the Free Software Foundation, Inc.,
18 * 59 Temple Place - Suite 330, Boston MA 02111-1307, USA.
19 *
20 */
22 #include <linux/errno.h>
23 #include <linux/dma-mapping.h>
24 #include <linux/module.h>
26 #include "cppi41.h"
28 #include "musb_core.h"
29 #include "musb_dma.h"
30 #include "cppi41_dma.h"
32 /* Configuration */
33 #define USB_CPPI41_DESC_SIZE_SHIFT 6
34 #define USB_CPPI41_DESC_ALIGN (1 << USB_CPPI41_DESC_SIZE_SHIFT)
35 #define USB_CPPI41_CH_NUM_PD 128 /* 4K bulk data at full speed */
36 #define USB_CPPI41_MAX_PD (USB_CPPI41_CH_NUM_PD * (USB_CPPI41_NUM_CH+1))
38 #undef DEBUG_CPPI_TD
39 #undef USBDRV_DEBUG
41 #ifdef USBDRV_DEBUG
42 #define dprintk(x, ...) printk(x, ## __VA_ARGS__)
43 #else
44 #define dprintk(x, ...)
45 #endif
47 /*
48 * Data structure definitions
49 */
51 /*
52 * USB Packet Descriptor
53 */
54 struct usb_pkt_desc;
56 struct usb_pkt_desc {
57 /* Hardware descriptor fields from this point */
58 struct cppi41_host_pkt_desc hw_desc; /* 40 bytes */
59 /* Protocol specific data */
60 dma_addr_t dma_addr; /* offs:44 byte */
61 struct usb_pkt_desc *next_pd_ptr; /* offs:48 byte*/
62 u8 ch_num;
63 u8 ep_num;
64 u8 eop;
65 u8 res1; /* offs:52 */
66 u8 res2[12]; /* offs:64 */
67 };
69 /**
70 * struct cppi41_channel - DMA Channel Control Structure
71 *
72 * Using the same for Tx/Rx.
73 */
74 struct cppi41_channel {
75 struct dma_channel channel;
77 struct cppi41_dma_ch_obj dma_ch_obj; /* DMA channel object */
78 struct cppi41_queue src_queue; /* Tx queue or Rx free descriptor/ */
79 /* buffer queue */
80 struct cppi41_queue_obj queue_obj; /* Tx queue object or Rx free */
81 /* descriptor/buffer queue object */
83 u32 tag_info; /* Tx PD Tag Information field */
85 /* Which direction of which endpoint? */
86 struct musb_hw_ep *end_pt;
87 u8 transmit;
88 u8 ch_num; /* Channel number of Tx/Rx 0..3 */
90 /* DMA mode: "transparent", RNDIS, CDC, or Generic RNDIS */
91 u8 dma_mode;
92 u8 autoreq;
94 /* Book keeping for the current transfer request */
95 dma_addr_t start_addr;
96 u32 length;
97 u32 curr_offset;
98 u16 pkt_size;
99 u8 transfer_mode;
100 u8 zlp_queued;
101 };
103 /**
104 * struct cppi41 - CPPI 4.1 DMA Controller Object
105 *
106 * Encapsulates all book keeping and data structures pertaining to
107 * the CPPI 1.4 DMA controller.
108 */
109 struct cppi41 {
110 struct dma_controller controller;
111 struct musb *musb;
113 struct cppi41_channel tx_cppi_ch[USB_CPPI41_NUM_CH];
114 struct cppi41_channel rx_cppi_ch[USB_CPPI41_NUM_CH];
116 struct usb_pkt_desc *pd_pool_head; /* Free PD pool head */
117 dma_addr_t pd_mem_phys; /* PD memory physical address */
118 void *pd_mem; /* PD memory pointer */
119 u8 pd_mem_rgn; /* PD memory region number */
121 u16 teardownQNum; /* Teardown completion queue number */
122 struct cppi41_queue_obj queue_obj; /* Teardown completion queue */
123 /* object */
124 u32 pkt_info; /* Tx PD Packet Information field */
125 struct usb_cppi41_info *cppi_info; /* cppi channel information */
126 u8 en_bd_intr; /* enable bd interrupt */
127 u32 automode_reg_offs; /* USB_AUTOREQ_REG offset */
128 u32 teardown_reg_offs; /* USB_TEARDOWN_REG offset */
129 u32 bd_size;
130 };
132 struct usb_cppi41_info usb_cppi41_info[2];
133 EXPORT_SYMBOL(usb_cppi41_info);
135 #ifdef DEBUG_CPPI_TD
136 static void print_pd_list(struct usb_pkt_desc *pd_pool_head)
137 {
138 struct usb_pkt_desc *curr_pd = pd_pool_head;
139 int cnt = 0;
141 while (curr_pd != NULL) {
142 if (cnt % 8 == 0)
143 dprintk("\n%02x ", cnt);
144 cnt++;
145 dprintk(" %p", curr_pd);
146 curr_pd = curr_pd->next_pd_ptr;
147 }
148 dprintk("\n");
149 }
150 #endif
152 static struct usb_pkt_desc *usb_get_free_pd(struct cppi41 *cppi)
153 {
154 struct usb_pkt_desc *free_pd = cppi->pd_pool_head;
156 if (free_pd != NULL) {
157 cppi->pd_pool_head = free_pd->next_pd_ptr;
158 free_pd->next_pd_ptr = NULL;
159 }
160 return free_pd;
161 }
163 static void usb_put_free_pd(struct cppi41 *cppi, struct usb_pkt_desc *free_pd)
164 {
165 free_pd->next_pd_ptr = cppi->pd_pool_head;
166 cppi->pd_pool_head = free_pd;
167 }
169 /**
170 * cppi41_controller_start - start DMA controller
171 * @controller: the controller
172 *
173 * This function initializes the CPPI 4.1 Tx/Rx channels.
174 */
175 static int __devinit cppi41_controller_start(struct dma_controller *controller)
176 {
177 struct cppi41 *cppi;
178 struct cppi41_channel *cppi_ch;
179 void __iomem *reg_base;
180 struct usb_pkt_desc *curr_pd;
181 unsigned long pd_addr;
182 int i;
183 struct usb_cppi41_info *cppi_info;
184 struct musb *musb;
186 cppi = container_of(controller, struct cppi41, controller);
187 cppi_info = cppi->cppi_info;
188 musb = cppi->musb;
190 if (cpu_is_ti81xx()) {
191 cppi->automode_reg_offs = TI81XX_USB_AUTOREQ_REG;
192 cppi->teardown_reg_offs = TI81XX_USB_TEARDOWN_REG;
193 } else {
194 cppi->automode_reg_offs = USB_AUTOREQ_REG;
195 cppi->teardown_reg_offs = USB_TEARDOWN_REG;
196 }
198 /*
199 * TODO: We may need to check USB_CPPI41_MAX_PD here since CPPI 4.1
200 * requires the descriptor count to be a multiple of 2 ^ 5 (i.e. 32).
201 * Similarly, the descriptor size should also be a multiple of 32.
202 */
204 /*
205 * Allocate free packet descriptor pool for all Tx/Rx endpoints --
206 * dma_alloc_coherent() will return a page aligned address, so our
207 * alignment requirement will be honored.
208 */
209 cppi->bd_size = USB_CPPI41_MAX_PD * sizeof(struct usb_pkt_desc);
210 cppi->pd_mem = dma_alloc_coherent(cppi->musb->controller,
211 cppi->bd_size,
212 &cppi->pd_mem_phys,
213 GFP_KERNEL | GFP_DMA);
214 if (cppi->pd_mem == NULL) {
215 dev_dbg(musb->controller, "ERROR: packet descriptor memory allocation failed\n");
216 return 0;
217 }
219 if (cppi41_mem_rgn_alloc(cppi_info->q_mgr, cppi->pd_mem_phys,
220 USB_CPPI41_DESC_SIZE_SHIFT,
221 get_count_order(USB_CPPI41_MAX_PD),
222 &cppi->pd_mem_rgn)) {
223 dev_dbg(musb->controller, "ERROR: queue manager memory region allocation "
224 "failed\n");
225 goto free_pds;
226 }
228 /* Allocate the teardown completion queue */
229 if (cppi41_queue_alloc(CPPI41_UNASSIGNED_QUEUE,
230 0, &cppi->teardownQNum)) {
231 dev_dbg(musb->controller, "ERROR: teardown completion queue allocation failed\n");
232 goto free_mem_rgn;
233 }
234 dev_dbg(musb->controller, "Allocated teardown completion queue %d in queue manager 0\n",
235 cppi->teardownQNum);
237 if (cppi41_queue_init(&cppi->queue_obj, 0, cppi->teardownQNum)) {
238 dev_dbg(musb->controller, "ERROR: teardown completion queue initialization "
239 "failed\n");
240 goto free_queue;
241 }
243 /*
244 * "Slice" PDs one-by-one from the big chunk and
245 * add them to the free pool.
246 */
247 curr_pd = (struct usb_pkt_desc *)cppi->pd_mem;
248 pd_addr = cppi->pd_mem_phys;
249 for (i = 0; i < USB_CPPI41_MAX_PD; i++) {
250 curr_pd->dma_addr = pd_addr;
252 usb_put_free_pd(cppi, curr_pd);
253 curr_pd = (struct usb_pkt_desc *)((char *)curr_pd +
254 USB_CPPI41_DESC_ALIGN);
255 pd_addr += USB_CPPI41_DESC_ALIGN;
256 }
258 /* Configure the Tx channels */
259 for (i = 0, cppi_ch = cppi->tx_cppi_ch;
260 i < ARRAY_SIZE(cppi->tx_cppi_ch); ++i, ++cppi_ch) {
261 const struct cppi41_tx_ch *tx_info;
263 memset(cppi_ch, 0, sizeof(struct cppi41_channel));
264 cppi_ch->transmit = 1;
265 cppi_ch->ch_num = i;
266 cppi_ch->channel.private_data = cppi;
268 /*
269 * Extract the CPPI 4.1 DMA Tx channel configuration and
270 * construct/store the Tx PD tag info field for later use...
271 */
272 tx_info = cppi41_dma_block[cppi_info->dma_block].tx_ch_info
273 + cppi_info->ep_dma_ch[i];
274 cppi_ch->src_queue = tx_info->tx_queue[0];
275 cppi_ch->tag_info = (tx_info->port_num <<
276 CPPI41_SRC_TAG_PORT_NUM_SHIFT) |
277 (tx_info->ch_num <<
278 CPPI41_SRC_TAG_CH_NUM_SHIFT) |
279 (tx_info->sub_ch_num <<
280 CPPI41_SRC_TAG_SUB_CH_NUM_SHIFT);
281 }
283 /* Configure the Rx channels */
284 for (i = 0, cppi_ch = cppi->rx_cppi_ch;
285 i < ARRAY_SIZE(cppi->rx_cppi_ch); ++i, ++cppi_ch) {
286 memset(cppi_ch, 0, sizeof(struct cppi41_channel));
287 cppi_ch->ch_num = i;
288 cppi_ch->channel.private_data = cppi;
289 }
291 /* Construct/store Tx PD packet info field for later use */
292 cppi->pkt_info = (CPPI41_PKT_TYPE_USB << CPPI41_PKT_TYPE_SHIFT) |
293 (CPPI41_RETURN_LINKED << CPPI41_RETURN_POLICY_SHIFT);
295 /* Do necessary configuartion in hardware to get started */
296 reg_base = cppi->musb->ctrl_base;
298 /* Disable auto request mode */
299 musb_writel(reg_base, cppi->automode_reg_offs, 0);
301 /* Disable the CDC/RNDIS modes */
302 musb_writel(reg_base, USB_TX_MODE_REG, 0);
303 musb_writel(reg_base, USB_RX_MODE_REG, 0);
305 return 1;
307 free_queue:
308 if (cppi41_queue_free(0, cppi->teardownQNum))
309 dev_dbg(musb->controller, "ERROR: failed to free teardown completion queue\n");
311 free_mem_rgn:
312 if (cppi41_mem_rgn_free(cppi_info->q_mgr, cppi->pd_mem_rgn))
313 dev_dbg(musb->controller, "ERROR: failed to free queue manager memory region\n");
315 free_pds:
316 dma_free_coherent(cppi->musb->controller,
317 USB_CPPI41_MAX_PD * USB_CPPI41_DESC_ALIGN,
318 cppi->pd_mem, cppi->pd_mem_phys);
320 return 0;
321 }
323 /**
324 * cppi41_controller_stop - stop DMA controller
325 * @controller: the controller
326 *
327 * De-initialize the DMA Controller as necessary.
328 */
329 static int cppi41_controller_stop(struct dma_controller *controller)
330 {
331 struct cppi41 *cppi;
332 void __iomem *reg_base;
333 struct usb_cppi41_info *cppi_info;
334 struct musb *musb;
336 cppi = container_of(controller, struct cppi41, controller);
337 cppi_info = cppi->cppi_info;
338 musb = cppi->musb;
340 /* Free the teardown completion queue */
341 if (cppi41_queue_free(cppi_info->q_mgr, cppi->teardownQNum))
342 dev_dbg(musb->controller, "ERROR: failed to free teardown completion queue\n");
344 /*
345 * Free the packet descriptor region allocated
346 * for all Tx/Rx channels.
347 */
348 if (cppi41_mem_rgn_free(cppi_info->q_mgr, cppi->pd_mem_rgn))
349 dev_dbg(musb->controller, "ERROR: failed to free queue manager memory region\n");
351 dma_free_coherent(cppi->musb->controller, cppi->bd_size,
352 cppi->pd_mem, cppi->pd_mem_phys);
354 cppi->pd_mem = 0;
355 cppi->pd_mem_phys = 0;
356 cppi->pd_pool_head = 0;
357 cppi->bd_size = 0;
359 reg_base = cppi->musb->ctrl_base;
361 /* Disable auto request mode */
362 musb_writel(reg_base, cppi->automode_reg_offs, 0);
364 /* Disable the CDC/RNDIS modes */
365 musb_writel(reg_base, USB_TX_MODE_REG, 0);
366 musb_writel(reg_base, USB_RX_MODE_REG, 0);
368 return 1;
369 }
371 /**
372 * cppi41_channel_alloc - allocate a CPPI channel for DMA.
373 * @controller: the controller
374 * @ep: the endpoint
375 * @is_tx: 1 for Tx channel, 0 for Rx channel
376 *
377 * With CPPI, channels are bound to each transfer direction of a non-control
378 * endpoint, so allocating (and deallocating) is mostly a way to notice bad
379 * housekeeping on the software side. We assume the IRQs are always active.
380 */
381 static struct dma_channel *cppi41_channel_alloc(struct dma_controller
382 *controller,
383 struct musb_hw_ep *ep, u8 is_tx)
384 {
385 struct cppi41 *cppi;
386 struct cppi41_channel *cppi_ch;
387 u32 ch_num, ep_num = ep->epnum;
388 struct usb_cppi41_info *cppi_info;
389 struct musb *musb;
391 cppi = container_of(controller, struct cppi41, controller);
392 cppi_info = cppi->cppi_info;
393 musb = cppi->musb;
395 /* Remember, ep_num: 1 .. Max_EP, and CPPI ch_num: 0 .. Max_EP - 1 */
396 ch_num = ep_num - 1;
398 if (ep_num > USB_CPPI41_NUM_CH) {
399 dev_dbg(musb->controller, "No %cx DMA channel for EP%d\n",
400 is_tx ? 'T' : 'R', ep_num);
401 return NULL;
402 }
404 cppi_ch = (is_tx ? cppi->tx_cppi_ch : cppi->rx_cppi_ch) + ch_num;
406 /* As of now, just return the corresponding CPPI 4.1 channel handle */
407 if (is_tx) {
408 /* Initialize the CPPI 4.1 Tx DMA channel */
409 if (cppi41_tx_ch_init(&cppi_ch->dma_ch_obj,
410 cppi_info->dma_block,
411 cppi_info->ep_dma_ch[ch_num])) {
412 dev_dbg(musb->controller, "ERROR: cppi41_tx_ch_init failed for "
413 "channel %d\n", ch_num);
414 return NULL;
415 }
416 /*
417 * Teardown descriptors will be pushed to the dedicated
418 * completion queue.
419 */
420 cppi41_dma_ch_default_queue(&cppi_ch->dma_ch_obj,
421 0, cppi->teardownQNum);
422 } else {
423 struct cppi41_rx_ch_cfg rx_cfg;
424 u8 q_mgr = cppi_info->q_mgr;
425 int i;
427 /* Initialize the CPPI 4.1 Rx DMA channel */
428 if (cppi41_rx_ch_init(&cppi_ch->dma_ch_obj,
429 cppi_info->dma_block,
430 cppi_info->ep_dma_ch[ch_num])) {
431 dev_dbg(musb->controller, "ERROR: cppi41_rx_ch_init failed\n");
432 return NULL;
433 }
435 if (cppi41_queue_alloc(CPPI41_FREE_DESC_BUF_QUEUE |
436 CPPI41_UNASSIGNED_QUEUE,
437 q_mgr, &cppi_ch->src_queue.q_num)) {
438 dev_dbg(musb->controller, "ERROR: cppi41_queue_alloc failed for "
439 "free descriptor/buffer queue\n");
440 return NULL;
441 }
442 dev_dbg(musb->controller, "Allocated free descriptor/buffer queue %d in "
443 "queue manager %d\n", cppi_ch->src_queue.q_num, q_mgr);
445 rx_cfg.default_desc_type = cppi41_rx_host_desc;
446 rx_cfg.sop_offset = 0;
447 rx_cfg.retry_starved = 1;
448 rx_cfg.rx_queue.q_mgr = cppi_ch->src_queue.q_mgr = q_mgr;
449 rx_cfg.rx_queue.q_num = cppi_info->rx_comp_q[ch_num];
450 for (i = 0; i < 4; i++)
451 rx_cfg.cfg.host_pkt.fdb_queue[i] = cppi_ch->src_queue;
452 cppi41_rx_ch_configure(&cppi_ch->dma_ch_obj, &rx_cfg);
453 }
455 /* Initialize the CPPI 4.1 DMA source queue */
456 if (cppi41_queue_init(&cppi_ch->queue_obj, cppi_ch->src_queue.q_mgr,
457 cppi_ch->src_queue.q_num)) {
458 dev_dbg(musb->controller, "ERROR: cppi41_queue_init failed for %s queue",
459 is_tx ? "Tx" : "Rx free descriptor/buffer");
460 if (is_tx == 0 &&
461 cppi41_queue_free(cppi_ch->src_queue.q_mgr,
462 cppi_ch->src_queue.q_num))
463 dev_dbg(musb->controller, "ERROR: failed to free Rx descriptor/buffer "
464 "queue\n");
465 return NULL;
466 }
468 /* Enable the DMA channel */
469 cppi41_dma_ch_enable(&cppi_ch->dma_ch_obj);
471 if (cppi_ch->end_pt)
472 dev_dbg(musb->controller, "Re-allocating DMA %cx channel %d (%p)\n",
473 is_tx ? 'T' : 'R', ch_num, cppi_ch);
475 cppi_ch->end_pt = ep;
476 cppi_ch->ch_num = ch_num;
477 cppi_ch->channel.status = MUSB_DMA_STATUS_FREE;
478 cppi_ch->channel.max_len = is_tx ?
479 CPPI41_TXDMA_MAXLEN : CPPI41_RXDMA_MAXLEN;
481 dev_dbg(musb->controller, "Allocated DMA %cx channel %d for EP%d\n", is_tx ? 'T' : 'R',
482 ch_num, ep_num);
484 return &cppi_ch->channel;
485 }
487 /**
488 * cppi41_channel_release - release a CPPI DMA channel
489 * @channel: the channel
490 */
491 static void cppi41_channel_release(struct dma_channel *channel)
492 {
493 struct cppi41_channel *cppi_ch;
495 /* REVISIT: for paranoia, check state and abort if needed... */
496 cppi_ch = container_of(channel, struct cppi41_channel, channel);
498 if (cppi_ch->end_pt == NULL)
499 printk(KERN_INFO "Releasing idle DMA channel %p\n", cppi_ch);
501 /* But for now, not its IRQ */
502 cppi_ch->end_pt = NULL;
503 channel->status = MUSB_DMA_STATUS_UNKNOWN;
505 cppi41_dma_ch_disable(&cppi_ch->dma_ch_obj);
507 /* De-allocate Rx free descriptior/buffer queue */
508 if (cppi_ch->transmit == 0 &&
509 cppi41_queue_free(cppi_ch->src_queue.q_mgr,
510 cppi_ch->src_queue.q_num))
511 printk(KERN_ERR "ERROR: failed to free Rx descriptor/buffer queue\n");
512 }
514 static void cppi41_mode_update(struct cppi41_channel *cppi_ch, u8 mode)
515 {
516 if (mode != cppi_ch->dma_mode) {
517 struct cppi41 *cppi = cppi_ch->channel.private_data;
518 void *__iomem reg_base = cppi->musb->ctrl_base;
519 u32 reg_val;
520 u8 ep_num = cppi_ch->ch_num + 1;
522 if (cppi_ch->transmit) {
523 reg_val = musb_readl(reg_base, USB_TX_MODE_REG);
524 reg_val &= ~USB_TX_MODE_MASK(ep_num);
525 reg_val |= mode << USB_TX_MODE_SHIFT(ep_num);
526 musb_writel(reg_base, USB_TX_MODE_REG, reg_val);
527 } else {
528 reg_val = musb_readl(reg_base, USB_RX_MODE_REG);
529 reg_val &= ~USB_RX_MODE_MASK(ep_num);
530 reg_val |= mode << USB_RX_MODE_SHIFT(ep_num);
531 musb_writel(reg_base, USB_RX_MODE_REG, reg_val);
532 }
533 cppi_ch->dma_mode = mode;
534 }
535 }
537 /*
538 * CPPI 4.1 Tx:
539 * ============
540 * Tx is a lot more reasonable than Rx: RNDIS mode seems to behave well except
541 * how it handles the exactly-N-packets case. It appears that there's a hiccup
542 * in that case (maybe the DMA completes before a ZLP gets written?) boiling
543 * down to not being able to rely on the XFER DMA writing any terminating zero
544 * length packet before the next transfer is started...
545 *
546 * The generic RNDIS mode does not have this misfeature, so we prefer using it
547 * instead. We then send the terminating ZLP *explictly* using DMA instead of
548 * doing it by PIO after an IRQ.
549 *
550 */
552 /**
553 * cppi41_next_tx_segment - DMA write for the next chunk of a buffer
554 * @tx_ch: Tx channel
555 *
556 * Context: controller IRQ-locked
557 */
558 static unsigned cppi41_next_tx_segment(struct cppi41_channel *tx_ch)
559 {
560 struct cppi41 *cppi = tx_ch->channel.private_data;
561 struct musb *musb = cppi->musb;
562 struct usb_pkt_desc *curr_pd;
563 u32 length = tx_ch->length - tx_ch->curr_offset;
564 u32 pkt_size = tx_ch->pkt_size;
565 unsigned num_pds, n;
566 struct usb_cppi41_info *cppi_info = cppi->cppi_info;
567 u16 q_mgr = cppi_info->q_mgr;
568 u16 tx_comp_q = cppi_info->tx_comp_q[tx_ch->ch_num];
569 u8 en_bd_intr = cppi->en_bd_intr;
571 /*
572 * Tx can use the generic RNDIS mode where we can probably fit this
573 * transfer in one PD and one IRQ. The only time we would NOT want
574 * to use it is when the hardware constraints prevent it...
575 */
576 if ((pkt_size & 0x3f) == 0 && length > pkt_size) {
577 num_pds = 1;
578 pkt_size = length;
579 cppi41_mode_update(tx_ch, USB_GENERIC_RNDIS_MODE);
580 } else {
581 num_pds = (length + pkt_size - 1) / pkt_size;
582 cppi41_mode_update(tx_ch, USB_TRANSPARENT_MODE);
583 }
585 /*
586 * If length of transmit buffer is 0 or a multiple of the endpoint size,
587 * then send the zero length packet.
588 */
589 if (!length || (tx_ch->transfer_mode && length % pkt_size == 0))
590 num_pds++;
592 dev_dbg(musb->controller, "TX DMA%u, %s, maxpkt %u, %u PDs, addr %#x, len %u\n",
593 tx_ch->ch_num, tx_ch->dma_mode ? "accelerated" : "transparent",
594 pkt_size, num_pds, tx_ch->start_addr + tx_ch->curr_offset, length);
596 for (n = 0; n < num_pds; n++) {
597 struct cppi41_host_pkt_desc *hw_desc;
599 /* Get Tx host packet descriptor from the free pool */
600 curr_pd = usb_get_free_pd(cppi);
601 if (curr_pd == NULL) {
602 dev_dbg(musb->controller, "No Tx PDs\n");
603 break;
604 }
606 if (length < pkt_size)
607 pkt_size = length;
609 hw_desc = &curr_pd->hw_desc;
610 hw_desc->desc_info = (CPPI41_DESC_TYPE_HOST <<
611 CPPI41_DESC_TYPE_SHIFT) | pkt_size;
612 hw_desc->tag_info = tx_ch->tag_info;
613 hw_desc->pkt_info = cppi->pkt_info;
614 hw_desc->pkt_info |= ((q_mgr << CPPI41_RETURN_QMGR_SHIFT) |
615 (tx_comp_q << CPPI41_RETURN_QNUM_SHIFT));
617 hw_desc->buf_ptr = tx_ch->start_addr + tx_ch->curr_offset;
618 hw_desc->buf_len = pkt_size;
619 hw_desc->next_desc_ptr = 0;
620 hw_desc->orig_buf_len = pkt_size;
622 curr_pd->ch_num = tx_ch->ch_num;
623 curr_pd->ep_num = tx_ch->end_pt->epnum;
625 tx_ch->curr_offset += pkt_size;
626 length -= pkt_size;
628 if (pkt_size == 0)
629 tx_ch->zlp_queued = 1;
631 if (en_bd_intr)
632 hw_desc->orig_buf_len |= CPPI41_PKT_INTR_FLAG;
634 dev_dbg(musb->controller, "TX PD %p: buf %08x, len %08x, pkt info %08x\n", curr_pd,
635 hw_desc->buf_ptr, hw_desc->buf_len, hw_desc->pkt_info);
637 cppi41_queue_push(&tx_ch->queue_obj, curr_pd->dma_addr,
638 USB_CPPI41_DESC_ALIGN, pkt_size);
639 }
641 return n;
642 }
644 static void cppi41_autoreq_update(struct cppi41_channel *rx_ch, u8 autoreq)
645 {
646 struct cppi41 *cppi = rx_ch->channel.private_data;
648 if (is_host_active(cppi->musb) &&
649 autoreq != rx_ch->autoreq) {
650 void *__iomem reg_base = cppi->musb->ctrl_base;
651 u32 reg_val = musb_readl(reg_base, cppi->automode_reg_offs);
652 u8 ep_num = rx_ch->ch_num + 1;
654 reg_val &= ~USB_RX_AUTOREQ_MASK(ep_num);
655 reg_val |= autoreq << USB_RX_AUTOREQ_SHIFT(ep_num);
657 musb_writel(reg_base, cppi->automode_reg_offs, reg_val);
658 rx_ch->autoreq = autoreq;
659 }
660 }
662 static void cppi41_set_ep_size(struct cppi41_channel *rx_ch, u32 pkt_size)
663 {
664 struct cppi41 *cppi = rx_ch->channel.private_data;
665 void *__iomem reg_base = cppi->musb->ctrl_base;
666 u8 ep_num = rx_ch->ch_num + 1;
668 musb_writel(reg_base, USB_GENERIC_RNDIS_EP_SIZE_REG(ep_num), pkt_size);
669 }
671 /*
672 * CPPI 4.1 Rx:
673 * ============
674 * Consider a 1KB bulk Rx buffer in two scenarios: (a) it's fed two 300 byte
675 * packets back-to-back, and (b) it's fed two 512 byte packets back-to-back.
676 * (Full speed transfers have similar scenarios.)
677 *
678 * The correct behavior for Linux is that (a) fills the buffer with 300 bytes,
679 * and the next packet goes into a buffer that's queued later; while (b) fills
680 * the buffer with 1024 bytes. How to do that with accelerated DMA modes?
681 *
682 * Rx queues in RNDIS mode (one single BD) handle (a) correctly but (b) loses
683 * BADLY because nothing (!) happens when that second packet fills the buffer,
684 * much less when a third one arrives -- which makes it not a "true" RNDIS mode.
685 * In the RNDIS protocol short-packet termination is optional, and it's fine if
686 * the peripherals (not hosts!) pad the messages out to end of buffer. Standard
687 * PCI host controller DMA descriptors implement that mode by default... which
688 * is no accident.
689 *
690 * Generic RNDIS mode is the only way to reliably make both cases work. This
691 * mode is identical to the "normal" RNDIS mode except for the case where the
692 * last packet of the segment matches the max USB packet size -- in this case,
693 * the packet will be closed when a value (0x10000 max) in the Generic RNDIS
694 * EP Size register is reached. This mode will work for the network drivers
695 * (CDC/RNDIS) as well as for the mass storage drivers where there is no short
696 * packet.
697 *
698 * BUT we can only use non-transparent modes when USB packet size is a multiple
699 * of 64 bytes. Let's see what happens when this is not the case...
700 *
701 * Rx queues (2 BDs with 512 bytes each) have converse problems to RNDIS mode:
702 * (b) is handled right but (a) loses badly. DMA doesn't stop after receiving
703 * a short packet and processes both of those PDs; so both packets are loaded
704 * into the buffer (with 212 byte gap between them), and the next buffer queued
705 * will NOT get its 300 bytes of data. Even in the case when there should be
706 * no short packets (URB_SHORT_NOT_OK is set), queueing several packets in the
707 * host mode doesn't win us anything since we have to manually "prod" the Rx
708 * process after each packet is received by setting ReqPkt bit in endpoint's
709 * RXCSR; in the peripheral mode without short packets, queueing could be used
710 * BUT we'll have to *teardown* the channel if a short packet still arrives in
711 * the peripheral mode, and to "collect" the left-over packet descriptors from
712 * the free descriptor/buffer queue in both cases...
713 *
714 * One BD at a time is the only way to make make both cases work reliably, with
715 * software handling both cases correctly, at the significant penalty of needing
716 * an IRQ per packet. (The lack of I/O overlap can be slightly ameliorated by
717 * enabling double buffering.)
718 *
719 * There seems to be no way to identify for sure the cases where the CDC mode
720 * is appropriate...
721 *
722 */
724 /**
725 * cppi41_next_rx_segment - DMA read for the next chunk of a buffer
726 * @rx_ch: Rx channel
727 *
728 * Context: controller IRQ-locked
729 *
730 * NOTE: In the transparent mode, we have to queue one packet at a time since:
731 * - we must avoid starting reception of another packet after receiving
732 * a short packet;
733 * - in host mode we have to set ReqPkt bit in the endpoint's RXCSR after
734 * receiving each packet but the last one... ugly!
735 */
736 static unsigned cppi41_next_rx_segment(struct cppi41_channel *rx_ch)
737 {
738 struct cppi41 *cppi = rx_ch->channel.private_data;
739 struct musb *musb = cppi->musb;
740 struct usb_pkt_desc *curr_pd;
741 struct cppi41_host_pkt_desc *hw_desc;
742 u32 length = rx_ch->length - rx_ch->curr_offset;
743 u32 pkt_size = rx_ch->pkt_size;
744 u32 max_rx_transfer_size = 64 * 1024;
745 u32 i, n_bd , pkt_len;
746 struct usb_gadget_driver *gadget_driver;
747 u8 en_bd_intr = cppi->en_bd_intr;
749 if (is_peripheral_active(cppi->musb)) {
750 /* TODO: temporary fix for CDC/RNDIS which needs to be in
751 * GENERIC_RNDIS mode. Without this RNDIS gadget taking
752 * more then 2K ms for a 64 byte pings.
753 */
754 #ifdef CONFIG_USB_GADGET_MUSB_HDRC
755 gadget_driver = cppi->musb->gadget_driver;
756 #endif
757 if (!strcmp(gadget_driver->driver.name, "g_ether")) {
758 cppi41_mode_update(rx_ch, USB_GENERIC_RNDIS_MODE);
759 } else {
760 max_rx_transfer_size = 512;
761 cppi41_mode_update(rx_ch, USB_TRANSPARENT_MODE);
762 }
763 pkt_len = 0;
764 if (rx_ch->length < max_rx_transfer_size)
765 pkt_len = rx_ch->length;
766 cppi41_set_ep_size(rx_ch, pkt_len);
767 } else {
768 /*
769 * Rx can use the generic RNDIS mode where we can
770 * probably fit this transfer in one PD and one IRQ
771 * (or two with a short packet).
772 */
773 if ((pkt_size & 0x3f) == 0 && length >= 2 * pkt_size) {
774 cppi41_mode_update(rx_ch, USB_GENERIC_RNDIS_MODE);
775 cppi41_autoreq_update(rx_ch, USB_AUTOREQ_ALL_BUT_EOP);
777 if (likely(length < 0x10000))
778 pkt_size = length - length % pkt_size;
779 else
780 pkt_size = 0x10000;
781 cppi41_set_ep_size(rx_ch, pkt_size);
782 } else {
783 cppi41_mode_update(rx_ch, USB_TRANSPARENT_MODE);
784 cppi41_autoreq_update(rx_ch, USB_NO_AUTOREQ);
785 }
786 }
788 dev_dbg(musb->controller, "RX DMA%u, %s, maxpkt %u, addr %#x, rec'd %u/%u\n",
789 rx_ch->ch_num, rx_ch->dma_mode ? "accelerated" : "transparent",
790 pkt_size, rx_ch->start_addr + rx_ch->curr_offset,
791 rx_ch->curr_offset, rx_ch->length);
793 /* calculate number of bd required */
794 n_bd = (length + max_rx_transfer_size - 1)/max_rx_transfer_size;
796 for (i = 0; i < n_bd ; ++i) {
797 /* Get Rx packet descriptor from the free pool */
798 curr_pd = usb_get_free_pd(cppi);
799 if (curr_pd == NULL) {
800 /* Shouldn't ever happen! */
801 dev_dbg(musb->controller, "No Rx PDs\n");
802 goto sched;
803 }
805 pkt_len =
806 (length > max_rx_transfer_size) ? max_rx_transfer_size : length;
808 hw_desc = &curr_pd->hw_desc;
809 hw_desc->desc_info = (CPPI41_DESC_TYPE_HOST <<
810 CPPI41_DESC_TYPE_SHIFT);
811 hw_desc->orig_buf_ptr = rx_ch->start_addr + rx_ch->curr_offset;
812 hw_desc->orig_buf_len = pkt_len;
814 /* buf_len field of buffer descriptor updated by dma
815 * after reception of data is completed
816 */
817 hw_desc->buf_len = 0;
819 curr_pd->ch_num = rx_ch->ch_num;
820 curr_pd->ep_num = rx_ch->end_pt->epnum;
822 curr_pd->eop = (length -= pkt_len) ? 0 : 1;
823 rx_ch->curr_offset += pkt_len;
825 if (en_bd_intr)
826 hw_desc->orig_buf_len |= CPPI41_PKT_INTR_FLAG;
827 /*
828 * Push the free Rx packet descriptor
829 * to the free descriptor/buffer queue.
830 */
831 cppi41_queue_push(&rx_ch->queue_obj, curr_pd->dma_addr,
832 USB_CPPI41_DESC_ALIGN, 0);
833 }
835 sched:
836 /*
837 * HCD arranged ReqPkt for the first packet.
838 * We arrange it for all but the last one.
839 */
840 if (is_host_active(cppi->musb) && rx_ch->channel.actual_len) {
841 void __iomem *epio = rx_ch->end_pt->regs;
842 u16 csr = musb_readw(epio, MUSB_RXCSR);
844 csr |= MUSB_RXCSR_H_REQPKT | MUSB_RXCSR_H_WZC_BITS;
845 musb_writew(epio, MUSB_RXCSR, csr);
846 }
848 /* enable schedular if not enabled */
849 if (is_peripheral_active(cppi->musb) && (n_bd > 0))
850 cppi41_schedtbl_add_dma_ch(0, 0, rx_ch->ch_num, 0);
851 return 1;
852 }
854 /**
855 * cppi41_channel_program - program channel for data transfer
856 * @channel: the channel
857 * @maxpacket: max packet size
858 * @mode: for Rx, 1 unless the USB protocol driver promised to treat
859 * all short reads as errors and kick in high level fault recovery;
860 * for Tx, 0 unless the protocol driver _requires_ short-packet
861 * termination mode
862 * @dma_addr: DMA address of buffer
863 * @length: length of buffer
864 *
865 * Context: controller IRQ-locked
866 */
867 static int cppi41_channel_program(struct dma_channel *channel, u16 maxpacket,
868 u8 mode, dma_addr_t dma_addr, u32 length)
869 {
870 struct cppi41_channel *cppi_ch;
871 unsigned queued;
873 cppi_ch = container_of(channel, struct cppi41_channel, channel);
875 switch (channel->status) {
876 case MUSB_DMA_STATUS_BUS_ABORT:
877 case MUSB_DMA_STATUS_CORE_ABORT:
878 /* Fault IRQ handler should have handled cleanup */
879 WARNING("%cx DMA%d not cleaned up after abort!\n",
880 cppi_ch->transmit ? 'T' : 'R', cppi_ch->ch_num);
881 break;
882 case MUSB_DMA_STATUS_BUSY:
883 WARNING("Program active channel? %cx DMA%d\n",
884 cppi_ch->transmit ? 'T' : 'R', cppi_ch->ch_num);
885 break;
886 case MUSB_DMA_STATUS_UNKNOWN:
887 WARNING("%cx DMA%d not allocated!\n",
888 cppi_ch->transmit ? 'T' : 'R', cppi_ch->ch_num);
889 return 0;
890 case MUSB_DMA_STATUS_FREE:
891 break;
892 }
894 channel->status = MUSB_DMA_STATUS_BUSY;
896 /* Set the transfer parameters, then queue up the first segment */
897 cppi_ch->start_addr = dma_addr;
898 cppi_ch->curr_offset = 0;
899 cppi_ch->pkt_size = maxpacket;
900 cppi_ch->length = length;
901 cppi_ch->transfer_mode = mode;
902 cppi_ch->zlp_queued = 0;
903 cppi_ch->channel.actual_len = 0;
905 /* Tx or Rx channel? */
906 if (cppi_ch->transmit)
907 queued = cppi41_next_tx_segment(cppi_ch);
908 else
909 queued = cppi41_next_rx_segment(cppi_ch);
911 return queued > 0;
912 }
914 static struct usb_pkt_desc *usb_get_pd_ptr(struct cppi41 *cppi,
915 unsigned long pd_addr)
916 {
917 if (pd_addr >= cppi->pd_mem_phys && pd_addr < cppi->pd_mem_phys +
918 USB_CPPI41_MAX_PD * USB_CPPI41_DESC_ALIGN)
919 return pd_addr - cppi->pd_mem_phys + cppi->pd_mem;
920 else
921 return NULL;
922 }
924 static int usb_check_teardown(struct cppi41_channel *cppi_ch,
925 unsigned long pd_addr)
926 {
927 u32 info;
928 struct cppi41 *cppi = cppi_ch->channel.private_data;
929 struct usb_cppi41_info *cppi_info = cppi->cppi_info;
930 struct musb *musb = cppi->musb;
932 if (cppi41_get_teardown_info(pd_addr, &info)) {
933 dev_dbg(musb->controller, "ERROR: not a teardown descriptor\n");
934 return 0;
935 }
937 if ((info & CPPI41_TEARDOWN_TX_RX_MASK) ==
938 (!cppi_ch->transmit << CPPI41_TEARDOWN_TX_RX_SHIFT) &&
939 (info & CPPI41_TEARDOWN_DMA_NUM_MASK) ==
940 (cppi_info->dma_block << CPPI41_TEARDOWN_DMA_NUM_SHIFT) &&
941 (info & CPPI41_TEARDOWN_CHAN_NUM_MASK) ==
942 (cppi_info->ep_dma_ch[cppi_ch->ch_num] <<
943 CPPI41_TEARDOWN_CHAN_NUM_SHIFT))
944 return 1;
946 dev_dbg(musb->controller, "ERROR: unexpected values in teardown descriptor\n");
947 return 0;
948 }
950 /*
951 * We can't handle the channel teardown via the default completion queue in
952 * context of the controller IRQ-locked, so we use the dedicated teardown
953 * completion queue which we can just poll for a teardown descriptor, not
954 * interfering with the Tx completion queue processing.
955 */
956 static void usb_tx_ch_teardown(struct cppi41_channel *tx_ch)
957 {
958 struct cppi41 *cppi = tx_ch->channel.private_data;
959 struct musb *musb = cppi->musb;
960 void __iomem *reg_base = musb->ctrl_base;
961 u32 td_reg, timeout = 0xfffff;
962 u8 ep_num = tx_ch->ch_num + 1;
963 unsigned long pd_addr;
964 struct cppi41_queue_obj tx_queue_obj;
965 struct usb_cppi41_info *cppi_info;
967 /* Initiate teardown for Tx DMA channel */
968 cppi41_dma_ch_teardown(&tx_ch->dma_ch_obj);
970 /* Wait for a descriptor to be queued and pop it... */
971 do {
972 td_reg = musb_readl(reg_base, cppi->teardown_reg_offs);
973 td_reg |= USB_TX_TDOWN_MASK(ep_num);
974 musb_writel(reg_base, cppi->teardown_reg_offs, td_reg);
976 pd_addr = cppi41_queue_pop(&cppi->queue_obj);
977 } while (!pd_addr && timeout--);
979 if (pd_addr) {
981 DBG(1, "Descriptor (%08lx) popped from teardown completion "
982 "queue\n", pd_addr);
984 if (usb_check_teardown(tx_ch, pd_addr)) {
985 DBG(1, "Teardown Desc (%lx) rcvd\n", pd_addr);
986 } else
987 ERR("Invalid PD(%08lx)popped from TearDn completion"
988 "queue\n", pd_addr);
989 } else {
990 if (timeout <= 0)
991 ERR("Teardown Desc not rcvd\n");
992 }
994 /* read the tx completion queue and remove
995 * completion bd if any
996 */
997 cppi_info = cppi->cppi_info;
998 if (cppi41_queue_init(&tx_queue_obj, cppi_info->q_mgr,
999 cppi_info->tx_comp_q[tx_ch->ch_num])) {
1000 ERR("ERROR: cppi41_queue_init failed for "
1001 "Tx completion queue");
1002 return;
1003 }
1005 while ((pd_addr = cppi41_queue_pop(&tx_queue_obj)) != 0) {
1006 struct usb_pkt_desc *curr_pd;
1008 curr_pd = usb_get_pd_ptr(cppi, pd_addr);
1009 if (curr_pd == NULL) {
1010 ERR("Invalid PD popped from Tx completion queue\n");
1011 continue;
1012 }
1014 DBG(1, "Tx-PD(%p) popped from completion queue\n", curr_pd);
1015 DBG(1, "ch(%d)epnum(%d)len(%d)\n", curr_pd->ch_num,
1016 curr_pd->ep_num, curr_pd->hw_desc.buf_len);
1018 usb_put_free_pd(cppi, curr_pd);
1019 }
1020 }
1022 /*
1023 * For Rx DMA channels, the situation is more complex: there's only a single
1024 * completion queue for all our needs, so we have to temporarily redirect the
1025 * completed descriptors to our teardown completion queue, with a possibility
1026 * of a completed packet landing there as well...
1027 */
1028 static void usb_rx_ch_teardown(struct cppi41_channel *rx_ch)
1029 {
1030 struct cppi41 *cppi = rx_ch->channel.private_data;
1031 struct usb_cppi41_info *cppi_info = cppi->cppi_info;
1032 u32 timeout = 0xfffff, pd_addr;
1033 struct cppi41_queue_obj rx_queue_obj;
1035 cppi41_dma_ch_default_queue(&rx_ch->dma_ch_obj, 0, cppi->teardownQNum);
1037 /* Initiate teardown for Rx DMA channel */
1038 cppi41_dma_ch_teardown(&rx_ch->dma_ch_obj);
1040 do {
1041 struct usb_pkt_desc *curr_pd;
1042 unsigned long pd_addr;
1044 /* Wait for a descriptor to be queued and pop it... */
1045 do {
1046 pd_addr = cppi41_queue_pop(&cppi->queue_obj);
1047 } while (!pd_addr && timeout--);
1049 if (timeout <= 0 || !pd_addr) {
1050 ERR("teardown Desc not found\n");
1051 break;
1052 }
1054 DBG(1, "Descriptor (%08lx) popped from teardown completion "
1055 "queue\n", pd_addr);
1057 /*
1058 * We might have popped a completed Rx PD, so check if the
1059 * physical address is within the PD region first. If it's
1060 * not the case, it must be a teardown descriptor...
1061 * */
1062 curr_pd = usb_get_pd_ptr(cppi, pd_addr);
1063 if (curr_pd == NULL) {
1064 if (usb_check_teardown(rx_ch, pd_addr))
1065 break;
1066 continue;
1067 }
1069 /* Paranoia: check if PD is from the right channel... */
1070 if (curr_pd->ch_num != rx_ch->ch_num) {
1071 ERR("Unexpected channel %d in Rx PD\n",
1072 curr_pd->ch_num);
1073 continue;
1074 }
1076 /* Extract the buffer length from the completed PD */
1077 rx_ch->channel.actual_len += curr_pd->hw_desc.buf_len;
1079 /*
1080 * Return Rx PDs to the software list --
1081 * this is protected by critical section.
1082 */
1083 usb_put_free_pd(cppi, curr_pd);
1084 } while (0);
1086 /* read the rx completion queue and remove
1087 * completion bd if any
1088 */
1089 if (cppi41_queue_init(&rx_queue_obj, cppi_info->q_mgr,
1090 cppi_info->rx_comp_q[rx_ch->ch_num])) {
1091 ERR("ERROR: cppi41_queue_init failed for "
1092 "Rx completion queue");
1093 return;
1094 }
1096 while ((pd_addr = cppi41_queue_pop(&rx_queue_obj)) != 0) {
1097 struct usb_pkt_desc *curr_pd;
1099 curr_pd = usb_get_pd_ptr(cppi, pd_addr);
1100 if (curr_pd == NULL) {
1101 ERR("Invalid PD popped from Rx completion queue\n");
1102 continue;
1103 }
1105 DBG(1, "Rx-PD(%p) popped from completion queue\n", curr_pd);
1106 DBG(1, "ch(%d)epnum(%d)len(%d)\n", curr_pd->ch_num,
1107 curr_pd->ep_num, curr_pd->hw_desc.buf_len);
1109 usb_put_free_pd(cppi, curr_pd);
1110 }
1112 /* Now restore the default Rx completion queue... */
1113 cppi41_dma_ch_default_queue(&rx_ch->dma_ch_obj, cppi_info->q_mgr,
1114 cppi_info->rx_comp_q[rx_ch->ch_num]);
1115 }
1117 /*
1118 * cppi41_channel_abort
1119 *
1120 * Context: controller IRQ-locked, endpoint selected.
1121 */
1122 static int cppi41_channel_abort(struct dma_channel *channel)
1123 {
1124 struct cppi41 *cppi;
1125 struct cppi41_channel *cppi_ch;
1126 struct musb *musb;
1127 void __iomem *reg_base, *epio;
1128 unsigned long pd_addr;
1129 u32 csr, td_reg;
1130 u8 ch_num, ep_num;
1132 cppi_ch = container_of(channel, struct cppi41_channel, channel);
1133 ch_num = cppi_ch->ch_num;
1134 cppi = cppi_ch->channel.private_data;
1135 musb = cppi->musb;
1137 switch (channel->status) {
1138 case MUSB_DMA_STATUS_BUS_ABORT:
1139 case MUSB_DMA_STATUS_CORE_ABORT:
1140 /* From Rx or Tx fault IRQ handler */
1141 case MUSB_DMA_STATUS_BUSY:
1142 /* The hardware needs shutting down... */
1143 dprintk("%s: DMA busy, status = %x\n",
1144 __func__, channel->status);
1145 break;
1146 case MUSB_DMA_STATUS_UNKNOWN:
1147 dev_dbg(musb->controller, "%cx DMA%d not allocated\n",
1148 cppi_ch->transmit ? 'T' : 'R', ch_num);
1149 /* FALLTHROUGH */
1150 case MUSB_DMA_STATUS_FREE:
1151 return 0;
1152 }
1154 reg_base = musb->ctrl_base;
1155 epio = cppi_ch->end_pt->regs;
1156 ep_num = ch_num + 1;
1158 #ifdef DEBUG_CPPI_TD
1159 printk("Before teardown:");
1160 print_pd_list(cppi->pd_pool_head);
1161 #endif
1163 if (cppi_ch->transmit) {
1164 dprintk("Tx channel teardown, cppi_ch = %p\n", cppi_ch);
1166 /* Tear down Tx DMA channel */
1167 usb_tx_ch_teardown(cppi_ch);
1169 /* Issue CPPI FIFO teardown for Tx channel */
1170 td_reg = musb_readl(reg_base, cppi->teardown_reg_offs);
1171 td_reg |= USB_TX_TDOWN_MASK(ep_num);
1172 musb_writel(reg_base, cppi->teardown_reg_offs, td_reg);
1174 /* Flush FIFO of the endpoint */
1175 csr = musb_readw(epio, MUSB_TXCSR);
1176 csr |= MUSB_TXCSR_FLUSHFIFO | MUSB_TXCSR_H_WZC_BITS;
1177 musb_writew(epio, MUSB_TXCSR, csr);
1178 musb_writew(epio, MUSB_TXCSR, csr);
1179 } else { /* Rx */
1180 dprintk("Rx channel teardown, cppi_ch = %p\n", cppi_ch);
1182 /* Flush FIFO of the endpoint */
1183 csr = musb_readw(epio, MUSB_RXCSR);
1184 csr |= MUSB_RXCSR_FLUSHFIFO | MUSB_RXCSR_H_WZC_BITS;
1185 musb_writew(epio, MUSB_RXCSR, csr);
1186 musb_writew(epio, MUSB_RXCSR, csr);
1188 /* Issue CPPI FIFO teardown for Rx channel */
1189 td_reg = musb_readl(reg_base, cppi->teardown_reg_offs);
1190 td_reg |= USB_RX_TDOWN_MASK(ep_num);
1191 musb_writel(reg_base, cppi->teardown_reg_offs, td_reg);
1193 /* Tear down Rx DMA channel */
1194 usb_rx_ch_teardown(cppi_ch);
1196 /*
1197 * NOTE: docs don't guarantee any of this works... we expect
1198 * that if the USB core stops telling the CPPI core to pull
1199 * more data from it, then it'll be safe to flush current Rx
1200 * DMA state iff any pending FIFO transfer is done.
1201 */
1203 /* For host, ensure ReqPkt is never set again */
1204 cppi41_autoreq_update(cppi_ch, USB_NO_AUTOREQ);
1206 /* For host, clear (just) ReqPkt at end of current packet(s) */
1207 if (is_host_active(cppi->musb))
1208 csr &= ~MUSB_RXCSR_H_REQPKT;
1209 csr |= MUSB_RXCSR_H_WZC_BITS;
1211 /* Clear DMA enable */
1212 csr &= ~MUSB_RXCSR_DMAENAB;
1213 musb_writew(epio, MUSB_RXCSR, csr);
1215 /* Flush the FIFO of endpoint once again */
1216 csr = musb_readw(epio, MUSB_RXCSR);
1217 csr |= MUSB_RXCSR_FLUSHFIFO | MUSB_RXCSR_H_WZC_BITS;
1218 musb_writew(epio, MUSB_RXCSR, csr);
1220 udelay(50);
1221 }
1223 /*
1224 * There might be PDs in the Rx/Tx source queue that were not consumed
1225 * by the DMA controller -- they need to be recycled properly.
1226 */
1227 while ((pd_addr = cppi41_queue_pop(&cppi_ch->queue_obj)) != 0) {
1228 struct usb_pkt_desc *curr_pd;
1230 curr_pd = usb_get_pd_ptr(cppi, pd_addr);
1231 if (curr_pd == NULL) {
1232 ERR("Invalid PD popped from source queue\n");
1233 continue;
1234 }
1236 /*
1237 * Return Rx/Tx PDs to the software list --
1238 * this is protected by critical section.
1239 */
1240 dprintk("Returning PD %p to the free PD list\n", curr_pd);
1241 usb_put_free_pd(cppi, curr_pd);
1242 }
1244 #ifdef DEBUG_CPPI_TD
1245 printk("After teardown:");
1246 print_pd_list(cppi->pd_pool_head);
1247 #endif
1249 /* Re-enable the DMA channel */
1250 cppi41_dma_ch_enable(&cppi_ch->dma_ch_obj);
1252 channel->status = MUSB_DMA_STATUS_FREE;
1254 return 0;
1255 }
1257 /**
1258 * cppi41_dma_controller_create -
1259 * instantiate an object representing DMA controller.
1260 */
1261 struct dma_controller * __devinit
1262 cppi41_dma_controller_create(struct musb *musb, void __iomem *mregs)
1263 {
1264 struct cppi41 *cppi;
1266 cppi = kzalloc(sizeof *cppi, GFP_KERNEL);
1267 if (!cppi)
1268 return NULL;
1270 /* Initialize the CPPI 4.1 DMA controller structure */
1271 cppi->musb = musb;
1272 cppi->controller.start = cppi41_controller_start;
1273 cppi->controller.stop = cppi41_controller_stop;
1274 cppi->controller.channel_alloc = cppi41_channel_alloc;
1275 cppi->controller.channel_release = cppi41_channel_release;
1276 cppi->controller.channel_program = cppi41_channel_program;
1277 cppi->controller.channel_abort = cppi41_channel_abort;
1278 cppi->cppi_info = (struct usb_cppi41_info *)&usb_cppi41_info[musb->id];;
1279 cppi->en_bd_intr = cppi->cppi_info->bd_intr_ctrl;
1281 return &cppi->controller;
1282 }
1283 EXPORT_SYMBOL(cppi41_dma_controller_create);
1285 /**
1286 * cppi41_dma_controller_destroy -
1287 * destroy a previously instantiated DMA controller
1288 * @controller: the controller
1289 */
1290 void cppi41_dma_controller_destroy(struct dma_controller *controller)
1291 {
1292 struct cppi41 *cppi;
1294 cppi = container_of(controller, struct cppi41, controller);
1296 /* Free the CPPI object */
1297 kfree(cppi);
1298 }
1299 EXPORT_SYMBOL(cppi41_dma_controller_destroy);
1301 static void usb_process_tx_queue(struct cppi41 *cppi, unsigned index)
1302 {
1303 struct cppi41_queue_obj tx_queue_obj;
1304 unsigned long pd_addr;
1305 struct usb_cppi41_info *cppi_info = cppi->cppi_info;
1306 struct musb *musb = cppi->musb;
1308 if (cppi41_queue_init(&tx_queue_obj, cppi_info->q_mgr,
1309 cppi_info->tx_comp_q[index])) {
1310 dev_dbg(musb->controller, "ERROR: cppi41_queue_init failed for "
1311 "Tx completion queue");
1312 return;
1313 }
1315 while ((pd_addr = cppi41_queue_pop(&tx_queue_obj)) != 0) {
1316 struct usb_pkt_desc *curr_pd;
1317 struct cppi41_channel *tx_ch;
1318 u8 ch_num, ep_num;
1319 u32 length;
1321 curr_pd = usb_get_pd_ptr(cppi, pd_addr);
1322 if (curr_pd == NULL) {
1323 ERR("Invalid PD popped from Tx completion queue\n");
1324 continue;
1325 }
1327 /* Extract the data from received packet descriptor */
1328 ch_num = curr_pd->ch_num;
1329 ep_num = curr_pd->ep_num;
1330 length = curr_pd->hw_desc.buf_len;
1332 tx_ch = &cppi->tx_cppi_ch[ch_num];
1333 tx_ch->channel.actual_len += length;
1335 /*
1336 * Return Tx PD to the software list --
1337 * this is protected by critical section
1338 */
1339 usb_put_free_pd(cppi, curr_pd);
1341 if ((tx_ch->curr_offset < tx_ch->length) ||
1342 (tx_ch->transfer_mode && !tx_ch->zlp_queued))
1343 cppi41_next_tx_segment(tx_ch);
1344 else if (tx_ch->channel.actual_len >= tx_ch->length) {
1345 tx_ch->channel.status = MUSB_DMA_STATUS_FREE;
1347 /*
1348 * We get Tx DMA completion interrupt even when
1349 * data is still in FIFO and not moved out to
1350 * USB bus. As we program the next request we
1351 * flush out and old data in FIFO which affects
1352 * USB functionality. So far, we have obsered
1353 * failure with iperf.
1354 */
1355 udelay(20);
1356 /* Tx completion routine callback */
1357 musb_dma_completion(cppi->musb, ep_num, 1);
1358 }
1359 }
1360 }
1362 static void usb_process_rx_queue(struct cppi41 *cppi, unsigned index)
1363 {
1364 struct cppi41_queue_obj rx_queue_obj;
1365 unsigned long pd_addr;
1366 struct usb_cppi41_info *cppi_info = cppi->cppi_info;
1367 struct musb *musb = cppi->musb;
1368 u8 en_bd_intr = cppi->en_bd_intr;
1370 if (cppi41_queue_init(&rx_queue_obj, cppi_info->q_mgr,
1371 cppi_info->rx_comp_q[index])) {
1372 dev_dbg(musb->controller, "ERROR: cppi41_queue_init failed for Rx queue\n");
1373 return;
1374 }
1376 while ((pd_addr = cppi41_queue_pop(&rx_queue_obj)) != 0) {
1377 struct usb_pkt_desc *curr_pd;
1378 struct cppi41_channel *rx_ch;
1379 u8 ch_num, ep_num;
1380 u32 length, orig_buf_len, timeout = 50;
1382 curr_pd = usb_get_pd_ptr(cppi, pd_addr);
1383 if (curr_pd == NULL) {
1384 ERR("Invalid PD popped from Rx completion queue\n");
1385 continue;
1386 }
1388 /* This delay is required to overcome the dma race condition
1389 * where software reads buffer descriptor before being updated
1390 * by dma as buffer descriptor's writes by dma still pending in
1391 * interconnect bridge.
1392 */
1393 while (timeout--) {
1394 length = curr_pd->hw_desc.desc_info &
1395 CPPI41_PKT_LEN_MASK;
1396 if (length != 0)
1397 break;
1398 udelay(1);
1399 }
1401 if (length == 0)
1402 ERR("!Race condtion: rxBD read before updated by dma");
1404 /* Extract the data from received packet descriptor */
1405 ch_num = curr_pd->ch_num;
1406 ep_num = curr_pd->ep_num;
1408 DBG(4, "Rx complete: dma channel(%d) ep%d len %d timeout %d\n",
1409 ch_num, ep_num, length, (50-timeout));
1411 rx_ch = &cppi->rx_cppi_ch[ch_num];
1412 rx_ch->channel.actual_len += length;
1414 if (curr_pd->eop) {
1415 curr_pd->eop = 0;
1416 /* disable the rx dma schedular */
1417 if (is_peripheral_active(cppi->musb))
1418 cppi41_schedtbl_remove_dma_ch(0, 0, ch_num, 0);
1419 }
1421 /*
1422 * Return Rx PD to the software list --
1423 * this is protected by critical section
1424 */
1425 usb_put_free_pd(cppi, curr_pd);
1427 orig_buf_len = curr_pd->hw_desc.orig_buf_len;
1428 if (en_bd_intr)
1429 orig_buf_len &= ~CPPI41_PKT_INTR_FLAG;
1431 if (unlikely(rx_ch->channel.actual_len >= rx_ch->length ||
1432 length < orig_buf_len)) {
1434 #ifdef CONFIG_SOC_OMAPTI81XX
1435 struct musb_hw_ep *ep;
1436 u8 isoc, next_seg = 0;
1438 /* Workaround for early rx completion of
1439 * cppi41 dma in Generic RNDIS mode for ti81xx
1440 */
1441 if (cpu_is_ti81xx() && is_host_enabled(cppi->musb)) {
1442 u32 pkt_size = rx_ch->pkt_size;
1443 ep = cppi->musb->endpoints + ep_num;
1444 isoc = musb_readb(ep->regs, MUSB_RXTYPE);
1445 isoc = (isoc >> 4) & 0x1;
1447 if (!isoc
1448 && (rx_ch->dma_mode == USB_GENERIC_RNDIS_MODE)
1449 && (rx_ch->channel.actual_len < rx_ch->length)
1450 && !(rx_ch->channel.actual_len % pkt_size))
1451 next_seg = 1;
1452 }
1453 if (next_seg) {
1454 rx_ch->curr_offset = rx_ch->channel.actual_len;
1455 cppi41_next_rx_segment(rx_ch);
1456 } else
1457 #endif
1458 {
1459 rx_ch->channel.status = MUSB_DMA_STATUS_FREE;
1461 /* Rx completion routine callback */
1462 musb_dma_completion(cppi->musb, ep_num, 0);
1463 }
1464 } else {
1465 if (is_peripheral_active(cppi->musb) &&
1466 ((rx_ch->length - rx_ch->curr_offset) > 0))
1467 cppi41_next_rx_segment(rx_ch);
1468 }
1469 }
1470 }
1472 /*
1473 * cppi41_completion - handle interrupts from the Tx/Rx completion queues
1474 *
1475 * NOTE: since we have to manually prod the Rx process in the transparent mode,
1476 * we certainly want to handle the Rx queues first.
1477 */
1478 void cppi41_completion(struct musb *musb, u32 rx, u32 tx)
1479 {
1480 struct cppi41 *cppi;
1481 unsigned index;
1483 cppi = container_of(musb->dma_controller, struct cppi41, controller);
1485 /* Process packet descriptors from the Rx queues */
1486 for (index = 0; rx != 0; rx >>= 1, index++)
1487 if (rx & 1)
1488 usb_process_rx_queue(cppi, index);
1490 /* Process packet descriptors from the Tx completion queues */
1491 for (index = 0; tx != 0; tx >>= 1, index++)
1492 if (tx & 1)
1493 usb_process_tx_queue(cppi, index);
1494 }
1495 EXPORT_SYMBOL(cppi41_completion);
1497 MODULE_DESCRIPTION("CPPI4.1 dma controller driver for musb");
1498 MODULE_LICENSE("GPL v2");
1500 static int __init cppi41_dma_init(void)
1501 {
1502 return 0;
1503 }
1504 module_init(cppi41_dma_init);
1506 static void __exit cppi41_dma__exit(void)
1507 {
1508 }
1509 module_exit(cppi41_dma__exit);