1 /*
2 * Copyright (c) 2011-2013, Texas Instruments Incorporated
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *
9 * * Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 *
12 * * Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * * Neither the name of Texas Instruments Incorporated nor the names of
17 * its contributors may be used to endorse or promote products derived
18 * from this software without specific prior written permission.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
22 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
24 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
25 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
26 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
27 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
28 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
29 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
30 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 */
32 /** ============================================================================
33 * @file VirtQueue.c
34 *
35 * @brief Virtio Queue implementation for BIOS
36 *
37 * Differences between BIOS version and Linux kernel (include/linux/virtio.h):
38 * - Renamed module from virtio.h to VirtQueue_Object.h to match the API prefixes;
39 * - BIOS (XDC) types and CamelCasing used;
40 * - virtio_device concept removed (i.e, assumes no containing device);
41 * - simplified scatterlist from Linux version;
42 * - VirtQueue_Objects are created statically here, so just added a VirtQueue_Object_init()
43 * fxn to take the place of the Virtio vring_new_virtqueue() API;
44 * - The notify function is implicit in the implementation, and not provided
45 * by the client, as it is in Linux virtio.
46 *
47 * All VirtQueue operations can be called in any context.
48 *
49 * The virtio header should be included in an application as follows:
50 * @code
51 * #include <ti/ipc/family/omap54xx/VirtQueue.h>
52 * @endcode
53 *
54 */
56 /* this define must precede inclusion of any xdc header file */
57 #define Registry_CURDESC ti_ipc_family_vayu__Desc
58 #define MODULE_NAME "ti.ipc.family.omap54xx.VirtQueue"
60 #include <xdc/std.h>
61 #include <xdc/runtime/System.h>
62 #include <xdc/runtime/Assert.h>
63 #include <xdc/runtime/Error.h>
64 #include <xdc/runtime/Memory.h>
65 #include <xdc/runtime/Registry.h>
66 #include <xdc/runtime/Log.h>
67 #include <xdc/runtime/Diags.h>
69 #include <ti/sysbios/hal/Hwi.h>
70 #include <ti/sysbios/knl/Clock.h>
71 #include <ti/sysbios/gates/GateHwi.h>
72 #include <ti/sysbios/BIOS.h>
73 #include <ti/sysbios/hal/Cache.h>
75 #include <ti/ipc/MultiProc.h>
77 #include <ti/ipc/rpmsg/virtio_ring.h>
78 #include <ti/pm/IpcPower.h>
79 #include <string.h>
81 #include <ti/ipc/rpmsg/_VirtQueue.h>
83 #include "InterruptProxy.h"
84 #include "VirtQueue.h"
87 /* Used for defining the size of the virtqueue registry */
88 #define NUM_QUEUES 4
90 /* Predefined device addresses */
91 #ifndef DSPC674
92 #define IPC_MEM_VRING0 0xA0000000
93 #define IPC_MEM_VRING1 0xA0004000
94 #else
95 #define IPC_MEM_VRING0 0x9FB00000
96 #define IPC_MEM_VRING1 0x9FB04000
97 #endif
98 #define IPC_MEM_VRING2 0xA0008000
99 #define IPC_MEM_VRING3 0xA000c000
101 /*
102 * Sizes of the virtqueues (expressed in number of buffers supported,
103 * and must be power of two)
104 */
105 #define VQ0_SIZE 256
106 #define VQ1_SIZE 256
107 #define VQ2_SIZE 256
108 #define VQ3_SIZE 256
110 /*
111 * enum - Predefined Mailbox Messages
112 *
113 * @RP_MSG_MBOX_READY: informs the M3's that we're up and running. will be
114 * followed by another mailbox message that carries the A9's virtual address
115 * of the shared buffer. This would allow the A9's drivers to send virtual
116 * addresses of the buffers.
117 *
118 * @RP_MSG_MBOX_STATE_CHANGE: informs the receiver that there is an inbound
119 * message waiting in its own receive-side vring. please note that currently
120 * this message is optional: alternatively, one can explicitly send the index
121 * of the triggered virtqueue itself. the preferred approach will be decided
122 * as we progress and experiment with those design ideas.
123 *
124 * @RP_MSG_MBOX_CRASH: this message indicates that the BIOS side is unhappy
125 *
126 * @RP_MBOX_ECHO_REQUEST: this message requests the remote processor to reply
127 * with RP_MBOX_ECHO_REPLY
128 *
129 * @RP_MBOX_ECHO_REPLY: this is a reply that is sent when RP_MBOX_ECHO_REQUEST
130 * is received.
131 *
132 * @RP_MBOX_ABORT_REQUEST: tells the M3 to crash on demand
133 *
134 * @RP_MBOX_BOOTINIT_DONE: this message indicates the BIOS side has reached a
135 * certain state during the boot process. This message is used to inform the
136 * host that the basic BIOS initialization is done, and lets the host use this
137 * notification to perform certain actions.
138 */
139 enum {
140 RP_MSG_MBOX_READY = (Int)0xFFFFFF00,
141 RP_MSG_MBOX_STATE_CHANGE = (Int)0xFFFFFF01,
142 RP_MSG_MBOX_CRASH = (Int)0xFFFFFF02,
143 RP_MBOX_ECHO_REQUEST = (Int)0xFFFFFF03,
144 RP_MBOX_ECHO_REPLY = (Int)0xFFFFFF04,
145 RP_MBOX_ABORT_REQUEST = (Int)0xFFFFFF05,
146 RP_MSG_FLUSH_CACHE = (Int)0xFFFFFF06,
147 RP_MSG_BOOTINIT_DONE = (Int)0xFFFFFF07,
148 RP_MSG_HIBERNATION = (Int)0xFFFFFF10,
149 RP_MSG_HIBERNATION_FORCE = (Int)0xFFFFFF11,
150 RP_MSG_HIBERNATION_ACK = (Int)0xFFFFFF12,
151 RP_MSG_HIBERNATION_CANCEL = (Int)0xFFFFFF13
152 };
154 #define DIV_ROUND_UP(n,d) (((n) + (d) - 1) / (d))
155 #define RP_MSG_NUM_BUFS (VQ0_SIZE) /* must be power of two */
156 #define RP_MSG_BUF_SIZE (512)
157 #define RP_MSG_BUFS_SPACE (RP_MSG_NUM_BUFS * RP_MSG_BUF_SIZE * 2)
159 #define PAGE_SIZE (4096)
160 /*
161 * The alignment to use between consumer and producer parts of vring.
162 * Note: this is part of the "wire" protocol. If you change this, you need
163 * to update your BIOS image as well
164 */
165 #define RP_MSG_VRING_ALIGN (4096)
167 /* With 256 buffers, our vring will occupy 3 pages */
168 #define RP_MSG_RING_SIZE ((DIV_ROUND_UP(vring_size(RP_MSG_NUM_BUFS, \
169 RP_MSG_VRING_ALIGN), PAGE_SIZE)) * PAGE_SIZE)
171 /* The total IPC space needed to communicate with a remote processor */
172 #define RPMSG_IPC_MEM (RP_MSG_BUFS_SPACE + 2 * RP_MSG_RING_SIZE)
174 #define ID_SYSM3_TO_A9 ID_SELF_TO_A9
175 #define ID_A9_TO_SYSM3 ID_A9_TO_SELF
176 #define ID_DSP_TO_A9 ID_SELF_TO_A9
177 #define ID_A9_TO_DSP ID_A9_TO_SELF
178 #define ID_APPM3_TO_A9 2
179 #define ID_A9_TO_APPM3 3
181 typedef struct VirtQueue_Object {
182 /* Id for this VirtQueue_Object */
183 UInt16 id;
185 /* The function to call when buffers are consumed (can be NULL) */
186 VirtQueue_callback callback;
188 /* Shared state */
189 struct vring vring;
191 /* Number of free buffers */
192 UInt16 num_free;
194 /* Last available index; updated by VirtQueue_getAvailBuf */
195 UInt16 last_avail_idx;
197 /* Last available index; updated by VirtQueue_addUsedBuf */
198 UInt16 last_used_idx;
200 /* Will eventually be used to kick remote processor */
201 UInt16 procId;
203 /* Gate to protect from multiple threads */
204 GateHwi_Handle gateH;
205 } VirtQueue_Object;
207 /* module diags mask */
208 Registry_Desc Registry_CURDESC;
210 static struct VirtQueue_Object *queueRegistry[NUM_QUEUES] = {NULL};
212 static UInt16 hostProcId;
213 #ifndef SMP
214 static UInt16 dspProcId;
215 static UInt16 sysm3ProcId;
216 static UInt16 appm3ProcId;
217 #endif
219 #if defined(M3_ONLY) && !defined(SMP)
220 extern Void OffloadM3_init();
221 extern Int OffloadM3_processSysM3Tasks(UArg msg);
222 #endif
224 /*!
225 * ======== _VirtQueue_init ========
226 *
227 * This function adds the VirtQueue "module" to the Registry so that
228 * DIAGS will work with this non-XDC module.
229 * Since VirtQueue_init is not called by XDC-VirtQueue module clients, this
230 * function is called in the first VirtQueue fxn called: VirtQueue_create.
231 */
232 static Void _VirtQueue_init()
233 {
234 static int initialized = 0;
236 if (!initialized) {
237 Registry_Result result;
239 /* register with xdc.runtime to get a diags mask */
240 result = Registry_addModule(&Registry_CURDESC, MODULE_NAME);
241 Assert_isTrue(result == Registry_SUCCESS, (Assert_Id)NULL);
243 initialized = 1;
244 }
245 }
247 static inline Void * mapPAtoVA(UInt pa)
248 {
249 return (Void *)((pa & 0x000fffffU) | IPC_MEM_VRING0);
250 }
252 static inline UInt mapVAtoPA(Void * va)
253 {
254 return ((UInt)va & 0x000fffffU) | 0x9cf00000U;
255 }
257 /*!
258 * ======== VirtQueue_kick ========
259 */
260 Void VirtQueue_kick(VirtQueue_Handle vq)
261 {
262 /* For now, simply interrupt remote processor */
263 if (vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT) {
264 Log_print0(Diags_USER1,
265 "VirtQueue_kick: no kick because of VRING_AVAIL_F_NO_INTERRUPT\n");
266 return;
267 }
269 Log_print2(Diags_USER1,
270 "VirtQueue_kick: Sending interrupt to proc %d with payload 0x%x\n",
271 (IArg)vq->procId, (IArg)vq->id);
272 InterruptProxy_intSend(vq->procId, vq->id);
273 }
275 /*!
276 * ======== VirtQueue_addUsedBuf ========
277 */
278 Int VirtQueue_addUsedBuf(VirtQueue_Handle vq, Int16 head, Int len)
279 {
280 struct vring_used_elem *used;
281 IArg key;
283 key = GateHwi_enter(vq->gateH);
284 if ((head > vq->vring.num) || (head < 0)) {
285 GateHwi_leave(vq->gateH, key);
286 Error_raise(NULL, Error_E_generic, 0, 0);
287 }
289 /*
290 * The virtqueue contains a ring of used buffers. Get a pointer to the
291 * next entry in that used ring.
292 */
293 used = &vq->vring.used->ring[vq->vring.used->idx % vq->vring.num];
294 used->id = head;
295 used->len = len;
297 vq->vring.used->idx++;
298 GateHwi_leave(vq->gateH, key);
300 return (0);
301 }
303 /*!
304 * ======== VirtQueue_addAvailBuf ========
305 */
306 Int VirtQueue_addAvailBuf(VirtQueue_Object *vq, Void *buf)
307 {
308 UInt16 avail;
309 IArg key;
311 if (vq->num_free == 0) {
312 /* There's no more space */
313 Error_raise(NULL, Error_E_generic, 0, 0);
314 }
316 vq->num_free--;
318 key = GateHwi_enter(vq->gateH);
319 avail = vq->vring.avail->idx++ % vq->vring.num;
321 vq->vring.desc[avail].addr = mapVAtoPA(buf);
322 vq->vring.desc[avail].len = RP_MSG_BUF_SIZE;
323 GateHwi_leave(vq->gateH, key);
325 return (vq->num_free);
326 }
328 /*!
329 * ======== VirtQueue_getUsedBuf ========
330 */
331 Void *VirtQueue_getUsedBuf(VirtQueue_Object *vq)
332 {
333 UInt16 head;
334 Void *buf;
335 IArg key;
337 key = GateHwi_enter(vq->gateH);
338 /* There's nothing available? */
339 if (vq->last_used_idx == vq->vring.used->idx) {
340 buf = NULL;
341 }
342 else {
343 head = vq->vring.used->ring[vq->last_used_idx % vq->vring.num].id;
344 vq->last_used_idx++;
346 buf = mapPAtoVA(vq->vring.desc[head].addr);
347 }
348 GateHwi_leave(vq->gateH, key);
350 return (buf);
351 }
353 /*!
354 * ======== VirtQueue_getAvailBuf ========
355 */
356 Int16 VirtQueue_getAvailBuf(VirtQueue_Handle vq, Void **buf, Int *len)
357 {
358 Int16 head;
359 IArg key;
361 key = GateHwi_enter(vq->gateH);
362 Log_print6(Diags_USER1, "getAvailBuf vq: 0x%x %d %d %d 0x%x 0x%x\n",
363 (IArg)vq, vq->last_avail_idx, vq->vring.avail->idx, vq->vring.num,
364 (IArg)&vq->vring.avail, (IArg)vq->vring.avail);
366 /* There's nothing available? */
367 if (vq->last_avail_idx == vq->vring.avail->idx) {
368 /* We need to know about added buffers */
369 vq->vring.used->flags &= ~VRING_USED_F_NO_NOTIFY;
370 head = (-1);
371 }
372 else {
373 /*
374 * Grab the next descriptor number they're advertising, and increment
375 * the index we've seen.
376 */
377 head = vq->vring.avail->ring[vq->last_avail_idx++ % vq->vring.num];
379 *buf = mapPAtoVA(vq->vring.desc[head].addr);
380 *len = vq->vring.desc[head].len;
381 }
382 GateHwi_leave(vq->gateH, key);
384 return (head);
385 }
387 /*!
388 * ======== VirtQueue_disableCallback ========
389 */
390 Void VirtQueue_disableCallback(VirtQueue_Object *vq)
391 {
392 /* TODO */
393 Log_print0(Diags_USER1, "VirtQueue_disableCallback called.");
394 }
396 /*!
397 * ======== VirtQueue_enableCallback ========
398 */
399 Bool VirtQueue_enableCallback(VirtQueue_Object *vq)
400 {
401 Log_print0(Diags_USER1, "VirtQueue_enableCallback called.");
403 /* TODO */
404 return (FALSE);
405 }
407 /*!
408 * ======== VirtQueue_isr ========
409 * Note 'arg' is ignored: it is the Hwi argument, not the mailbox argument.
410 */
411 Void VirtQueue_isr(UArg msg)
412 {
413 VirtQueue_Object *vq;
415 Log_print1(Diags_USER1, "VirtQueue_isr received msg = 0x%x\n", msg);
417 #ifndef SMP
418 if (MultiProc_self() == sysm3ProcId || MultiProc_self() == dspProcId) {
419 #endif
420 switch(msg) {
421 case (UInt)RP_MSG_MBOX_READY:
422 return;
424 case (UInt)RP_MBOX_ECHO_REQUEST:
425 InterruptProxy_intSend(hostProcId, (UInt)(RP_MBOX_ECHO_REPLY));
426 return;
428 case (UInt)RP_MBOX_ABORT_REQUEST:
429 {
430 /* Suppress Coverity Error: FORWARD_NULL: */
431 /* coverity[assign_zero] */
432 Fxn f = (Fxn)0x0;
433 Log_print0(Diags_USER1, "Crash on demand ...\n");
434 /* coverity[var_deref_op] */
435 f();
436 }
437 return;
439 case (UInt)RP_MSG_FLUSH_CACHE:
440 Cache_wbAll();
441 return;
443 #ifndef DSPC674
444 case (UInt)RP_MSG_HIBERNATION:
445 if (IpcPower_canHibernate() == FALSE) {
446 InterruptProxy_intSend(hostProcId,
447 (UInt)RP_MSG_HIBERNATION_CANCEL);
448 return;
449 }
451 /* Fall through */
452 case (UInt)RP_MSG_HIBERNATION_FORCE:
453 #ifndef SMP
454 /* Core0 should notify Core1 */
455 if (MultiProc_self() == sysm3ProcId) {
456 InterruptProxy_intSend(appm3ProcId,
457 (UInt)(RP_MSG_HIBERNATION));
458 }
459 #endif
460 /* Ack request */
461 InterruptProxy_intSend(hostProcId,
462 (UInt)RP_MSG_HIBERNATION_ACK);
463 IpcPower_suspend();
464 return;
465 #endif
466 default:
467 #if defined(M3_ONLY) && !defined(SMP)
468 /* Check and process any Inter-M3 Offload messages */
469 if (OffloadM3_processSysM3Tasks(msg))
470 return;
471 #endif
473 /*
474 * If the message isn't one of the above, it's either part of the
475 * 2-message synchronization sequence or it a virtqueue message
476 */
477 break;
478 }
479 #ifndef SMP
480 }
481 #ifndef DSPC674
482 else if (msg & 0xFFFF0000) {
483 if (msg == (UInt)RP_MSG_HIBERNATION) {
484 IpcPower_suspend();
485 }
486 return;
487 }
489 if (MultiProc_self() == sysm3ProcId && (msg == ID_A9_TO_APPM3 || msg == ID_APPM3_TO_A9)) {
490 InterruptProxy_intSend(appm3ProcId, (UInt)msg);
491 }
492 else {
493 #endif
494 #endif
495 /* Don't let unknown messages to pass as a virtqueue index */
496 if (msg >= NUM_QUEUES) {
497 /* Adding print here deliberately, we should never see this */
498 System_printf("VirtQueue_isr: Invalid mailbox message 0x%x "
499 "received\n", msg);
500 return;
501 }
503 vq = queueRegistry[msg];
504 if (vq) {
505 vq->callback(vq);
506 }
507 #ifndef SMP
508 #ifndef DSPC674
509 }
510 #endif
511 #endif
512 }
515 /*!
516 * ======== VirtQueue_create ========
517 */
518 VirtQueue_Handle VirtQueue_create(UInt16 remoteProcId, VirtQueue_Params *params,
519 Error_Block *eb)
520 {
521 VirtQueue_Object *vq;
522 Void *vringAddr;
524 /* Perform initialization we can't do in Instance_init (being non-XDC): */
525 _VirtQueue_init();
527 vq = Memory_alloc(NULL, sizeof(VirtQueue_Object), 0, eb);
528 if (NULL == vq) {
529 return (NULL);
530 }
532 /* Create the thread protection gate */
533 vq->gateH = GateHwi_create(NULL, eb);
534 if (Error_check(eb)) {
535 Log_error0("VirtQueue_create: could not create gate object");
536 Memory_free(NULL, vq, sizeof(VirtQueue_Object));
537 return (NULL);
538 }
540 vq->callback = params->callback;
541 vq->id = params->vqId;
542 vq->procId = remoteProcId;
543 vq->last_avail_idx = 0;
545 #ifndef SMP
546 if (MultiProc_self() == appm3ProcId) {
547 /* vqindices that belong to AppM3 should be big so they don't
548 * collide with SysM3's virtqueues */
549 vq->id += 2;
550 }
551 #endif
553 switch (vq->id) {
554 /* IPC transport vrings */
555 case ID_SELF_TO_A9:
556 /* IPU/DSP -> A9 */
557 vringAddr = (struct vring *) IPC_MEM_VRING0;
558 break;
559 case ID_A9_TO_SELF:
560 /* A9 -> IPU/DSP */
561 vringAddr = (struct vring *) IPC_MEM_VRING1;
562 break;
563 #ifndef SMP
564 case ID_APPM3_TO_A9:
565 /* APPM3 -> A9 */
566 vringAddr = (struct vring *) IPC_MEM_VRING2;
567 break;
568 case ID_A9_TO_APPM3:
569 /* A9 -> APPM3 */
570 vringAddr = (struct vring *) IPC_MEM_VRING3;
571 break;
572 #endif
573 default:
574 GateHwi_delete(&vq->gateH);
575 Memory_free(NULL, vq, sizeof(VirtQueue_Object));
576 return (NULL);
577 }
579 Log_print3(Diags_USER1,
580 "vring: %d 0x%x (0x%x)\n", vq->id, (IArg)vringAddr,
581 RP_MSG_RING_SIZE);
583 /* See coverity related comment in vring_init() */
584 /* coverity[overrun-call] */
585 vring_init(&(vq->vring), RP_MSG_NUM_BUFS, vringAddr, RP_MSG_VRING_ALIGN);
587 /*
588 * Don't trigger a mailbox message every time MPU makes another buffer
589 * available
590 */
591 if (vq->procId == hostProcId) {
592 vq->vring.used->flags &= ~VRING_USED_F_NO_NOTIFY;
593 }
595 queueRegistry[vq->id] = vq;
597 return (vq);
598 }
600 /*!
601 * ======== VirtQueue_startup ========
602 */
603 Void VirtQueue_startup()
604 {
605 hostProcId = MultiProc_getId("HOST");
606 #ifndef SMP
607 dspProcId = MultiProc_getId("DSP");
608 sysm3ProcId = MultiProc_getId("CORE0");
609 appm3ProcId = MultiProc_getId("CORE1");
610 #endif
612 #ifndef DSPC674
613 /* Initilize the IpcPower module */
614 IpcPower_init();
615 #endif
617 #if defined(M3_ONLY) && !defined(SMP)
618 if (MultiProc_self() == sysm3ProcId) {
619 OffloadM3_init();
620 }
621 #endif
623 InterruptProxy_intRegister(VirtQueue_isr);
624 }
626 /*!
627 * ======== VirtQueue_postCrashToMailbox ========
628 */
629 Void VirtQueue_postCrashToMailbox(Void)
630 {
631 InterruptProxy_intSend(0, (UInt)RP_MSG_MBOX_CRASH);
632 }
634 #define CACHE_WB_TICK_PERIOD 5
636 /*!
637 * ======== VirtQueue_cacheWb ========
638 *
639 * Used for flushing SysMin trace buffer.
640 */
641 Void VirtQueue_cacheWb()
642 {
643 static UInt32 oldticks = 0;
644 UInt32 newticks;
646 newticks = Clock_getTicks();
647 if (newticks - oldticks < (UInt32)CACHE_WB_TICK_PERIOD) {
648 /* Don't keep flushing cache */
649 return;
650 }
652 oldticks = newticks;
654 /* Flush the cache */
655 Cache_wbAll();
656 }