1 /*
2 * Copyright (c) 2011-2019 Texas Instruments Incorporated - http://www.ti.com
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *
9 * * Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 *
12 * * Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * * Neither the name of Texas Instruments Incorporated nor the names of
17 * its contributors may be used to endorse or promote products derived
18 * from this software without specific prior written permission.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
22 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
24 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
25 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
26 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
27 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
28 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
29 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
30 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 */
33 /** ============================================================================
34 * @file VirtQueue.c
35 *
36 * @brief Virtio Queue implementation for BIOS
37 *
38 * Differences between BIOS version and Linux kernel (include/linux/virtio.h):
39 * - Renamed module from virtio.h to VirtQueue_Object.h to match the API
40 * prefixes;
41 * - BIOS (XDC) types and CamelCasing used;
42 * - virtio_device concept removed (i.e, assumes no containing device);
43 * - simplified scatterlist from Linux version;
44 * - VirtQueue_Objects are created statically here, so just added a
45 * VirtQueue_Object_init()
46 * fxn to take the place of the Virtio vring_new_virtqueue() API;
47 * - The notify function is implicit in the implementation, and not provided
48 * by the client, as it is in Linux virtio.
49 *
50 * All VirtQueue operations can be called in any context.
51 *
52 * The virtio header should be included in an application as follows:
53 * @code
54 * #include <ti/ipc/family/vayu/VirtQueue.h>
55 * @endcode
56 *
57 */
59 /* this define must precede inclusion of any xdc header file */
60 #define Registry_CURDESC ti_ipc_family_vayu__Desc
61 #define MODULE_NAME "ti.ipc.family.vayu.VirtQueue"
63 #include <string.h>
65 #include <xdc/std.h>
66 #include <xdc/runtime/System.h>
67 #include <xdc/runtime/Assert.h>
68 #include <xdc/runtime/Error.h>
69 #include <xdc/runtime/Memory.h>
70 #include <xdc/runtime/Registry.h>
71 #include <xdc/runtime/Log.h>
72 #include <xdc/runtime/Diags.h>
74 #include <ti/sysbios/hal/Hwi.h>
75 #include <ti/sysbios/knl/Clock.h>
76 #include <ti/sysbios/gates/GateHwi.h>
77 #include <ti/sysbios/hal/Cache.h>
79 #include <ti/ipc/MultiProc.h>
80 #include <ti/ipc/remoteproc/Resource.h>
81 #include <ti/ipc/remoteproc/rsc_types.h>
82 #include <ti/ipc/rpmsg/virtio_ring.h>
83 #include <ti/ipc/rpmsg/_VirtQueue.h>
84 #include <ti/pm/IpcPower.h>
85 #include <ti/sdo/ipc/notifyDrivers/IInterrupt.h>
87 #include "InterruptProxy.h"
88 #include "VirtQueue.h"
90 /*
91 * The following three VIRTIO_* defines must match those in
92 * <Linux_kernel>/include/uapi/linux/virtio_config.h
93 */
94 #define VIRTIO_CONFIG_S_ACKNOWLEDGE 1
95 #define VIRTIO_CONFIG_S_DRIVER 2
96 #define VIRTIO_CONFIG_S_DRIVER_OK 4
98 #define VRING_BUFS_PRIMED (VIRTIO_CONFIG_S_ACKNOWLEDGE | \
99 VIRTIO_CONFIG_S_DRIVER | VIRTIO_CONFIG_S_DRIVER_OK)
101 /* Used for defining the size of the virtqueue registry */
102 #define NUM_QUEUES 2
104 /*
105 * Size of the virtqueues (expressed in number of buffers supported,
106 * and must be power of two)
107 */
108 #define VQ_SIZE 256
110 /*
111 * enum - Predefined Mailbox Messages
112 *
113 * @RP_MSG_MBOX_READY: informs the slave that we're up and running. will be
114 * followed by another mailbox message that carries the HOST's virtual address
115 * of the shared buffer. This would allow the HOST's drivers to send virtual
116 * addresses of the buffers.
117 *
118 * @RP_MSG_MBOX_STATE_CHANGE: informs the receiver that there is an inbound
119 * message waiting in its own receive-side vring. please note that currently
120 * this message is optional: alternatively, one can explicitly send the index
121 * of the triggered virtqueue itself. the preferred approach will be decided
122 * as we progress and experiment with those design ideas.
123 *
124 * @RP_MSG_MBOX_CRASH: this message indicates that the BIOS side is unhappy
125 *
126 * @RP_MBOX_ECHO_REQUEST: this message requests the remote processor to reply
127 * with RP_MBOX_ECHO_REPLY
128 *
129 * @RP_MBOX_ECHO_REPLY: this is a reply that is sent when RP_MBOX_ECHO_REQUEST
130 * is received.
131 *
132 * @RP_MBOX_ABORT_REQUEST: tells the M3 to crash on demand
133 *
134 * @RP_MBOX_BOOTINIT_DONE: this message indicates the BIOS side has reached a
135 * certain state during the boot process. This message is used to inform the
136 * host that the basic BIOS initialization is done, and lets the host use this
137 * notification to perform certain actions.
138 */
139 enum {
140 RP_MSG_MBOX_READY = (Int)0xFFFFFF00,
141 RP_MSG_MBOX_STATE_CHANGE = (Int)0xFFFFFF01,
142 RP_MSG_MBOX_CRASH = (Int)0xFFFFFF02,
143 RP_MBOX_ECHO_REQUEST = (Int)0xFFFFFF03,
144 RP_MBOX_ECHO_REPLY = (Int)0xFFFFFF04,
145 RP_MBOX_ABORT_REQUEST = (Int)0xFFFFFF05,
146 RP_MSG_FLUSH_CACHE = (Int)0xFFFFFF06,
147 RP_MSG_BOOTINIT_DONE = (Int)0xFFFFFF07,
148 RP_MSG_HIBERNATION = (Int)0xFFFFFF10,
149 RP_MSG_HIBERNATION_FORCE = (Int)0xFFFFFF11,
150 RP_MSG_HIBERNATION_ACK = (Int)0xFFFFFF12,
151 RP_MSG_HIBERNATION_CANCEL = (Int)0xFFFFFF13
152 };
154 #define DIV_ROUND_UP(n,d) (((n) + (d) - 1) / (d))
155 #define RP_MSG_NUM_BUFS (VQ_SIZE) /* must be power of two */
156 #define RP_MSG_BUF_SIZE (512)
157 #define RP_MSG_BUFS_SPACE (RP_MSG_NUM_BUFS * RP_MSG_BUF_SIZE * 2)
159 #define PAGE_SIZE (4096)
160 /*
161 * The alignment to use between consumer and producer parts of vring.
162 * Note: this is part of the "wire" protocol. If you change this, you need
163 * to update your BIOS image as well
164 */
165 #define RP_MSG_VRING_ALIGN (4096)
167 /* With 256 buffers, our vring will occupy 3 pages */
168 #define RP_MSG_RING_SIZE ((DIV_ROUND_UP(vring_size(RP_MSG_NUM_BUFS, \
169 RP_MSG_VRING_ALIGN), PAGE_SIZE)) * PAGE_SIZE)
171 /* The total IPC space needed to communicate with a remote processor */
172 #define RPMSG_IPC_MEM (RP_MSG_BUFS_SPACE + 2 * RP_MSG_RING_SIZE)
174 typedef struct VirtQueue_Object {
175 /* Id for this VirtQueue_Object */
176 UInt16 id;
178 /* The function to call when buffers are consumed (can be NULL) */
179 VirtQueue_callback callback;
181 /* Shared state */
182 struct vring vring;
184 /* Number of free buffers */
185 UInt16 num_free;
187 /* Last available index; updated by VirtQueue_getAvailBuf */
188 UInt16 last_avail_idx;
190 /* Will eventually be used to kick remote processor */
191 UInt16 procId;
193 /* Gate to protect from multiple threads */
194 GateHwi_Handle gateH;
196 /* Base phys addr - used for quick pa/va translations */
197 UInt32 basePa;
199 /* Base virt addr - used for quick pa/va translations */
200 UInt32 baseVa;
201 } VirtQueue_Object;
203 /* module diags mask */
204 Registry_Desc Registry_CURDESC;
206 static struct VirtQueue_Object *queueRegistry[NUM_QUEUES] = {NULL};
208 static UInt16 hostProcId;
210 #define DSPEVENTID 5
211 IInterrupt_IntInfo intInfo;
213 /*!
214 * ======== _VirtQueue_init ========
215 *
216 * This function adds the VirtQueue "module" to the Registry so that
217 * DIAGS will work with this non-XDC module.
218 * Since VirtQueue_init is not called by XDC-VirtQueue module clients, this
219 * function is called in the first VirtQueue fxn called: VirtQueue_create.
220 */
221 static Void _VirtQueue_init()
222 {
223 static int initialized = 0;
225 if (!initialized) {
226 Registry_Result result;
228 /* register with xdc.runtime to get a diags mask */
229 result = Registry_addModule(&Registry_CURDESC, MODULE_NAME);
230 Assert_isTrue(result == Registry_SUCCESS, (Assert_Id)NULL);
231 /* Double check , In case Assert is disabled */
232 if (result != Registry_SUCCESS) {
233 return;
234 }
236 while (Resource_getVdevStatus(VIRTIO_ID_RPMSG) != VRING_BUFS_PRIMED);
238 initialized = 1;
239 }
240 }
242 static inline Void * _VirtQueue_getVA(VirtQueue_Handle vq, UInt32 pa)
243 {
244 return (Void *)(pa - vq->basePa + vq->baseVa);
245 }
247 /*!
248 * ======== VirtQueue_kick ========
249 */
250 Void VirtQueue_kick(VirtQueue_Handle vq)
251 {
252 /* For now, simply interrupt remote processor */
253 if (vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT) {
254 Log_print0(Diags_USER1,
255 "VirtQueue_kick: no kick because of VRING_AVAIL_F_NO_INTERRUPT\n");
256 return;
257 }
259 Log_print2(Diags_USER1,
260 "VirtQueue_kick: Sending interrupt to proc %d with payload 0x%x\n",
261 (IArg)vq->procId, (IArg)vq->id);
262 InterruptProxy_intSend(vq->procId, NULL, vq->id);
263 }
265 /*!
266 * ======== VirtQueue_addUsedBuf ========
267 */
268 Int VirtQueue_addUsedBuf(VirtQueue_Handle vq, Int16 head, Int len)
269 {
270 struct vring_used_elem *used;
271 IArg key;
273 key = GateHwi_enter(vq->gateH);
274 if (((unsigned int)head > vq->vring.num) || (head < 0)) {
275 GateHwi_leave(vq->gateH, key);
276 Error_raise(NULL, Error_E_generic, 0, 0);
277 }
279 /*
280 * The virtqueue contains a ring of used buffers. Get a pointer to the
281 * next entry in that used ring.
282 */
283 used = &vq->vring.used->ring[vq->vring.used->idx % vq->vring.num];
284 used->id = head;
285 used->len = len;
287 vq->vring.used->idx++;
288 GateHwi_leave(vq->gateH, key);
290 return (0);
291 }
293 /*!
294 * ======== VirtQueue_getAvailBuf ========
295 */
296 Int16 VirtQueue_getAvailBuf(VirtQueue_Handle vq, Void **buf, Int *len)
297 {
298 Int16 head;
299 IArg key;
301 key = GateHwi_enter(vq->gateH);
302 Log_print6(Diags_USER1, "getAvailBuf vq: 0x%x %d %d %d 0x%x 0x%x\n",
303 (IArg)vq, vq->last_avail_idx, vq->vring.avail->idx, vq->vring.num,
304 (IArg)&vq->vring.avail, (IArg)vq->vring.avail);
306 /* Clear flag here to avoid race condition with remote processor.
307 * This is a negative flag, clearing it means that we want to
308 * receive an interrupt when a buffer has been added to the pool.
309 */
310 vq->vring.used->flags &= ~VRING_USED_F_NO_NOTIFY;
312 /* There's nothing available? */
313 if (vq->last_avail_idx == vq->vring.avail->idx) {
314 head = (-1);
315 }
316 else {
317 /* No need to be kicked about added buffers anymore */
318 vq->vring.used->flags |= VRING_USED_F_NO_NOTIFY;
320 /*
321 * Grab the next descriptor number they're advertising, and increment
322 * the index we've seen.
323 */
324 head = vq->vring.avail->ring[vq->last_avail_idx++ % vq->vring.num];
326 *buf = _VirtQueue_getVA(vq, vq->vring.desc[head].addr);
327 *len = vq->vring.desc[head].len;
328 }
329 GateHwi_leave(vq->gateH, key);
331 return (head);
332 }
334 /*!
335 * ======== VirtQueue_disableCallback ========
336 */
337 Void VirtQueue_disableCallback(VirtQueue_Object *vq)
338 {
339 /* TODO */
340 Log_print0(Diags_USER1, "VirtQueue_disableCallback called.");
341 }
343 /*!
344 * ======== VirtQueue_enableCallback ========
345 */
346 Bool VirtQueue_enableCallback(VirtQueue_Object *vq)
347 {
348 Log_print0(Diags_USER1, "VirtQueue_enableCallback called.");
350 /* TODO */
351 return (FALSE);
352 }
354 /*!
355 * ======== VirtQueue_isr ========
356 * Note 'arg' is ignored: it is the Hwi argument, not the mailbox argument.
357 */
358 Void VirtQueue_isr(UArg msg)
359 {
360 VirtQueue_Object *vq;
362 msg = InterruptProxy_intClear(hostProcId, &intInfo);
364 Log_print1(Diags_USER1, "VirtQueue_isr received msg = 0x%x\n", msg);
366 switch(msg) {
367 case (UInt)RP_MSG_MBOX_READY:
368 return;
370 case (UInt)RP_MBOX_ECHO_REQUEST:
371 InterruptProxy_intSend(hostProcId, NULL, (UInt)(RP_MBOX_ECHO_REPLY));
372 return;
374 case (UInt)RP_MBOX_ABORT_REQUEST:
375 {
376 /* Suppress Coverity Error: FORWARD_NULL: */
377 /* coverity[assign_zero] */
378 Fxn f = (Fxn)0x0;
379 Log_print0(Diags_USER1, "Crash on demand ...\n");
380 /* coverity[var_deref_op] */
381 f();
382 }
383 return;
385 case (UInt)RP_MSG_FLUSH_CACHE:
386 Cache_wbAll();
387 return;
389 case (UInt)RP_MSG_HIBERNATION:
390 if (IpcPower_canHibernate() == FALSE) {
391 InterruptProxy_intSend(hostProcId, NULL,
392 (UInt)RP_MSG_HIBERNATION_CANCEL);
393 return;
394 }
396 /* Fall through */
397 case (UInt)RP_MSG_HIBERNATION_FORCE:
398 /* Ack request */
399 InterruptProxy_intSend(hostProcId, NULL,
400 (UInt)RP_MSG_HIBERNATION_ACK);
401 IpcPower_suspend();
402 return;
404 default:
405 /*
406 * If the message isn't one of the above, it's either part of the
407 * 2-message synchronization sequence or it a virtqueue message
408 */
409 break;
410 }
412 /* Don't let unknown messages to pass as a virtqueue index */
413 if (msg >= NUM_QUEUES) {
414 /* Adding print here deliberately, we should never see this */
415 System_printf("VirtQueue_isr: Invalid mailbox message 0x%x "
416 "received\n", msg);
417 return;
418 }
420 vq = queueRegistry[msg];
421 if (vq) {
422 vq->callback(vq);
423 }
424 }
427 /*!
428 * ======== VirtQueue_create ========
429 */
430 VirtQueue_Handle VirtQueue_create(UInt16 remoteProcId, VirtQueue_Params *params,
431 Error_Block *eb)
432 {
433 VirtQueue_Object *vq;
434 Void *vringAddr;
435 Int result;
437 /* Perform initialization we can't do in Instance_init (being non-XDC): */
438 _VirtQueue_init();
440 vq = Memory_alloc(NULL, sizeof(VirtQueue_Object), 0, eb);
441 if (NULL == vq) {
442 return (NULL);
443 }
445 /* Create the thread protection gate */
446 vq->gateH = GateHwi_create(NULL, eb);
447 if (Error_check(eb)) {
448 Log_error0("VirtQueue_create: could not create gate object");
449 Memory_free(NULL, vq, sizeof(VirtQueue_Object));
450 return (NULL);
451 }
453 vq->callback = params->callback;
454 vq->id = params->vqId;
455 vq->procId = remoteProcId;
456 vq->last_avail_idx = 0;
458 switch (vq->id) {
459 /* IPC transport vrings */
460 case ID_SELF_TO_HOST:
461 case ID_HOST_TO_SELF:
462 vq->basePa = (UInt32)Resource_getVringDA(vq->id);
463 Assert_isTrue(vq->basePa != 0, NULL);
465 result = Resource_physToVirt(vq->basePa, &(vq->baseVa));
466 Assert_isTrue(result == Resource_S_SUCCESS, (Assert_Id)NULL);
467 if (result != Resource_S_SUCCESS) {
468 return NULL;
469 }
471 vringAddr = (Void *)vq->baseVa;
472 break;
473 default:
474 GateHwi_delete(&vq->gateH);
475 Memory_free(NULL, vq, sizeof(VirtQueue_Object));
476 return (NULL);
477 }
479 Log_print3(Diags_USER1,
480 "vring: %d 0x%x (0x%x)\n", vq->id, (IArg)vringAddr,
481 RP_MSG_RING_SIZE);
483 /* See coverity related comment in vring_init() */
484 /* coverity[overrun-call] */
485 vring_init(&(vq->vring), RP_MSG_NUM_BUFS, vringAddr, RP_MSG_VRING_ALIGN);
487 /*
488 * Don't trigger a mailbox message every time MPU makes another buffer
489 * available
490 */
491 if (vq->procId == hostProcId) {
492 vq->vring.used->flags &= ~VRING_USED_F_NO_NOTIFY;
493 }
495 queueRegistry[vq->id] = vq;
497 return (vq);
498 }
500 /*!
501 * ======== VirtQueue_startup ========
502 */
503 Void VirtQueue_startup()
504 {
505 hostProcId = MultiProc_getId("HOST");
507 /* Note that "64P" matches 64P, 674, 66 and others. We prefer 66 on vayu,
508 * but technically vayu DSPs support any of these.
509 */
510 #if defined(xdc_target__isaCompatible_64P)
511 intInfo.intVectorId = DSPEVENTID;
512 #endif
514 /* Initilize the IpcPower module */
515 IpcPower_init();
517 /*
518 * Wait for HLOS (Virtio device) to indicate that priming of host's receive
519 * buffers is complete, indicating that host is ready to send.
520 *
521 * Though this is a Linux Virtio configuration status, it must be
522 * implemented by each non-Linux HLOS as well.
523 */
524 Log_print1(Diags_USER1, "VirtQueue_startup: VDEV status: 0x%x\n",
525 Resource_getVdevStatus(VIRTIO_ID_RPMSG));
526 Log_print0(Diags_USER1, "VirtQueue_startup: Polling VDEV status...\n");
527 while (Resource_getVdevStatus(VIRTIO_ID_RPMSG) != VRING_BUFS_PRIMED);
528 Log_print1(Diags_USER1, "VirtQueue_startup: VDEV status: 0x%x\n",
529 Resource_getVdevStatus(VIRTIO_ID_RPMSG));
531 InterruptProxy_intRegister(hostProcId, &intInfo, (Fxn)VirtQueue_isr,
532 (UArg)NULL);
533 Log_print0(Diags_USER1, "Passed VirtQueue_startup\n");
534 }
536 /*!
537 * ======== VirtQueue_postCrashToMailbox ========
538 */
539 Void VirtQueue_postCrashToMailbox(Void)
540 {
541 InterruptProxy_intSend(0, NULL, (UInt)RP_MSG_MBOX_CRASH);
542 }
544 #define CACHE_WB_TICK_PERIOD 5
546 /*!
547 * ======== ti_ipc_family_vayu_VirtQueue_cacheWb ========
548 *
549 * Used for flushing SysMin trace buffer.
550 */
551 Void ti_ipc_family_vayu_VirtQueue_cacheWb()
552 {
553 static UInt32 oldticks = 0;
554 UInt32 newticks;
556 newticks = Clock_getTicks();
557 if (newticks - oldticks < (UInt32)CACHE_WB_TICK_PERIOD) {
558 /* Don't keep flushing cache */
559 return;
560 }
562 oldticks = newticks;
564 /* Flush the cache */
565 Cache_wbAll();
566 }