Add initial support to start/stop cores after launching QNX IPC resource manager
[ipc/ipcdev.git] / qnx / src / ipc3x_dev / ti / syslink / build / Qnx / resmgr / syslink_main.c
index 7024d4d1096dcf210d2292d0e1ebf6728afbc8e8..d1ac04a1257e128849a230f4d7af5d597bdf9521 100644 (file)
@@ -98,6 +98,7 @@ static int numAttach = 0;
 
 #if defined(SYSLINK_PLATFORM_VAYU)
 static bool gatempEnabled = false;
+static Int32 sr0OwnerProcId = -1;
 #endif
 
 // Syslink hibernation global variables
@@ -119,12 +120,26 @@ typedef struct syslink_firmware_info_t {
     char * proc;
     char * firmware;
     bool attachOnly;
+    int  procState;  /* state of processor - index into procStateNames array */
+    int  readProcState; /* state that is currently being printed */
+    bool reload;     /* reload core during recovery */
+    bool freeString; /* Need to free previously allocated firmware string */
 } syslink_firmware_info;
 static syslink_firmware_info syslink_firmware[MultiProc_MAXPROCESSORS];
+
+/* Number of valid entries in syslink_firmware array */
 static unsigned int syslink_num_cores = 0;
 
-int init_ipc(syslink_dev_t * dev, syslink_firmware_info * firmware, bool recover);
-int deinit_ipc(syslink_dev_t * dev, bool recover);
+int init_ipc(syslink_dev_t * dev, syslink_firmware_info * firmware,
+    bool recover);
+int deinit_ipc(syslink_dev_t * dev, syslink_firmware_info * firmware,
+    bool recover);
+int init_syslink_trace_device(syslink_dev_t *dev);
+int deinit_syslink_trace_device(syslink_dev_t *dev);
+
+Int syslink_error_cb (UInt16 procId, ProcMgr_Handle handle,
+                      ProcMgr_State fromState, ProcMgr_State toState,
+                      ProcMgr_EventStatus status, Ptr args);
 
 static RscTable_Handle rscHandle[MultiProc_MAXPROCESSORS];
 
@@ -135,6 +150,13 @@ static ProcMgr_State errStates[] = {ProcMgr_State_Mmu_Fault,
                                     ProcMgr_State_Watchdog,
                                     ProcMgr_State_EndValue};
 
+/* Processor states */
+#define RESET_STATE  0
+#define RUNNING_STATE 1
+
+static String procStateNames[] = { "In reset",
+                                   "Running" };
+
 typedef struct syslink_trace_info_t {
     uintptr_t   va;
     uint32_t    len;
@@ -145,6 +167,544 @@ typedef struct syslink_trace_info_t {
 
 static syslink_trace_info proc_traces[MultiProc_MAXPROCESSORS];
 
+static int runSlave(syslink_dev_t *dev, uint16_t procId,
+    syslink_firmware_info * firmware_info)
+{
+    int status = 0;
+    ProcMgr_AttachParams attachParams;
+
+    if (firmware_info->firmware) {
+        rscHandle[procId] = RscTable_alloc(firmware_info->firmware, procId);
+        if (rscHandle[procId] == NULL) {
+            status = -1;
+            return status;
+        }
+    }
+
+    status = ProcMgr_open(&procH[procId], procId);
+    if (status < 0 || procH[procId] == NULL) {
+        goto procmgropen_fail;
+    }
+
+    /* Load and start the remote processor. */
+    ProcMgr_getAttachParams(procH[procId], &attachParams);
+    if (firmware_info->attachOnly) {
+        attachParams.bootMode = ProcMgr_BootMode_NoBoot;
+    }
+    status = ProcMgr_attach(procH[procId], &attachParams);
+    if (status < 0) {
+        GT_setFailureReason(curTrace,
+                            GT_4CLASS,
+                            "init_ipc",
+                            status,
+                            "ProcMgr_attach failed!");
+        goto procmgrattach_fail;
+    }
+
+    if ((firmware_info->firmware) &&
+        (!firmware_info->attachOnly)) {
+        status = ProcMgr_load(procH[procId],
+            (String)firmware_info->firmware, 0, NULL,
+             NULL, &procH_fileId[procId]);
+        if (status < 0) {
+            GT_setFailureReason(curTrace,
+                                GT_4CLASS,
+                                "init_ipc",
+                                status,
+                                "ProcMgr_load failed!");
+            goto procmgrload_fail;
+        }
+    }
+
+    status = Ipc_attach(procId);
+    if (status < 0) {
+        GT_setFailureReason(curTrace,
+                             GT_4CLASS,
+                             "init_ipc",
+                             status,
+                             "Ipc_attach failed!");
+        goto ipcattach_fail;
+    }
+
+    status = ProcMgr_registerNotify(procH[procId], syslink_error_cb, (Ptr)dev,
+        -1, errStates);
+    if (status < 0) {
+        goto procmgrreg_fail;
+    }
+
+    if (!firmware_info->attachOnly) {
+        status = ProcMgr_start(procH[procId], NULL);
+        if (status < 0) {
+            GT_setFailureReason(curTrace,
+                                GT_4CLASS,
+                                "init_ipc",
+                                status,
+                                "ProcMgr_start failed!");
+            goto procmgrstart_fail;
+        }
+    }
+
+    Osal_printf("runSlave successful for core %d\n", procId);
+
+    return 0;
+
+procmgrstart_fail:
+    ProcMgr_unregisterNotify(procH[procId], syslink_error_cb,
+        (Ptr)dev, errStates);
+procmgrreg_fail:
+    Ipc_detach(procId);
+ipcattach_fail:
+    if ((firmware_info->firmware) &&
+        (!firmware_info->attachOnly)) {
+        ProcMgr_unload(procH[procId], procH_fileId[procId]);
+    }
+procmgrload_fail:
+    ProcMgr_detach(procH[procId]);
+procmgrattach_fail:
+    ProcMgr_close(&procH[procId]);
+    procH[procId] = NULL;
+procmgropen_fail:
+    RscTable_free(&rscHandle[procId]);
+
+    return -1;
+}
+
+static int resetSlave(syslink_dev_t *dev, uint16_t procId)
+{
+    if ((procH[procId]) && (procH_fileId[procId])) {
+        GT_1trace(curTrace, GT_4CLASS, "stopping %s", MultiProc_getName(procId));
+        ProcMgr_stop(procH[procId]);
+    }
+
+    if (procH[procId]) {
+        ProcMgr_unregisterNotify(procH[procId], syslink_error_cb, (Ptr)dev,
+            errStates);
+        Ipc_detach(procId);
+        if (procH_fileId[procId]) {
+            ProcMgr_unload(procH[procId], procH_fileId[procId]);
+            procH_fileId[procId] = 0;
+        }
+        ProcMgr_detach(procH[procId]);
+        ProcMgr_close(&procH[procId]);
+        procH[procId] = NULL;
+        RscTable_free(&rscHandle[procId]);
+        rscHandle[procId] = NULL;
+    }
+
+    Osal_printf("resetSlave successful for core %d\n", procId);
+
+    return 0;
+}
+
+static int slave_state_read(resmgr_context_t *ctp, io_read_t *msg,
+    syslink_ocb_t *ocb)
+{
+    int             nbytes;
+    int             nparts;
+    int             status;
+    int             nleft;
+    int             i;
+    uint16_t        procid = ocb->ocb.attr->procid;
+    syslink_dev_t * dev = ocb->ocb.attr->dev;
+
+    if ((status = iofunc_read_verify(ctp, msg, &ocb->ocb, NULL)) != EOK) {
+        return (status);
+    }
+
+    if ((msg->i.xtype & _IO_XTYPE_MASK) != _IO_XTYPE_NONE) {
+        return (ENOSYS);
+    }
+
+    pthread_mutex_lock(&dev->firmwareLock);
+
+    for (i = 0; i < syslink_num_cores; i++) {
+        if (syslink_firmware[i].proc_id == procid) {
+            break;
+        }
+    }
+    if (i == syslink_num_cores) {
+        if ((syslink_num_cores < MultiProc_MAXPROCESSORS)) {
+            syslink_firmware[syslink_num_cores].proc =
+                MultiProc_getName(procid);
+            syslink_firmware[syslink_num_cores].proc_id = procid;
+            syslink_firmware[syslink_num_cores].attachOnly = false;
+            syslink_firmware[syslink_num_cores].reload = false;
+            syslink_firmware[syslink_num_cores].procState = RESET_STATE;
+            syslink_firmware[syslink_num_cores].freeString = false;
+            syslink_firmware[syslink_num_cores++].firmware = NULL;
+        }
+        else {
+            pthread_mutex_unlock(&dev->firmwareLock);
+            return (EBADSLT);
+        }
+    }
+
+    if (ocb->ocb.offset == 0) {
+        /* latch onto new state, so that we print out complete strings */
+        syslink_firmware[i].readProcState = syslink_firmware[i].procState;
+    }
+
+    nleft = strlen(procStateNames[syslink_firmware[i].readProcState])
+        - ocb->ocb.offset; /* the state is expressed in one byte */
+    nbytes = min(msg->i.nbytes, nleft);
+
+    /* Make sure the user has supplied a big enough buffer */
+    if (nbytes > 0) {
+        /* set up the return data IOV */
+        SETIOV(ctp->iov,
+            (char *)procStateNames[syslink_firmware[i].readProcState]
+            + ocb->ocb.offset, nbytes);
+
+        pthread_mutex_unlock(&dev->firmwareLock);
+
+        /* set up the number of bytes (returned by client's read()) */
+        _IO_SET_READ_NBYTES(ctp, nbytes);
+
+        ocb->ocb.offset += nbytes;
+
+        nparts = 1;
+    }
+    else {
+        pthread_mutex_unlock(&dev->firmwareLock);
+
+        _IO_SET_READ_NBYTES (ctp, 0);
+
+        /* reset offset */
+        ocb->ocb.offset = 0;
+
+        nparts = 0;
+    }
+
+    /* mark the access time as invalid (we just accessed it) */
+
+    if (msg->i.nbytes > 0) {
+        ocb->ocb.attr->attr.flags |= IOFUNC_ATTR_ATIME;
+    }
+
+    return (_RESMGR_NPARTS(nparts));
+}
+
+static int slave_state_write(resmgr_context_t *ctp, io_write_t *msg,
+    syslink_ocb_t *ocb)
+{
+    int             status;
+    char *          buf;
+    uint16_t        procid = ocb->ocb.attr->procid;
+    int             i;
+    char *          ptr;
+    syslink_dev_t * dev = ocb->ocb.attr->dev;
+    Int32           sr0ProcId;
+
+    if ((status = iofunc_write_verify(ctp, msg, &ocb->ocb, NULL)) != EOK) {
+        return (status);
+    }
+
+    if ((msg->i.xtype & _IO_XTYPE_MASK) != _IO_XTYPE_NONE) {
+        return (ENOSYS);
+    }
+
+    /* set up the number of bytes (returned by client's write()) */
+    _IO_SET_WRITE_NBYTES (ctp, msg->i.nbytes);
+
+    buf = (char *) malloc(msg->i.nbytes + 1);
+    if (buf == NULL) {
+        return (ENOMEM);
+    }
+
+    /*
+     *  Read the data from the sender's message buffer.
+     */
+    resmgr_msgread(ctp, buf, msg->i.nbytes, sizeof(msg->i));
+    buf[msg->i.nbytes] = '\0'; /* just in case the text is not NULL terminated */
+    if((ptr = strchr(buf, '\n')) != NULL) { /* Remove line feed if any */
+        *ptr = '\0';
+    }
+
+    pthread_mutex_lock(&dev->firmwareLock);
+    for (i = 0; i < syslink_num_cores; i++) {
+        if (syslink_firmware[i].proc_id == procid) {
+            break;
+        }
+    }
+    if (i == syslink_num_cores) {
+        if ((syslink_num_cores < MultiProc_MAXPROCESSORS)) {
+            syslink_firmware[syslink_num_cores].proc =
+                MultiProc_getName(procid);
+            syslink_firmware[syslink_num_cores].proc_id = procid;
+            syslink_firmware[syslink_num_cores].attachOnly = false;
+            syslink_firmware[syslink_num_cores].reload = false;
+            syslink_firmware[syslink_num_cores].procState = RESET_STATE;
+            syslink_firmware[syslink_num_cores].freeString = false;
+            syslink_firmware[syslink_num_cores++].firmware = NULL;
+        }
+        else {
+            pthread_mutex_unlock(&dev->firmwareLock);
+            return (EBADSLT);
+        }
+    }
+
+    if (strcmp("1", buf) == 0) {
+        if ((syslink_firmware[i].procState == RESET_STATE) &&
+           (syslink_firmware[i].firmware != NULL)) {
+            runSlave(ocb->ocb.attr->dev, procid, &syslink_firmware[i]);
+#if defined(SYSLINK_PLATFORM_VAYU)
+            if (gatempEnabled) {
+                if (sr0OwnerProcId == -1) {
+                    /* Set up GateMP */
+                    status = GateMP_setup(&sr0ProcId);
+                    if ((status < 0) && (status != GateMP_E_NOTFOUND)) {
+                        resetSlave(ocb->ocb.attr->dev, procid);
+                        pthread_mutex_unlock(&dev->firmwareLock);
+                        free(buf);
+                        return (EIO);
+                    }
+                    else if (status == 0) {
+                        /* We have discovered an owner for SR0 */
+                        sr0OwnerProcId = sr0ProcId;
+                    }
+                }
+                else {
+                    /*
+                     * We have already identified SR0 owner and setup GateMP.
+                     * Do nothing.
+                     */
+                }
+            }
+#endif
+            printf("Core is now running with image '%s'\n",
+                syslink_firmware[i].firmware);
+            syslink_firmware[i].procState = RUNNING_STATE;
+            syslink_firmware[i].reload = true;
+            status = init_syslink_trace_device(dev);
+            if (status < 0) {
+                pthread_mutex_unlock(&dev->firmwareLock);
+                free(buf);
+                return (EIO);
+            }
+            printf("Core %s has been started.\n", MultiProc_getName(procid));
+        }
+    }
+    else if (strcmp("0", buf) == 0) {
+        if (syslink_firmware[i].procState == RUNNING_STATE) {
+            resetSlave(ocb->ocb.attr->dev, procid);
+            syslink_firmware[i].procState = RESET_STATE;
+            syslink_firmware[i].reload = false;
+            status = deinit_syslink_trace_device(dev);
+            if (status < 0) {
+                pthread_mutex_unlock(&dev->firmwareLock);
+                free(buf);
+                Osal_printf("IPC: deinit_syslink_trace_device failed %d",
+                    status);
+                return (EIO);
+            }
+#if defined(SYSLINK_PLATFORM_VAYU)
+            if ((gatempEnabled) && (procid == sr0OwnerProcId)) {
+                sr0OwnerProcId = -1;
+                GateMP_destroy();
+            }
+#endif
+            printf("Core %s has been reset.\n", MultiProc_getName(procid));
+        }
+    }
+    else {
+        /* ignore the input as it is not recognized */
+        fprintf(stderr, "Unrecognized input\n");
+    }
+
+    pthread_mutex_unlock(&dev->firmwareLock);
+
+    free(buf);
+
+    if (msg->i.nbytes > 0) {
+        ocb->ocb.attr->attr.flags |= IOFUNC_ATTR_MTIME | IOFUNC_ATTR_CTIME;
+    }
+
+    return (_RESMGR_NPARTS(0));
+}
+
+static int slave_file_read(resmgr_context_t *ctp, io_read_t *msg,
+    syslink_ocb_t *ocb)
+{
+    int             nbytes;
+    int             nparts;
+    int             status;
+    int             nleft;
+    int             i;
+    uint16_t        procid = ocb->ocb.attr->procid;
+    syslink_dev_t * dev = ocb->ocb.attr->dev;
+
+    if ((status = iofunc_read_verify (ctp, msg, &ocb->ocb, NULL)) != EOK) {
+        return (status);
+    }
+
+    if ((msg->i.xtype & _IO_XTYPE_MASK) != _IO_XTYPE_NONE) {
+        return (ENOSYS);
+    }
+
+    pthread_mutex_lock(&dev->firmwareLock);
+    for (i = 0; i < syslink_num_cores; i++) {
+        if (syslink_firmware[i].proc_id == procid) {
+            break;
+        }
+    }
+
+    if (i == syslink_num_cores) {
+        if ((syslink_num_cores < MultiProc_MAXPROCESSORS)) {
+            syslink_firmware[syslink_num_cores].proc =
+                MultiProc_getName(procid);
+            syslink_firmware[syslink_num_cores].proc_id = procid;
+            syslink_firmware[syslink_num_cores].attachOnly = false;
+            syslink_firmware[syslink_num_cores].reload = false;
+            syslink_firmware[syslink_num_cores].procState = RESET_STATE;
+            syslink_firmware[syslink_num_cores].freeString = false;
+            syslink_firmware[syslink_num_cores++].firmware = NULL;
+        }
+        else {
+            pthread_mutex_unlock(&dev->firmwareLock);
+            return (EBADSLT);
+        }
+    }
+
+    if (syslink_firmware[i].firmware == NULL) {
+        nbytes = 0;
+    }
+    else {
+        nleft = strlen(syslink_firmware[i].firmware)
+            - ocb->ocb.offset; /* the state is expressed in one byte */
+        nbytes = min(msg->i.nbytes, nleft);
+    }
+
+    /* Make sure the user has supplied a big enough buffer */
+    if (nbytes > 0) {
+        /* set up the return data IOV */
+        SETIOV(ctp->iov, (char *)syslink_firmware[i].firmware
+            + ocb->ocb.offset, nbytes);
+
+        /* set up the number of bytes (returned by client's read()) */
+        _IO_SET_READ_NBYTES(ctp, nbytes);
+
+        ocb->ocb.offset += nbytes;
+
+        nparts = 1;
+    }
+    else {
+        _IO_SET_READ_NBYTES (ctp, 0);
+
+        /* reset offset */
+        ocb->ocb.offset = 0;
+
+        nparts = 0;
+    }
+
+    pthread_mutex_unlock(&dev->firmwareLock);
+
+    /* mark the access time as invalid (we just accessed it) */
+
+    if (msg->i.nbytes > 0) {
+        ocb->ocb.attr->attr.flags |= IOFUNC_ATTR_ATIME;
+    }
+
+    return (_RESMGR_NPARTS(nparts));
+}
+
+static int slave_file_write(resmgr_context_t *ctp, io_write_t *msg,
+    syslink_ocb_t *ocb)
+{
+    int             status;
+    char *          buf;
+    uint16_t        procid = ocb->ocb.attr->procid;
+    int             i;
+    char *          absPath;
+    char *          ptr;
+    syslink_dev_t * dev = ocb->ocb.attr->dev;
+
+    if ((status = iofunc_write_verify(ctp, msg, &ocb->ocb, NULL)) != EOK) {
+        return (status);
+    }
+
+    if ((msg->i.xtype & _IO_XTYPE_MASK) != _IO_XTYPE_NONE) {
+        return (ENOSYS);
+    }
+
+    /* set up the number of bytes (returned by client's write()) */
+    _IO_SET_WRITE_NBYTES (ctp, msg->i.nbytes);
+
+    buf = (char *) malloc(msg->i.nbytes + 1);
+    if (buf == NULL) {
+        return (ENOMEM);
+    }
+
+    /*
+     *  Read the data from the sender's message buffer.
+     */
+    resmgr_msgread(ctp, buf, msg->i.nbytes, sizeof(msg->i));
+    buf[msg->i.nbytes] = '\0'; /* just in case the text is not NULL terminated */
+    if((ptr = strchr(buf, '\n')) != NULL) { /* Remove line feed if any */
+        *ptr = '\0';
+    }
+
+    /* Get the abs path for all firmware files */
+    absPath = calloc(1, PATH_MAX + 1);
+    if (absPath == NULL) {
+        free(buf);
+        return (ENOMEM);
+    }
+    if (NULL == realpath(buf, absPath)) {
+        fprintf(stderr, "invalid path to executable: %d\n", errno);
+        fprintf(stderr, "make sure you are specifying the full path to the "
+            "file.\n");
+        free(absPath);
+        free(buf);
+        return (ENOENT);
+    }
+    free(buf);
+
+    pthread_mutex_lock(&dev->firmwareLock);
+
+    /*
+     * Check if an entry in syslink_firmware already exists for this core.
+     * If not, create one. Otherwise just update the firmware path.
+     */
+    for (i = 0; i < syslink_num_cores; i++) {
+        if (syslink_firmware[i].proc_id == procid) {
+            break;
+        }
+    }
+    if (i == syslink_num_cores) {
+        if (syslink_num_cores < MultiProc_MAXPROCESSORS) {
+            syslink_firmware[syslink_num_cores].proc =
+                MultiProc_getName(procid);
+            syslink_firmware[syslink_num_cores].proc_id = procid;
+            syslink_firmware[syslink_num_cores].attachOnly = false;
+            syslink_firmware[syslink_num_cores].reload = false;
+            syslink_firmware[syslink_num_cores].procState = RESET_STATE;
+            syslink_firmware[syslink_num_cores].freeString = true;
+            syslink_firmware[syslink_num_cores++].firmware = absPath;
+        }
+        else {
+            pthread_mutex_unlock(&dev->firmwareLock);
+            free(absPath);
+            return (EBADSLT);
+        }
+    }
+    else {
+        /* Free previously allocated string */
+        if ((syslink_firmware[syslink_num_cores].freeString) &&
+           (syslink_firmware[syslink_num_cores].firmware)) {
+            free(syslink_firmware[i].firmware);
+        }
+        syslink_firmware[i].firmware = absPath;
+    }
+
+    pthread_mutex_unlock(&dev->firmwareLock);
+
+    if (msg->i.nbytes > 0) {
+        ocb->ocb.attr->attr.flags |= IOFUNC_ATTR_MTIME | IOFUNC_ATTR_CTIME;
+    }
+
+    return (_RESMGR_NPARTS(0));
+}
+
 int syslink_read(resmgr_context_t *ctp, io_read_t *msg, syslink_ocb_t *ocb)
 {
     int         nbytes;
@@ -277,6 +837,95 @@ syslink_ocb_free (IOFUNC_OCB_T * i_ocb)
     }
 }
 
+int init_slave_devices(syslink_dev_t *dev)
+{
+    resmgr_attr_t    resmgr_attr;
+    int              i;
+    syslink_attr_t * slave_attr;
+    int              status = 0;
+
+    memset(&resmgr_attr, 0, sizeof resmgr_attr);
+    resmgr_attr.nparts_max = 1;
+    resmgr_attr.msg_max_size = _POSIX_PATH_MAX;
+
+    for (i = 1; i < MultiProc_getNumProcessors(); i++) {
+        iofunc_func_init(_RESMGR_CONNECT_NFUNCS, &dev->syslink.cfuncs_state[i],
+                         _RESMGR_IO_NFUNCS, &dev->syslink.iofuncs_state[i]);
+        slave_attr = &dev->syslink.cattr_slave[i];
+        iofunc_attr_init(&slave_attr->attr,
+                         S_IFCHR | 0777, NULL, NULL);
+        slave_attr->attr.mount = &dev->syslink.mattr;
+        slave_attr->procid = i;
+        slave_attr->dev = (Ptr)dev;
+        iofunc_time_update(&slave_attr->attr);
+        snprintf(dev->syslink.device_name, _POSIX_PATH_MAX,
+                  "%s-state/%s", IPC_DEVICE_PATH, MultiProc_getName(i));
+        dev->syslink.iofuncs_state[i].read = slave_state_read;
+        dev->syslink.iofuncs_state[i].write = slave_state_write;
+
+        if (-1 == (dev->syslink.resmgr_id_state[i] =
+            resmgr_attach(dev->dpp, &resmgr_attr,
+                dev->syslink.device_name, _FTYPE_ANY, 0,
+                &dev->syslink.cfuncs_state[i],
+                &dev->syslink.iofuncs_state[i],
+                &slave_attr->attr))) {
+            GT_setFailureReason(curTrace, GT_4CLASS, "init_slave_devices",
+                status, "resmgr_attach failed");
+            return (-1);
+        }
+    }
+
+    for (i = 1; i < MultiProc_getNumProcessors(); i++) {
+        iofunc_func_init(_RESMGR_CONNECT_NFUNCS, &dev->syslink.cfuncs_file[i],
+                         _RESMGR_IO_NFUNCS, &dev->syslink.iofuncs_file[i]);
+        slave_attr = &dev->syslink.cattr_slave[i];
+        iofunc_attr_init(&slave_attr->attr,
+                         S_IFCHR | 0777, NULL, NULL);
+        slave_attr->attr.mount = &dev->syslink.mattr;
+        slave_attr->procid = i;
+        slave_attr->dev = (Ptr)dev;
+        iofunc_time_update(&slave_attr->attr);
+        snprintf(dev->syslink.device_name, _POSIX_PATH_MAX,
+                  "%s-file/%s", IPC_DEVICE_PATH, MultiProc_getName(i));
+        dev->syslink.iofuncs_file[i].read = slave_file_read;
+        dev->syslink.iofuncs_file[i].write = slave_file_write;
+
+        if (-1 == (dev->syslink.resmgr_id_file[i] =
+            resmgr_attach(dev->dpp, &resmgr_attr,
+                dev->syslink.device_name, _FTYPE_ANY, 0,
+                &dev->syslink.cfuncs_file[i],
+                &dev->syslink.iofuncs_file[i],
+                &slave_attr->attr))) {
+            GT_setFailureReason(curTrace, GT_4CLASS, "init_slave_devices",
+                status, "resmgr_attach failed");
+            return (-1);
+        }
+    }
+
+    return (status);
+}
+
+int deinit_slave_devices(syslink_dev_t *dev)
+{
+    int status = EOK;
+    int i = 0;
+
+    for (i = 1; i < MultiProc_getNumProcessors(); i++) {
+        status = resmgr_detach(dev->dpp, dev->syslink.resmgr_id_state[i], 0);
+        if (status < 0) {
+            Osal_printf("IPC: resmgr_detach of state device %d failed: %d",
+                i, errno);
+        }
+        status = resmgr_detach(dev->dpp, dev->syslink.resmgr_id_file[i], 0);
+        if (status < 0) {
+            Osal_printf("IPC: resmgr_detach of file device %d failed: %d",
+                i, errno);
+        }
+    }
+
+    return (status);
+}
+
 int init_syslink_trace_device(syslink_dev_t *dev)
 {
     resmgr_attr_t    resmgr_attr;
@@ -292,64 +941,78 @@ int init_syslink_trace_device(syslink_dev_t *dev)
     resmgr_attr.msg_max_size = 2048;
 
     for (i = 0; i < syslink_num_cores; i++) {
-        iofunc_func_init(_RESMGR_CONNECT_NFUNCS, &dev->syslink.cfuncs_trace[i],
-                         _RESMGR_IO_NFUNCS, &dev->syslink.iofuncs_trace[i]);
-        trace_attr = &dev->syslink.cattr_trace[i];
-        iofunc_attr_init(&trace_attr->attr,
+        /*
+         * Initialize trace device only for cores that are running and their
+         * device is not yet setup.
+         */
+        if ((syslink_firmware[i].procState == RUNNING_STATE) &&
+            (proc_traces[i].va == NULL)) {
+            iofunc_func_init(_RESMGR_CONNECT_NFUNCS,
+                &dev->syslink.cfuncs_trace[i],
+                _RESMGR_IO_NFUNCS, &dev->syslink.iofuncs_trace[i]);
+            trace_attr = &dev->syslink.cattr_trace[i];
+            iofunc_attr_init(&trace_attr->attr,
                          S_IFCHR | 0777, NULL, NULL);
-        trace_attr->attr.mount = &dev->syslink.mattr;
-        trace_attr->procid = i;
-        iofunc_time_update(&trace_attr->attr);
-        snprintf (dev->syslink.device_name, _POSIX_PATH_MAX,
+            trace_attr->attr.mount = &dev->syslink.mattr;
+            trace_attr->procid = i;
+            iofunc_time_update(&trace_attr->attr);
+            snprintf (dev->syslink.device_name, _POSIX_PATH_MAX,
                   "/dev/ipc-trace%d", syslink_firmware[i].proc_id);
-        dev->syslink.iofuncs_trace[i].read = syslink_read;
-        snprintf (trace_name, _POSIX_PATH_MAX, "%d", 0);
-        pa = 0;
-        status = RscTable_getInfo(syslink_firmware[i].proc_id, TYPE_TRACE, 0, &da, &pa, &len);
-        if (status == 0) {
-            /* last 8 bytes are for writeIdx/readIdx */
-            proc_traces[i].len = len - (sizeof(uint32_t) * 2);
-            if (da && !pa) {
-                /* need to translate da->pa */
-                status = ProcMgr_translateAddr (procH[syslink_firmware[i].proc_id],
-                                                (Ptr *) &pa,
-                                                ProcMgr_AddrType_MasterPhys,
-                                                (Ptr) da,
-                                                ProcMgr_AddrType_SlaveVirt);
-            }
-            else {
-                GT_setFailureReason(curTrace, GT_4CLASS, "init_syslink_trace_device",
-                                    status, "not performing ProcMgr_translate");
-            }
-            /* map length aligned to page size */
-            proc_traces[i].va =
+            dev->syslink.iofuncs_trace[i].read = syslink_read;
+            snprintf (trace_name, _POSIX_PATH_MAX, "%d", 0);
+            pa = 0;
+            status = RscTable_getInfo(syslink_firmware[i].proc_id, TYPE_TRACE,
+                0, &da, &pa, &len);
+            if (status == 0) {
+                /* last 8 bytes are for writeIdx/readIdx */
+                proc_traces[i].len = len - (sizeof(uint32_t) * 2);
+                if (da && !pa) {
+                    /* need to translate da->pa */
+                    status = ProcMgr_translateAddr(
+                        procH[syslink_firmware[i].proc_id],
+                        (Ptr *) &pa,
+                        ProcMgr_AddrType_MasterPhys,
+                        (Ptr) da,
+                        ProcMgr_AddrType_SlaveVirt);
+                }
+                else {
+                    GT_setFailureReason(curTrace, GT_4CLASS,
+                        "init_syslink_trace_device",
+                        status, "not performing ProcMgr_translate");
+                }
+                /* map length aligned to page size */
+                proc_traces[i].va =
                     mmap_device_io (((len + 0x1000 - 1) / 0x1000) * 0x1000, pa);
-            proc_traces[i].widx = (uint32_t *)(proc_traces[i].va + \
+                proc_traces[i].widx = (uint32_t *)(proc_traces[i].va + \
                                                proc_traces[i].len);
-            proc_traces[i].ridx = (uint32_t *)((uint32_t)proc_traces[i].widx + \
-                                               sizeof(uint32_t));
-            if (proc_traces[i].va == MAP_DEVICE_FAILED) {
-                GT_setFailureReason(curTrace, GT_4CLASS, "init_syslink_trace_device",
-                                    status, "mmap_device_io failed");
-                GT_1trace(curTrace, GT_4CLASS, "errno %d", errno);
+                proc_traces[i].ridx = (uint32_t *)((uint32_t)proc_traces[i].widx
+                    + sizeof(uint32_t));
+                if (proc_traces[i].va == MAP_DEVICE_FAILED) {
+                    GT_setFailureReason(curTrace, GT_4CLASS,
+                        "init_syslink_trace_device",
+                        status, "mmap_device_io failed");
+                    GT_1trace(curTrace, GT_4CLASS, "errno %d", errno);
+                    proc_traces[i].va = NULL;
+                }
+                proc_traces[i].firstRead = TRUE;
+            }
+            else {
+                GT_setFailureReason(curTrace, GT_4CLASS,
+                    "init_syslink_trace_device",
+                    status, "RscTable_getInfo failed");
                 proc_traces[i].va = NULL;
             }
-            proc_traces[i].firstRead = TRUE;
-        }
-        else {
-            GT_setFailureReason(curTrace, GT_4CLASS, "init_syslink_trace_device",
-                                status, "RscTable_getInfo failed");
-            proc_traces[i].va = NULL;
-        }
-        if (-1 == (dev->syslink.resmgr_id_trace[i] =
+            if (-1 == (dev->syslink.resmgr_id_trace[i] =
                        resmgr_attach(dev->dpp, &resmgr_attr,
                                      dev->syslink.device_name, _FTYPE_ANY, 0,
                                      &dev->syslink.cfuncs_trace[i],
                                      &dev->syslink.iofuncs_trace[i],
                                      &trace_attr->attr))) {
-            GT_setFailureReason(curTrace, GT_4CLASS, "init_syslink_trace_device",
-                                status, "resmgr_attach failed");
-            return(-1);
+                GT_setFailureReason(curTrace, GT_4CLASS,
+                    "init_syslink_trace_device",
+                    status, "resmgr_attach failed");
+                return(-1);
+            }
         }
     }
 
@@ -362,16 +1025,22 @@ int deinit_syslink_trace_device(syslink_dev_t *dev)
     int i = 0;
 
     for (i = 0; i < syslink_num_cores; i++) {
-        status = resmgr_detach(dev->dpp, dev->syslink.resmgr_id_trace[i], 0);
-        if (status < 0) {
-            Osal_printf("IPC: resmgr_detach failed %d", errno);
-            status = errno;
-        }
-        if (proc_traces[i].va && proc_traces[i].va != MAP_DEVICE_FAILED) {
-            munmap((void *)proc_traces[i].va,
+        /* Only disable trace device on cores in RESET state */
+        if ((syslink_firmware[i].procState == RESET_STATE) &&
+            (proc_traces[i].va != NULL)) {
+            status = resmgr_detach(dev->dpp, dev->syslink.resmgr_id_trace[i],
+                0);
+            if (status < 0) {
+                Osal_printf("IPC: resmgr_detach of trace device %d failed: %d",
+                    i, errno);
+                status = errno;
+            }
+            if (proc_traces[i].va && proc_traces[i].va != MAP_DEVICE_FAILED) {
+                munmap((void *)proc_traces[i].va,
                    ((proc_traces[i].len + 8 + 0x1000 - 1) / 0x1000) * 0x1000);
+            }
+            proc_traces[i].va = NULL;
         }
-        proc_traces[i].va = NULL;
     }
 
     return (status);
@@ -385,6 +1054,7 @@ int init_syslink_device(syslink_dev_t *dev)
     int              status = 0;
 
     pthread_mutex_init(&dev->lock, NULL);
+    pthread_mutex_init(&dev->firmwareLock, NULL);
 
     memset(&resmgr_attr, 0, sizeof resmgr_attr);
     resmgr_attr.nparts_max = 10;
@@ -421,6 +1091,11 @@ int init_syslink_device(syslink_dev_t *dev)
         return(-1);
     }
 
+    status = init_slave_devices(dev);
+    if (status < 0) {
+        return status;
+    }
+
     status = init_syslink_trace_device(dev);
     if (status < 0) {
         return status;
@@ -436,11 +1111,20 @@ int deinit_syslink_device(syslink_dev_t *dev)
 
     status = resmgr_detach(dev->dpp, dev->syslink.resmgr_id, 0);
     if (status < 0) {
-        Osal_printf("IPC: resmgr_detach failed %d", errno);
+        Osal_printf("IPC: resmgr_detach of %s failed: %d",
+            IPC_DEVICE_PATH, errno);
         status = errno;
     }
 
     status = deinit_syslink_trace_device(dev);
+    if (status < 0) {
+        Osal_printf("IPC: deinit_syslink_trace_device failed %d", status);
+    }
+
+    status = deinit_slave_devices(dev);
+    if (status < 0) {
+        Osal_printf("IPC: deinit_slave_devices failed %d", status);
+    }
 
     return(status);
 }
@@ -476,10 +1160,13 @@ static void ipc_recover(Ptr args)
     syslink_dev_t * dev = (syslink_dev_t *)args;
 
     if (!disableRecovery) {
-        deinit_ipc(dev, TRUE);
-        init_ipc(dev, syslink_firmware, TRUE);
+        /* Protect the syslink_firmware array as we recover */
+        pthread_mutex_lock(&dev->firmwareLock);
+        deinit_ipc(dev, syslink_firmware, TRUE);
         deinit_syslink_trace_device(dev);
+        init_ipc(dev, syslink_firmware, TRUE);
         init_syslink_trace_device(dev);
+        pthread_mutex_unlock(&dev->firmwareLock);
     }
     else {
         GT_0trace(curTrace, GT_4CLASS,
@@ -617,99 +1304,26 @@ int init_ipc(syslink_dev_t * dev, syslink_firmware_info * firmware, bool recover
                  * if we are in recovery, we load the cores we previously
                  * attached to
                  */
-                syslink_firmware[i].attachOnly = false;
+                firmware[i].attachOnly = false;
             }
 
-            if (syslink_firmware[i].firmware) {
-                rscHandle[procId] = RscTable_alloc(firmware[i].firmware, procId);
-                if (rscHandle[procId] == NULL) {
-                    status = -1;
-                    break;
+            if ((!recover) || (firmware[i].reload)) {
+                status = runSlave(dev, procId, &firmware[i]);
+                if (status == 0) {
+                    firmware[i].procState = RUNNING_STATE;
+                    continue;
                 }
-            }
-
-            status = ProcMgr_open(&procH[procId], procId);
-            if (status < 0 || procH[procId] == NULL)
-                goto procmgropen_fail;
-
-            /* Load and start the remote processor. */
-            ProcMgr_getAttachParams (procH[procId], &attachParams);
-            if (syslink_firmware[i].attachOnly) {
-                attachParams.bootMode = ProcMgr_BootMode_NoBoot;
-            }
-            status = ProcMgr_attach (procH[procId], &attachParams);
-            if (status < 0) {
-                GT_setFailureReason (curTrace,
-                                     GT_4CLASS,
-                                     "init_ipc",
-                                     status,
-                                     "ProcMgr_attach failed!");
-                goto procmgrattach_fail;
-            }
-
-            if ((syslink_firmware[i].firmware) &&
-                (!syslink_firmware[i].attachOnly)) {
-                status = ProcMgr_load (procH[procId],
-                                       (String)firmware[i].firmware, 0, NULL,
-                                        NULL, &procH_fileId[procId]);
-                if (status < 0) {
-                    GT_setFailureReason (curTrace,
-                                         GT_4CLASS,
-                                         "init_ipc",
-                                         status,
-                                         "ProcMgr_load failed!");
-                    goto procmgrload_fail;
-                }
-            }
-
-            status = Ipc_attach (procId);
-            if (status < 0) {
-                GT_setFailureReason (curTrace,
-                                     GT_4CLASS,
-                                     "init_ipc",
-                                     status,
-                                     "Ipc_attach failed!");
-                goto ipcattach_fail;
-            }
-
-            status = ProcMgr_registerNotify(procH[procId], syslink_error_cb, (Ptr)dev,
-                                            -1, errStates);
-            if (status < 0) {
-                goto procmgrreg_fail;
-            }
-
-            if (!syslink_firmware[i].attachOnly) {
-                status = ProcMgr_start(procH[procId], NULL);
-                if (status < 0) {
-                    GT_setFailureReason (curTrace,
-                                     GT_4CLASS,
-                                     "init_ipc",
-                                     status,
-                                     "ProcMgr_start failed!");
-                    goto procmgrstart_fail;
+                else {
+                    break;
                 }
             }
-
-            continue;
-
-procmgrstart_fail:
-            ProcMgr_unregisterNotify(procH[procId], syslink_error_cb,
-                                (Ptr)dev, errStates);
-procmgrreg_fail:
-            Ipc_detach(procId);
-ipcattach_fail:
-            if ((syslink_firmware[i].firmware) &&
-                (!syslink_firmware[i].attachOnly)) {
-                ProcMgr_unload(procH[procId], procH_fileId[procId]);
+            else {
+                /*
+                 * During recovery, do not run cores unless they were previously
+                 * running
+                 */
+                continue;
             }
-procmgrload_fail:
-            ProcMgr_detach(procH[procId]);
-procmgrattach_fail:
-            ProcMgr_close(&procH[procId]);
-            procH[procId] = NULL;
-procmgropen_fail:
-            RscTable_free(&rscHandle[procId]);
-            break;
         }
 
         if (status < 0)
@@ -727,6 +1341,8 @@ procmgropen_fail:
 
 #if defined(SYSLINK_PLATFORM_VAYU)
         if (gatempEnabled) {
+            Int32 sr0ProcId;
+
             /* Set up NameServer for resource manager process */
             status = NameServer_setup();
             if (status < 0) {
@@ -734,10 +1350,20 @@ procmgropen_fail:
             }
 
             /* Set up GateMP */
-            status = GateMP_setup();
-            if (status < 0) {
+            status = GateMP_setup(&sr0ProcId);
+            if ((status < 0) && (status != GateMP_E_NOTFOUND)) {
                 goto gatempsetup_fail;
             }
+            else if (status == 0) {
+                sr0OwnerProcId = sr0ProcId;
+            }
+            else {
+                /*
+                 * If we did not find the default gate, perhaps SR0 is
+                 * not yet loaded. This is ok.
+                 */
+                status = 0;
+            }
         }
 #endif
 
@@ -760,7 +1386,7 @@ tiipcsetup_fail:
         }
         ProcMgr_unregisterNotify(procH[procId], syslink_error_cb,
                                 (Ptr)dev, errStates);
-        if (!syslink_firmware[i].attachOnly) {
+        if (!firmware[i].attachOnly) {
             ProcMgr_stop(procH[procId]);
             if (procH_fileId[procId]) {
                 ProcMgr_unload(procH[procId], procH_fileId[procId]);
@@ -783,7 +1409,8 @@ exit:
     return status;
 }
 
-int deinit_ipc(syslink_dev_t * dev, bool recover)
+int deinit_ipc(syslink_dev_t * dev, syslink_firmware_info * firmware,
+    bool recover)
 {
     int status = EOK;
     uint32_t i = 0, id = 0;
@@ -795,10 +1422,8 @@ int deinit_ipc(syslink_dev_t * dev, bool recover)
 
     // Stop the remote cores right away
     for (i = 0; i < MultiProc_MAXPROCESSORS; i++) {
-        /* Only stop the ones that we have loaded */
-        if ((procH[i]) && (procH_fileId[i])) {
-            GT_1trace(curTrace, GT_4CLASS, "stopping %s", MultiProc_getName(i));
-            ProcMgr_stop(procH[i]);
+        if (procH[i]) {
+            resetSlave(dev, i);
         }
     }
 
@@ -814,23 +1439,6 @@ int deinit_ipc(syslink_dev_t * dev, bool recover)
 
     ti_ipc_destroy(recover);
 
-    for (i = 0; i < MultiProc_MAXPROCESSORS; i++) {
-        if (procH[i]) {
-            ProcMgr_unregisterNotify (procH[i], syslink_error_cb, (Ptr)dev,
-                                      errStates);
-            Ipc_detach(i);
-            if (procH_fileId[i]) {
-                ProcMgr_unload(procH[i], procH_fileId[i]);
-                procH_fileId[i] = 0;
-            }
-            ProcMgr_detach(procH[i]);
-            ProcMgr_close(&procH[i]);
-            procH[i] = NULL;
-            RscTable_free(&rscHandle[i]);
-            rscHandle[i] = NULL;
-        }
-    }
-
     if (!recover && dev->ipc_recovery_work != NULL) {
         OsalThread_delete (&dev->ipc_recovery_work);
         dev->ipc_recovery_work = NULL;
@@ -844,21 +1452,23 @@ int deinit_ipc(syslink_dev_t * dev, bool recover)
             log = fopen(logFilename, "a+");
             if (log) {
                 for (id = 0; id < syslink_num_cores; id++) {
-                    if (proc_traces[id].va) {
-                        /* print traces */
-                        fprintf(log, "*************************************\n");
-                        fprintf(log, "***       CORE%d TRACE DUMP        ***\n",
-                            syslink_firmware[i].proc_id);
-                        fprintf(log, "*************************************\n");
-                        for (i = (*proc_traces[id].widx + 1);
-                            i < (proc_traces[id].len - 8);
-                            i++) {
-                            fprintf(log, "%c",
-                                *(char *)((uint32_t)proc_traces[id].va + i));
-                        }
-                        for (i = 0; i < *proc_traces[id].widx; i++) {
-                            fprintf(log, "%c",
-                                *(char *)((uint32_t)proc_traces[id].va + i));
+                    if (firmware[id].procState == RUNNING_STATE) {
+                        if (proc_traces[id].va) {
+                            /* print traces */
+                            fprintf(log, "*************************************\n");
+                            fprintf(log, "***       CORE%d TRACE DUMP        ***\n",
+                                firmware[i].proc_id);
+                            fprintf(log, "*************************************\n");
+                            for (i = (*proc_traces[id].widx + 1);
+                                i < (proc_traces[id].len - 8);
+                                i++) {
+                                fprintf(log, "%c",
+                                    *(char *)((uint32_t)proc_traces[id].va + i));
+                            }
+                            for (i = 0; i < *proc_traces[id].widx; i++) {
+                                fprintf(log, "%c",
+                                    *(char *)((uint32_t)proc_traces[id].va + i));
+                            }
                         }
                     }
                 }
@@ -873,6 +1483,11 @@ int deinit_ipc(syslink_dev_t * dev, bool recover)
         }
     }
 
+    /* After printing trace, set all processor states to RESET */
+    for (id = 0; id < syslink_num_cores; id++) {
+        firmware[id].procState = RESET_STATE;
+    }
+
     status = Ipc_destroy();
     if (status < 0) {
         printf("Ipc_destroy() failed 0x%x", status);
@@ -926,7 +1541,7 @@ int main(int argc, char *argv[])
     syslink_dev_t * dev = NULL;
     thread_pool_attr_t tattr;
     int status;
-    int error = 0;
+    int error = EOK;
     sigset_t set;
     int channelid = 0;
     int c;
@@ -1000,6 +1615,8 @@ int main(int argc, char *argv[])
         syslink_firmware[syslink_num_cores].proc = argv [optind];
         syslink_firmware[syslink_num_cores].attachOnly =
             ((numAttach-- > 0) ? true : false);
+        syslink_firmware[syslink_num_cores].reload = true;
+        syslink_firmware[syslink_num_cores].freeString = false;
         syslink_firmware[syslink_num_cores++].firmware = argv [optind+1];
     }
 
@@ -1189,11 +1806,20 @@ done:
     GT_0trace(curTrace, GT_4CLASS, "IPC resource manager exiting \n");
 
     error = thread_pool_destroy(dev->tpool);
-    if (error < 0)
+    if (error < 0) {
         Osal_printf("IPC: thread_pool_destroy returned an error");
-    deinit_ipc(dev, FALSE);
+    }
+    deinit_ipc(dev, syslink_firmware, FALSE);
     deinit_devices(dev);
     free(dev);
 
-    return (EOK);
+    /* Free the abs path of firmware files if necessary */
+    for (i = 0; i < syslink_num_cores; i++) {
+        if ((syslink_firmware[i].freeString) &&
+            (syslink_firmware[i].firmware)) {
+            free(syslink_firmware[i].firmware);
+        }
+    }
+
+    return (error);
 }