fix authentication issues

Fix a few issues with the re-worked authentication.  If x11/wayland
auth fails, we want to preserve the original fd, and *not* call
drmOpen() again if dce_set_fd() has been called.

3 files changed, 26 insertions, 13 deletions

diff --git a/libdce-wayland.c b/libdce-wayland.c
index 891a9e4..093f7de 100644
--- a/libdce-wayland.c
+++ b/libdce-wayland.c
@@ -77,7 +77,7 @@ drm_handle_device(void *data, struct wl_drm *drm, const char *device)
         d->fd = open(device, O_RDWR | O_CLOEXEC);
         if (d->fd == -1) {
             ERROR("could not open %s: %m", device);
-            d->error = 1;
+            d->error = -1;
             return;
         }
     }
@@ -120,14 +120,19 @@ display_handle_global (struct wl_display *display, uint32_t id,
     }
 }
 
-int dce_auth_wayland(int fd)
+int dce_auth_wayland(int *fd)
 {
     struct display disp = {0};
     struct display *d = &disp;
 
-    d->fd = fd;
+    d->fd = *fd;
 
+    INFO("attempting to open wayland connection");
     d->display = wl_display_connect(NULL);
+    if (!d->display) {
+        ERROR("Could not open display");
+        return -1;
+    }
     wl_display_add_global_listener (d->display,
             display_handle_global, d);
 
@@ -141,5 +146,8 @@ int dce_auth_wayland(int fd)
     wl_display_flush (d->display);
     wl_display_disconnect (d->display);
 
-    return d->fd;
+    if (!d->error)
+        *fd = d->fd;
+
+    return d->error;
 }
diff --git a/libdce-x11.c b/libdce-x11.c
index be7ad03..d2d5056 100644
--- a/libdce-x11.c
+++ b/libdce-x11.c
@@ -51,13 +51,15 @@
 
 #include <ti/dce/dce_priv.h>
 
-int dce_auth_x11(int fd)
+int dce_auth_x11(int *pfd)
 {
     Display *dpy;
     Window root;
     drm_magic_t magic;
     int eventBase, errorBase, major, minor;
     char *driver, *device;
+    int ret = -1;
+    int fd = *pfd;
 
     INFO("attempting to open X11 connection");
     dpy = XOpenDisplay(NULL);
@@ -118,9 +120,12 @@ int dce_auth_x11(int fd)
         goto no_x11_free;
     }
 
+    ret = 0;
+    *pfd = fd;
+
 no_x11_free:
     XFree(driver);
     XFree(device);
 no_x11:
-    return fd;
+    return ret;
 }
diff --git a/libdce.c b/libdce.c
index a06fb2c..4cc641c 100644
--- a/libdce.c
+++ b/libdce.c
@@ -51,10 +51,10 @@
 #include <ti/sdo/ce/video2/videnc2.h>
 
 #ifdef HAVE_X11
-int dce_auth_x11(int fd);
+int dce_auth_x11(int *fd);
 #endif
 #ifdef HAVE_WAYLAND
-int dce_auth_wayland(int fd);
+int dce_auth_wayland(int *fd);
 #endif
 
 static int init(void);
@@ -430,20 +430,20 @@ static int init(void)
 
 #ifdef HAVE_X11
     if (!authenticated) {
-        fd = dce_auth_x11(fd);
-        if (fd != -1)
+        int ret = dce_auth_x11(&fd);
+        if (!ret)
             authenticated = 1;
     }
 #endif
 #ifdef HAVE_WAYLAND
     if (!authenticated) {
-        fd = dce_auth_wayland(fd);
-        if (fd != -1)
+        int ret = dce_auth_wayland(&fd);
+        if (!ret)
             authenticated = 1;
     }
 #endif
 
-    if (!authenticated) {
+    if ((fd == -1) && !authenticated) {
         INFO("no X11/wayland, fallback to opening DRM device directly");
         fd = drmOpen("omapdrm", "platform:omapdrm:00");
     }