From 0b918ec6f136a94a6f6bbb67fb378d2cf7bc2b58 Mon Sep 17 00:00:00 2001 From: HuJK Date: Thu, 30 Jul 2026 19:55:25 +0800 Subject: [PATCH 1/2] Use poll() instead of select() in the threaded client and listener loops select() cannot express a file descriptor numbered at or above FD_SETSIZE, which is 1024 and not adjustable -- fd_set is a fixed-size bitmap. On bionic FD_SET is FORTIFY-checked, so exceeding it is not a corrupted bitmap but an immediate abort: FORTIFY: FD_SET: file descriptor 1073 >= FD_SETSIZE 1024 Fatal signal 6 (SIGABRT) in tid 18004 (v_gpu), pid 15615 (ubuntu 26 drm2k) Reproduced from a running system, not theorised. This library is linked into crosvm, whose GPU device holds one dma-buf descriptor per guest buffer object; a game in a guest reaches a thousand of them, so an accepted VNC socket is routinely numbered above 1023. clientInput() then aborts the process on its first loop iteration, and on that platform a dying VMM resets the whole device. What the limit is on matters: the fd NUMBER, not the count. Raising RLIMIT_NOFILE does not help, and neither does closing descriptors after the fact -- once accept() has returned 1073, that client's thread is going to abort. Both reachable sites converted, with the mapping select() implies for a socket: readable -> POLLIN | POLLPRI exception -> POLLERR | POLLHUP | POLLNVAL (select's efds) writable -> POLLOUT EINTR is now retried rather than logged and treated as failure, which select() here did not distinguish either. listenerRun's accept branch matches the ready socket by fd rather than by a fixed pollfd index, because either listen socket may be absent and the slots are filled conditionally. --- src/libvncserver/main.c | 134 +++++++++++++++++++++++++++------------- 1 file changed, 90 insertions(+), 44 deletions(-) diff --git a/src/libvncserver/main.c b/src/libvncserver/main.c index 9e4bd3c..b1ed8ae 100644 --- a/src/libvncserver/main.c +++ b/src/libvncserver/main.c @@ -15,6 +15,10 @@ #endif #include #include +#ifndef WIN32 +#include +#include +#endif #include "private.h" #include @@ -528,42 +532,56 @@ clientInput(void *data) uintptr_t output_thread = _beginthread(clientOutput, 0, cl); #endif + /* DroidVM: poll(), not select(). + * + * select() cannot express a file descriptor numbered at or above FD_SETSIZE, which is 1024 and + * not adjustable -- fd_set is a fixed-size bitmap. On bionic, FD_SET is FORTIFY-checked, so + * exceeding it is not a silently corrupted bitmap but an immediate abort: + * + * FORTIFY: FD_SET: file descriptor 1073 >= FD_SETSIZE 1024 + * Fatal signal 6 (SIGABRT) in tid ... (v_gpu), pid ... (crosvm) + * + * That is not hypothetical here. This library is linked into a VMM whose GPU device holds one + * dma-buf descriptor per guest buffer object -- over a thousand of them in a running game -- + * so an accepted VNC socket is routinely numbered above 1023. The abort then takes the VMM + * down, and on this platform a dying VMM resets the whole phone. + * + * Note what the limit is on: the fd NUMBER, not how many are open. Raising RLIMIT_NOFILE does + * not help, and neither does closing other descriptors, once a high number has been handed out. + * poll() has no such ceiling, and the mapping is exact: + * readable -> POLLIN | POLLPRI + * exception -> POLLERR | POLLHUP | POLLNVAL (select's efds, for a socket) + * writable -> POLLOUT + */ while (cl->state != RFB_SHUTDOWN) { - fd_set rfds, wfds, efds; - struct timeval tv; - int n; + struct pollfd pfd[2]; + int npfd, n; if (cl->sock == RFB_INVALID_SOCKET) { /* Client has disconnected. */ break; } - FD_ZERO(&rfds); - FD_SET(cl->sock, &rfds); -#ifndef WIN32 - FD_SET(cl->pipe_notify_client_thread[0], &rfds); -#endif - FD_ZERO(&efds); - FD_SET(cl->sock, &efds); - + pfd[0].fd = cl->sock; + pfd[0].events = POLLIN | POLLPRI; /* Are we transferring a file in the background? */ - FD_ZERO(&wfds); if ((cl->fileTransfer.fd!=-1) && (cl->fileTransfer.sending==1)) - FD_SET(cl->sock, &wfds); - + pfd[0].events |= POLLOUT; + pfd[0].revents = 0; + npfd = 1; #ifndef WIN32 - int nfds = cl->pipe_notify_client_thread[0] > cl->sock ? cl->pipe_notify_client_thread[0] : cl->sock; -#else - int nfds = cl->sock; + pfd[1].fd = cl->pipe_notify_client_thread[0]; + pfd[1].events = POLLIN; + pfd[1].revents = 0; + npfd = 2; #endif - tv.tv_sec = 60; /* 1 minute */ - tv.tv_usec = 0; - - n = select(nfds + 1, &rfds, &wfds, &efds, &tv); + n = poll(pfd, npfd, 60 * 1000); /* 1 minute */ if (n < 0) { - rfbLogPerror("ReadExact: select"); + if (errno == EINTR) + continue; + rfbLogPerror("ReadExact: poll"); break; } if (n == 0) /* timeout */ @@ -573,7 +591,7 @@ clientInput(void *data) } #ifndef WIN32 - if (FD_ISSET(cl->pipe_notify_client_thread[0], &rfds)) + if (pfd[1].revents & (POLLIN | POLLERR | POLLHUP | POLLNVAL)) { /* Reset the pipe */ char buf; @@ -583,10 +601,10 @@ clientInput(void *data) #endif /* We have some space on the transmit queue, send some data */ - if (FD_ISSET(cl->sock, &wfds)) + if (pfd[0].revents & POLLOUT) rfbSendFileTransferChunk(cl); - if (FD_ISSET(cl->sock, &rfds) || FD_ISSET(cl->sock, &efds)) + if (pfd[0].revents & (POLLIN | POLLPRI | POLLERR | POLLHUP | POLLNVAL)) { #ifdef LIBVNCSERVER_WITH_WEBSOCKETS do { @@ -622,8 +640,8 @@ listenerRun(void *data) struct sockaddr_storage peer; rfbClientPtr cl = NULL; socklen_t len; - fd_set listen_fds; /* temp file descriptor list for select() */ - struct timeval tv; + struct pollfd lpfd[3]; /* listenSock, listen6Sock, notify pipe */ + int npfd, notify_idx = -1; /* Only checking socket state here and not using rfbIsActive() @@ -638,25 +656,39 @@ listenerRun(void *data) while (screen->socketState != RFB_SOCKET_SHUTDOWN) { client_fd = -1; cl = NULL; - FD_ZERO(&listen_fds); - if(screen->listenSock != RFB_INVALID_SOCKET) - FD_SET(screen->listenSock, &listen_fds); - if(screen->listen6Sock != RFB_INVALID_SOCKET) - FD_SET(screen->listen6Sock, &listen_fds); + /* poll() here for the same reason as clientInput above: the listen sockets and the notify + * pipe are opened early and are usually low-numbered, but nothing guarantees that, and a + * single high fd is an abort rather than a degradation. */ + npfd = 0; + if(screen->listenSock != RFB_INVALID_SOCKET) { + lpfd[npfd].fd = screen->listenSock; + lpfd[npfd].events = POLLIN; + lpfd[npfd].revents = 0; + npfd++; + } + if(screen->listen6Sock != RFB_INVALID_SOCKET) { + lpfd[npfd].fd = screen->listen6Sock; + lpfd[npfd].events = POLLIN; + lpfd[npfd].revents = 0; + npfd++; + } #ifndef WIN32 - FD_SET(screen->pipe_notify_listener_thread[0], &listen_fds); - screen->maxFd = rfbMax(screen->maxFd, screen->pipe_notify_listener_thread[0]); + notify_idx = npfd; + lpfd[npfd].fd = screen->pipe_notify_listener_thread[0]; + lpfd[npfd].events = POLLIN; + lpfd[npfd].revents = 0; + npfd++; #endif - tv.tv_sec = 0; - tv.tv_usec = screen->select_timeout_usec; - if (select(screen->maxFd+1, &listen_fds, NULL, NULL, &tv) == -1) { - rfbLogPerror("listenerRun: error in select"); + if (poll(lpfd, npfd, (int)(screen->select_timeout_usec / 1000)) == -1) { + if (errno == EINTR) + continue; + rfbLogPerror("listenerRun: error in poll"); return THREAD_ROUTINE_RETURN_VALUE; } #ifndef WIN32 - if (FD_ISSET(screen->pipe_notify_listener_thread[0], &listen_fds)) + if (lpfd[notify_idx].revents & (POLLIN | POLLERR | POLLHUP | POLLNVAL)) { /* Reset the pipe */ char buf; @@ -666,12 +698,26 @@ listenerRun(void *data) } #endif - /* If there is something on the listening sockets, handle new connections */ + /* If there is something on the listening sockets, handle new connections. + * The pollfd slots were filled in the same order the sockets are tested here, so match on + * fd rather than on a fixed index -- either listen socket may be absent. */ len = sizeof (peer); - if (screen->listenSock != RFB_INVALID_SOCKET && FD_ISSET(screen->listenSock, &listen_fds)) - client_fd = accept(screen->listenSock, (struct sockaddr*)&peer, &len); - else if (screen->listen6Sock != RFB_INVALID_SOCKET && FD_ISSET(screen->listen6Sock, &listen_fds)) - client_fd = accept(screen->listen6Sock, (struct sockaddr*)&peer, &len); + { + int i, ready4 = 0, ready6 = 0; + + for (i = 0; i < npfd; i++) { + if (!(lpfd[i].revents & (POLLIN | POLLERR | POLLHUP | POLLNVAL))) + continue; + if (screen->listenSock != RFB_INVALID_SOCKET && lpfd[i].fd == screen->listenSock) + ready4 = 1; + else if (screen->listen6Sock != RFB_INVALID_SOCKET && lpfd[i].fd == screen->listen6Sock) + ready6 = 1; + } + if (ready4) + client_fd = accept(screen->listenSock, (struct sockaddr*)&peer, &len); + else if (ready6) + client_fd = accept(screen->listen6Sock, (struct sockaddr*)&peer, &len); + } if(client_fd >= 0) cl = rfbNewClient(screen,client_fd); From 6ab9b320cca502d2abec740eeb89aa3ef14c0f24 Mon Sep 17 00:00:00 2001 From: HuJK Date: Thu, 30 Jul 2026 20:23:26 +0800 Subject: [PATCH 2/2] Handle file descriptors at or above FD_SETSIZE instead of aborting on them fd_set cannot represent a descriptor numbered at or above FD_SETSIZE, which is 1024 and fixed -- fds_bits is a fixed-size array. On bionic FD_SET/FD_CLR/FD_ISSET are FORTIFY-checked, so exceeding it is not a corrupted bitmap but an immediate abort: FORTIFY: FD_SET: file descriptor 1073 >= FD_SETSIZE 1024 Fatal signal 6 (SIGABRT) in tid 18004 (v_gpu), pid 15615 Observed on a running system. This library is linked into a VMM whose GPU device holds several dma-buf descriptors per guest buffer object, so a few hundred live objects push the fd table past 1023 and an accepted VNC socket lands above it. The abort takes the VMM down, and on that platform a dying VMM resets the whole device. What the limit is on matters: the fd NUMBER, not the count -- RLIMIT_NOFILE and closing other descriptors are both irrelevant once accept() has returned 1073. Two kinds of site, treated differently. Real I/O waits become poll(), which has no such ceiling: clientInput and listenerRun in main.c, and rfbReadExactTimeout and rfbWriteExact in sockets.c. rfbWriteExact is the one that matters most in practice -- it is reached on every back-pressured framebuffer write. Mapping is the one select() implies for a socket: POLLIN|POLLPRI readable, POLLERR|POLLHUP|POLLNVAL exception, POLLOUT writable. EINTR is now retried rather than reported as an error. allFds/maxFd bookkeeping is guarded instead: rfbNewTCPOrUDPClient (the actual crash site -- it runs on the listener thread right after accept(), before any client thread exists), rfbClientConnectionGone, and rfbCloseClient. Nothing reads allFds or maxFd except rfbCheckFds and rfbProcessNewConnection, which are only reached from rfbProcessEvents, so with a listener thread they are write-only and skipping a high fd costs nothing. On the rfbProcessEvents path a skipped fd would not be serviced; that is a real limitation, which is why it logs once rather than passing silently. --- src/libvncserver/rfbserver.c | 50 ++++++++++++++++++++++++++++++++++-- src/libvncserver/sockets.c | 43 +++++++++++++++++++++---------- 2 files changed, 77 insertions(+), 16 deletions(-) diff --git a/src/libvncserver/rfbserver.c b/src/libvncserver/rfbserver.c index 7bf8368..82fd18f 100644 --- a/src/libvncserver/rfbserver.c +++ b/src/libvncserver/rfbserver.c @@ -115,6 +115,52 @@ static const int tight2turbo_subsamp[10] = { }; #endif +/* DroidVM: FD_SETSIZE-safe bookkeeping helpers. + * + * fd_set cannot represent a descriptor numbered at or above FD_SETSIZE (1024, fixed -- fds_bits is + * a fixed-size array), and on bionic FD_SET/FD_CLR/FD_ISSET are FORTIFY-checked, so exceeding it + * aborts the process rather than corrupting a bitmap: + * + * FORTIFY: FD_SET: file descriptor 1073 >= FD_SETSIZE 1024 + * + * That is reachable here. This library is linked into a VMM whose GPU device holds several dma-buf + * descriptors per guest buffer object, so an accepted VNC socket is routinely numbered above 1023, + * and rfbNewTCPOrUDPClient's FD_SET on it is the first thing to touch that number. The abort takes + * the VMM down, and on that platform a dying VMM resets the device. + * + * allFds and maxFd are only ever READ by rfbCheckFds/rfbProcessNewConnection, i.e. only on the + * rfbProcessEvents polling path. In background mode (rfbRunEventLoop with a listener thread, which + * is how the VMM drives this) they are write-only, so skipping a high fd loses nothing there. On + * the polling path a skipped fd would not be serviced -- that is a real limitation, and it is why + * this warns rather than failing silently. + */ +static int rfbFdSetSafe(int fd, fd_set *set, const char *what) +{ + if (fd < 0 || fd >= FD_SETSIZE) { + static int warned = 0; + if (!warned) { + warned = 1; + rfbLog("%s: fd %d is at or above FD_SETSIZE %d; not tracking it in the fd_set. " + "This is harmless with a listener thread (nothing reads the set) but the " + "rfbProcessEvents path would not service it.\n", what, fd, (int)FD_SETSIZE); + } + return 0; + } + FD_SET(fd, set); + return 1; +} + +static void rfbFdClrSafe(int fd, fd_set *set) +{ + if (fd >= 0 && fd < FD_SETSIZE) + FD_CLR(fd, set); +} + +static int rfbFdIsSetSafe(int fd, fd_set *set) +{ + return (fd >= 0 && fd < FD_SETSIZE) ? FD_ISSET(fd, set) : 0; +} + static void rfbProcessClientProtocolVersion(rfbClientPtr cl); static void rfbProcessClientNormalMessage(rfbClientPtr cl); static void rfbProcessClientInitMessage(rfbClientPtr cl); @@ -369,7 +415,7 @@ rfbNewTCPOrUDPClient(rfbScreenInfoPtr rfbScreen, rfbLogPerror("setsockopt failed: can't set TCP_NODELAY flag, non TCP socket?"); } - FD_SET(sock,&(rfbScreen->allFds)); + rfbFdSetSafe(sock, &(rfbScreen->allFds), "rfbNewClient"); rfbScreen->maxFd = rfbMax(sock,rfbScreen->maxFd); #endif @@ -601,7 +647,7 @@ rfbClientConnectionGone(rfbClientPtr cl) free(cl->afterEncBuf); if(cl->sock != RFB_INVALID_SOCKET) - FD_CLR(cl->sock,&(cl->screen->allFds)); + rfbFdClrSafe(cl->sock, &(cl->screen->allFds)); cl->clientGoneHook(cl); diff --git a/src/libvncserver/sockets.c b/src/libvncserver/sockets.c index c18493c..273a787 100644 --- a/src/libvncserver/sockets.c +++ b/src/libvncserver/sockets.c @@ -52,6 +52,10 @@ #endif #include +#ifndef WIN32 +#include +#include +#endif #ifdef LIBVNCSERVER_HAVE_SYS_TYPES_H #include @@ -561,10 +565,14 @@ rfbCloseClient(rfbClientPtr cl) if (cl->sock != RFB_INVALID_SOCKET) #endif { - /* Remove client sock from allFds and adapt maxFd */ - FD_CLR(cl->sock,&(cl->screen->allFds)); + /* Remove client sock from allFds and adapt maxFd. + * Guarded rather than polled: this is bookkeeping, nothing is being waited on. A socket + * above FD_SETSIZE was never added (see rfbserver.c), so there is nothing to clear. */ + if (cl->sock >= 0 && cl->sock < FD_SETSIZE) + FD_CLR(cl->sock,&(cl->screen->allFds)); if(cl->sock==cl->screen->maxFd) while(cl->screen->maxFd>0 + && cl->screen->maxFd < FD_SETSIZE && !FD_ISSET(cl->screen->maxFd,&(cl->screen->allFds))) cl->screen->maxFd--; #ifdef LIBVNCSERVER_WITH_WEBSOCKETS @@ -658,6 +666,7 @@ const uint8_t *fuzz_data; int rfbReadExactTimeout(rfbClientPtr cl, char* buf, int len, int timeout) { + struct pollfd pfd; #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION if (fuzz_offset + len <= fuzz_size) { memcpy(buf, fuzz_data + fuzz_offset, len); @@ -713,17 +722,21 @@ rfbReadExactTimeout(rfbClientPtr cl, char* buf, int len, int timeout) continue; } #endif - FD_ZERO(&fds); - FD_SET(sock, &fds); - tv.tv_sec = timeout / 1000; - tv.tv_usec = (timeout % 1000) * 1000; - n = select(sock+1, &fds, NULL, &fds, &tv); + /* DroidVM: poll(), not select() -- see the note in rfbserver.c. A client socket here + * can be numbered above FD_SETSIZE, and FD_SET on it aborts the process under bionic's + * FORTIFY. This is on the read path of every partial RFB message. */ + pfd.fd = sock; + pfd.events = POLLIN | POLLPRI; + pfd.revents = 0; + n = poll(&pfd, 1, timeout); if (n < 0) { - rfbLogPerror("ReadExact: select"); + if (errno == EINTR) + continue; + rfbLogPerror("ReadExact: poll"); return n; } if (n == 0) { - rfbErr("ReadExact: select timeout\n"); + rfbErr("ReadExact: poll timeout\n"); errno = ETIMEDOUT; return -1; } @@ -844,6 +857,7 @@ rfbWriteExact(rfbClientPtr cl, const char *buf, int len) { + struct pollfd pfd; #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION return 1; #endif @@ -922,11 +936,12 @@ rfbWriteExact(rfbClientPtr cl, need to do this because select doesn't necessarily return immediately when the other end has gone away */ - FD_ZERO(&fds); - FD_SET(sock, &fds); - tv.tv_sec = 5; - tv.tv_usec = 0; - n = select(sock+1, NULL, &fds, NULL /* &fds */, &tv); + /* DroidVM: poll(), not select(). This one is hit on every back-pressured update, so + * at any real resolution it is the site a high-numbered socket reaches first. */ + pfd.fd = sock; + pfd.events = POLLOUT; + pfd.revents = 0; + n = poll(&pfd, 1, 5 * 1000); if (n < 0) { #ifdef WIN32 errno=WSAGetLastError();