From 082e869d0e6e2441b810fc9877bce1acd30f9393 Mon Sep 17 00:00:00 2001 From: HuJK Date: Sun, 30 Aug 2026 17:21:09 +0800 Subject: [PATCH] sockets: survive a descriptor numbered at or above FD_SETSIZE fd_set cannot represent a descriptor at or above FD_SETSIZE, which is 1024 and fixed: fds_bits is a fixed-size array. On bionic FD_SET/FD_CLR/FD_ISSET are FORTIFY-checked, so exceeding it is not a quietly corrupted bitmap but an immediate abort: FORTIFY: FD_SET: file descriptor 1073 >= FD_SETSIZE 1024 Fatal signal 6 (SIGABRT) in tid 18004 (v_gpu), pid 15615 Observed on a running system. This library gets linked into a VMM whose GPU device holds several dma-buf descriptors per guest buffer object, so a few hundred live objects push the fd table past 1023 and an accepted VNC socket lands above it. The abort takes the VMM down, and on that platform a dying VMM resets the device. What the limit is on is worth stating: the descriptor NUMBER, not how many are open. Raising RLIMIT_NOFILE does not help and neither does closing other descriptors, because accept() has already returned 1073. Two kinds of site, treated differently. Real I/O waits move to poll(), which has no such ceiling -- clientInput and listenerRun in main.c, rfbReadExactTimeout and rfbWriteExact in sockets.c. rfbWriteExact matters most in practice: it is reached on every back-pressured write. The remaining select() sites, which multiplex over the whole client set, keep select() but skip descriptors the set cannot hold rather than aborting on them, so one unrepresentable client costs that client and not the process. --- src/libvncserver/main.c | 134 +++++++++++++++++++++++------------ src/libvncserver/rfbserver.c | 50 ++++++++++++- src/libvncserver/sockets.c | 43 +++++++---- 3 files changed, 167 insertions(+), 60 deletions(-) diff --git a/src/libvncserver/main.c b/src/libvncserver/main.c index 9e4bd3c..b1ed8ae 100644 --- a/src/libvncserver/main.c +++ b/src/libvncserver/main.c @@ -15,6 +15,10 @@ #endif #include #include +#ifndef WIN32 +#include +#include +#endif #include "private.h" #include @@ -528,42 +532,56 @@ clientInput(void *data) uintptr_t output_thread = _beginthread(clientOutput, 0, cl); #endif + /* DroidVM: poll(), not select(). + * + * select() cannot express a file descriptor numbered at or above FD_SETSIZE, which is 1024 and + * not adjustable -- fd_set is a fixed-size bitmap. On bionic, FD_SET is FORTIFY-checked, so + * exceeding it is not a silently corrupted bitmap but an immediate abort: + * + * FORTIFY: FD_SET: file descriptor 1073 >= FD_SETSIZE 1024 + * Fatal signal 6 (SIGABRT) in tid ... (v_gpu), pid ... (crosvm) + * + * That is not hypothetical here. This library is linked into a VMM whose GPU device holds one + * dma-buf descriptor per guest buffer object -- over a thousand of them in a running game -- + * so an accepted VNC socket is routinely numbered above 1023. The abort then takes the VMM + * down, and on this platform a dying VMM resets the whole phone. + * + * Note what the limit is on: the fd NUMBER, not how many are open. Raising RLIMIT_NOFILE does + * not help, and neither does closing other descriptors, once a high number has been handed out. + * poll() has no such ceiling, and the mapping is exact: + * readable -> POLLIN | POLLPRI + * exception -> POLLERR | POLLHUP | POLLNVAL (select's efds, for a socket) + * writable -> POLLOUT + */ while (cl->state != RFB_SHUTDOWN) { - fd_set rfds, wfds, efds; - struct timeval tv; - int n; + struct pollfd pfd[2]; + int npfd, n; if (cl->sock == RFB_INVALID_SOCKET) { /* Client has disconnected. */ break; } - FD_ZERO(&rfds); - FD_SET(cl->sock, &rfds); -#ifndef WIN32 - FD_SET(cl->pipe_notify_client_thread[0], &rfds); -#endif - FD_ZERO(&efds); - FD_SET(cl->sock, &efds); - + pfd[0].fd = cl->sock; + pfd[0].events = POLLIN | POLLPRI; /* Are we transferring a file in the background? */ - FD_ZERO(&wfds); if ((cl->fileTransfer.fd!=-1) && (cl->fileTransfer.sending==1)) - FD_SET(cl->sock, &wfds); - + pfd[0].events |= POLLOUT; + pfd[0].revents = 0; + npfd = 1; #ifndef WIN32 - int nfds = cl->pipe_notify_client_thread[0] > cl->sock ? cl->pipe_notify_client_thread[0] : cl->sock; -#else - int nfds = cl->sock; + pfd[1].fd = cl->pipe_notify_client_thread[0]; + pfd[1].events = POLLIN; + pfd[1].revents = 0; + npfd = 2; #endif - tv.tv_sec = 60; /* 1 minute */ - tv.tv_usec = 0; - - n = select(nfds + 1, &rfds, &wfds, &efds, &tv); + n = poll(pfd, npfd, 60 * 1000); /* 1 minute */ if (n < 0) { - rfbLogPerror("ReadExact: select"); + if (errno == EINTR) + continue; + rfbLogPerror("ReadExact: poll"); break; } if (n == 0) /* timeout */ @@ -573,7 +591,7 @@ clientInput(void *data) } #ifndef WIN32 - if (FD_ISSET(cl->pipe_notify_client_thread[0], &rfds)) + if (pfd[1].revents & (POLLIN | POLLERR | POLLHUP | POLLNVAL)) { /* Reset the pipe */ char buf; @@ -583,10 +601,10 @@ clientInput(void *data) #endif /* We have some space on the transmit queue, send some data */ - if (FD_ISSET(cl->sock, &wfds)) + if (pfd[0].revents & POLLOUT) rfbSendFileTransferChunk(cl); - if (FD_ISSET(cl->sock, &rfds) || FD_ISSET(cl->sock, &efds)) + if (pfd[0].revents & (POLLIN | POLLPRI | POLLERR | POLLHUP | POLLNVAL)) { #ifdef LIBVNCSERVER_WITH_WEBSOCKETS do { @@ -622,8 +640,8 @@ listenerRun(void *data) struct sockaddr_storage peer; rfbClientPtr cl = NULL; socklen_t len; - fd_set listen_fds; /* temp file descriptor list for select() */ - struct timeval tv; + struct pollfd lpfd[3]; /* listenSock, listen6Sock, notify pipe */ + int npfd, notify_idx = -1; /* Only checking socket state here and not using rfbIsActive() @@ -638,25 +656,39 @@ listenerRun(void *data) while (screen->socketState != RFB_SOCKET_SHUTDOWN) { client_fd = -1; cl = NULL; - FD_ZERO(&listen_fds); - if(screen->listenSock != RFB_INVALID_SOCKET) - FD_SET(screen->listenSock, &listen_fds); - if(screen->listen6Sock != RFB_INVALID_SOCKET) - FD_SET(screen->listen6Sock, &listen_fds); + /* poll() here for the same reason as clientInput above: the listen sockets and the notify + * pipe are opened early and are usually low-numbered, but nothing guarantees that, and a + * single high fd is an abort rather than a degradation. */ + npfd = 0; + if(screen->listenSock != RFB_INVALID_SOCKET) { + lpfd[npfd].fd = screen->listenSock; + lpfd[npfd].events = POLLIN; + lpfd[npfd].revents = 0; + npfd++; + } + if(screen->listen6Sock != RFB_INVALID_SOCKET) { + lpfd[npfd].fd = screen->listen6Sock; + lpfd[npfd].events = POLLIN; + lpfd[npfd].revents = 0; + npfd++; + } #ifndef WIN32 - FD_SET(screen->pipe_notify_listener_thread[0], &listen_fds); - screen->maxFd = rfbMax(screen->maxFd, screen->pipe_notify_listener_thread[0]); + notify_idx = npfd; + lpfd[npfd].fd = screen->pipe_notify_listener_thread[0]; + lpfd[npfd].events = POLLIN; + lpfd[npfd].revents = 0; + npfd++; #endif - tv.tv_sec = 0; - tv.tv_usec = screen->select_timeout_usec; - if (select(screen->maxFd+1, &listen_fds, NULL, NULL, &tv) == -1) { - rfbLogPerror("listenerRun: error in select"); + if (poll(lpfd, npfd, (int)(screen->select_timeout_usec / 1000)) == -1) { + if (errno == EINTR) + continue; + rfbLogPerror("listenerRun: error in poll"); return THREAD_ROUTINE_RETURN_VALUE; } #ifndef WIN32 - if (FD_ISSET(screen->pipe_notify_listener_thread[0], &listen_fds)) + if (lpfd[notify_idx].revents & (POLLIN | POLLERR | POLLHUP | POLLNVAL)) { /* Reset the pipe */ char buf; @@ -666,12 +698,26 @@ listenerRun(void *data) } #endif - /* If there is something on the listening sockets, handle new connections */ + /* If there is something on the listening sockets, handle new connections. + * The pollfd slots were filled in the same order the sockets are tested here, so match on + * fd rather than on a fixed index -- either listen socket may be absent. */ len = sizeof (peer); - if (screen->listenSock != RFB_INVALID_SOCKET && FD_ISSET(screen->listenSock, &listen_fds)) - client_fd = accept(screen->listenSock, (struct sockaddr*)&peer, &len); - else if (screen->listen6Sock != RFB_INVALID_SOCKET && FD_ISSET(screen->listen6Sock, &listen_fds)) - client_fd = accept(screen->listen6Sock, (struct sockaddr*)&peer, &len); + { + int i, ready4 = 0, ready6 = 0; + + for (i = 0; i < npfd; i++) { + if (!(lpfd[i].revents & (POLLIN | POLLERR | POLLHUP | POLLNVAL))) + continue; + if (screen->listenSock != RFB_INVALID_SOCKET && lpfd[i].fd == screen->listenSock) + ready4 = 1; + else if (screen->listen6Sock != RFB_INVALID_SOCKET && lpfd[i].fd == screen->listen6Sock) + ready6 = 1; + } + if (ready4) + client_fd = accept(screen->listenSock, (struct sockaddr*)&peer, &len); + else if (ready6) + client_fd = accept(screen->listen6Sock, (struct sockaddr*)&peer, &len); + } if(client_fd >= 0) cl = rfbNewClient(screen,client_fd); diff --git a/src/libvncserver/rfbserver.c b/src/libvncserver/rfbserver.c index 7bf8368..82fd18f 100644 --- a/src/libvncserver/rfbserver.c +++ b/src/libvncserver/rfbserver.c @@ -115,6 +115,52 @@ static const int tight2turbo_subsamp[10] = { }; #endif +/* DroidVM: FD_SETSIZE-safe bookkeeping helpers. + * + * fd_set cannot represent a descriptor numbered at or above FD_SETSIZE (1024, fixed -- fds_bits is + * a fixed-size array), and on bionic FD_SET/FD_CLR/FD_ISSET are FORTIFY-checked, so exceeding it + * aborts the process rather than corrupting a bitmap: + * + * FORTIFY: FD_SET: file descriptor 1073 >= FD_SETSIZE 1024 + * + * That is reachable here. This library is linked into a VMM whose GPU device holds several dma-buf + * descriptors per guest buffer object, so an accepted VNC socket is routinely numbered above 1023, + * and rfbNewTCPOrUDPClient's FD_SET on it is the first thing to touch that number. The abort takes + * the VMM down, and on that platform a dying VMM resets the device. + * + * allFds and maxFd are only ever READ by rfbCheckFds/rfbProcessNewConnection, i.e. only on the + * rfbProcessEvents polling path. In background mode (rfbRunEventLoop with a listener thread, which + * is how the VMM drives this) they are write-only, so skipping a high fd loses nothing there. On + * the polling path a skipped fd would not be serviced -- that is a real limitation, and it is why + * this warns rather than failing silently. + */ +static int rfbFdSetSafe(int fd, fd_set *set, const char *what) +{ + if (fd < 0 || fd >= FD_SETSIZE) { + static int warned = 0; + if (!warned) { + warned = 1; + rfbLog("%s: fd %d is at or above FD_SETSIZE %d; not tracking it in the fd_set. " + "This is harmless with a listener thread (nothing reads the set) but the " + "rfbProcessEvents path would not service it.\n", what, fd, (int)FD_SETSIZE); + } + return 0; + } + FD_SET(fd, set); + return 1; +} + +static void rfbFdClrSafe(int fd, fd_set *set) +{ + if (fd >= 0 && fd < FD_SETSIZE) + FD_CLR(fd, set); +} + +static int rfbFdIsSetSafe(int fd, fd_set *set) +{ + return (fd >= 0 && fd < FD_SETSIZE) ? FD_ISSET(fd, set) : 0; +} + static void rfbProcessClientProtocolVersion(rfbClientPtr cl); static void rfbProcessClientNormalMessage(rfbClientPtr cl); static void rfbProcessClientInitMessage(rfbClientPtr cl); @@ -369,7 +415,7 @@ rfbNewTCPOrUDPClient(rfbScreenInfoPtr rfbScreen, rfbLogPerror("setsockopt failed: can't set TCP_NODELAY flag, non TCP socket?"); } - FD_SET(sock,&(rfbScreen->allFds)); + rfbFdSetSafe(sock, &(rfbScreen->allFds), "rfbNewClient"); rfbScreen->maxFd = rfbMax(sock,rfbScreen->maxFd); #endif @@ -601,7 +647,7 @@ rfbClientConnectionGone(rfbClientPtr cl) free(cl->afterEncBuf); if(cl->sock != RFB_INVALID_SOCKET) - FD_CLR(cl->sock,&(cl->screen->allFds)); + rfbFdClrSafe(cl->sock, &(cl->screen->allFds)); cl->clientGoneHook(cl); diff --git a/src/libvncserver/sockets.c b/src/libvncserver/sockets.c index c18493c..273a787 100644 --- a/src/libvncserver/sockets.c +++ b/src/libvncserver/sockets.c @@ -52,6 +52,10 @@ #endif #include +#ifndef WIN32 +#include +#include +#endif #ifdef LIBVNCSERVER_HAVE_SYS_TYPES_H #include @@ -561,10 +565,14 @@ rfbCloseClient(rfbClientPtr cl) if (cl->sock != RFB_INVALID_SOCKET) #endif { - /* Remove client sock from allFds and adapt maxFd */ - FD_CLR(cl->sock,&(cl->screen->allFds)); + /* Remove client sock from allFds and adapt maxFd. + * Guarded rather than polled: this is bookkeeping, nothing is being waited on. A socket + * above FD_SETSIZE was never added (see rfbserver.c), so there is nothing to clear. */ + if (cl->sock >= 0 && cl->sock < FD_SETSIZE) + FD_CLR(cl->sock,&(cl->screen->allFds)); if(cl->sock==cl->screen->maxFd) while(cl->screen->maxFd>0 + && cl->screen->maxFd < FD_SETSIZE && !FD_ISSET(cl->screen->maxFd,&(cl->screen->allFds))) cl->screen->maxFd--; #ifdef LIBVNCSERVER_WITH_WEBSOCKETS @@ -658,6 +666,7 @@ const uint8_t *fuzz_data; int rfbReadExactTimeout(rfbClientPtr cl, char* buf, int len, int timeout) { + struct pollfd pfd; #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION if (fuzz_offset + len <= fuzz_size) { memcpy(buf, fuzz_data + fuzz_offset, len); @@ -713,17 +722,21 @@ rfbReadExactTimeout(rfbClientPtr cl, char* buf, int len, int timeout) continue; } #endif - FD_ZERO(&fds); - FD_SET(sock, &fds); - tv.tv_sec = timeout / 1000; - tv.tv_usec = (timeout % 1000) * 1000; - n = select(sock+1, &fds, NULL, &fds, &tv); + /* DroidVM: poll(), not select() -- see the note in rfbserver.c. A client socket here + * can be numbered above FD_SETSIZE, and FD_SET on it aborts the process under bionic's + * FORTIFY. This is on the read path of every partial RFB message. */ + pfd.fd = sock; + pfd.events = POLLIN | POLLPRI; + pfd.revents = 0; + n = poll(&pfd, 1, timeout); if (n < 0) { - rfbLogPerror("ReadExact: select"); + if (errno == EINTR) + continue; + rfbLogPerror("ReadExact: poll"); return n; } if (n == 0) { - rfbErr("ReadExact: select timeout\n"); + rfbErr("ReadExact: poll timeout\n"); errno = ETIMEDOUT; return -1; } @@ -844,6 +857,7 @@ rfbWriteExact(rfbClientPtr cl, const char *buf, int len) { + struct pollfd pfd; #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION return 1; #endif @@ -922,11 +936,12 @@ rfbWriteExact(rfbClientPtr cl, need to do this because select doesn't necessarily return immediately when the other end has gone away */ - FD_ZERO(&fds); - FD_SET(sock, &fds); - tv.tv_sec = 5; - tv.tv_usec = 0; - n = select(sock+1, NULL, &fds, NULL /* &fds */, &tv); + /* DroidVM: poll(), not select(). This one is hit on every back-pressured update, so + * at any real resolution it is the site a high-numbered socket reaches first. */ + pfd.fd = sock; + pfd.events = POLLOUT; + pfd.revents = 0; + n = poll(&pfd, 1, 5 * 1000); if (n < 0) { #ifdef WIN32 errno=WSAGetLastError();