int link_read(struct link *link, char *data, size_t count, time_t stoptime) { ssize_t total = 0; ssize_t chunk = 0; if(count == 0) return 0; /* If this is a small read, attempt to fill the buffer */ if(count < BUFFER_SIZE) { chunk = fill_buffer(link, stoptime); if(chunk <= 0) return chunk; } /* Then, satisfy the read from the buffer, if any. */ if(link->buffer_length > 0) { chunk = MIN(link->buffer_length, count); memcpy(data, &link->buffer[link->buffer_start], chunk); data += chunk; total += chunk; count -= chunk; link->buffer_start += chunk; link->buffer_length -= chunk; } /* Otherwise, pull it all off the wire. */ while(count > 0) { chunk = read(link->fd, data, count); if(chunk < 0) { if(errno_is_temporary(errno)) { if(link_sleep(link, stoptime, 1, 0)) { continue; } else { break; } } else { break; } } else if(chunk == 0) { break; } else { total += chunk; count -= chunk; data += chunk; } } if(total > 0) { return total; } else { if(chunk == 0) { return 0; } else { return -1; } } }
static ssize_t fill_buffer(struct link *link, time_t stoptime) { if(link->buffer_length > 0) return link->buffer_length; while(1) { ssize_t chunk = read(link->fd, link->buffer, sizeof(link->buffer)); if(chunk > 0) { link->read += chunk; link->buffer_start = link->buffer; link->buffer_length = chunk; return chunk; } else if(chunk == 0) { link->buffer_start = link->buffer; link->buffer_length = 0; return 0; } else { if(errno_is_temporary(errno)) { if(link_sleep(link, stoptime, 1, 0)) { continue; } else { return -1; } } else { return -1; } } } }
static int fill_buffer(struct link *link, time_t stoptime) { int chunk; if(link->buffer_length > 0) return link->buffer_length; while(1) { chunk = read(link->fd, link->buffer, BUFFER_SIZE); if(chunk > 0) { link->buffer_start = 0; link->buffer_length = chunk; return chunk; } else if(chunk == 0) { link->buffer_start = 0; link->buffer_length = 0; return 0; } else { if(errno_is_temporary(errno)) { if(link_sleep(link, stoptime, 1, 0)) { continue; } else { return -1; } } else { return -1; } } } }
static int link_internal_sleep(struct link *link, struct timeval *timeout, int reading, int writing) { int result; struct pollfd pfd; int msec = (timeout->tv_sec * 1000.0) + (timeout->tv_usec/1000.0); while (1) { pfd.fd = link->fd; pfd.revents = 0; if (reading) pfd.events = POLLIN; if (writing) pfd.events = POLLOUT; result = poll(&pfd, 1, msec); if (result > 0) { if (reading && (pfd.revents & POLLIN)) { return 1; } if (writing && (pfd.revents & POLLOUT)) { return 1; } if (pfd.revents & POLLHUP) { return 0; } continue; } else if (result == 0) { return 0; } else if (errno_is_temporary(errno)) { continue; } else { return 0; } } }
ssize_t link_read_avail(struct link *link, char *data, size_t count, time_t stoptime) { ssize_t total = 0; ssize_t chunk = 0; /* First, satisfy anything from the buffer. */ if(link->buffer_length > 0) { chunk = MIN(link->buffer_length, count); memcpy(data, link->buffer_start, chunk); data += chunk; total += chunk; count -= chunk; link->buffer_start += chunk; link->buffer_length -= chunk; } /* Next, read what is available off the wire */ while(count > 0) { chunk = read(link->fd, data, count); if(chunk < 0) { /* ONLY BLOCK IF NOTHING HAS BEEN READ */ if(errno_is_temporary(errno) && total == 0) { if(link_sleep(link, stoptime, 1, 0)) { continue; } else { break; } } else { break; } } else if(chunk == 0) { break; } else { link->read += chunk; total += chunk; count -= chunk; data += chunk; } } if(total > 0) { return total; } else { if(chunk == 0) { return 0; } else { return -1; } } }
static int link_internal_sleep(struct link *link, struct timeval *timeout, sigset_t *mask, int reading, int writing) { int result; struct pollfd pfd; int msec; sigset_t cmask; if(timeout) { msec = (timeout->tv_sec * 1000.0) + (timeout->tv_usec/1000.0); } else { msec = -1; } if(reading && link->buffer_length) { return 1; } while (1) { pfd.fd = link->fd; pfd.revents = 0; if (reading) pfd.events = POLLIN; if (writing) pfd.events = POLLOUT; sigprocmask(SIG_UNBLOCK, mask, &cmask); result = poll(&pfd, 1, msec); sigprocmask(SIG_SETMASK, &cmask, NULL); if (result > 0) { if (reading && (pfd.revents & POLLIN)) { return 1; } if (writing && (pfd.revents & POLLOUT)) { return 1; } if (pfd.revents & POLLHUP) { return 0; } continue; } else if (result == 0) { return 0; } else if (mask && errno == EINTR) { return 0; } else if (errno_is_temporary(errno)) { continue; } else { return 0; } } }
ssize_t link_write(struct link *link, const char *data, size_t count, time_t stoptime) { ssize_t total = 0; ssize_t chunk = 0; if (!link) return errno = EINVAL, -1; while(count > 0) { chunk = write(link->fd, data, count); if(chunk < 0) { if(errno_is_temporary(errno)) { if(link_sleep(link, stoptime, 0, 1)) { continue; } else { break; } } else { break; } } else if(chunk == 0) { break; } else { link->written += chunk; total += chunk; count -= chunk; data += chunk; } } if(total > 0) { return total; } else { if(chunk == 0) { return 0; } else { return -1; } } }
int datagram_recv(struct datagram *d, char *data, int length, char *addr, int *port, int timeout) { int result; struct sockaddr_in iaddr; SOCKLEN_T iaddr_length; fd_set fds; struct timeval tm; while(1) { tm.tv_sec = timeout / 1000000; tm.tv_usec = timeout % 1000000; FD_ZERO(&fds); FD_SET(d->fd, &fds); result = select(d->fd + 1, &fds, 0, 0, &tm); if(result > 0) { if(FD_ISSET(d->fd, &fds)) break; } else if(result < 0 && errno_is_temporary(errno)) { continue; } else { return -1; } } iaddr_length = sizeof(iaddr); result = recvfrom(d->fd, data, length, 0, (struct sockaddr *) &iaddr, &iaddr_length); if(result < 0) return result; addr_to_string(&iaddr.sin_addr, addr); *port = ntohs(iaddr.sin_port); return result; }
struct link *link_connect(const char *addr, int port, time_t stoptime) { struct sockaddr_in address; struct link *link = 0; int result; int save_errno; link = link_create(); if(!link) goto failure; link_squelch(); memset(&address, 0, sizeof(address)); #if defined(CCTOOLS_OPSYS_DARWIN) address.sin_len = sizeof(address); #endif address.sin_family = AF_INET; address.sin_port = htons(port); if(!string_to_ip_address(addr, (unsigned char *) &address.sin_addr)) goto failure; link->fd = socket(AF_INET, SOCK_STREAM, 0); if(link->fd < 0) goto failure; link_window_configure(link); /* sadly, cygwin does not do non-blocking connect correctly */ #ifdef CCTOOLS_OPSYS_CYGWIN if(!link_nonblocking(link, 0)) goto failure; #else if(!link_nonblocking(link, 1)) goto failure; #endif debug(D_TCP, "connecting to %s:%d", addr, port); do { result = connect(link->fd, (struct sockaddr *) &address, sizeof(address)); /* On some platforms, errno is not set correctly. */ /* If the remote address can be found, then we are really connected. */ /* Also, on bsd-derived systems, failure to connect is indicated by a second connect returning EINVAL. */ if(result < 0 && !errno_is_temporary(errno)) { if(errno == EINVAL) errno = ECONNREFUSED; break; } if(link_address_remote(link, link->raddr, &link->rport)) { debug(D_TCP, "made connection to %s:%d", link->raddr, link->rport); #ifdef CCTOOLS_OPSYS_CYGWIN link_nonblocking(link, 1); #endif return link; } } while(link_sleep(link, stoptime, 0, 1)); debug(D_TCP, "connection to %s:%d failed (%s)", addr, port, strerror(errno)); failure: save_errno = errno; if(link) link_close(link); errno = save_errno; return 0; }
struct link *link_connect(const char *addr, int port, time_t stoptime) { struct sockaddr_in address; struct link *link = 0; int result; int save_errno; link = link_create(); if(!link) goto failure; link_squelch(); memset(&address, 0, sizeof(address)); #if defined(CCTOOLS_OPSYS_DARWIN) address.sin_len = sizeof(address); #endif address.sin_family = AF_INET; address.sin_port = htons(port); if(!string_to_ip_address(addr, (unsigned char *) &address.sin_addr)) goto failure; link->fd = socket(AF_INET, SOCK_STREAM, 0); if(link->fd < 0) goto failure; link_window_configure(link); /* sadly, cygwin does not do non-blocking connect correctly */ #ifdef CCTOOLS_OPSYS_CYGWIN if(!link_nonblocking(link, 0)) goto failure; #else if(!link_nonblocking(link, 1)) goto failure; #endif debug(D_TCP, "connecting to %s:%d", addr, port); while(1) { // First attempt a non-blocking connect result = connect(link->fd, (struct sockaddr *) &address, sizeof(address)); // On many platforms, non-blocking connect sets errno in unexpected ways: // On OSX, result=-1 and errno==EISCONN indicates a successful connection. if(result<0 && errno==EISCONN) result=0; // On BSD-derived systems, failure to connect is indicated by errno = EINVAL. // Set it to something more explanatory. if(result<0 && errno==EINVAL) errno=ECONNREFUSED; // Otherwise, a non-temporary errno should cause us to bail out. if(result<0 && !errno_is_temporary(errno)) break; // If the remote address is valid, we are connected no matter what. if(link_address_remote(link, link->raddr, &link->rport)) { debug(D_TCP, "made connection to %s:%d", link->raddr, link->rport); #ifdef CCTOOLS_OPSYS_CYGWIN link_nonblocking(link, 1); #endif return link; } // if the time has expired, bail out if( time(0) >= stoptime ) { errno = ETIMEDOUT; break; } // wait for some activity on the socket. link_sleep(link, stoptime, 0, 1); // No matter how the sleep ends, we want to go back to the top // and call connect again to get a proper errno. } debug(D_TCP, "connection to %s:%d failed (%s)", addr, port, strerror(errno)); failure: save_errno = errno; if(link) link_close(link); errno = save_errno; return 0; }
/* Generic read/write wrappers for condor. These function emulate-ish the * read/write system calls under unix except that they are portable, use * a timeout, and make sure that all data is read or written. * * A few notes on the behavior differing from POSIX: * - These will never fail due to EINTR. * - If in non_blocking mode, there may be a short read or write returned. * - The corresponding POSIX functon returns 0 bytes read when the peer closed * the socket; these return -2. * - If zero bytes were read/written in non-blocking mode, this will return 0. This differs * from POSIX. * - Providing a zero-sized argument to this function will cause the program to abort(). * * Returns < 0 on failure. -1 = general error, -2 = peer closed socket */ int condor_read( char const *peer_description, SOCKET fd, char *buf, int sz, int timeout, int flags, bool non_blocking ) { Selector selector; int nr = 0, nro; unsigned int start_time=0, cur_time=0; char sinbuf[SINFUL_STRING_BUF_SIZE]; if( IsDebugLevel(D_NETWORK) ) { dprintf(D_NETWORK, "condor_read(fd=%d %s,,size=%d,timeout=%d,flags=%d,non_blocking=%d)\n", fd, not_null_peer_description(peer_description,fd,sinbuf), sz, timeout, flags, non_blocking); } /* PRE Conditions. */ ASSERT(fd >= 0); /* Need valid file descriptor */ ASSERT(buf != NULL); /* Need real memory to put data into */ ASSERT(sz > 0); /* Need legit size on buffer */ if (non_blocking) { #ifdef WIN32 unsigned long mode = 1; // nonblocking mode if (ioctlsocket(fd, FIONBIO, &mode) < 0) return -1; #else int fcntl_flags; if ( (fcntl_flags=fcntl(fd, F_GETFL)) < 0 ) return -1; // set nonblocking mode if ( ((fcntl_flags & O_NONBLOCK) == 0) && fcntl(fd, F_SETFL, fcntl_flags | O_NONBLOCK) == -1 ) return -1; #endif nr = -2; while (nr == -2 || (nr == -1 && errno == EINTR)) { nr = recv(fd, buf, sz, flags); } if ( nr <= 0 ) { int the_error; char const *the_errorstr; #ifdef WIN32 the_error = WSAGetLastError(); the_errorstr = ""; #else the_error = errno; the_errorstr = strerror(the_error); #endif if ( nr == 0 && !(flags & MSG_PEEK)) { nr = -2; dprintf( D_FULLDEBUG, "condor_read(): " "Socket closed when trying to read %d bytes from %s in non-blocking mode\n", sz, not_null_peer_description(peer_description,fd,sinbuf) ); } else if ( !errno_is_temporary(the_error) ) { dprintf( D_ALWAYS, "condor_read() failed: recv() %d bytes from %s " "returned %d, " "timeout=%d, errno=%d %s.\n", sz, not_null_peer_description(peer_description,fd,sinbuf), nr, timeout, the_error, the_errorstr ); } else { nr = 0; } } #ifdef WIN32 mode = 0; // reset blocking mode if (ioctlsocket(fd, FIONBIO, &mode) < 0) return -1; #else // reset flags to prior value if ( ((fcntl_flags & O_NONBLOCK) == 0) && (fcntl(fd, F_SETFL, fcntl_flags) == -1) ) return -1; #endif return nr; } selector.add_fd( fd, Selector::IO_READ ); if ( timeout > 0 ) { start_time = time(NULL); cur_time = start_time; } while( nr < sz ) { if( timeout > 0 ) { if( cur_time == 0 ) { cur_time = time(NULL); } // If it hasn't yet been longer then we said we would wait... if( start_time + timeout > cur_time ) { selector.set_timeout( (start_time + timeout) - cur_time ); } else { dprintf( D_ALWAYS, "condor_read(): timeout reading %d bytes from %s.\n", sz, not_null_peer_description(peer_description,fd,sinbuf) ); return -1; } cur_time = 0; if( IsDebugVerbose( D_NETWORK ) ) { dprintf(D_NETWORK, "condor_read(): fd=%d\n", fd); } selector.execute(); if( IsDebugVerbose( D_NETWORK ) ) { dprintf(D_NETWORK, "condor_read(): select returned %d\n", selector.select_retval()); } if ( selector.timed_out() ) { dprintf( D_ALWAYS, "condor_read(): timeout reading %d bytes from %s.\n", sz, not_null_peer_description(peer_description,fd,sinbuf) ); return -1; } else if ( selector.signalled() ) { continue; } else if ( !selector.has_ready() ) { int the_error; char const *the_errorstr; #ifdef WIN32 the_error = WSAGetLastError(); the_errorstr = ""; #else the_error = errno; the_errorstr = strerror(the_error); #endif dprintf( D_ALWAYS, "condor_read() failed: select() " "returns %d, reading %d bytes from %s (errno=%d %s).\n", selector.select_retval(), sz, not_null_peer_description(peer_description,fd,sinbuf), the_error, the_errorstr ); return -1; } } start_thread_safe("recv"); nro = recv(fd, &buf[nr], sz - nr, flags); // Save the error value before stop_thread_safe(), as that may // overwrite it. int the_error; #ifdef WIN32 the_error = WSAGetLastError(); #else the_error = errno; #endif stop_thread_safe("recv"); if( nro <= 0 ) { // If timeout > 0, and we made it here, then // we know we were woken by select(). Now, if // select() wakes up on a read fd, and then recv() // subsequently returns 0, that means that the // socket has been closed by our peer. // If timeout == 0, then recv() should have // blocked until 1 or more bytes arrived. // Thus no matter what, if nro==0, then the // socket must be closed. if ( nro == 0 ) { dprintf( D_FULLDEBUG, "condor_read(): " "Socket closed when trying to read %d bytes from %s\n", sz, not_null_peer_description(peer_description,fd,sinbuf) ); return -2; } char const *the_errorstr; #ifdef WIN32 the_errorstr = ""; #else the_errorstr = strerror(the_error); #endif if ( errno_is_temporary(the_error) ) { dprintf( D_FULLDEBUG, "condor_read(): " "recv() returned temporary error %d %s," "still trying to read from %s\n", the_error,the_errorstr, not_null_peer_description(peer_description,fd,sinbuf) ); continue; } dprintf( D_ALWAYS, "condor_read() failed: recv(fd=%d) returned %d, " "errno = %d %s, reading %d bytes from %s.\n", fd, nro, the_error, the_errorstr, sz, not_null_peer_description(peer_description,fd,sinbuf) ); if( the_error == ETIMEDOUT ) { if( timeout <= 0 ) { dprintf( D_ALWAYS, "condor_read(): read timeout during blocking read from %s\n", not_null_peer_description(peer_description,fd,sinbuf)); } else { int lapse = (int)(time(NULL)-start_time); dprintf( D_ALWAYS, "condor_read(): UNEXPECTED read timeout after %ds during non-blocking read from %s (desired timeout=%ds)\n", lapse, not_null_peer_description(peer_description,fd,sinbuf), timeout); } } return -1; } nr += nro; } /* Post Conditions */ ASSERT( nr == sz ); // we should have read *ALL* the data return nr; }
int condor_write( char const *peer_description, SOCKET fd, const char *buf, int sz, int timeout, int flags, bool non_blocking ) { int nw = 0, nwo = 0; unsigned int start_time = 0, cur_time = 0; char tmpbuf[1]; int nro; bool select_for_read = true; bool needs_select = true; char sinbuf[SINFUL_STRING_BUF_SIZE]; if( IsDebugLevel( D_NETWORK ) ) { dprintf(D_NETWORK, "condor_write(fd=%d %s,,size=%d,timeout=%d,flags=%d,non_blocking=%d)\n", fd, not_null_peer_description(peer_description,fd,sinbuf), sz, timeout, flags, non_blocking); } /* Pre-conditions. */ ASSERT(sz > 0); /* Can't write buffers that are have no data */ ASSERT(fd >= 0); /* Need valid file descriptor */ ASSERT(buf != NULL); /* Need valid buffer to write */ if (non_blocking) { #ifdef WIN32 unsigned long mode = 1; // nonblocking mode if (ioctlsocket(fd, FIONBIO, &mode) < 0) return -1; #else int fcntl_flags; if ( (fcntl_flags=fcntl(fd, F_GETFL)) < 0 ) return -1; // set nonblocking mode if ( ((fcntl_flags & O_NONBLOCK) == 0) && fcntl(fd, F_SETFL, fcntl_flags | O_NONBLOCK) == -1 ) return -1; #endif nw = -2; while (nw == -2 || (nw == -1 && errno == EINTR)) { nw = send(fd, buf, sz, flags); } if ( nw <= 0 ) { int the_error; char const *the_errorstr; #ifdef WIN32 the_error = WSAGetLastError(); the_errorstr = ""; #else the_error = errno; the_errorstr = strerror(the_error); #endif if ( !errno_is_temporary(the_error) ) { dprintf( D_ALWAYS, "condor_write() failed: send() %d bytes to %s " "returned %d, " "timeout=%d, errno=%d %s.\n", sz, not_null_peer_description(peer_description,fd,sinbuf), nw, timeout, the_error, the_errorstr ); } else { nw = 0; } } if (nw < 0) { dprintf(D_NETWORK, "condor_write (non-blocking) wrote %d bytes.\n", nw); } #ifdef WIN32 mode = 0; // reset blocking mode if (ioctlsocket(fd, FIONBIO, &mode) < 0) return -1; #else // reset flags to prior value if ( ((fcntl_flags & O_NONBLOCK) == 0) && fcntl(fd, F_SETFL, fcntl_flags) == -1 ) return -1; #endif return nw; } Selector selector; selector.add_fd( fd, Selector::IO_READ ); selector.add_fd( fd, Selector::IO_WRITE ); selector.add_fd( fd, Selector::IO_EXCEPT ); if(timeout > 0) { start_time = time(NULL); cur_time = start_time; } while( nw < sz ) { needs_select = true; if( timeout > 0 ) { while( needs_select ) { if( cur_time == 0 ) { cur_time = time(NULL); } if( start_time + timeout > cur_time ) { selector.set_timeout( (start_time + timeout) - cur_time ); } else { dprintf( D_ALWAYS, "condor_write(): " "timed out writing %d bytes to %s\n", sz, not_null_peer_description(peer_description,fd,sinbuf) ); return -1; } cur_time = 0; // The write and except sets are added at the top of // this function, since we always want to select on // them. if( select_for_read ) { // Also, put it in the read fds, so we'll wake // up if the socket is closed selector.add_fd( fd, Selector::IO_READ ); } else { selector.delete_fd( fd, Selector::IO_READ ); } selector.execute(); // unless we decide we need to select() again, we // want to break out of our while() loop now that // we've actually performed a select() needs_select = false; if ( selector.timed_out() ) { dprintf( D_ALWAYS, "condor_write(): " "timed out writing %d bytes to %s\n", sz, not_null_peer_description(peer_description,fd,sinbuf) ); return -1; } else if ( selector.signalled() ) { needs_select = true; continue; } else if ( selector.has_ready() ) { if ( selector.fd_ready( fd, Selector::IO_READ ) ) { dprintf(D_NETWORK, "condor_write(): socket %d is readable\n", fd); // see if the socket was closed nro = recv(fd, tmpbuf, 1, MSG_PEEK); if( nro == -1 ) { int the_error; char const *the_errorstr; #ifdef WIN32 the_error = WSAGetLastError(); the_errorstr = ""; #else the_error = errno; the_errorstr = strerror(the_error); #endif if(errno_is_temporary( the_error )) { continue; } dprintf( D_ALWAYS, "condor_write(): " "Socket closed when trying " "to write %d bytes to %s, fd is %d, " "errno=%d %s\n", sz, not_null_peer_description(peer_description,fd,sinbuf), fd, the_error, the_errorstr ); return -1; } if( ! nro ) { dprintf( D_ALWAYS, "condor_write(): " "Socket closed when trying " "to write %d bytes to %s, fd is %d\n", sz, not_null_peer_description(peer_description,fd,sinbuf), fd ); return -1; } /* otherwise, there's real data to consume on the read side, and we don't want to put our fd in the readfds anymore or select() will never block. also, we need to re-do the select() */ needs_select = true; select_for_read = false; } } else { dprintf( D_ALWAYS, "condor_write() failed: select() " "returns %d, " "writing %d bytes to %s.\n", selector.select_retval(), sz, not_null_peer_description(peer_description,fd,sinbuf) ); return -1; } } } start_thread_safe("send"); nwo = send(fd, &buf[nw], sz - nw, flags); // Save the error value before stop_thread_safe(), as that may // overwrite it. int the_error; #ifdef WIN32 the_error = WSAGetLastError(); #else the_error = errno; #endif stop_thread_safe("send"); if( nwo <= 0 ) { char const *the_errorstr; #ifdef WIN32 the_errorstr = ""; #else the_errorstr = strerror(the_error); #endif if ( errno_is_temporary(the_error) ) { dprintf( D_FULLDEBUG, "condor_write(): " "send() returned temporary error %d %s," "still trying to write %d bytes to %s\n", the_error, the_errorstr, sz, not_null_peer_description(peer_description,fd,sinbuf) ); continue; } dprintf( D_ALWAYS, "condor_write() failed: send() %d bytes to %s " "returned %d, " "timeout=%d, errno=%d %s.\n", sz, not_null_peer_description(peer_description,fd,sinbuf), nwo, timeout, the_error, the_errorstr ); return -1; } nw += nwo; } /* POST conditions. */ ASSERT( nw == sz ); /* Make sure that we wrote everything */ return nw; }