int64 stream_in_module(FILE *fp)
{
int64 count=0;
MODULE *m;
while (GETBT && B(MODULE) && T(BEGIN))
{
char name[1024];
memset(name,0,sizeof(name));
OK;
while (GETBT && B(MODULE) && !T(END))
{
OK;
if T(NAME)
{
GETD(name,sizeof(name));
m = module_load(name,0,NULL);
if (m==NULL)
return stream_error("module %s version is not found", name);
}
else if T(VERSION)
{
unsigned short major, minor;
GETS(major);
GETS(minor);
if (m->major!=major || m->minor!=minor)
return stream_error("module %s version %d.%02d specified does not match version %d.%02d found", name, major, minor, m->major, m->minor);
}
else
stream_warning("ignoring token %d in module stream", t);
}
int dyad_listenEx(
dyad_Stream *stream, const char *host, int port, int backlog
) {
struct addrinfo hints, *ai = NULL;
int err, optval;
char buf[64];
dyad_Event e;
/* Get addrinfo */
memset(&hints, 0, sizeof(hints));
hints.ai_family = AF_UNSPEC;
hints.ai_socktype = SOCK_STREAM;
hints.ai_flags = AI_PASSIVE;
sprintf(buf, "%d", port);
err = getaddrinfo(host, buf, &hints, &ai);
if (err) {
stream_error(stream, "could not get addrinfo", errno);
goto fail;
}
/* Init socket */
err = stream_initSocket(stream, ai->ai_family, ai->ai_socktype,
ai->ai_protocol);
if (err) goto fail;
/* Set SO_REUSEADDR so that the socket can be immediately bound without
* having to wait for any closed socket on the same port to timeout */
optval = 1;
setsockopt(stream->sockfd, SOL_SOCKET, SO_REUSEADDR,
&optval, sizeof(optval));
/* Bind and listen */
err = bind(stream->sockfd, ai->ai_addr, ai->ai_addrlen);
if (err) {
stream_error(stream, "could not bind socket", errno);
goto fail;
}
err = listen(stream->sockfd, backlog);
if (err) {
stream_error(stream, "socket failed on listen", errno);
goto fail;
}
stream->state = DYAD_STATE_LISTENING;
stream->port = port;
stream_initAddress(stream);
/* Emit listening event */
e = createEvent(DYAD_EVENT_LISTEN);
e.msg = "socket is listening";
stream_emitEvent(stream, &e);
freeaddrinfo(ai);
return 0;
fail:
if (ai) freeaddrinfo(ai);
return -1;
}
/* fcopy(instream, outstream)
* Copies data from one stream to the other until EOF on the input.
*/
void fcopy(FILE * fin, FILE * fout) {
unsigned char buf[4096];
size_t len;
while ((len = fread(buf, 1, sizeof(buf), fin)) > 0) {
if (fwrite(buf, 1, len, fout) < len)
break;
}
if (ferror(fin)) {
stream_error("fread", fin);
}
if (ferror(fout)) {
stream_error("fwrite", fout);
}
}
static void stream_acceptPendingConnections(dyad_Stream *stream) {
for (;;) {
dyad_Stream *remote;
dyad_Event e;
int err = 0;
dyad_Socket sockfd = accept(stream->sockfd, NULL, NULL);
if (sockfd == INVALID_SOCKET) {
err = errno;
if (err == EWOULDBLOCK) {
/* No more waiting sockets */
return;
}
}
/* Create client stream */
remote = dyad_newStream();
remote->state = DYAD_STATE_CONNECTED;
/* Set stream's socket */
stream_setSocket(remote, sockfd);
/* Emit accept event */
e = createEvent(DYAD_EVENT_ACCEPT);
e.msg = "accepted connection";
e.remote = remote;
stream_emitEvent(stream, &e);
/* Handle invalid socket -- the stream is still made and the ACCEPT event
* is still emitted, but its shut immediately with an error */
if (remote->sockfd == INVALID_SOCKET) {
stream_error(remote, "failed to create socket on accept", err);
return;
}
}
}
int dyad_connect(dyad_Stream *stream, const char *host, int port) {
struct addrinfo hints, *ai = NULL;
int err;
char buf[64];
/* Resolve host */
memset(&hints, 0, sizeof(hints));
hints.ai_family = AF_UNSPEC;
hints.ai_socktype = SOCK_STREAM;
sprintf(buf, "%d", port);
err = getaddrinfo(host, buf, &hints, &ai);
if (err) {
stream_error(stream, "could not resolve host", 0);
goto fail;
}
/* Start connecting */
err = stream_initSocket(stream, ai->ai_family, ai->ai_socktype,
ai->ai_protocol);
if (err) goto fail;
connect(stream->sockfd, ai->ai_addr, ai->ai_addrlen);
stream->state = DYAD_STATE_CONNECTING;
freeaddrinfo(ai);
return 0;
fail:
if (ai) freeaddrinfo(ai);
return -1;
}
void attach(shared_ptr<stream> source_new)
{
if(source_)
throw stream_error("A source is already attached to this octet_stream_reader");
source_ = source_new;
}
/* read_stream_write_blocksums(data_stream, zsync_stream)
* Reads the data stream and writes to the zsync stream the blocksums for the
* given data. No compression handling.
*/
void read_stream_write_blocksums(FILE * fin, FILE * fout) {
unsigned char *buf = malloc(blocksize);
if (!buf) {
fprintf(stderr, "out of memory\n");
exit(1);
}
while (!feof(fin)) {
int got = fread(buf, 1, blocksize, fin);
if (got > 0) {
if (!no_look_inside && len == 0 && buf[0] == 0x1f && buf[1] == 0x8b) {
do_zstream(fin, fout, (char *)buf, got);
break;
}
/* The SHA-1 sum, unlike our internal block-based sums, is on the whole file and nothing else - no padding */
SHA1Update(&shactx, buf, got);
write_block_sums(buf, got, fout);
len += got;
}
else {
if (ferror(fin))
stream_error("fread", fin);
}
}
free(buf);
}
void attach(shared_ptr<stream> new_sink)
{
if(sink_)
throw stream_error("A sink is already attached to the octet_stream_writer");
sink_ = new_sink;
cap_ = default_initial_bucket_size;
buffer_.reset(bucket_data_mem::create(cap_, 0));
read_in_buffer_();
}
/* fcopy_hashes(hash_stream, zsync_stream, rsum_bytes, hash_bytes)
* Copy the full block checksums from their temporary store file to the .zsync,
* stripping the hashes down to the desired lengths specified by the last 2
* parameters.
*/
void fcopy_hashes(FILE * fin, FILE * fout, size_t rsum_bytes, size_t hash_bytes) {
unsigned char buf[20];
size_t len;
while ((len = fread(buf, 1, sizeof(buf), fin)) > 0) {
/* write trailing rsum_bytes of the rsum (trailing because the second part of the rsum is more useful in practice for hashing), and leading checksum_bytes of the checksum */
if (fwrite(buf + 4 - rsum_bytes, 1, rsum_bytes, fout) < rsum_bytes)
break;
if (fwrite(buf + 4, 1, hash_bytes, fout) < hash_bytes)
break;
}
if (ferror(fin)) {
stream_error("fread", fin);
}
if (ferror(fout)) {
stream_error("fwrite", fout);
}
}
static int stream_initSocket(
dyad_Stream *stream, int domain, int type, int protocol
) {
stream->sockfd = socket(domain, type, protocol);
if (stream->sockfd == INVALID_SOCKET) {
stream_error(stream, "could not create socket", errno);
return -1;
}
stream_setSocket(stream, stream->sockfd);
return 0;
}
/* write_block_sums(buffer[], num_bytes, output_stream)
* Given one block of data, calculate the checksums for this block and write
* them (as raw bytes) to the given output stream */
static void write_block_sums(unsigned char *buf, size_t got, FILE * f) {
struct rsum r;
unsigned char checksum[CHECKSUM_SIZE];
/* Pad for our checksum, if this is a short last block */
if (got < blocksize)
memset(buf + got, 0, blocksize - got);
/* Do rsum and checksum, and convert to network endian */
r = rcksum_calc_rsum_block(buf, blocksize);
rcksum_calc_checksum(&checksum[0], buf, blocksize);
r.a = htons(r.a);
r.b = htons(r.b);
/* Write them raw to the stream */
if (fwrite(&r, sizeof r, 1, f) != 1)
stream_error("fwrite", f);
if (fwrite(checksum, sizeof checksum, 1, f) != 1)
stream_error("fwrite", f);
}
void dyad_update(void) {
dyad_Stream *stream;
struct timeval tv;
destroyClosedStreams();
updateTickTimer();
updateStreamTimeouts();
/* Create fd sets for select() */
select_zero(&dyad_selectSet);
stream = dyad_streams;
while (stream) {
switch (stream->state) {
case DYAD_STATE_CONNECTED:
select_add(&dyad_selectSet, SELECT_READ, stream->sockfd);
if (!(stream->flags & DYAD_FLAG_READY) ||
stream->writeBuffer.length != 0
) {
select_add(&dyad_selectSet, SELECT_WRITE, stream->sockfd);
}
break;
case DYAD_STATE_CLOSING:
select_add(&dyad_selectSet, SELECT_WRITE, stream->sockfd);
break;
case DYAD_STATE_CONNECTING:
select_add(&dyad_selectSet, SELECT_WRITE, stream->sockfd);
select_add(&dyad_selectSet, SELECT_EXCEPT, stream->sockfd);
break;
case DYAD_STATE_LISTENING:
select_add(&dyad_selectSet, SELECT_READ, stream->sockfd);
break;
}
stream = stream->next;
}
/* Init timeout value and do select */
#ifdef _MSC_VER
#pragma warning(push)
/* Disable double to long implicit conversion warning,
* because the type of timeval's fields don't agree across platforms */
#pragma warning(disable: 4244)
#endif
tv.tv_sec = dyad_updateTimeout;
tv.tv_usec = (dyad_updateTimeout - tv.tv_sec) * 1e6;
#ifdef _MSC_VER
#pragma warning(pop)
#endif
select(dyad_selectSet.maxfd + 1,
dyad_selectSet.fds[SELECT_READ],
dyad_selectSet.fds[SELECT_WRITE],
dyad_selectSet.fds[SELECT_EXCEPT],
&tv);
/* Handle streams */
stream = dyad_streams;
while (stream) {
switch (stream->state) {
case DYAD_STATE_CONNECTED:
if (select_has(&dyad_selectSet, SELECT_READ, stream->sockfd)) {
stream_handleReceivedData(stream);
if (stream->state == DYAD_STATE_CLOSED) {
break;
}
}
/* Fall through */
case DYAD_STATE_CLOSING:
if (select_has(&dyad_selectSet, SELECT_WRITE, stream->sockfd)) {
stream_flushWriteBuffer(stream);
}
break;
case DYAD_STATE_CONNECTING:
if (select_has(&dyad_selectSet, SELECT_WRITE, stream->sockfd)) {
/* Check socket for error */
int optval = 0;
socklen_t optlen = sizeof(optval);
dyad_Event e;
getsockopt(stream->sockfd, SOL_SOCKET, SO_ERROR, &optval, &optlen);
if (optval != 0) goto connectFailed;
/* Handle succeselful connection */
stream->state = DYAD_STATE_CONNECTED;
stream->lastActivity = dyad_getTime();
stream_initAddress(stream);
/* Emit connect event */
e = createEvent(DYAD_EVENT_CONNECT);
e.msg = "connected to server";
stream_emitEvent(stream, &e);
} else if (
select_has(&dyad_selectSet, SELECT_EXCEPT, stream->sockfd)
) {
/* Handle failed connection */
connectFailed:
stream_error(stream, "could not connect to server", 0);
}
break;
case DYAD_STATE_LISTENING:
if (select_has(&dyad_selectSet, SELECT_READ, stream->sockfd)) {
stream_acceptPendingConnections(stream);
}
break;
}
/* If data was just now written to the stream we should immediately try to
* send it */
if (
stream->flags & DYAD_FLAG_WRITTEN &&
stream->state != DYAD_STATE_CLOSED
) {
stream_flushWriteBuffer(stream);
}
stream = stream->next;
}
}
void check_throw()
{
if(state.any(throw_on.as_integer()))
throw stream_error(0);
}
/** stream_compress
Format of compressed stream
[US/len] [bit15 runlen flag, bit 0-14 data len]
Bit 15 clear : raw data follows for len given
[UC/data ... ]
Bit 15 set : compressed data
[SC/delta] differential to apply to each value
[SC/value] initial value
**/
size_t stream_decompress(FILE *fp, const char *buf, const size_t len)
{
char *ptr = buf;
size_t count=0, buflen=0;
size_t confirm=0;
struct {
unsigned int runlen:15;
unsigned int is_compressed:1;
} runstate;
for (buflen=0; buflen<len; buflen+=runstate.runlen)
{
if (fread(&runstate,2,1,fp)!=1)
return stream_error("stream_decompress(): failed to read runlen");
else
count+=2;
// check for end of compressed stream
if (runstate.runlen==0)
break;
// handle run data
if (runstate.is_compressed) // compression flag set
{
char delta;
unsigned char value;
if (fread(&delta,1,1,fp)!=1 || fread(&value,1,1,fp)!=1)
return stream_error("stream_decompress(): failed to read delta/value");
else
count += 2;
if (delta==0)
{
memset(ptr,value,runstate.runlen+1);
ptr += runstate.runlen+1;
}
else
{
unsigned short run = runstate.runlen+1;
while (run-->0)
{
*ptr++ = value;
value += delta;
}
}
}
else // no compression
{
if (fread(ptr,1,runstate.runlen,fp)!=runstate.runlen)
return stream_error("stream_decompress(): failed to read raw data");
else
{
ptr += runstate.runlen;
count += runstate.runlen;
}
}
}
// check for overrun
if (buflen>len)
return stream_error("stream_decompress(): stream overrun--possible invalid stream");
// read confirmation code
if (fread(&confirm,sizeof(confirm),1,fp)==1 && confirm==count)
return count;
else
return stream_error("stream_decompress(): stream confirmation code mismatched--probable invalid stream");
}