/** * Reads and returns a templated array encoded in the Watchman binary protocol * format, starting at `ptr` and finishing at or before `end` * * Templated arrays are arrays of hashes which have repetitive key information * pulled out into a separate "headers" prefix. * * @see https://github.com/facebook/watchman/blob/master/BSER.markdown */ VALUE watchman_load_template(char **ptr, char *end) { int64_t header_items_count, i, row_count; VALUE array, hash, header, key, value; *ptr += sizeof(int8_t); // caller has already verified the marker // process template header array header_items_count = watchman_load_array_header(ptr, end); header = rb_ary_new2(header_items_count); for (i = 0; i < header_items_count; i++) { rb_ary_push(header, watchman_load_string(ptr, end)); } // process row items row_count = watchman_load_int(ptr, end); array = rb_ary_new2(header_items_count); while (row_count--) { hash = rb_hash_new(); for (i = 0; i < header_items_count; i++) { if (*ptr >= end) { rb_raise(rb_eArgError, "unexpected end of input"); } if (*ptr[0] == WATCHMAN_SKIP_MARKER) { *ptr += sizeof(uint8_t); } else { value = watchman_load(ptr, end); key = rb_ary_entry(header, i); rb_hash_aset(hash, key, value); } } rb_ary_push(array, hash); } return array; }
/** * Reads and returns a string encoded in the Watchman binary protocol format, * starting at `ptr` and finishing at or before `end` */ VALUE watchman_load_string(char **ptr, char *end) { int64_t len; VALUE string; if (*ptr >= end) { rb_raise(rb_eArgError, "unexpected end of input"); } if (*ptr[0] != WATCHMAN_STRING_MARKER) { rb_raise(rb_eArgError, "not a number"); } *ptr += sizeof(int8_t); if (*ptr >= end) { rb_raise(rb_eArgError, "invalid string header"); } len = watchman_load_int(ptr, end); if (len == 0) { // special case for zero-length strings return rb_str_new2(""); } else if (*ptr + len > end) { rb_raise(rb_eArgError, "insufficient string storage"); } string = rb_str_new(*ptr, len); *ptr += len; return string; }
/** * CommandT::Watchman::Utils.load(serialized) * * Converts the binary object, `serialized`, from the Watchman binary protocol * format into a normal Ruby object. */ VALUE CommandTWatchmanUtils_load(VALUE self, VALUE serialized) { char *ptr, *end; long len; uint64_t payload_size; VALUE loaded; serialized = StringValue(serialized); len = RSTRING_LEN(serialized); ptr = RSTRING_PTR(serialized); end = ptr + len; // expect at least the binary marker and a int8_t length counter if ((size_t)len < sizeof(WATCHMAN_BINARY_MARKER) - 1 + sizeof(int8_t) * 2) { rb_raise(rb_eArgError, "undersized header"); } if (memcmp(ptr, WATCHMAN_BINARY_MARKER, sizeof(WATCHMAN_BINARY_MARKER) - 1)) { rb_raise(rb_eArgError, "missing binary marker"); } // get size marker ptr += sizeof(WATCHMAN_BINARY_MARKER) - 1; payload_size = watchman_load_int(&ptr, end); if (!payload_size) { rb_raise(rb_eArgError, "empty payload"); } // sanity check length if (ptr + payload_size != end) { rb_raise( rb_eArgError, "payload size mismatch (%lu)", (unsigned long)(end - (ptr + payload_size)) ); } loaded = watchman_load(&ptr, end); // one more sanity check if (ptr != end) { rb_raise( rb_eArgError, "payload termination mismatch (%lu)", (unsigned long)(end - ptr) ); } return loaded; }
/** * Helper method which returns length of the array encoded in the Watchman * binary protocol format, starting at `ptr` and finishing at or before `end` */ int64_t watchman_load_array_header(char **ptr, char *end) { if (*ptr >= end) { rb_raise(rb_eArgError, "unexpected end of input"); } // verify and consume marker if (*ptr[0] != WATCHMAN_ARRAY_MARKER) { rb_raise(rb_eArgError, "not an array"); } *ptr += sizeof(int8_t); // expect a count if (*ptr + sizeof(int8_t) * 2 > end) { rb_raise(rb_eArgError, "incomplete array header"); } return watchman_load_int(ptr, end); }
/** * Reads and returns a hash encoded in the Watchman binary protocol format, * starting at `ptr` and finishing at or before `end` */ VALUE watchman_load_hash(char **ptr, char *end) { int64_t count, i; VALUE hash, key, value; *ptr += sizeof(int8_t); // caller has already verified the marker // expect a count if (*ptr + sizeof(int8_t) * 2 > end) { rb_raise(rb_eArgError, "incomplete hash header"); } count = watchman_load_int(ptr, end); hash = rb_hash_new(); for (i = 0; i < count; i++) { key = watchman_load_string(ptr, end); value = watchman_load(ptr, end); rb_hash_aset(hash, key, value); } return hash; }
/** * Reads and returns an object encoded in the Watchman binary protocol format, * starting at `ptr` and finishing at or before `end` */ VALUE watchman_load(char **ptr, char *end) { if (*ptr >= end) { rb_raise(rb_eArgError, "unexpected end of input"); } switch (*ptr[0]) { case WATCHMAN_ARRAY_MARKER: return watchman_load_array(ptr, end); case WATCHMAN_HASH_MARKER: return watchman_load_hash(ptr, end); case WATCHMAN_STRING_MARKER: return watchman_load_string(ptr, end); case WATCHMAN_INT8_MARKER: case WATCHMAN_INT16_MARKER: case WATCHMAN_INT32_MARKER: case WATCHMAN_INT64_MARKER: return LL2NUM(watchman_load_int(ptr, end)); case WATCHMAN_FLOAT_MARKER: return rb_float_new(watchman_load_double(ptr, end)); case WATCHMAN_TRUE: *ptr += 1; return Qtrue; case WATCHMAN_FALSE: *ptr += 1; return Qfalse; case WATCHMAN_NIL: *ptr += 1; return Qnil; case WATCHMAN_TEMPLATE_MARKER: return watchman_load_template(ptr, end); default: rb_raise(rb_eTypeError, "unsupported type"); } return Qnil; // keep the compiler happy }
/** * CommandT::Watchman::Utils.query(query, socket) * * Converts `query`, a Watchman query comprising Ruby objects, into the Watchman * binary protocol format, transmits it over socket, and unserializes and * returns the result. */ VALUE CommandTWatchmanUtils_query(VALUE self, VALUE query, VALUE socket) { char *payload; int fileno, flags; int8_t peek[WATCHMAN_PEEK_BUFFER_SIZE]; int8_t sizes[] = { 0, 0, 0, 1, 2, 4, 8 }; int8_t sizes_idx; int8_t *pdu_size_ptr; int64_t payload_size; long query_len; ssize_t peek_size, sent, received; void *buffer; VALUE loaded, serialized; fileno = NUM2INT(rb_funcall(socket, rb_intern("fileno"), 0)); // do blocking I/O to simplify the following logic flags = fcntl(fileno, F_GETFL); if (fcntl(fileno, F_SETFL, flags & ~O_NONBLOCK) == -1) { rb_raise(rb_eRuntimeError, "unable to clear O_NONBLOCK flag"); } // send the message serialized = CommandTWatchmanUtils_dump(self, query); query_len = RSTRING_LEN(serialized); sent = send(fileno, RSTRING_PTR(serialized), query_len, 0); if (sent == -1) { watchman_raise_system_call_error(errno); } else if (sent != query_len) { rb_raise(rb_eRuntimeError, "expected to send %ld bytes but sent %ld", query_len, sent); } // sniff to see how large the header is received = recv(fileno, peek, WATCHMAN_SNIFF_BUFFER_SIZE, MSG_PEEK | MSG_WAITALL); if (received == -1) { watchman_raise_system_call_error(errno); } else if (received != WATCHMAN_SNIFF_BUFFER_SIZE) { rb_raise(rb_eRuntimeError, "failed to sniff PDU header"); } // peek at size of PDU sizes_idx = peek[sizeof(WATCHMAN_BINARY_MARKER) - 1]; if (sizes_idx < WATCHMAN_INT8_MARKER || sizes_idx > WATCHMAN_INT64_MARKER) { rb_raise(rb_eRuntimeError, "bad PDU size marker"); } peek_size = sizeof(WATCHMAN_BINARY_MARKER) - 1 + sizeof(int8_t) + sizes[sizes_idx]; received = recv(fileno, peek, peek_size, MSG_PEEK); if (received == -1) { watchman_raise_system_call_error(errno); } else if (received != peek_size) { rb_raise(rb_eRuntimeError, "failed to peek at PDU header"); } pdu_size_ptr = peek + sizeof(WATCHMAN_BINARY_MARKER) - sizeof(int8_t); payload_size = peek_size + watchman_load_int((char **)&pdu_size_ptr, (char *)peek + peek_size); // actually read the PDU buffer = xmalloc(payload_size); if (!buffer) { rb_raise( rb_eNoMemError, "failed to allocate %lld bytes", (long long int)payload_size ); } received = recv(fileno, buffer, payload_size, MSG_WAITALL); if (received == -1) { watchman_raise_system_call_error(errno); } else if (received != payload_size) { rb_raise(rb_eRuntimeError, "failed to load PDU"); } payload = (char *)buffer + peek_size; loaded = watchman_load(&payload, payload + payload_size); free(buffer); return loaded; }
/** * RubyWatchman.query(query, socket) * * Converts `query`, a Watchman query comprising Ruby objects, into the Watchman * binary protocol format, transmits it over socket, and unserializes and * returns the result. */ VALUE RubyWatchman_query(VALUE self, VALUE query, VALUE socket) { VALUE error = Qnil; VALUE errorClass = Qnil; VALUE loaded = Qnil; void *buffer = NULL; int fileno = NUM2INT(rb_funcall(socket, rb_intern("fileno"), 0)); // do blocking I/O to simplify the following logic int flags = fcntl(fileno, F_GETFL); if ( !(flags & O_NONBLOCK) && fcntl(fileno, F_SETFL, flags & ~O_NONBLOCK) == -1 ) { error = rb_str_new2("unable to clear O_NONBLOCK flag"); goto cleanup; } // send the message VALUE serialized = RubyWatchman_dump(self, query); long query_len = RSTRING_LEN(serialized); ssize_t sent = send(fileno, RSTRING_PTR(serialized), query_len, 0); if (sent == -1) { goto system_call_fail; } else if (sent != query_len) { error = rb_str_new2("sent byte count mismatch"); goto cleanup; } // sniff to see how large the header is int8_t peek[WATCHMAN_PEEK_BUFFER_SIZE]; ssize_t received = recv(fileno, peek, WATCHMAN_SNIFF_BUFFER_SIZE, MSG_PEEK | MSG_WAITALL); if (received == -1) { goto system_call_fail; } else if (received != WATCHMAN_SNIFF_BUFFER_SIZE) { error = rb_str_new2("failed to sniff PDU header"); goto cleanup; } // peek at size of PDU int8_t sizes[] = { 0, 0, 0, 1, 2, 4, 8 }; int8_t sizes_idx = peek[sizeof(WATCHMAN_BINARY_MARKER) - 1]; if (sizes_idx < WATCHMAN_INT8_MARKER || sizes_idx > WATCHMAN_INT64_MARKER) { error = rb_str_new2("bad PDU size marker"); goto cleanup; } ssize_t peek_size = sizeof(WATCHMAN_BINARY_MARKER) - 1 + sizeof(int8_t) + sizes[sizes_idx]; received = recv(fileno, peek, peek_size, MSG_PEEK); if (received == -1) { goto system_call_fail; } else if (received != peek_size) { error = rb_str_new2("failed to peek at PDU header"); goto cleanup; } int8_t *pdu_size_ptr = peek + sizeof(WATCHMAN_BINARY_MARKER) - sizeof(int8_t); int64_t payload_size = peek_size + watchman_load_int((char **)&pdu_size_ptr, (char *)peek + peek_size); // actually read the PDU buffer = xmalloc(payload_size); if (!buffer) { errorClass = rb_eNoMemError; error = rb_str_new2("failed to allocate"); goto cleanup; } received = recv(fileno, buffer, payload_size, MSG_WAITALL); if (received == -1) { goto system_call_fail; } else if (received != payload_size) { error = rb_str_new2("failed to load PDU"); goto cleanup; } if (!(flags & O_NONBLOCK) && fcntl(fileno, F_SETFL, flags) == -1) { error = rb_str_new2("unable to restore fnctl flags"); goto cleanup; } char *payload = buffer + peek_size; loaded = watchman_load(&payload, payload + payload_size); goto cleanup; system_call_fail: errorClass = rb_eSystemCallError; error = INT2FIX(errno); cleanup: if (buffer) { xfree(buffer); } if (!(flags & O_NONBLOCK) && fcntl(fileno, F_SETFL, flags) == -1) { rb_raise(rb_eRuntimeError, "unable to restore fnctl flags"); } if (NIL_P(errorClass)) { errorClass = rb_eRuntimeError; } if (!NIL_P(error)) { rb_exc_raise(rb_class_new_instance(1, &error, errorClass)); } return loaded; }