int msg_consume(rd_kafka_message_t *rkmessage,
	void *opaque) {
	if (rkmessage->err) {
		if (rkmessage->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) {
			// 			fprintf(stderr,
			// 				"%% Consumer reached end of %s [%"PRId32"] "
			// 				"message queue at offset %"PRId64"\n",
			// 				rd_kafka_topic_name(rkmessage->rkt),
			// 				rkmessage->partition, rkmessage->offset);
			return PULL_DATA_SUCCESS;
		}

		// 		fprintf(stderr, "%% Consume error for topic \"%s\" [%"PRId32"] "
		// 			"offset %"PRId64": %s\n",
		// 			rd_kafka_topic_name(rkmessage->rkt),
		// 			rkmessage->partition,
		// 			rkmessage->offset,
		// 			rd_kafka_message_errstr(rkmessage));

		if (rkmessage->err == RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION ||
			rkmessage->err == RD_KAFKA_RESP_ERR__UNKNOWN_TOPIC)
			return rkmessage->err;
		return PULL_DATA_FAILED;
	}
	consume_data((const char*)rkmessage->payload, (const int)rkmessage->len);

	return PULL_DATA_SUCCESS;
}
예제 #2
0
consume_data_multi_scan(j_decompress_ptr cinfo) {
    huffman_index *index = cinfo->entropy->index;
    int i, retcode, ci;
    int mcu = cinfo->input_iMCU_row;
    jinit_phuff_decoder(cinfo);
    for (i = 0; i < index->scan_count; i++) {
        (*cinfo->inputctl->finish_input_pass)(cinfo);
        jset_input_stream_position(cinfo, index->scan[i].bitstream_offset);
        cinfo->output_iMCU_row = mcu;
        cinfo->unread_marker = 0;
        // Consume SOS and DHT headers
        retcode = (*cinfo->inputctl->consume_markers)(cinfo, index, i);
        cinfo->input_iMCU_row = mcu;
        cinfo->input_scan_number = i;
        cinfo->entropy->index = index;
        // Consume scan block data
        consume_data(cinfo);
    }
    cinfo->input_iMCU_row = mcu + 1;
    cinfo->input_scan_number = 0;
    cinfo->output_scan_number = 0;
    return JPEG_ROW_COMPLETED;
}
int amqp_handle_input(amqp_connection_state_t state,
                      amqp_bytes_t received_data,
                      amqp_frame_t *decoded_frame)
{
  size_t bytes_consumed;
  void *raw_frame;

  /* Returning frame_type of zero indicates either insufficient input,
     or a complete, ignored frame was read. */
  decoded_frame->frame_type = 0;

  if (received_data.len == 0) {
    return AMQP_STATUS_OK;
  }

  if (state->state == CONNECTION_STATE_IDLE) {
    state->state = CONNECTION_STATE_HEADER;
  }

  bytes_consumed = consume_data(state, &received_data);

  /* do we have target_size data yet? if not, return with the
     expectation that more will arrive */
  if (state->inbound_offset < state->target_size) {
    return bytes_consumed;
  }

  raw_frame = state->inbound_buffer.bytes;

  switch (state->state) {
  case CONNECTION_STATE_INITIAL:
    /* check for a protocol header from the server */
    if (memcmp(raw_frame, "AMQP", 4) == 0) {
      decoded_frame->frame_type = AMQP_PSEUDOFRAME_PROTOCOL_HEADER;
      decoded_frame->channel = 0;

      decoded_frame->payload.protocol_header.transport_high
        = amqp_d8(raw_frame, 4);
      decoded_frame->payload.protocol_header.transport_low
        = amqp_d8(raw_frame, 5);
      decoded_frame->payload.protocol_header.protocol_version_major
        = amqp_d8(raw_frame, 6);
      decoded_frame->payload.protocol_header.protocol_version_minor
        = amqp_d8(raw_frame, 7);

      return_to_idle(state);
      return bytes_consumed;
    }

    /* it's not a protocol header; fall through to process it as a
       regular frame header */

  case CONNECTION_STATE_HEADER: {
    amqp_channel_t channel;
    amqp_pool_t *channel_pool;
    /* frame length is 3 bytes in */
    channel = amqp_d16(raw_frame, 1);

    state->target_size
      = amqp_d32(raw_frame, 3) + HEADER_SIZE + FOOTER_SIZE;

    if ((size_t)state->frame_max < state->target_size) {
      return AMQP_STATUS_BAD_AMQP_DATA;
    }

    channel_pool = amqp_get_or_create_channel_pool(state, channel);
    if (NULL == channel_pool) {
      return AMQP_STATUS_NO_MEMORY;
    }

    amqp_pool_alloc_bytes(channel_pool, state->target_size, &state->inbound_buffer);
    if (NULL == state->inbound_buffer.bytes) {
      return AMQP_STATUS_NO_MEMORY;
    }
    memcpy(state->inbound_buffer.bytes, state->header_buffer, HEADER_SIZE);
    raw_frame = state->inbound_buffer.bytes;

    state->state = CONNECTION_STATE_BODY;

    bytes_consumed += consume_data(state, &received_data);

    /* do we have target_size data yet? if not, return with the
       expectation that more will arrive */
    if (state->inbound_offset < state->target_size) {
      return bytes_consumed;
    }

  }
    /* fall through to process body */

  case CONNECTION_STATE_BODY: {
    amqp_bytes_t encoded;
    int res;
    amqp_pool_t *channel_pool;

    /* Check frame end marker (footer) */
    if (amqp_d8(raw_frame, state->target_size - 1) != AMQP_FRAME_END) {
      return AMQP_STATUS_BAD_AMQP_DATA;
    }

    decoded_frame->frame_type = amqp_d8(raw_frame, 0);
    decoded_frame->channel = amqp_d16(raw_frame, 1);

    channel_pool = amqp_get_or_create_channel_pool(state, decoded_frame->channel);
    if (NULL == channel_pool) {
      return AMQP_STATUS_NO_MEMORY;
    }

    switch (decoded_frame->frame_type) {
    case AMQP_FRAME_METHOD:
      decoded_frame->payload.method.id = amqp_d32(raw_frame, HEADER_SIZE);
      encoded.bytes = amqp_offset(raw_frame, HEADER_SIZE + 4);
      encoded.len = state->target_size - HEADER_SIZE - 4 - FOOTER_SIZE;

      res = amqp_decode_method(decoded_frame->payload.method.id,
                               channel_pool, encoded,
                               &decoded_frame->payload.method.decoded);
      if (res < 0) {
        return res;
      }

      break;

    case AMQP_FRAME_HEADER:
      decoded_frame->payload.properties.class_id
        = amqp_d16(raw_frame, HEADER_SIZE);
      /* unused 2-byte weight field goes here */
      decoded_frame->payload.properties.body_size
        = amqp_d64(raw_frame, HEADER_SIZE + 4);
      encoded.bytes = amqp_offset(raw_frame, HEADER_SIZE + 12);
      encoded.len = state->target_size - HEADER_SIZE - 12 - FOOTER_SIZE;
      decoded_frame->payload.properties.raw = encoded;

      res = amqp_decode_properties(decoded_frame->payload.properties.class_id,
                                   channel_pool, encoded,
                                   &decoded_frame->payload.properties.decoded);
      if (res < 0) {
        return res;
      }

      break;

    case AMQP_FRAME_BODY:
      decoded_frame->payload.body_fragment.len
        = state->target_size - HEADER_SIZE - FOOTER_SIZE;
      decoded_frame->payload.body_fragment.bytes
        = amqp_offset(raw_frame, HEADER_SIZE);
      break;

    case AMQP_FRAME_HEARTBEAT:
      break;

    default:
      /* Ignore the frame */
      decoded_frame->frame_type = 0;
      break;
    }

    return_to_idle(state);
    return bytes_consumed;
  }

  default:
    amqp_abort("Internal error: invalid amqp_connection_state_t->state %d", state->state);
    return bytes_consumed;
  }
}
예제 #4
0
int amqp_handle_input(amqp_connection_state_t state,
		      amqp_bytes_t received_data,
		      amqp_frame_t *decoded_frame)
{
  size_t bytes_consumed;
  void *raw_frame;

  /* Returning frame_type of zero indicates either insufficient input,
     or a complete, ignored frame was read. */
  decoded_frame->frame_type = 0;

  if (received_data.len == 0)
    return 0;

  if (state->state == CONNECTION_STATE_IDLE) {
    state->inbound_buffer.bytes = amqp_pool_alloc(&state->frame_pool,
						  state->inbound_buffer.len);
    if (state->inbound_buffer.bytes == NULL)
      /* state->inbound_buffer.len is always nonzero, because it
	 corresponds to frame_max, which is not permitted to be less
	 than AMQP_FRAME_MIN_SIZE (currently 4096 bytes). */
      return -ERROR_NO_MEMORY;

    state->state = CONNECTION_STATE_HEADER;
  }

  bytes_consumed = consume_data(state, &received_data);

  /* do we have target_size data yet? if not, return with the
     expectation that more will arrive */
  if (state->inbound_offset < state->target_size)
    return bytes_consumed;

  raw_frame = state->inbound_buffer.bytes;

  switch (state->state) {
  case CONNECTION_STATE_INITIAL:
    /* check for a protocol header from the server */
    if (memcmp(raw_frame, "AMQP", 4) == 0) {
      decoded_frame->frame_type = AMQP_PSEUDOFRAME_PROTOCOL_HEADER;
      decoded_frame->channel = 0;

      decoded_frame->payload.protocol_header.transport_high
                                        = amqp_d8(raw_frame, 4);
      decoded_frame->payload.protocol_header.transport_low
                                        = amqp_d8(raw_frame, 5);
      decoded_frame->payload.protocol_header.protocol_version_major
                                        = amqp_d8(raw_frame, 6);
      decoded_frame->payload.protocol_header.protocol_version_minor
                                        = amqp_d8(raw_frame, 7);

      return_to_idle(state);
      return bytes_consumed;
    }

    /* it's not a protocol header; fall through to process it as a
       regular frame header */

  case CONNECTION_STATE_HEADER:
    /* frame length is 3 bytes in */
    state->target_size
           = amqp_d32(raw_frame, 3) + HEADER_SIZE + FOOTER_SIZE;
    state->state = CONNECTION_STATE_BODY;

    bytes_consumed += consume_data(state, &received_data);

    /* do we have target_size data yet? if not, return with the
       expectation that more will arrive */
    if (state->inbound_offset < state->target_size)
      return bytes_consumed;

    /* fall through to process body */

  case CONNECTION_STATE_BODY: {
    amqp_bytes_t encoded;
    int res;

    /* Check frame end marker (footer) */
    if (amqp_d8(raw_frame, state->target_size - 1) != AMQP_FRAME_END)
      return -ERROR_BAD_AMQP_DATA;

    decoded_frame->frame_type = amqp_d8(raw_frame, 0);
    decoded_frame->channel = amqp_d16(raw_frame, 1);

    switch (decoded_frame->frame_type) {
    case AMQP_FRAME_METHOD:
      decoded_frame->payload.method.id = amqp_d32(raw_frame, HEADER_SIZE);
      encoded.bytes = amqp_offset(raw_frame, HEADER_SIZE + 4);
      encoded.len = state->target_size - HEADER_SIZE - 4 - FOOTER_SIZE;

      res = amqp_decode_method(decoded_frame->payload.method.id,
			       &state->decoding_pool, encoded,
			       &decoded_frame->payload.method.decoded);
      if (res < 0)
	return res;

      break;

    case AMQP_FRAME_HEADER:
      decoded_frame->payload.properties.class_id
                                          = amqp_d16(raw_frame, HEADER_SIZE);
      /* unused 2-byte weight field goes here */
      decoded_frame->payload.properties.body_size
                                      = amqp_d64(raw_frame, HEADER_SIZE + 4);
      encoded.bytes = amqp_offset(raw_frame, HEADER_SIZE + 12);
      encoded.len = state->target_size - HEADER_SIZE - 12 - FOOTER_SIZE;
      decoded_frame->payload.properties.raw = encoded;

      res = amqp_decode_properties(decoded_frame->payload.properties.class_id,
                                   &state->decoding_pool, encoded,
                                   &decoded_frame->payload.properties.decoded);
      if (res < 0)
        return res;

      break;

    case AMQP_FRAME_BODY:
      decoded_frame->payload.body_fragment.len
                            = state->target_size - HEADER_SIZE - FOOTER_SIZE;
      decoded_frame->payload.body_fragment.bytes
                                       = amqp_offset(raw_frame, HEADER_SIZE);
      break;

    case AMQP_FRAME_HEARTBEAT:
      break;

    default:
      /* Ignore the frame */
      decoded_frame->frame_type = 0;
      break;
    }

    return_to_idle(state);

    if (decoded_frame->frame_type == AMQP_FRAME_METHOD) {
        if (decoded_frame->payload.method.id == AMQP_BASIC_RETURN_METHOD) {
	  amqp_basic_return_t *m = decoded_frame->payload.method.decoded;
	  if(state->basic_return_callback)
	      state->basic_return_callback(decoded_frame->channel, m,
					   state->basic_return_callback_data);
	}
	else if (decoded_frame->payload.method.id == AMQP_CHANNEL_CLOSE_METHOD) {
	    amqp_channel_close_t *m = decoded_frame->payload.method.decoded;
	    if (state->channel_close_callback)
		state->channel_close_callback(decoded_frame->channel, m,
					      state->channel_close_callback_data);
	}
    }
      
    return bytes_consumed;
  }

  default:
    amqp_abort("Internal error: invalid amqp_connection_state_t->state %d", state->state);
    return bytes_consumed;
  }
}
예제 #5
0
/*
 * sgs_connection_do_work()
 */
int sgs_connection_do_work(sgs_connection_impl *connection) {
    fd_set readset, writeset, exceptset;
    struct timeval timeout_tv;
    int result;
    sgs_socket_t sockfd;
    socklen_t optlen;

    if (connection->state == SGS_CONNECTION_IMPL_DISCONNECTED) {
        /** Error: should not call do_io() when disconnected. */
        errno = ENOTCONN;
        return -1;
    }

    sockfd = connection->socket_fd;
    
    FD_ZERO(&readset);
    FD_ZERO(&writeset);
    FD_ZERO(&exceptset);
    
    FD_SET(sockfd, &readset);
    FD_SET(sockfd, &writeset);
    FD_SET(sockfd, &exceptset);
    
    timeout_tv.tv_sec = 0;
    timeout_tv.tv_usec = 0;
    
    
    result = select(sockfd + 1, &readset, &writeset, &exceptset, &timeout_tv);
    if (result <= 0) {
        return result;  /** -1 or 0 */
    }
    
    if (FD_ISSET(sockfd, &exceptset)) {
        optlen = sizeof(errno);  /* SO_ERROR should return an int */
        if (sgs_socket_getsockopt(sockfd, SOL_SOCKET, SO_ERROR, &errno, &optlen) == -1) {
            return -1;
        }
        
        conn_closed(connection);
        return -1;
    }
    
    if (FD_ISSET(sockfd, &readset)) {
        /** Read stuff off the socket and write it to the in-buffer. */
        result = sgs_impl_read_from_fd(connection->inbuf, sockfd);
        if (result == -1) {
            return -1;
        }
        
        /* Return value of 0 may or may not mean that EOF was read. */
        if ((result == 0) && (sgs_buffer_remaining(connection->inbuf) > 0)) {
            conn_closed(connection);   /** The server closed the socket. */
            return 0;
        }
        
        /** Try to pull out messages from the inbuf and process them. */
        if (consume_data(connection) == -1) return -1;
    }
    
    if (FD_ISSET(sockfd, &writeset)) {
        /** If we are waiting for connect() to complete, its done now. */
        if (connection->state == SGS_CONNECTION_IMPL_CONNECTING) {
            connection->state = SGS_CONNECTION_IMPL_CONNECTED;
        }
        
        /** Read stuff out of the out-buffer and write it to the socket. */
        result = sgs_impl_write_to_fd(connection->outbuf, sockfd);
        if (result == -1) {
            return -1;
        }
    }
    
    /** If there is room in inbuf, then register interest in socket reads. */
    if (connection->state == SGS_CONNECTION_IMPL_CONNECTED) {
        if (sgs_buffer_remaining(connection->inbuf) > 0){
            if (connection->input_enabled == 0){
                connection->ctx->reg_fd_cb(connection, sockfd, POLLIN);
                connection->input_enabled = 1;
            }
        } else if (connection->input_enabled == 1){
            connection->ctx->unreg_fd_cb(connection, sockfd, POLLIN);
            connection->input_enabled = 0;
        }

        /** If there is data in outbuf, then register interest in socket writes. */
        if (sgs_buffer_size(connection->outbuf) > 0) {
            if (connection->output_enabled == 0) {
                connection->ctx->reg_fd_cb(connection, sockfd, POLLOUT);
                connection->output_enabled = 1;
            }
        } else if (connection->output_enabled == 1){
            connection->ctx->unreg_fd_cb(connection, sockfd, POLLOUT);
            connection->output_enabled = 0;
        }
    }
    return 0;
}