Beispiel #1
0
/**
 * Determines whether inbound parsing needs to continue or stop. In
 * case the data appears to be plain text HTTP, we try to continue.
 *
 * @param[in] connp
 * @return HTP_OK if the parser can resume parsing, HTP_DATA_BUFFER if
 *         we need more data.
 */
htp_status_t htp_connp_REQ_CONNECT_PROBE_DATA(htp_connp_t *connp) {
    for (;;) {//;i < max_read; i++) {
        IN_PEEK_NEXT(connp);
        // Have we reached the end of the line? For some reason
        // we can't test after IN_COPY_BYTE_OR_RETURN */
        if (connp->in_next_byte == LF || connp->in_next_byte == 0x00)
            break;

        IN_COPY_BYTE_OR_RETURN(connp);

    }

    unsigned char *data;
    size_t len;
    if (htp_connp_req_consolidate_data(connp, &data, &len) != HTP_OK) {
        fprintf(stderr, "htp_connp_req_consolidate_data fail");
        return HTP_ERROR;
    }
#ifdef HTP_DEBUG
    fprint_raw_data(stderr, "PROBING", data, len);
#endif

    size_t pos = 0;
    size_t mstart = 0;
    // skip past leading whitespace. IIS allows this
    while ((pos < len) && htp_is_space(data[pos]))
        pos++;
    if (pos)
        mstart = pos;
    // The request method starts at the beginning of the
    // line and ends with the first whitespace character.
    while ((pos < len) && (!htp_is_space(data[pos])))
        pos++;

    int methodi = HTP_M_UNKNOWN;
    bstr *method = bstr_dup_mem(data + mstart, pos - mstart);
    if (method) {
        methodi = htp_convert_method_to_number(method);
        bstr_free(method);
    }
    if (methodi != HTP_M_UNKNOWN) {
#ifdef HTP_DEBUG
        fprint_raw_data(stderr, "htp_connp_REQ_CONNECT_PROBE_DATA: tunnel contains plain text HTTP", data, len);
#endif
        connp->in_state = htp_connp_REQ_IDLE;
    } else {
#ifdef HTP_DEBUG
        fprint_raw_data(stderr, "htp_connp_REQ_CONNECT_PROBE_DATA: tunnel is not HTTP", data, len);
#endif
        connp->in_status = HTP_STREAM_TUNNEL;
        connp->out_status = HTP_STREAM_TUNNEL;
    }

    // not calling htp_connp_req_clear_buffer, we're not consuming the data

    return HTP_OK;
}
Beispiel #2
0
htp_status_t htp_tx_res_process_body_data_ex(htp_tx_t *tx, const void *data, size_t len) {
    if (tx == NULL) return HTP_ERROR;

    // NULL data is allowed in this private function; it's
    // used to indicate the end of response body.

    #ifdef HTP_DEBUG
    fprint_raw_data(stderr, __FUNCTION__, data, len);
    #endif

    htp_tx_data_t d;

    d.tx = tx;
    d.data = (unsigned char *) data;
    d.len = len;

    // Keep track of body size before decompression.
    tx->response_message_len += d.len;

    switch (tx->response_content_encoding_processing) {
        case HTP_COMPRESSION_GZIP:
        case HTP_COMPRESSION_DEFLATE:
            // Send data buffer to the decompressor.
            tx->connp->out_decompressor->decompress(tx->connp->out_decompressor, &d);

            if (data == NULL) {
                // Shut down the decompressor, if we used one.
                tx->connp->out_decompressor->destroy(tx->connp->out_decompressor);
                tx->connp->out_decompressor = NULL;
            }
            break;

        case HTP_COMPRESSION_NONE:
            // When there's no decompression, response_entity_len.
            // is identical to response_message_len.
            tx->response_entity_len += d.len;

            htp_status_t rc = htp_res_run_hook_body_data(tx->connp, &d);
            if (rc != HTP_OK) return HTP_ERROR;
            break;

        default:
            // Internal error.
            htp_log(tx->connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0,
                    "[Internal Error] Invalid tx->response_content_encoding_processing value: %d",
                    tx->response_content_encoding_processing);
            return HTP_ERROR;
            break;
    }

    return HTP_OK;
}
Beispiel #3
0
static htp_status_t htp_tx_res_process_body_data_decompressor_callback(htp_tx_data_t *d) {
    #if HTP_DEBUG
    fprint_raw_data(stderr, __FUNCTION__, d->data, d->len);
    #endif

    // Keep track of actual response body length.
    d->tx->response_entity_len += d->len;

    // Invoke all callbacks.
    int rc = htp_res_run_hook_body_data(d->tx->connp, d);
    if (rc != HTP_OK) return HTP_ERROR;

    return HTP_OK;
}
Beispiel #4
0
/**
 * Parses request line.
 *
 * @param[in] connp
 * @returns HTP_OK on state change, HTP_ERROR on error, or HTP_DATA when more data is needed.
 */
htp_status_t htp_connp_REQ_LINE(htp_connp_t *connp) {
    for (;;) {
        // Get one byte
        IN_COPY_BYTE_OR_RETURN(connp);       

        // Have we reached the end of the line?
        if (connp->in_next_byte == LF) {
            unsigned char *data;
            size_t len;

            htp_connp_req_consolidate_data(connp, &data, &len);

            #ifdef HTP_DEBUG
            fprint_raw_data(stderr, __FUNCTION__, data, len);
            #endif

            // Is this a line that should be ignored?
            if (htp_connp_is_line_ignorable(connp, data, len)) {
                // We have an empty/whitespace line, which we'll note, ignore and move on.
                connp->in_tx->request_ignored_lines++;

                // TODO How many empty lines are we willing to accept?

                htp_connp_req_clear_buffer(connp);

                return HTP_OK;
            }

            // Process request line.

            htp_chomp(data, &len);

            connp->in_tx->request_line = bstr_dup_mem(data, len);
            if (connp->in_tx->request_line == NULL) return HTP_ERROR;

            if (connp->cfg->parse_request_line(connp) != HTP_OK) return HTP_ERROR;

            // Finalize request line parsing.

            if (htp_tx_state_request_line(connp->in_tx) != HTP_OK) return HTP_ERROR;

            htp_connp_req_clear_buffer(connp);

            return HTP_OK;
        }
    }

    return HTP_ERROR;
}
Beispiel #5
0
/**
 * Extracts chunk length.
 *
 * @param[in] connp
 * @returns HTP_OK on state change, HTP_ERROR on error, or HTP_DATA when more data is needed.
 */
htp_status_t htp_connp_RES_BODY_CHUNKED_LENGTH(htp_connp_t *connp) {
    for (;;) {
        OUT_COPY_BYTE_OR_RETURN(connp);
        
        // Have we reached the end of the line?
        if (connp->out_next_byte == LF) {
            unsigned char *data;
            size_t len;

            if (htp_connp_res_consolidate_data(connp, &data, &len) != HTP_OK) {
                return HTP_ERROR;
            }

            connp->out_tx->response_message_len += len;

            #ifdef HTP_DEBUG
            fprint_raw_data(stderr, "Chunk length line", data, len);
            #endif

            htp_chomp(data, &len);
            
            connp->out_chunked_length = htp_parse_chunked_length(data, len);
            
            htp_connp_res_clear_buffer(connp);

            // Handle chunk length
            if (connp->out_chunked_length > 0) {
                // More data available                
                connp->out_state = htp_connp_RES_BODY_CHUNKED_DATA;
            } else if (connp->out_chunked_length == 0) {
                // End of data
                connp->out_state = htp_connp_RES_HEADERS;
                connp->out_tx->response_progress = HTP_RESPONSE_TRAILER;
            } else {
                // Invalid chunk length
                htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0,
                        "Response chunk encoding: Invalid chunk length: %d", connp->out_chunked_length);
                return HTP_ERROR;
            }

            return HTP_OK;
        }
    }

    return HTP_ERROR;
}
Beispiel #6
0
/**
 * Extracts chunk length.
 *
 * @param[in] connp
 * @returns HTP_OK on state change, HTP_ERROR on error, or HTP_DATA when more data is needed.
 */
htp_status_t htp_connp_REQ_BODY_CHUNKED_LENGTH(htp_connp_t *connp) {
    for (;;) {
        IN_COPY_BYTE_OR_RETURN(connp);

        // Have we reached the end of the line?
        if (connp->in_next_byte == LF) {
            unsigned char *data;
            size_t len;

            htp_connp_req_consolidate_data(connp, &data, &len);

            connp->in_tx->request_message_len += len;

            #ifdef HTP_DEBUG
            fprint_raw_data(stderr, "Chunk length line", data, len);
            #endif

            htp_chomp(data, &len);

            connp->in_chunked_length = htp_parse_chunked_length(data, len);

            htp_connp_req_clear_buffer(connp);

            // Handle chunk length.
            if (connp->in_chunked_length > 0) {
                // More data available.
                // TODO Add a check (flag) for excessive chunk length.
                connp->in_state = htp_connp_REQ_BODY_CHUNKED_DATA;
            } else if (connp->in_chunked_length == 0) {
                // End of data
                connp->in_state = htp_connp_REQ_HEADERS;
                connp->in_tx->request_progress = HTP_REQUEST_TRAILER;
            } else {
                // Invalid chunk length.
                htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Request chunk encoding: Invalid chunk length");
                return HTP_ERROR;
            }

            return HTP_OK;
        }
    }

    return HTP_ERROR;
}
Beispiel #7
0
/**
 * Parse the request line.
 *
 * @param[in] connp
 * @returns HTP_OK on succesful parse, HTP_ERROR on error.
 */
htp_status_t htp_connp_REQ_LINE_complete(htp_connp_t *connp) {
    unsigned char *data;
    size_t len;

    if (htp_connp_req_consolidate_data(connp, &data, &len) != HTP_OK) {
        return HTP_ERROR;
    }

    #ifdef HTP_DEBUG
    fprint_raw_data(stderr, __FUNCTION__, data, len);
    #endif

    // Is this a line that should be ignored?
    if (htp_connp_is_line_ignorable(connp, data, len)) {
        // We have an empty/whitespace line, which we'll note, ignore and move on.
        connp->in_tx->request_ignored_lines++;

        htp_connp_req_clear_buffer(connp);

        return HTP_OK;
    }

    // Process request line.

    htp_chomp(data, &len);

    connp->in_tx->request_line = bstr_dup_mem(data, len);
    if (connp->in_tx->request_line == NULL)
        return HTP_ERROR;

    if (connp->cfg->parse_request_line(connp) != HTP_OK)
        return HTP_ERROR;

    // Finalize request line parsing.

    if (htp_tx_state_request_line(connp->in_tx) != HTP_OK)
        return HTP_ERROR;

    htp_connp_req_clear_buffer(connp);

    return HTP_OK;
}
Beispiel #8
0
/**
 * This method is invoked whenever a piece of data, belonging to a single field (name or value)
 * becomes available. It will either create a new parameter or store the transient information
 * until a parameter can be created.
 *
 * @param urlenp
 * @param data
 * @param startpos
 * @param endpos
 * @param c Should contain -1 if the reason this function is called is because the end of
 *          the current data chunk is reached.
 */
static void htp_urlenp_add_field_piece(htp_urlenp_t *urlenp, unsigned char *data, size_t startpos, size_t endpos, int c) {
    // Add field if we know it ended or if we know that
    // we've used all of the input data
    if ((c != -1) || (urlenp->_complete)) {
        // Add field
        bstr *field = NULL;

        // Did we use the string builder for this field?
        if (bstr_builder_size(urlenp->_bb) > 0) {
            // The current field consists of more than once piece,
            // we have to use the string builder

            // Add current piece to string builder
            if (endpos - startpos > 0) {
                bstr_builder_append_mem(urlenp->_bb, (char *) data + startpos, endpos - startpos);
            }

            // Generate the field and clear the string builder
            field = bstr_builder_to_str(urlenp->_bb);
            if (field == NULL) return;
            bstr_builder_clear(urlenp->_bb);
        } else {
            // We only have the current piece to work with, so
            // no need to involve the string builder
            field = bstr_dup_mem((char *) data + startpos, endpos - startpos);
            if (field == NULL) return;
        }

        // Process the field differently, depending on the current state
        if (urlenp->_state == HTP_URLENP_STATE_KEY) {
            // Store the name for later
            urlenp->_name = field;

            if (urlenp->_complete) {
                // Param with key but no value
                bstr *name = urlenp->_name;
                bstr *value = bstr_dup_c("");

                if (urlenp->decode_url_encoding) {                    
                    // htp_uriencoding_normalize_inplace(name);
                    htp_decode_urlencoded_inplace(urlenp->tx->connp->cfg, urlenp->tx, name);
                }
                
                table_addn(urlenp->params, name, value);
                urlenp->_name = NULL;

                #ifdef HTP_DEBUG
                fprint_raw_data(stderr, "NAME", (unsigned char *) bstr_ptr(name), bstr_len(name));
                fprint_raw_data(stderr, "VALUE", (unsigned char *) bstr_ptr(value), bstr_len(value));
                #endif
            }
        } else {
            // Param with key and value                        
            bstr *name = urlenp->_name;
            bstr *value = field;
            
            if (urlenp->decode_url_encoding) {                
                htp_decode_urlencoded_inplace(urlenp->tx->connp->cfg, urlenp->tx, name);
                htp_decode_urlencoded_inplace(urlenp->tx->connp->cfg, urlenp->tx, value);
            }

            table_addn(urlenp->params, name, value);
            urlenp->_name = NULL;

            #ifdef HTP_DEBUG
            fprint_raw_data(stderr, "NAME", (unsigned char *) bstr_ptr(name), bstr_len(name));
            fprint_raw_data(stderr, "VALUE", (unsigned char *) bstr_ptr(value), bstr_len(value));
            #endif
        }
    } else {
        // Make a copy of the data and store it in an array for later
        if (endpos - startpos > 0) {
            bstr_builder_append_mem(urlenp->_bb, (char *) data + startpos, endpos - startpos);
        }
    }
}
htp_status_t htp_parse_request_line_generic_ex(htp_connp_t *connp, int nul_terminates) {
    htp_tx_t *tx = connp->in_tx;
    unsigned char *data = bstr_ptr(tx->request_line);
    size_t len = bstr_len(tx->request_line);
    size_t pos = 0;
    size_t mstart = 0;
    size_t start;
    size_t bad_delim;

    if (nul_terminates) {
        // The line ends with the first NUL byte.
        
        size_t newlen = 0;
        while ((pos < len) && (data[pos] != '\0')) {
            pos++;
            newlen++;
        }

        // Start again, with the new length.
        len = newlen;
        pos = 0;
    }

    // skip past leading whitespace. IIS allows this
    while ((pos < len) && htp_is_space(data[pos])) pos++;
    if (pos) {
        htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "Request line: leading whitespace");
        mstart = pos;

        if (connp->cfg->requestline_leading_whitespace_unwanted != HTP_UNWANTED_IGNORE) {
            // reset mstart so that we copy the whitespace into the method
            mstart = 0;
            // set expected response code to this anomaly
            tx->response_status_expected_number = connp->cfg->requestline_leading_whitespace_unwanted;
        }
    }

    // The request method starts at the beginning of the
    // line and ends with the first whitespace character.
    while ((pos < len) && (!htp_is_space(data[pos]))) pos++;

    // No, we don't care if the method is empty.

    tx->request_method = bstr_dup_mem(data + mstart, pos - mstart);
    if (tx->request_method == NULL) return HTP_ERROR;

    #ifdef HTP_DEBUG
    fprint_raw_data(stderr, __FUNCTION__, bstr_ptr(tx->request_method), bstr_len(tx->request_method));
    #endif

    tx->request_method_number = htp_convert_method_to_number(tx->request_method);

    bad_delim = 0;
    // Ignore whitespace after request method. The RFC allows
    // for only one SP, but then suggests any number of SP and HT
    // should be permitted. Apache uses isspace(), which is even
    // more permitting, so that's what we use here.
    while ((pos < len) && (isspace(data[pos]))) {
        if (!bad_delim && data[pos] != 0x20) {
            bad_delim++;
        }
        pos++;
    }
    if (bad_delim) {
        htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "Request line: non-compliant delimiter between Method and URI");
    }

    // Is there anything after the request method?
    if (pos == len) {
        // No, this looks like a HTTP/0.9 request.

        tx->is_protocol_0_9 = 1;
        tx->request_protocol_number = HTP_PROTOCOL_0_9;

        return HTP_OK;
    }

    start = pos;
    bad_delim = 0;

    // The URI ends with the first whitespace.
    while ((pos < len) && (data[pos] != 0x20)) {
        if (!bad_delim && htp_is_space(data[pos])) {
            bad_delim++;
        }
        pos++;
    }
    /* if we've seen some 'bad' delimiters, we retry with those */
    if (bad_delim && pos == len) {
        // special case: even though RFC's allow only SP (0x20), many
        // implementations allow other delimiters, like tab or other
        // characters that isspace() accepts.
        pos = start;
        while ((pos < len) && (!htp_is_space(data[pos]))) pos++;
    }
    if (bad_delim) {
        // warn regardless if we've seen non-compliant chars
        htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "Request line: URI contains non-compliant delimiter");
    }

    tx->request_uri = bstr_dup_mem(data + start, pos - start);
    if (tx->request_uri == NULL) return HTP_ERROR;

    #ifdef HTP_DEBUG
    fprint_raw_data(stderr, __FUNCTION__, bstr_ptr(tx->request_uri), bstr_len(tx->request_uri));
    #endif

    // Ignore whitespace after URI.
    while ((pos < len) && (htp_is_space(data[pos]))) pos++;

    // Is there protocol information available?
    if (pos == len) {
        // No, this looks like a HTTP/0.9 request.

        tx->is_protocol_0_9 = 1;
        tx->request_protocol_number = HTP_PROTOCOL_0_9;

        return HTP_OK;
    }

    // The protocol information continues until the end of the line.
    tx->request_protocol = bstr_dup_mem(data + pos, len - pos);
    if (tx->request_protocol == NULL) return HTP_ERROR;

    tx->request_protocol_number = htp_parse_protocol(tx->request_protocol);

    #ifdef HTP_DEBUG
    fprint_raw_data(stderr, __FUNCTION__, bstr_ptr(tx->request_protocol), bstr_len(tx->request_protocol));
    #endif

    return HTP_OK;
}
/**
 * Parse request line as Apache 2.2 does.
 *
 * @param connp
 * @return HTP_OK or HTP_ERROR
 */
int htp_parse_request_line_apache_2_2(htp_connp_t *connp) {
    htp_tx_t *tx = connp->in_tx;
    unsigned char *data = (unsigned char *) bstr_ptr(tx->request_line);
    size_t len = bstr_len(tx->request_line);
    size_t pos = 0;

    // In this implementation we assume the
    // line ends with the first NUL byte.
    if (tx->request_line_nul_offset != -1) {
        len = tx->request_line_nul_offset - 1;
    }

    // The request method starts at the beginning of the
    // line and ends with the first whitespace character.
    while ((pos < len) && (!htp_is_space(data[pos]))) {
        pos++;
    }

    // No, we don't care if the method is empty.

    tx->request_method = bstr_memdup((char *) data, pos);

#ifdef HTP_DEBUG
    fprint_raw_data(stderr, __FUNCTION__, (unsigned char *)bstr_ptr(tx->request_method), bstr_len(tx->request_method));
#endif

    tx->request_method_number = htp_convert_method_to_number(tx->request_method);

    // Ignore whitespace after request method. The RFC allows
    // for only one SP, but then suggests any number of SP and HT
    // should be permitted. Apache uses isspace(), which is even
    // more permitting, so that's what we use here.
    while ((pos < len) && (isspace(data[pos]))) {
        pos++;
    }

    size_t start = pos;

    // The URI ends with the first whitespace.
    while ((pos < len) && (!htp_is_space(data[pos]))) {
        pos++;
    }

    tx->request_uri = bstr_memdup((char *) data + start, pos - start);
    if (tx->request_uri == NULL) {
        return HTP_ERROR;
    }

#ifdef HTP_DEBUG
    fprint_raw_data(stderr, __FUNCTION__, (unsigned char *)bstr_ptr(tx->request_uri), bstr_len(tx->request_uri));
#endif

    // Ignore whitespace after URI
    while ((pos < len) && (htp_is_space(data[pos]))) {
        pos++;
    }

    // Is there protocol information available?
    if (pos == len) {
        // No, this looks like a HTTP/0.9 request.
        tx->protocol_is_simple = 1;
        return HTP_OK;
    }

    // The protocol information spreads until the end of the line.
    tx->request_protocol = bstr_memdup((char *) data + pos, len - pos);
    tx->request_protocol_number = htp_parse_protocol(tx->request_protocol);

#ifdef HTP_DEBUG
    fprint_raw_data(stderr, __FUNCTION__, (unsigned char *)bstr_ptr(tx->request_protocol), bstr_len(tx->request_protocol));
#endif

    return HTP_OK;
}
Beispiel #11
0
/**
 * Parses request headers.
 *
 * @param connp
 * @returns HTP_OK on state change, HTTP_ERROR on error, or HTP_DATA when more data is needed.
 */
int htp_connp_REQ_HEADERS(htp_connp_t *connp) {
    for (;;) {
        IN_COPY_BYTE_OR_RETURN(connp);

        if (connp->in_header_line == NULL) {
            connp->in_header_line = calloc(1, sizeof (htp_header_line_t));
            if (connp->in_header_line == NULL) return HTP_ERROR;
            connp->in_header_line->first_nul_offset = -1;
        }

        // Keep track of NUL bytes
        if (connp->in_next_byte == 0) {
            // Store the offset of the first NUL
            if (connp->in_header_line->has_nulls == 0) {
                connp->in_header_line->first_nul_offset = connp->in_line_len;
            }

            // Remember how many NULs there were
            connp->in_header_line->flags |= HTP_FIELD_NUL_BYTE;
            connp->in_header_line->has_nulls++;
        }

        // Have we reached the end of the line?
        if (connp->in_next_byte == LF) {
            #ifdef HTP_DEBUG
            fprint_raw_data(stderr, __FUNCTION__, connp->in_line, connp->in_line_len);
            #endif

            // Should we terminate headers?
            if (htp_connp_is_line_terminator(connp, connp->in_line, connp->in_line_len)) {
                // Terminator line

                // Parse previous header, if any
                if (connp->in_header_line_index != -1) {
                    if (connp->cfg->process_request_header(connp) != HTP_OK) {
                        // Note: downstream responsible for error logging
                        return HTP_ERROR;
                    }

                    // Reset index
                    connp->in_header_line_index = -1;
                }

                // Cleanup
                free(connp->in_header_line);
                connp->in_line_len = 0;
                connp->in_header_line = NULL;                

                // We've seen all request headers
                if (connp->in_chunk_count != connp->in_chunk_request_index) {
                    connp->in_tx->flags |= HTP_MULTI_PACKET_HEAD;
                }

                // Move onto the next processing phase
                if (connp->in_tx->progress[0] == TX_PROGRESS_REQ_HEADERS) {
                    // Determine if this request has a body
                    //connp->in_state = htp_connp_REQ_BODY_DETERMINE;
                    connp->in_state = htp_connp_REQ_CONNECT_CHECK;
                } else {
                    // Run hook REQUEST_TRAILER
                    int rc = hook_run_all(connp->cfg->hook_request_trailer, connp);
                    if (rc != HOOK_OK) {
                        htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0,
                            "Request trailer callback returned error (%d)", rc);
                        return HTP_ERROR;
                    }

                    // We've completed parsing this request
                    connp->in_state = htp_connp_REQ_IDLE;
                    connp->in_tx->progress[0] = TX_PROGRESS_WAIT;
                }

                return HTP_OK;
            }

            // Prepare line for consumption
            size_t raw_in_line_len = connp->in_line_len;
            htp_chomp(connp->in_line, &connp->in_line_len);

            // Check for header folding
            if (htp_connp_is_line_folded(connp->in_line, connp->in_line_len) == 0) {
                // New header line

                // Parse previous header, if any
                if (connp->in_header_line_index != -1) {
                    if (connp->cfg->process_request_header(connp) != HTP_OK) {
                        // Note: downstream responsible for error logging
                        return HTP_ERROR;
                    }

                    // Reset index
                    connp->in_header_line_index = -1;
                }

                // Remember the index of the fist header line
                connp->in_header_line_index = connp->in_header_line_counter;
            } else {
                // Folding; check that there's a previous header line to add to
                if (connp->in_header_line_index == -1) {
                    if (!(connp->in_tx->flags & HTP_INVALID_FOLDING)) {
                        connp->in_tx->flags |= HTP_INVALID_FOLDING;
                        htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0,
                            "Invalid request field folding");
                    }
                }
            }

            // Add the raw header line to the list
            if (raw_in_line_len > connp->in_line_len) {
                if (raw_in_line_len - connp->in_line_len == 2 &&
                        connp->in_line[connp->in_line_len] == 0x0d &&
                        connp->in_line[connp->in_line_len + 1] == 0x0a) {
                    connp->in_header_line->terminators = NULL;
                } else {
                    connp->in_header_line->terminators =
                        bstr_memdup((char *) connp->in_line + connp->in_line_len,
                                raw_in_line_len - connp->in_line_len);
                    if (connp->in_header_line->terminators == NULL) {
                        return HTP_ERROR;
                    }
                }
            } else {
                connp->in_header_line->terminators = NULL;
            }

            connp->in_header_line->line = bstr_memdup((char *) connp->in_line, connp->in_line_len);
            if (connp->in_header_line->line == NULL) {
                return HTP_ERROR;
            }
            list_add(connp->in_tx->request_header_lines, connp->in_header_line);
            connp->in_header_line = NULL;

            // Cleanup for the next line
            connp->in_line_len = 0;
            if (connp->in_header_line_index == -1) {
                connp->in_header_line_index = connp->in_header_line_counter;
            }

            connp->in_header_line_counter++;
        }
    }
    return HTP_ERROR;
}
Beispiel #12
0
/**
 * This method is invoked whenever a piece of data, belonging to a single field (name or value)
 * becomes available. It will either create a new parameter or store the transient information
 * until a parameter can be created.
 *
 * @param[in] urlenp
 * @param[in] data
 * @param[in] startpos
 * @param[in] endpos
 * @param[in] c Should contain -1 if the reason this function is called is because the end of
 *          the current data chunk is reached.
 */
static void htp_urlenp_add_field_piece(htp_urlenp_t *urlenp, const unsigned char *data, size_t startpos, size_t endpos, int last_char) {    
    // Add field if we know it ended (last_char is something other than -1)
    // or if we know that there won't be any more input data (urlenp->_complete is true).
    if ((last_char != -1) || (urlenp->_complete)) {       
        // Prepare the field value, assembling from multiple pieces as necessary.
    
        bstr *field = NULL;

        // Did we use the string builder for this field?
        if (bstr_builder_size(urlenp->_bb) > 0) {            
            // The current field consists of more than once piece, we have to use the string builder.

            // Add current piece to string builder.
            if ((data != NULL) && (endpos - startpos > 0)) {
                bstr_builder_append_mem(urlenp->_bb, data + startpos, endpos - startpos);
            }

            // Generate the field and clear the string builder.
            field = bstr_builder_to_str(urlenp->_bb);
            if (field == NULL) return;

            bstr_builder_clear(urlenp->_bb);
        } else {            
            // We only have the current piece to work with, so no need to involve the string builder.
            if ((data != NULL) && (endpos - startpos > 0)) {
                field = bstr_dup_mem(data + startpos, endpos - startpos);
                if (field == NULL) return;
            }
        }

        // Process field as key or value, as appropriate.
        
        if (urlenp->_state == HTP_URLENP_STATE_KEY) {
            // Key.

            // If there is no more work left to do, then we have a single key. Add it.
            if ((urlenp->_complete)||(last_char == urlenp->argument_separator)) {                
                
                // Handling empty pairs is tricky. We don't want to create a pair for
                // an entirely empty input, but in some cases it may be appropriate
                // (e.g., /index.php?&q=2).
                if ((field != NULL)||(last_char == urlenp->argument_separator)) {
                    // Add one pair, with an empty value and possibly empty key too.

                    bstr *name = field;
                    if (name == NULL) {
                        name = bstr_dup_c("");
                        if (name == NULL) return;
                    }

                    bstr *value = bstr_dup_c("");
                    if (value == NULL) {
                        bstr_free(name);
                        return;
                    }

                    if (urlenp->decode_url_encoding) {
                        htp_tx_urldecode_params_inplace(urlenp->tx, name);
                    }

                    htp_table_addn(urlenp->params, name, value);

                    urlenp->_name = NULL;

                    #ifdef HTP_DEBUG
                    fprint_raw_data(stderr, "NAME", bstr_ptr(name), bstr_len(name));
                    fprint_raw_data(stderr, "VALUE", bstr_ptr(value), bstr_len(value));
                    #endif
                }
            } else {                
                // This key will possibly be followed by a value, so keep it for later.
                urlenp->_name = field;
            }
        } else {            
            // Value (with a key remembered from before).

            bstr *name = urlenp->_name;
            urlenp->_name = NULL;

            if (name == NULL) {
                name = bstr_dup_c("");
                if (name == NULL) {
                    bstr_free(field);
                    return;
                }
            }

            bstr *value = field;
            if (value == NULL) {
                value = bstr_dup_c("");
                if (value == NULL) {
                    bstr_free(name);
                    return;
                }
            }

            if (urlenp->decode_url_encoding) {
                htp_tx_urldecode_params_inplace(urlenp->tx, name);
                htp_tx_urldecode_params_inplace(urlenp->tx, value);
            }

            htp_table_addn(urlenp->params, name, value);           

            #ifdef HTP_DEBUG
            fprint_raw_data(stderr, "NAME", bstr_ptr(name), bstr_len(name));
            fprint_raw_data(stderr, "VALUE", bstr_ptr(value), bstr_len(value));
            #endif
        }        
    } else {
        // The field has not ended. We'll make a copy of of the available data for later.
        if ((data != NULL) && (endpos - startpos > 0)) {
            bstr_builder_append_mem(urlenp->_bb, data + startpos, endpos - startpos);
        }
    }
}
Beispiel #13
0
/**
 * Parses request line.
 *
 * @param connp
 * @returns HTP_OK on state change, HTTP_ERROR on error, or HTP_DATA when more data is needed.
 */
int htp_connp_REQ_LINE(htp_connp_t *connp) {
    for (;;) {
        // Get one byte
        IN_COPY_BYTE_OR_RETURN(connp);

        // Keep track of NUL bytes
        if (connp->in_next_byte == 0) {
            // Remember how many NULs there were
            connp->in_tx->request_line_nul++;

            // Store the offset of the first NUL byte
            if (connp->in_tx->request_line_nul_offset == -1) {
                connp->in_tx->request_line_nul_offset = connp->in_line_len;
            }
        }

        // Have we reached the end of the line?
        if (connp->in_next_byte == LF) {
            #ifdef HTP_DEBUG
            fprint_raw_data(stderr, __FUNCTION__, connp->in_line, connp->in_line_len);
            #endif

            // Is this a line that should be ignored?
            if (htp_connp_is_line_ignorable(connp, connp->in_line, connp->in_line_len)) {
                // We have an empty/whitespace line, which we'll note, ignore and move on
                connp->in_tx->request_ignored_lines++;

                // TODO How many empty lines are we willing to accept?

                // Start again
                connp->in_line_len = 0;

                return HTP_OK;
            }

            // Process request line

            connp->in_tx->request_line_raw = bstr_dup_mem((char *) connp->in_line, connp->in_line_len);
            if (connp->in_tx->request_line_raw == NULL) {
                return HTP_ERROR;
            }

            /// @todo Would be nice to reference request_line_raw data
            htp_chomp(connp->in_line, &connp->in_line_len);
            connp->in_tx->request_line = bstr_dup_ex(connp->in_tx->request_line_raw, 0, connp->in_line_len);
            if (connp->in_tx->request_line == NULL) {
                return HTP_ERROR;
            }

            // Parse request line
            if (connp->cfg->parse_request_line(connp) != HTP_OK) {
                // Note: downstream responsible for error logging
                return HTP_ERROR;
            }

            if (connp->in_tx->request_method_number == M_CONNECT) {
                // Parse authority
                if (htp_parse_authority(connp, connp->in_tx->request_uri, &(connp->in_tx->parsed_uri_incomplete)) != HTP_OK) {
                    // Note: downstream responsible for error logging
                    return HTP_ERROR;
                }
            } else {
                // Parse the request URI
                if (htp_parse_uri(connp->in_tx->request_uri, &(connp->in_tx->parsed_uri_incomplete)) != HTP_OK) {
                    // Note: downstream responsible for error logging
                    return HTP_ERROR;
                }

                // Keep the original URI components, but
                // create a copy which we can normalize and use internally
                if (htp_normalize_parsed_uri(connp, connp->in_tx->parsed_uri_incomplete, connp->in_tx->parsed_uri)) {
                    // Note: downstream responsible for error logging
                    return HTP_ERROR;
                }

                // Run hook REQUEST_URI_NORMALIZE
                int rc = hook_run_all(connp->cfg->hook_request_uri_normalize, connp);
                if (rc != HOOK_OK) {
                    switch (rc) {
                        case HOOK_STOP:
                            return HTP_STOP;
                        case HOOK_ERROR:
                        case HOOK_DECLINED:
                        default:
                            htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0,
                                "Request headers callback returned error (%d)", rc);
                            return HTP_ERROR;
                    }
                }

                // Now is a good time to generate request_uri_normalized, before we finalize
                // parsed_uri (and lose the information which parts were provided in the request and
                // which parts we added).
                if (connp->cfg->generate_request_uri_normalized) {
                    connp->in_tx->request_uri_normalized = htp_unparse_uri_noencode(connp->in_tx->parsed_uri);

                    if (connp->in_tx->request_uri_normalized == NULL) {
                        // There's no sense in logging anything on a memory allocation failure
                        return HTP_ERROR;
                    }

                    #ifdef HTP_DEBUG
                    fprint_raw_data(stderr, "request_uri_normalized",
                        (unsigned char *) bstr_ptr(connp->in_tx->request_uri_normalized),
                        bstr_len(connp->in_tx->request_uri_normalized));
                    #endif
                }

                // Finalize parsed_uri

                // Scheme
                if (connp->in_tx->parsed_uri->scheme != NULL) {
                    if (bstr_cmp_c(connp->in_tx->parsed_uri->scheme, "http") != 0) {
                        // TODO Invalid scheme
                    }
                } else {
                    connp->in_tx->parsed_uri->scheme = bstr_dup_c("http");
                    if (connp->in_tx->parsed_uri->scheme == NULL) {
                        return HTP_ERROR;
                    }
                }

                // Port
                if (connp->in_tx->parsed_uri->port != NULL) {
                    if (connp->in_tx->parsed_uri->port_number != -1) {
                        // Check that the port in the URI is the same
                        // as the port on which the client is talking
                        // to the server
                        if (connp->conn->use_local_port) {
                            if (connp->in_tx->parsed_uri->port_number != connp->conn->local_port) {
                                // Incorrect port; use the real port instead
                                connp->in_tx->parsed_uri->port_number = connp->conn->local_port;
                                // TODO Log
                            }
                        } else {
                            connp->in_tx->parsed_uri->port_number = connp->conn->remote_port;
                        }
                    } else {
                        // Invalid port; use the real port instead
                        if (connp->conn->use_local_port) {
                            connp->in_tx->parsed_uri->port_number = connp->conn->local_port;
                        } else {
                            connp->in_tx->parsed_uri->port_number = connp->conn->remote_port;
                        }
                        // TODO Log
                    }
                } else {
                    if (connp->conn->use_local_port) {
                        connp->in_tx->parsed_uri->port_number = connp->conn->local_port;
                    } else {
                        connp->in_tx->parsed_uri->port_number = connp->conn->remote_port;
                    }
                }

                // Path
                if (connp->in_tx->parsed_uri->path == NULL) {
                    connp->in_tx->parsed_uri->path = bstr_dup_c("/");
                    if (connp->in_tx->parsed_uri->path == NULL) {
                        return HTP_ERROR;
                    }
                }
            }

            // Run hook REQUEST_LINE
            int rc = hook_run_all(connp->cfg->hook_request_line, connp);
            if (rc != HOOK_OK) {
                switch (rc) {
                    case HOOK_STOP:
                        return HTP_STOP;
                    case HOOK_ERROR:
                    case HOOK_DECLINED:
                    default:
                        htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0,
                            "Request headers callback returned error (%d)", rc);
                        return HTP_ERROR;
                }
            }

            // Clean up.
            connp->in_line_len = 0;

            // Move on to the next phase.
            connp->in_state = htp_connp_REQ_PROTOCOL;

            return HTP_OK;
        }
    }
}
Beispiel #14
0
/**
 * Parses request headers.
 *
 * @param connp
 * @returns HTP_OK on state change, HTTP_ERROR on error, or HTP_DATA when more data is needed.
 */
int htp_connp_REQ_HEADERS(htp_connp_t *connp) {
    for (;;) {
        IN_COPY_BYTE_OR_RETURN(connp);

        // Allocate structure to hold one header line
        if (connp->in_header_line == NULL) {
            connp->in_header_line = calloc(1, sizeof (htp_header_line_t));
            if (connp->in_header_line == NULL) return HTP_ERROR;
            connp->in_header_line->first_nul_offset = -1;
        }

        // Keep track of NUL bytes
        if (connp->in_next_byte == 0) {
            // Store the offset of the first NUL
            if (connp->in_header_line->has_nulls == 0) {
                connp->in_header_line->first_nul_offset = connp->in_line_len;
            }

            // Remember how many NULs there were
            connp->in_header_line->flags |= HTP_FIELD_NUL_BYTE;
            connp->in_header_line->has_nulls++;
        }

        // Have we reached the end of the line?
        if (connp->in_next_byte == LF) {
            #ifdef HTP_DEBUG
            fprint_raw_data(stderr, __FUNCTION__, connp->in_line, connp->in_line_len);
            #endif

            // Should we terminate headers?
            if (htp_connp_is_line_terminator(connp, connp->in_line, connp->in_line_len)) {
                // Terminator line
                connp->in_tx->request_headers_sep = bstr_dup_mem((char *)connp->in_line, connp->in_line_len);
                if (connp->in_tx->request_headers_sep == NULL) {
                    return HTP_ERROR;
                }

                // Parse previous header, if any
                if (connp->in_header_line_index != -1) {
                    if (connp->cfg->process_request_header(connp) != HTP_OK) {
                        // Note: downstream responsible for error logging
                        return HTP_ERROR;
                    }

                    // Reset index
                    connp->in_header_line_index = -1;
                }

                // Cleanup
                free(connp->in_header_line);
                connp->in_line_len = 0;
                connp->in_header_line = NULL;

                // We've seen all request headers

                // Did this request arrive in multiple chunks?
                if (connp->in_chunk_count != connp->in_chunk_request_index) {
                    connp->in_tx->flags |= HTP_MULTI_PACKET_HEAD;
                }

                // Move onto the next processing phase
                if (connp->in_tx->progress == TX_PROGRESS_REQ_HEADERS) {
                    // Remember how many header lines there were before trailers
                    connp->in_tx->request_header_lines_no_trailers = list_size(connp->in_tx->request_header_lines);

                    // Run hook REQUEST_HEADERS_RAW
                    //if (connp->cfg->hook_request_headers_raw != NULL) {
                    //    htp_req_run_hook_request_headers_raw(connp, 0,
                    //        connp->in_tx->request_header_lines_no_trailers);
                    //}

                    // Determine if this request has a body                    
                    connp->in_state = htp_connp_REQ_CONNECT_CHECK;
                } else {
                    // Run hook REQUEST_HEADERS_RAW
                    //if ((connp->cfg->hook_request_headers_raw != NULL)
                    //    && (list_size(connp->in_tx->request_header_lines) > connp->in_tx->request_header_lines_no_trailers)) {
                    //    htp_req_run_hook_request_headers_raw(connp,
                    //        connp->in_tx->request_header_lines_no_trailers,
                    //        list_size(connp->in_tx->request_header_lines));
                    //}

                    // Run hook REQUEST_TRAILER
                    int rc = hook_run_all(connp->cfg->hook_request_trailer, connp);
                    if (rc != HOOK_OK) {
                        switch (rc) {
                            case HOOK_STOP:
                                return HTP_STOP;
                            case HOOK_ERROR:
                            case HOOK_DECLINED:
                            default:
                                htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0,
                                    "Request headers callback returned error (%d)", rc);
                                return HTP_ERROR;
                        }
                    }

                    // We've completed parsing this request
                    connp->in_state = htp_connp_REQ_IDLE;
                    connp->in_tx->progress = TX_PROGRESS_WAIT;
                }

                return HTP_OK;
            }

            // Prepare line for consumption
            int chomp_result = htp_chomp(connp->in_line, &connp->in_line_len);

            // Check for header folding
            if (htp_connp_is_line_folded(connp->in_line, connp->in_line_len) == 0) {
                // New header line

                // Parse previous header, if any
                if (connp->in_header_line_index != -1) {
                    if (connp->cfg->process_request_header(connp) != HTP_OK) {
                        // Note: downstream responsible for error logging
                        return HTP_ERROR;
                    }

                    // Reset index
                    connp->in_header_line_index = -1;
                }

                // Remember the index of the fist header line
                connp->in_header_line_index = connp->in_header_line_counter;
            } else {
                // Folding; check that there's a previous header line to add to
                if (connp->in_header_line_index == -1) {
                    if (!(connp->in_tx->flags & HTP_INVALID_FOLDING)) {
                        connp->in_tx->flags |= HTP_INVALID_FOLDING;
                        htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0,
                            "Invalid request field folding");
                    }
                }
            }

            // Add the raw header line to the list
            connp->in_header_line->line = bstr_dup_mem((char *) connp->in_line, connp->in_line_len + chomp_result);
            if (connp->in_header_line->line == NULL) {
                return HTP_ERROR;
            }
            
            list_add(connp->in_tx->request_header_lines, connp->in_header_line);
            connp->in_header_line = NULL;

            // Cleanup for the next line
            connp->in_line_len = 0;
            if (connp->in_header_line_index == -1) {
                connp->in_header_line_index = connp->in_header_line_counter;
            }

            connp->in_header_line_counter++;
        }
    }
}
Beispiel #15
0
/**
 * Process a chunk of outbound (server or response) data.
 *
 * @param connp
 * @param timestamp
 * @param data
 * @param len
 * @return HTP_OK on state change, HTTP_ERROR on error, or HTP_DATA when more data is needed
 */
int htp_connp_res_data(htp_connp_t *connp, htp_time_t *timestamp, unsigned char *data, size_t len) {
    #ifdef HTP_DEBUG
    fprintf(stderr, "htp_connp_res_data(connp->out_status %x)\n", connp->out_status);
    fprint_raw_data(stderr, __FUNCTION__, data, len);
    #endif

    // Return if the connection has had a fatal error
    if (connp->out_status == STREAM_STATE_ERROR) {
        htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Outbound parser is in STREAM_STATE_ERROR");

        #ifdef HTP_DEBUG
        fprintf(stderr, "htp_connp_res_data: returning STREAM_STATE_DATA (previous error)\n");
        #endif

        return STREAM_STATE_ERROR;
    }

    // If the length of the supplied data chunk is zero, proceed
    // only if the stream has been closed. We do not allow zero-sized
    // chunks in the API, but we use it internally to force the parsers
    // to finalize parsing.
    if ((len == 0) && (connp->out_status != STREAM_STATE_CLOSED)) {
        htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Zero-length data chunks are not allowed");

        #ifdef HTP_DEBUG
        fprintf(stderr, "htp_connp_res_data: returning STREAM_STATE_DATA (zero-length chunk)\n");
        #endif

        return STREAM_STATE_CLOSED;
    }

    // Store the current chunk information
    memcpy(&connp->out_timestamp, timestamp, sizeof(*timestamp));
    connp->out_current_data = data;
    connp->out_current_len = len;
    connp->out_current_offset = 0;
    connp->conn->out_data_counter += len;
    connp->conn->out_packet_counter++;

    // Return without processing any data if the stream is in tunneling
    // mode (which it would be after an initial CONNECT transaction.
    if (connp->out_status == STREAM_STATE_TUNNEL) {
        #ifdef HTP_DEBUG
        fprintf(stderr, "htp_connp_res_data: returning STREAM_STATE_TUNNEL\n");
        #endif

        return STREAM_STATE_TUNNEL;
    }

    // Invoke a processor, in a loop, until an error
    // occurs or until we run out of data. Many processors
    // will process a request, each pointing to the next
    // processor that needs to run.
    for (;;) {
        #ifdef HTP_DEBUG
        fprintf(stderr, "htp_connp_res_data: out state=%s, progress=%s\n",
            htp_connp_out_state_as_string(connp),
            htp_tx_progress_as_string(connp->out_tx));
        #endif

        // Return if there's been an error
        // or if we've run out of data. We are relying
        // on processors to add error messages, so we'll
        // keep quiet here.
        int rc = connp->out_state(connp);
        if (rc == HTP_OK) {
            if (connp->out_status == STREAM_STATE_TUNNEL) {
                #ifdef HTP_DEBUG
                fprintf(stderr, "htp_connp_res_data: returning STREAM_STATE_TUNNEL\n");
                #endif

                return STREAM_STATE_TUNNEL;
            }
        } else {
            // Do we need more data?
            if (rc == HTP_DATA) {
                #ifdef HTP_DEBUG
                fprintf(stderr, "htp_connp_res_data: returning STREAM_STATE_DATA\n");
                #endif

                connp->out_status = STREAM_STATE_DATA;

                return STREAM_STATE_DATA;
            }

            // Check for suspended parsing
            if (rc == HTP_DATA_OTHER) {
                // We might have actually consumed the entire data chunk?
                if (connp->out_current_offset >= connp->out_current_len) {
                    #ifdef HTP_DEBUG
                    fprintf(stderr, "htp_connp_res_data: returning STREAM_STATE_DATA (suspended parsing)\n");
                    #endif

                    connp->out_status = STREAM_STATE_DATA;

                    // Do not send STREAM_DATE_DATA_OTHER if we've
                    // consumed the entire chunk
                    return STREAM_STATE_DATA;
                } else {
                    #ifdef HTP_DEBUG
                    fprintf(stderr, "htp_connp_res_data: returning STREAM_STATE_DATA_OTHER\n");
                    #endif

                    connp->out_status = STREAM_STATE_DATA_OTHER;

                    // Partial chunk consumption
                    return STREAM_STATE_DATA_OTHER;
                }
            }

            #ifdef HTP_DEBUG
            fprintf(stderr, "htp_connp_res_data: returning STREAM_STATE_ERROR\n");
            #endif

            // Remember that we've had an error. Errors are
            // not possible to recover from.
            connp->out_status = STREAM_STATE_ERROR;

            return STREAM_STATE_ERROR;
        }
    }
}
Beispiel #16
0
/**
 * Parses response line.
 *
 * @param connp
 * @returns HTP_OK on state change, HTTP_ERROR on error, or HTP_DATA when more data is needed.
 */
int htp_connp_RES_LINE(htp_connp_t * connp) {
    for (;;) {
        // Get one byte
        OUT_COPY_BYTE_OR_RETURN(connp);

        // Have we reached the end of the line?
        if (connp->out_next_byte == LF) {
            #ifdef HTP_DEBUG
            fprint_raw_data(stderr, __FUNCTION__, connp->out_line, connp->out_line_len);
            #endif

            // Is this a line that should be ignored?
            if (htp_connp_is_line_ignorable(connp, connp->out_line, connp->out_line_len)) {
                // We have an empty/whitespace line, which we'll note, ignore and move on
                connp->out_tx->response_ignored_lines++;

                // TODO How many lines are we willing to accept?

                // Start again
                connp->out_line_len = 0;

                return HTP_OK;
            }

            // Process response line

            // Deallocate previous response line allocations, which we would have on a 100 response
            // TODO Consider moving elsewhere; no need to make these checks on every response
            if (connp->out_tx->response_line != NULL) {
                bstr_free(&connp->out_tx->response_line);
            }

            if (connp->out_tx->response_protocol != NULL) {
                bstr_free(&connp->out_tx->response_protocol);
            }

            if (connp->out_tx->response_status != NULL) {
                bstr_free(&connp->out_tx->response_status);
            }

            if (connp->out_tx->response_message != NULL) {
                bstr_free(&connp->out_tx->response_message);
            }

            connp->out_tx->response_line_raw = bstr_dup_mem((char *) connp->out_line, connp->out_line_len);
            if (connp->out_tx->response_line_raw == NULL) {
                return HTP_ERROR;
            }

            /// @todo Would be nice to reference response_line_raw data
            int chomp_result = htp_chomp(connp->out_line, &connp->out_line_len);
            connp->out_tx->response_line = bstr_dup_ex(connp->out_tx->response_line_raw, 0, connp->out_line_len);
            if (connp->out_tx->response_line == NULL) {
                return HTP_ERROR;
            }

            // Parse response line
            if (connp->cfg->parse_response_line(connp) != HTP_OK) {
                // Note: downstream responsible for error logging
                return HTP_ERROR;
            }

            // Is the response line valid?
            if ((connp->out_tx->response_protocol_number < 0)
                || (connp->out_tx->response_status_number < 0)
                || (connp->out_tx->response_status_number < HTP_VALID_STATUS_MIN)
                || (connp->out_tx->response_status_number > HTP_VALID_STATUS_MAX)) {
                // Response line is invalid
                htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "Invalid response line");

                connp->out_tx->flags |= HTP_STATUS_LINE_INVALID;
            }

            // Even when the response line is invalid, determine if it looks like
            // a response line (which is what browsers do).
            if (htp_resembles_response_line(connp->out_tx) == 0) {
                // Process this line as response body data
                htp_tx_data_t d;

                d.tx = connp->out_tx;
                d.data = connp->out_line;
                d.len = connp->out_line_len + chomp_result;

                connp->out_tx->response_message_len += d.len;
                connp->out_tx->response_entity_len += d.len;
                
                int rc = htp_res_run_hook_body_data(connp, &d);
                if (rc != HOOK_OK) {
                    htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0,
                        "Response body data callback returned error (%d)", rc);
                    return HTP_ERROR;
                }

                // Continue to process response body
                connp->out_tx->response_transfer_coding = IDENTITY;
                connp->out_state = htp_connp_RES_BODY_IDENTITY;
                connp->out_tx->progress = TX_PROGRESS_RES_BODY;

                return HTP_OK;
            }

            // Run hook RESPONSE_LINE
            int rc = hook_run_all(connp->cfg->hook_response_line, connp);
            if (rc != HOOK_OK) {
                htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0,
                    "Response line callback returned error (%d)", rc);
                return HTP_ERROR;
            }

            // Clean up.
            connp->out_line_len = 0;

            // Move on to the next phase.
            connp->out_state = htp_connp_RES_HEADERS;
            connp->out_tx->progress = TX_PROGRESS_RES_HEADERS;

            return HTP_OK;
        }
    }
}
Beispiel #17
0
/**
 * Parses response headers.
 *
 * @param connp
 * @returns HTP_OK on state change, HTTP_ERROR on error, or HTP_DATA when more data is needed.
 */
int htp_connp_RES_HEADERS(htp_connp_t * connp) {
    for (;;) {
        OUT_COPY_BYTE_OR_RETURN(connp);

        if (connp->out_header_line == NULL) {
            connp->out_header_line = calloc(1, sizeof (htp_header_line_t));
            if (connp->out_header_line == NULL) return HTP_ERROR;
            connp->out_header_line->first_nul_offset = -1;
        }

        // Keep track of NUL bytes
        if (connp->out_next_byte == 0) {
            // Store the offset of the first NUL
            if (connp->out_header_line->has_nulls == 0) {
                connp->out_header_line->first_nul_offset = connp->out_line_len;
            }

            // Remember how many NULs there were
            connp->out_header_line->flags |= HTP_FIELD_NUL_BYTE;
            connp->out_header_line->has_nulls++;
        }

        // Have we reached the end of the line?
        if (connp->out_next_byte == LF) {
            #ifdef HTP_DEBUG
            fprint_raw_data(stderr, __FUNCTION__, connp->out_line, connp->out_line_len);
            #endif

            // Should we terminate headers?
            if (htp_connp_is_line_terminator(connp, connp->out_line, connp->out_line_len)) {
                // Terminator line
                connp->out_tx->response_headers_sep = bstr_dup_mem((char *)connp->out_line, connp->out_line_len);
                if (connp->out_tx->response_headers_sep == NULL) {
                    return HTP_ERROR;
                }

                // Parse previous header, if any
                if (connp->out_header_line_index != -1) {
                    // Only try to parse a header, but ignore
                    // any problems. That's what browsers do.
                    connp->cfg->process_response_header(connp);

                    // Reset index
                    connp->out_header_line_index = -1;
                }

                // Cleanup
                free(connp->out_header_line);
                connp->out_line_len = 0;
                connp->out_header_line = NULL;

                // We've seen all response headers
                if (connp->out_tx->progress == TX_PROGRESS_RES_HEADERS) {
                    // Determine if this response has a body
                    connp->out_state = htp_connp_RES_BODY_DETERMINE;
                } else {
                    // Run hook response_TRAILER
                    int rc = hook_run_all(connp->cfg->hook_response_trailer, connp);
                    if (rc != HOOK_OK) {
                        htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0,
                            "Response trailer callback returned error (%d)", rc);
                        return HTP_ERROR;
                    }

                    // We've completed parsing this response
                    connp->out_state = htp_connp_RES_IDLE;
                }

                return HTP_OK;
            }

            // Prepare line for consumption
            int chomp_result = htp_chomp(connp->out_line, &connp->out_line_len);

            // Check for header folding
            if (htp_connp_is_line_folded(connp->out_line, connp->out_line_len) == 0) {
                // New header line

                // Parse previous header, if any
                if (connp->out_header_line_index != -1) {
                    // Only try to parse a header, but ignore
                    // any problems. That's what browsers do.
                    connp->cfg->process_response_header(connp);

                    // Reset index
                    connp->out_header_line_index = -1;
                }

                // Remember the index of the fist header line
                connp->out_header_line_index = connp->out_header_line_counter;
            } else {
                // Folding; check that there's a previous header line to add to
                if (connp->out_header_line_index == -1) {
                    if (!(connp->out_tx->flags & HTP_INVALID_FOLDING)) {
                        connp->out_tx->flags |= HTP_INVALID_FOLDING;
                        htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "Invalid response field folding");
                    }
                }
            }

            // Add the raw header line to the list
            connp->out_header_line->line = bstr_dup_mem((char *) connp->out_line, connp->out_line_len + chomp_result);
            if (connp->out_header_line->line == NULL) {
                return HTP_ERROR;
            }

            list_add(connp->out_tx->response_header_lines, connp->out_header_line);
            connp->out_header_line = NULL;

            // Cleanup for the next line
            connp->out_line_len = 0;
            if (connp->out_header_line_index == -1) {

                connp->out_header_line_index = connp->out_header_line_counter;
            }

            connp->out_header_line_counter++;
        }
    }
}
Beispiel #18
0
/**
 * Parses response headers.
 *
 * @param[in] connp
 * @returns HTP_OK on state change, HTP_ERROR on error, or HTP_DATA when more data is needed.
 */
htp_status_t htp_connp_RES_HEADERS(htp_connp_t *connp) {
    for (;;) {
        OUT_COPY_BYTE_OR_RETURN(connp);       
        
        // Have we reached the end of the line?
        if (connp->out_next_byte == LF) {
            unsigned char *data;
            size_t len;

            htp_connp_res_consolidate_data(connp, &data, &len);

            #ifdef HTP_DEBUG
            fprint_raw_data(stderr, __FUNCTION__, data, len);
            #endif

            // Should we terminate headers?
            if (htp_connp_is_line_terminator(connp, data, len)) {
                // Parse previous header, if any.
                if (connp->out_header != NULL) {
                    if (connp->cfg->process_response_header(connp, bstr_ptr(connp->out_header),
                            bstr_len(connp->out_header)) != HTP_OK) return HTP_ERROR;

                    bstr_free(connp->out_header);
                    connp->out_header = NULL;
                }
                
                htp_connp_res_clear_buffer(connp);               

                // We've seen all response headers.
                if (connp->out_tx->progress == HTP_RESPONSE_HEADERS) {
                    // Response headers.

                    // The next step is to determine if this response has a body.
                    connp->out_state = htp_connp_RES_BODY_DETERMINE;
                } else {
                    // Response trailer.

                    // Finalize sending raw trailer data.
                    htp_status_t rc = htp_connp_res_receiver_finalize_clear(connp);
                    if (rc != HTP_OK) return rc;

                    // Run hook response_TRAILER.
                    rc = htp_hook_run_all(connp->cfg->hook_response_trailer, connp);
                    if (rc != HTP_OK) return rc;

                    // The next step is to finalize this response.
                    connp->out_state = htp_connp_RES_FINALIZE;
                }

                return HTP_OK;
            }

            htp_chomp(data, &len);

            // Check for header folding.
            if (htp_connp_is_line_folded(data, len) == 0) {
                // New header line.

                // Parse previous header, if any.
                if (connp->out_header != NULL) {
                    if (connp->cfg->process_response_header(connp, bstr_ptr(connp->out_header),
                            bstr_len(connp->out_header)) != HTP_OK) return HTP_ERROR;

                    bstr_free(connp->out_header);
                    connp->out_header = NULL;
                }

                OUT_PEEK_NEXT(connp);

                if (htp_is_folding_char(connp->out_next_byte) == 0) {
                    // Because we know this header is not folded, we can process the buffer straight away.
                    if (connp->cfg->process_response_header(connp, data, len) != HTP_OK) return HTP_ERROR;
                } else {
                    // Keep the partial header data for parsing later.
                    connp->out_header = bstr_dup_mem(data, len);
                    if (connp->out_header == NULL) return HTP_ERROR;
                }
            } else {
                // Folding; check that there's a previous header line to add to.
                if (connp->out_header == NULL) {
                    // Invalid folding.

                    // Warn only once per transaction.
                    if (!(connp->out_tx->flags & HTP_INVALID_FOLDING)) {
                        connp->out_tx->flags |= HTP_INVALID_FOLDING;
                        htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "Invalid response field folding");
                    }

                    // Keep the header data for parsing later.
                    connp->out_header = bstr_dup_mem(data, len);
                    if (connp->out_header == NULL) return HTP_ERROR;
                } else {
                    // Add to the existing header.                    
                    bstr *new_out_header = bstr_add_mem(connp->out_header, data, len);
                    if (new_out_header == NULL) return HTP_ERROR;
                    connp->out_header = new_out_header;
                }
            }

            htp_connp_res_clear_buffer(connp);
        }
    }

    return HTP_ERROR;
}
Beispiel #19
0
/**
 * Parses response line.
 *
 * @param[in] connp
 * @returns HTP_OK on state change, HTP_ERROR on error, or HTP_DATA when more data is needed.
 */
htp_status_t htp_connp_RES_LINE(htp_connp_t *connp) {
    for (;;) {
        // Get one byte
        OUT_COPY_BYTE_OR_RETURN(connp);

        // Have we reached the end of the line?
        if (connp->out_next_byte == LF) {
            unsigned char *data;
            size_t len;

            htp_connp_res_consolidate_data(connp, &data, &len);

            #ifdef HTP_DEBUG
            fprint_raw_data(stderr, __FUNCTION__, data, len);
            #endif

            // Is this a line that should be ignored?
            if (htp_connp_is_line_ignorable(connp, data, len)) {
                // We have an empty/whitespace line, which we'll note, ignore and move on
                connp->out_tx->response_ignored_lines++;

                // TODO How many lines are we willing to accept?

                // Start again
                htp_connp_res_clear_buffer(connp);

                return HTP_OK;
            }
           
            // Deallocate previous response line allocations, which we would have on a 100 response.

            if (connp->out_tx->response_line != NULL) {
                bstr_free(connp->out_tx->response_line);
                connp->out_tx->response_line = NULL;
            }           

            if (connp->out_tx->response_protocol != NULL) {
                bstr_free(connp->out_tx->response_protocol);
                connp->out_tx->response_protocol = NULL;
            }

            if (connp->out_tx->response_status != NULL) {
                bstr_free(connp->out_tx->response_status);
                connp->out_tx->response_status = NULL;
            }

            if (connp->out_tx->response_message != NULL) {
                bstr_free(connp->out_tx->response_message);
                connp->out_tx->response_message = NULL;
            }

            // Process response line.           

            int chomp_result = htp_chomp(data, &len);

            connp->out_tx->response_line = bstr_dup_mem(data, len);
            if (connp->out_tx->response_line == NULL) return HTP_ERROR;            

            if (connp->cfg->parse_response_line(connp) != HTP_OK) return HTP_ERROR;

            // If the response line is invalid, determine if it _looks_ like
            // a response line. If it does not look like a line, process the
            // data as a response body because that is what browsers do.

            if (htp_treat_response_line_as_body(connp->out_tx)) {
                connp->out_tx->response_content_encoding_processing = HTP_COMPRESSION_NONE;
                
                int rc = htp_tx_res_process_body_data(connp->out_tx, data, len + chomp_result);
                if (rc != HTP_OK) return rc;
               
                // Continue to process response body. Because we don't have
                // any headers to parse, we assume the body continues until
                // the end of the stream.
                connp->out_tx->response_transfer_coding = HTP_CODING_IDENTITY;
                connp->out_tx->progress = HTP_RESPONSE_BODY;
                connp->out_state = htp_connp_RES_BODY_IDENTITY_STREAM_CLOSE;
                connp->out_body_data_left = -1;
                
                return HTP_OK;
            }
           
            int rc = htp_tx_state_response_line(connp->out_tx);
            if (rc != HTP_OK) return rc;
            
            htp_connp_res_clear_buffer(connp);

            // Move on to the next phase.
            connp->out_state = htp_connp_RES_HEADERS;
            connp->out_tx->progress = HTP_RESPONSE_HEADERS;

            return HTP_OK;
        }
    }

    return HTP_ERROR;
}
Beispiel #20
0
htp_status_t htp_tx_state_request_line(htp_tx_t *tx) {
    htp_connp_t *connp = tx->connp;

    if (connp->in_tx->request_method_number == HTP_M_CONNECT) {
        // Parse authority
        if (htp_parse_uri_hostport(connp, connp->in_tx->request_uri, &(connp->in_tx->parsed_uri_incomplete)) != HTP_OK) {
            // Note: downstream responsible for error logging.
            return HTP_ERROR;
        }
    } else {
        // Parse the request URI
        if (htp_parse_uri(connp->in_tx->request_uri, &(connp->in_tx->parsed_uri_incomplete)) != HTP_OK) {
            // Note: downstream responsible for error logging.
            return HTP_ERROR;
        }

        // Keep the original URI components, but
        // create a copy which we can normalize and use internally.
        if (htp_normalize_parsed_uri(connp, connp->in_tx->parsed_uri_incomplete, connp->in_tx->parsed_uri) != HTP_OK) {
            // Note: downstream responsible for error logging.
            return HTP_ERROR;
        }

        // Run hook REQUEST_URI_NORMALIZE.
        int rc = htp_hook_run_all(connp->cfg->hook_request_uri_normalize, connp);
        if (rc != HTP_OK) return rc;

        // Now is a good time to generate request_uri_normalized, before we finalize
        // parsed_uri (and lose the information which parts were provided in the request and
        // which parts we added).
        if (connp->cfg->generate_request_uri_normalized) {
            connp->in_tx->request_uri_normalized = htp_unparse_uri_noencode(connp->in_tx->parsed_uri);
            if (connp->in_tx->request_uri_normalized == NULL) return HTP_ERROR;

            #ifdef HTP_DEBUG
            fprint_raw_data(stderr, "request_uri_normalized",
                    (unsigned char *) bstr_ptr(connp->in_tx->request_uri_normalized),
                    bstr_len(connp->in_tx->request_uri_normalized));
            #endif
        }

        // Finalize parsed_uri.

        // Scheme.
        if (connp->in_tx->parsed_uri->scheme != NULL) {
            if (bstr_cmp_c(connp->in_tx->parsed_uri->scheme, "http") != 0) {
                // TODO Invalid scheme.
            }
        } else {
            connp->in_tx->parsed_uri->scheme = bstr_dup_c("http");
            if (connp->in_tx->parsed_uri->scheme == NULL) {
                return HTP_ERROR;
            }
        }

        // Path.
        if (connp->in_tx->parsed_uri->path == NULL) {
            connp->in_tx->parsed_uri->path = bstr_dup_c("/");
            if (connp->in_tx->parsed_uri->path == NULL) {
                return HTP_ERROR;
            }
        }
    }

    // Run hook REQUEST_LINE.
    int rc = htp_hook_run_all(connp->cfg->hook_request_line, connp);
    if (rc != HTP_OK) return rc;

    // Move on to the next phase.
    connp->in_state = htp_connp_REQ_PROTOCOL;

    return HTP_OK;
}
Beispiel #21
0
/**
 * Parses request headers.
 *
 * @param[in] connp
 * @returns HTP_OK on state change, HTP_ERROR on error, or HTP_DATA when more data is needed.
 */
htp_status_t htp_connp_REQ_HEADERS(htp_connp_t *connp) {
    for (;;) {
        IN_COPY_BYTE_OR_RETURN(connp);

        // Have we reached the end of the line?
        if (connp->in_next_byte == LF) {
            unsigned char *data;
            size_t len;

            htp_connp_req_consolidate_data(connp, &data, &len);

            #ifdef HTP_DEBUG
            fprint_raw_data(stderr, __FUNCTION__, data, len);
            #endif           

            // Should we terminate headers?
            if (htp_connp_is_line_terminator(connp, data, len)) {
                // Parse previous header, if any.
                if (connp->in_header != NULL) {
                    if (connp->cfg->process_request_header(connp, bstr_ptr(connp->in_header),
                            bstr_len(connp->in_header)) != HTP_OK) return HTP_ERROR;

                    bstr_free(connp->in_header);
                    connp->in_header = NULL;
                }

                htp_connp_req_clear_buffer(connp);

                // We've seen all the request headers.
                return htp_tx_state_request_headers(connp->in_tx);
            }

            htp_chomp(data, &len);

            // Check for header folding.
            if (htp_connp_is_line_folded(data, len) == 0) {
                // New header line.

                // Parse previous header, if any.
                if (connp->in_header != NULL) {
                    if (connp->cfg->process_request_header(connp, bstr_ptr(connp->in_header),
                            bstr_len(connp->in_header)) != HTP_OK) return HTP_ERROR;

                    bstr_free(connp->in_header);
                    connp->in_header = NULL;
                }

                IN_PEEK_NEXT(connp);

                if (htp_is_folding_char(connp->in_next_byte) == 0) {
                    // Because we know this header is not folded, we can process the buffer straight away.
                    if (connp->cfg->process_request_header(connp, data, len) != HTP_OK) return HTP_ERROR;
                } else {
                    // Keep the partial header data for parsing later.
                    connp->in_header = bstr_dup_mem(data, len);
                    if (connp->in_header == NULL) return HTP_ERROR;
                }
            } else {
                // Folding; check that there's a previous header line to add to.
                if (connp->in_header == NULL) {
                    // Invalid folding.

                    // Warn only once per transaction.
                    if (!(connp->in_tx->flags & HTP_INVALID_FOLDING)) {
                        connp->in_tx->flags |= HTP_INVALID_FOLDING;
                        htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "Invalid request field folding");
                    }

                    // Keep the header data for parsing later.
                    connp->in_header = bstr_dup_mem(data, len);
                    if (connp->in_header == NULL) return HTP_ERROR;
                } else {
                    // Add to the existing header.                    
                    bstr *new_in_header = bstr_add_mem(connp->in_header, data, len);
                    if (new_in_header == NULL) return HTP_ERROR;
                    connp->in_header = new_in_header;
                }
            }

            htp_connp_req_clear_buffer(connp);
        }
    }

    return HTP_ERROR;
}
Beispiel #22
0
/**
 * Generic response line parser.
 * 
 * @param[in] connp
 * @return HTP status
 */
htp_status_t htp_parse_response_line_generic(htp_connp_t *connp) {
    htp_tx_t *tx = connp->out_tx;
    unsigned char *data = bstr_ptr(tx->response_line);
    size_t len = bstr_len(tx->response_line);
    size_t pos = 0;

    tx->response_protocol = NULL;
    tx->response_protocol_number = HTP_PROTOCOL_INVALID;
    tx->response_status = NULL;
    tx->response_status_number = HTP_STATUS_INVALID;
    tx->response_message = NULL;

    // Ignore whitespace at the beginning of the line.
    while ((pos < len) && (htp_is_space(data[pos]))) pos++;

    size_t start = pos;

    // Find the end of the protocol string.
    while ((pos < len) && (!htp_is_space(data[pos]))) pos++;    
    if (pos - start == 0) return HTP_OK;
    
    tx->response_protocol = bstr_dup_mem(data + start, pos - start);
    if (tx->response_protocol == NULL) return HTP_ERROR;    

    tx->response_protocol_number = htp_parse_protocol(tx->response_protocol);

    #ifdef HTP_DEBUG
    fprint_raw_data(stderr, "Response protocol", bstr_ptr(tx->response_protocol), bstr_len(tx->response_protocol));
    fprintf(stderr, "Response protocol number: %d\n", tx->response_protocol_number);
    #endif

    // Ignore whitespace after the response protocol.
    while ((pos < len) && (htp_is_space(data[pos]))) pos++;
    if (pos == len) return HTP_OK;

    start = pos;

    // Find the next whitespace character.
    while ((pos < len) && (!htp_is_space(data[pos]))) pos++;    
    if (pos - start == 0) return HTP_OK;
    
    tx->response_status = bstr_dup_mem(data + start, pos - start);
    if (tx->response_status == NULL) return HTP_ERROR;

    tx->response_status_number = htp_parse_status(tx->response_status);

    #ifdef HTP_DEBUG
    fprint_raw_data(stderr, "Response status (as text)", bstr_ptr(tx->response_status), bstr_len(tx->response_status));
    fprintf(stderr, "Response status number: %d\n", tx->response_status_number);
    #endif

    // Ignore whitespace that follows the status code.
    while ((pos < len) && (isspace(data[pos]))) pos++;
    if (pos == len) return HTP_OK;

    // Assume the message stretches until the end of the line.    
    tx->response_message = bstr_dup_mem(data + pos, len - pos);
    if (tx->response_message == NULL) return HTP_ERROR;    

    #ifdef HTP_DEBUG
    fprint_raw_data(stderr, "Response status message", bstr_ptr(tx->response_message), bstr_len(tx->response_message));
    #endif

    return HTP_OK;
}
Beispiel #23
0
int htp_connp_req_data(htp_connp_t *connp, const htp_time_t *timestamp, const void *data, size_t len) {
    #ifdef HTP_DEBUG
    fprintf(stderr, "htp_connp_req_data(connp->in_status %x)\n", connp->in_status);
    fprint_raw_data(stderr, __FUNCTION__, data, len);
    #endif

    // Return if the connection is in stop state.
    if (connp->in_status == HTP_STREAM_STOP) {
        htp_log(connp, HTP_LOG_MARK, HTP_LOG_INFO, 0, "Inbound parser is in HTP_STREAM_STOP");

        return HTP_STREAM_STOP;
    }

    // Return if the connection had a fatal error earlier
    if (connp->in_status == HTP_STREAM_ERROR) {
        htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Inbound parser is in HTP_STREAM_ERROR");

        #ifdef HTP_DEBUG
        fprintf(stderr, "htp_connp_req_data: returning HTP_STREAM_DATA (previous error)\n");
        #endif

        return HTP_STREAM_ERROR;
    }

    // If the length of the supplied data chunk is zero, proceed
    // only if the stream has been closed. We do not allow zero-sized
    // chunks in the API, but we use them internally to force the parsers
    // to finalize parsing.
    if (((data == NULL)||(len == 0)) && (connp->in_status != HTP_STREAM_CLOSED)) {
        htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Zero-length data chunks are not allowed");

        #ifdef HTP_DEBUG
        fprintf(stderr, "htp_connp_req_data: returning HTP_STREAM_DATA (zero-length chunk)\n");
        #endif

        return HTP_STREAM_CLOSED;
    }

    // Remember the timestamp of the current request data chunk
    if (timestamp != NULL) {
        memcpy(&connp->in_timestamp, timestamp, sizeof (*timestamp));
    }

    // Store the current chunk information    
    connp->in_current_data = (unsigned char *) data;
    connp->in_current_len = len;
    connp->in_current_read_offset = 0;
    connp->in_current_consume_offset = 0;
    connp->in_current_receiver_offset = 0;
    connp->in_chunk_count++;

    htp_conn_track_inbound_data(connp->conn, len, timestamp);


    // Return without processing any data if the stream is in tunneling
    // mode (which it would be after an initial CONNECT transaction).
    if (connp->in_status == HTP_STREAM_TUNNEL) {
        #ifdef HTP_DEBUG
        fprintf(stderr, "htp_connp_req_data: returning HTP_STREAM_TUNNEL\n");
        #endif

        return HTP_STREAM_TUNNEL;
    }

    if (connp->out_status == HTP_STREAM_DATA_OTHER) {
        connp->out_status = HTP_STREAM_DATA;
    }

    // Invoke a processor, in a loop, until an error
    // occurs or until we run out of data. Many processors
    // will process a request, each pointing to the next
    // processor that needs to run.
    for (;;) {
        #ifdef HTP_DEBUG
        fprintf(stderr, "htp_connp_req_data: in state=%s, progress=%s\n",
                htp_connp_in_state_as_string(connp),
                htp_tx_request_progress_as_string(connp->in_tx));
        #endif

        // Return if there's been an error or if we've run out of data. We are relying
        // on processors to supply error messages, so we'll keep quiet here.
        htp_status_t rc = connp->in_state(connp);
        if (rc == HTP_OK) {
            if (connp->in_status == HTP_STREAM_TUNNEL) {
                #ifdef HTP_DEBUG
                fprintf(stderr, "htp_connp_req_data: returning HTP_STREAM_TUNNEL\n");
                #endif

                return HTP_STREAM_TUNNEL;
            }

            rc = htp_req_handle_state_change(connp);
        }

        if (rc != HTP_OK) {
            // Do we need more data?
            if ((rc == HTP_DATA) || (rc == HTP_DATA_BUFFER)) {
                htp_connp_req_receiver_send_data(connp, 0 /* not last */);

                if (rc == HTP_DATA_BUFFER) {
                    htp_connp_req_buffer(connp);
                }

                #ifdef HTP_DEBUG
                fprintf(stderr, "htp_connp_req_data: returning HTP_STREAM_DATA\n");
                #endif

                connp->in_status = HTP_STREAM_DATA;

                return HTP_STREAM_DATA;
            }

            // Check for suspended parsing.
            if (rc == HTP_DATA_OTHER) {
                // We might have actually consumed the entire data chunk?
                if (connp->in_current_read_offset >= connp->in_current_len) {
                    // Do not send STREAM_DATE_DATA_OTHER if we've consumed the entire chunk.

                    #ifdef HTP_DEBUG
                    fprintf(stderr, "htp_connp_req_data: returning HTP_STREAM_DATA (suspended parsing)\n");
                    #endif

                    connp->in_status = HTP_STREAM_DATA;

                    return HTP_STREAM_DATA;
                } else {
                    // Partial chunk consumption.

                    #ifdef HTP_DEBUG
                    fprintf(stderr, "htp_connp_req_data: returning HTP_STREAM_DATA_OTHER\n");
                    #endif

                    connp->in_status = HTP_STREAM_DATA_OTHER;

                    return HTP_STREAM_DATA_OTHER;
                }
            }

            // Check for the stop signal.
            if (rc == HTP_STOP) {
                #ifdef HTP_DEBUG
                fprintf(stderr, "htp_connp_req_data: returning HTP_STREAM_STOP\n");
                #endif

                connp->in_status = HTP_STREAM_STOP;

                return HTP_STREAM_STOP;
            }

            #ifdef HTP_DEBUG
            fprintf(stderr, "htp_connp_req_data: returning HTP_STREAM_ERROR\n");
            #endif

            // Permanent stream error.
            connp->in_status = HTP_STREAM_ERROR;

            return HTP_STREAM_ERROR;
        }
    }

    // Permanent stream error.
    connp->in_status = HTP_STREAM_ERROR;
    
    return HTP_STREAM_ERROR;
}
/**
 * Generic response line parser.
 * 
 * @param[in] connp
 * @return HTP status
 */
int htp_parse_response_line_generic(htp_connp_t *connp) {
    htp_tx_t *tx = connp->out_tx;
    unsigned char *data = (unsigned char *) bstr_ptr(tx->response_line);
    size_t len = bstr_len(tx->response_line);
    size_t pos = 0;
    
    // Ignore whitespace at the beginning of the line
    while ((pos < len) && (htp_is_space(data[pos]))) {
        pos++;
    }

    size_t start = pos;

    // Find the end of the protocol string
    while ((pos < len) && (!htp_is_space(data[pos]))) {
        pos++;
    }

    tx->response_protocol = bstr_dup_mem(data + start, pos - start);
    if (tx->response_protocol == NULL) {
        return HTP_ERROR;
    }

    tx->response_protocol_number = htp_parse_protocol(tx->response_protocol);        

    #ifdef HTP_DEBUG
    fprint_raw_data(stderr, __FUNCTION__, (unsigned char *) bstr_ptr(tx->response_protocol), bstr_len(tx->response_protocol));
    #endif

    // Ignore whitespace after response protocol
    // TODO Why use both isspace (below) and htp_is_space (above)?
    while ((pos < len) && (isspace(data[pos]))) {
        pos++;
    }

    start = pos;

    // Find the next whitespace character
    while ((pos < len) && (!htp_is_space(data[pos]))) {
        pos++;
    }

    tx->response_status = bstr_dup_mem(data + start, pos - start);
    if (tx->response_status == NULL) {
        return HTP_ERROR;
    }

    tx->response_status_number = htp_parse_status(tx->response_status);    

    #ifdef HTP_DEBUG
    fprint_raw_data(stderr, __FUNCTION__, (unsigned char *) bstr_ptr(tx->response_status), bstr_len(tx->response_status));
    #endif

    // Ignore whitespace that follows
    while ((pos < len) && (isspace(data[pos]))) {
        pos++;
    }

    tx->response_message = bstr_dup_mem(data + pos, len - pos);
    if (tx->response_message == NULL) {
        return HTP_ERROR;
    }

    #ifdef HTP_DEBUG
    fprint_raw_data(stderr, __FUNCTION__, (unsigned char *) bstr_ptr(tx->response_message), bstr_len(tx->response_message));
    #endif

    return HTP_OK;
}