Пример #1
0
// Should only be called when we actually have the start of an array
// Otherwise it is an internal error
bool JSON::parse_json_array() {
  NOT_PRODUCT(const char* prev_pos);
  int c;

  mark_pos();
  // Check that we are not called in error
  if (expect_any("[", "array start character", INTERNAL_ERROR) <= 0) {
    return false;
  }

  if (!callback(JSON_ARRAY_BEGIN, NULL, level++)) {
    return false;
  }

  for (;;) {
    mark_pos();
    c = skip_to_token();
    if (c == 0) {
      error(SYNTAX_ERROR, "EOS when expecting a json value or array end");
      return false;
    } else if (c < 0) {
      return false;
    } else if (c == ']') {
      // We got here from either empty array "[]" or ending comma "[1,]"
      next();
      break;
    }

    mark_pos();
    NOT_PRODUCT(prev_pos = pos);
    if (parse_json_value() == false) {
      return false;
    }
    assert(pos > prev_pos, "parsing stalled");

    c = skip_to_token();
    mark_pos();
    if (expect_any(",]", "value separator or array end") <= 0) {
      return false;
    }
    if (c == ']') {
      break;
    }
  }

  assert(c == ']', "array parsing ended without array end token (']')");
  return callback(JSON_ARRAY_END, NULL, --level);
}
Пример #2
0
/*
 * Should only be called when we actually have a block comment to skip.
 * Otherwise it is an internal error.
 *
 * Returns the first token after the block comment without consuming it.
 * Returns -1 if EOS is encountered in the middle of a comment.
 */
int JSON::skip_block_comment() {
  const char* current;

  // Check that we are not called in error.
  if (peek() != '/' || peek(1) != '*') {
    // Let expect handle EOS.
    expect_string("/*", "block comment start", INTERNAL_ERROR);
    return 0;
  }

  current = pos;
  for (;;) {
    current = strchrnul_(current, '*');

    if (current[0] == 0 || current[1] == 0) {
      // Advance error marker to start of block comment
      mark_pos();
      error(SYNTAX_ERROR, "Block comment started here never ended. Expected \"*/\" before EOS.");
      return -1;
    }

    if (current[1] == '/') {
      pos = current;
      if (expect_string("*/", "block comment end", INTERNAL_ERROR) == false) {
        return -1;
      }
      // Found block comment end
      return peek();
    }
    current++;
  }
}
Пример #3
0
bool JSON::parse_json_symbol(const char* name, JSON_TYPE symbol) {
  if (expect_string(name, "maybe you forgot to quote your strings?") == false) {
    mark_pos();
    return false;
  }
  return callback(symbol, NULL, level);
}
Пример #4
0
bool JSON::parse_json_number() {
  double double_value;
  int tokens, read;
  JSON_VAL v;

  mark_pos();

  // Parsing number - for simplicity ints are limited to 2**53
  // sscanf as a double and check if part is 0.
  tokens = sscanf(pos, "%lf%n", &double_value, &read);
  assert(tokens <= 1, "scanf implementation that counts as a token, parsing json numbers will always fail");
  if (tokens == 1) {
    assert(read > 0, "sanity");

    if (floor(double_value) == double_value) {
      // No exponent - treat as an int
      v.int_value = (int)double_value;
      if (!callback(JSON_NUMBER_INT, &v, level)) {
        return false;
      }
    } else {
      v.double_value = double_value;
      if (!callback(JSON_NUMBER_FLOAT, &v, level)) {
        return false;
      }
    }
    skip(read);
    return true;
  }

  error(SYNTAX_ERROR, "Couldn't parse json number (note that exponents are not supported).");
  return false;
}
Пример #5
0
bool JSON::parse_json_string(bool key) {
  const char* end;
  JSON_VAL v;

  mark_pos();
  if (expect_any("\"", "string start character", INTERNAL_ERROR) <= 0) {
    return false;
  }

  end = strchr(pos, '"'); // TODO: escapes
  if (end == NULL) {
    error(SYNTAX_ERROR, "String started here never ended. Expected \'\"\' before EOS.");
    return false;
  }

  v.str.start = pos;
  v.str.length = end - pos;
  skip(end - pos);

  if (expect_any("\"", "string end character", INTERNAL_ERROR) <= 0) {
    return false;
  }

  if (key == true) {
    return callback(JSON_KEY, &v, level);
  } else {
    return callback(JSON_STRING, &v, level);
  }
}
Пример #6
0
	int Tokenizer::tokenize( const string &raw_sent, V2_STR &data, const size_t init_offset ) 
	{
		string	trimmed_sent = trim_ws( raw_sent );		

		if( trimmed_sent == "") {
			return 0;
		}else {
			int	n_tokens = splitter( trimmed_sent, data );
			mark_pos( raw_sent, data, init_offset );

			return	n_tokens;
		}
	}
Пример #7
0
// Allow object keys to be without quotation,
// but then restrict to ([a-zA-Z0-9_])+
bool JSON::parse_json_key() {
  const char* begin;
  JSON_VAL v;
  u_char c;

  mark_pos();
  c = peek();
  if (c == '"') {
    return parse_json_string(true);
  }

  begin = pos;
  c = peek();
  if (c == 0) {
    error(SYNTAX_ERROR, "Got EOS when expecting an object key.");
    return false;
  } else if (is_word(c) == false) {
    error(SYNTAX_ERROR, "Expected an object key, which can be a double-quoted (\") string or a simple string (only alphanumeric characters and underscore, separated by whitespace) that doesn't need to be quoted.");
    return false;
  }

  for (;;) {
    c = peek();
    // Allow the key to be delimited by control characters and the object key-value separator ':'
    if (c <= ' ' || c == ':') {
      break;
    } else if (is_word(c) == false) {
      error(SYNTAX_ERROR, "Object key need to be quoted, or consist entirely of alphanumeric characters and underscores.");
      return false;
    }
    next();
  }

  v.str.start = begin;
  v.str.length = pos - begin;
  return callback(JSON_KEY, &v, level);
}
Пример #8
0
bool JSON::parse_json_value() {
  int c;

  c = skip_to_token();
  if (c == -1) {
    return false;
  }

  // Must start with object or array
  if (level == 0) {

    switch (c) {
    case '{':
      if (parse_json_object() == false) {
        return false;
      }
      c = skip_to_token();
      if (c > 0) {
        mark_pos();
        error(SYNTAX_ERROR, "Only one top level object/array is allowed.");
        return false;
      } else if (c < 0) {
        return false;
      }
      return true;

    case '[':
      if (parse_json_array() == false) {
        return false;
      }
      c = skip_to_token();
      if (c > 0) {
        mark_pos();
        error(SYNTAX_ERROR, "Only one top level object/array is allowed.");
        return false;
      } else if (c < 0) {
        return false;
      }
      return true;

    case 0:
      error(SYNTAX_ERROR, "EOS was encountered before any json declarations");
      return false;

    default:
      error(SYNTAX_ERROR, "Json must start with an object or an array.");
      return false;
    }
  } else { // level > 0
    switch (c) {
    case '{':
      return parse_json_object();

    case '[':
      return parse_json_array();

    case '"':
      return parse_json_string();

    case '-': case '0':
    case '1': case '2': case '3':
    case '4': case '5': case '6':
    case '7': case '8': case '9':
      return parse_json_number();

    case 't':
      return parse_json_symbol("true", JSON_TRUE);

    case 'f':
      return parse_json_symbol("false", JSON_FALSE);

    case 'n':
      return parse_json_symbol("null", JSON_NULL);

    case 0:
      error(SYNTAX_ERROR, "EOS was encountered when expecting a json value.");
      return false;

    default:
      error(SYNTAX_ERROR, "Could not parse as a json value (did you forget to quote your strings?).");
      return false;
    }
  }
}
Пример #9
0
// Should only be called when we actually have the start of an object
// Otherwise it is an internal error
bool JSON::parse_json_object() {
  NOT_PRODUCT(const char* prev_pos);
  int c;

  mark_pos();
  // Check that we are not called in error
  if (expect_any("{", "object start", INTERNAL_ERROR) <= 0) {
    return false;
  }

  if (!callback(JSON_OBJECT_BEGIN, NULL, level++)) {
    return false;
  }

  for (;;) {
    mark_pos();
    c = skip_to_token();
    if (c == 0) {
      error(SYNTAX_ERROR, "EOS when expecting an object key or object end");
      return false;
    } else if (c < 0) {
      return false;
    } else if (c == '}') {
      // We got here from either empty object "{}" or ending comma "{a:1,}"
      next();
      break;
    }

    NOT_PRODUCT(prev_pos = pos);
    if (parse_json_key() == false) {
      return false;
    }
    assert(pos > prev_pos, "parsing stalled");

    skip_to_token();
    mark_pos();
    if (expect_any(":", "object key-value separator") <= 0) {
      return false;
    }

    skip_to_token();
    mark_pos();
    NOT_PRODUCT(prev_pos = pos);
    if (parse_json_value() == false) {
      return false;
    }
    assert(pos > prev_pos, "parsing stalled");

    c = skip_to_token();
    mark_pos();
    if (expect_any(",}", "value separator or object end") <= 0) {
      return false;
    }
    if (c == '}') {
      break;
    }
  }

  assert(c == '}', "array parsing ended without object end token ('}')");
  return callback(JSON_OBJECT_END, NULL, --level);
}
Пример #10
0
void *process(void *path) {
  char i_char[23];
  char j_char[23];
  char k_char[23];
  char t_char = '0';
  char line[512] = "0";
  int i = 0;
  FILE *f;

  getxattr(path, XATTR_ANALYZER, &t_char, 1);
  printf("\nProcessing file %s \n", path); 
  printf("  attr %s is %d\n", XATTR_ANALYZER, t_char - '0');
  switch(t_char - '0') {
    case X_A_DONE:
      mark_done(path);
      break;
    case X_A_INPROGRESS:
    case X_A_NONE:
      memset(i_char, 0, 23);
      memset(j_char, 0, 23);
      memset(k_char, 0, 23);
      getxattr(path, XATTR_ANALYZER_POS, i_char, 23);
      getxattr(path, XATTR_ANALYZER_POS_TS, j_char, 23);
      printf("  attr %s is %s\n", XATTR_ANALYZER_POS, i_char);
      printf("  attr %s is %s\n", XATTR_ANALYZER_POS_TS, j_char);
      f = fopen(path, "r");
      if(f == NULL) {
        fprintf(stderr, "Error when openning file %s\n", path);
        perror("fopen");
        return;
      }
      if(strlen(i_char) != 0) {
        fseek(f, atol(i_char), SEEK_SET);
        printf("  Moved to pos %d\n", atol(i_char));
        if(fgets(line, 512, f) == NULL) {
          fclose(f);
          return;
        }
        if(strlen(j_char) != 0 && strncmp(line, j_char, strlen(j_char)) !=0) {
          printf("  The position is not matched: %s, %s", j_char, line);
          fclose(f);
          return;
        } 
      }
      if(fgets(line, 512, f) == NULL) {
        printf("  End of the file or error occured, will read this file again after 1 min!\n");
        for(i = 0; i < 60; i++) {
          sleep(1);
          if(killed) {
            fclose(f);
            return;
          }
        }
        if(fgets(line, 512, f) == NULL) {
          printf("  Got EOF in second read, marking this file as finished!\n");
          mark_done(path);
          fclose(f);
          return;
        }
      }
      do {
        if(read_line(line, j_char) == -1) {
          continue; 
        }
        strcpy(k_char, i_char);
        sprintf(i_char, "%d", ftell(f));
        printf("  Finished line start with %s, location was %s, location is %s, killed is %d\n", j_char, k_char, i_char, killed);
        if(i++ == 600) {
          mark_pos(path, k_char, j_char);
          i = 0;
        }
        //sleep(1);
      } while(fgets(line, 512, f) != NULL && (killed == 0));
      mark_pos(path, k_char, j_char);

      MYSQL *conn = pthread_getspecific(my_con);
      if(conn != NULL) mysql_close(conn);
      mysql_thread_end();
      fclose(f);
      return;
  }
}