C++ (Cpp) hs_lookup_input_checkpoint Examples

Example #1

File: hs_output_plugins.c Project: wxdublin/hindsight

static void add_to_output_plugins(hs_output_plugins* plugins,
                                  hs_output_plugin* p)
{
  bool added = false;
  int idx = -1;
  pthread_mutex_lock(&plugins->list_lock);
  for (int i = 0; i < plugins->list_cap; ++i) {
    if (!plugins->list[i]) {
      idx = i;
    } else if (strcmp(plugins->list[i]->sb->name, p->sb->name) == 0) {
      idx = i;
      remove_plugin(plugins, idx);
      add_plugin(plugins, p, idx);
      added = true;
      break;
    }
  }
  if (!added && idx != -1) add_plugin(plugins, p, idx);

  if (idx == -1) {
    // todo probably don't want to grow it by 1
    ++plugins->list_cap;
    hs_output_plugin** tmp = realloc(plugins->list,
                                     sizeof(hs_output_plugin*)
                                     * plugins->list_cap);
    idx = plugins->list_cap - 1;

    if (tmp) {
      plugins->list = tmp;
      add_plugin(plugins, p, idx);
    } else {
      hs_log(g_module, 0, "plugins realloc failed");
      exit(EXIT_FAILURE);
    }
  }
  pthread_mutex_unlock(&plugins->list_lock);
  assert(p->list_index >= 0);

  hs_config* cfg = p->plugins->cfg;
  // sync the output and read checkpoints
  // the read and output checkpoints can differ to allow for batching
  hs_lookup_input_checkpoint(&cfg->cp_reader,
                             hs_input_dir,
                             p->sb->name,
                             cfg->output_path,
                             &p->input.ib.cp);
  p->cur.input.id = p->cp.input.id = p->input.ib.cp.id;
  p->cur.input.offset = p->cp.input.offset = p->input.ib.cp.offset;

  hs_lookup_input_checkpoint(&cfg->cp_reader,
                             hs_analysis_dir,
                             p->sb->name,
                             cfg->output_path,
                             &p->analysis.ib.cp);
  p->cur.analysis.id = p->cp.analysis.id = p->analysis.ib.cp.id;
  p->cur.analysis.offset = p->cp.analysis.offset = p->analysis.ib.cp.offset;

  int ret = pthread_create(&p->thread, NULL, input_thread, (void*)p);
  if (ret) {
    perror("pthread_create failed");
    exit(EXIT_FAILURE);
  }
}

Example #2

File: hs_analysis_plugins.c Project: bbinet/hindsight

static void* input_thread(void* arg)
{
  hs_analysis_thread* at = (hs_analysis_thread*)arg;
  hs_log(g_module, 6, "starting input thread: %d", at->tid);

  hs_heka_message msg;
  hs_init_heka_message(&msg, 8);

  hs_config* cfg = at->plugins->cfg;
  hs_lookup_input_checkpoint(&cfg->cp_reader,
                             hs_input_dir,
                             at->input.name,
                             cfg->output_path,
                             &at->input.ib.cp);
  at->cp.id = at->input.ib.cp.id;
  at->cp.offset = at->input.ib.cp.offset;

  size_t bytes_read = 0;
#ifdef HINDSIGHT_CLI
  bool input_stop = false;
  while (!(at->plugins->stop && input_stop)) {
#else
  while (!at->plugins->stop) {
#endif
    if (at->input.fh) {
      if (hs_find_message(&msg, &at->input.ib)) {
        at->msg = &msg;
        at->current_t = time(NULL);
        analyze_message(at);

        // advance the checkpoint
        pthread_mutex_lock(&at->cp_lock);
        at->plugins->sample = false;
        at->cp.id = at->input.ib.cp.id;
        at->cp.offset = at->input.ib.cp.offset -
          (at->input.ib.readpos - at->input.ib.scanpos);
        pthread_mutex_unlock(&at->cp_lock);
      } else {
        bytes_read = hs_read_file(&at->input);
      }

      if (!bytes_read) {
#ifdef HINDSIGHT_CLI
        size_t cid = at->input.ib.cp.id;
#endif
        // see if the next file is there yet
        hs_open_file(&at->input, hs_input_dir, at->input.ib.cp.id + 1);
#ifdef HINDSIGHT_CLI
        if (cid == at->input.ib.cp.id && at->plugins->stop) {
          input_stop = true;
        }
#endif
      }
    } else { // still waiting on the first file
      hs_open_file(&at->input, hs_input_dir, at->input.ib.cp.id);
#ifdef HINDSIGHT_CLI
      if (!at->input.fh && at->plugins->stop) {
        input_stop = true;
      }
#endif
    }

    if (bytes_read || at->msg) {
      at->msg = NULL;
    } else {
      // trigger any pending timer events
      hs_clear_heka_message(&msg); // create an idle/empty message
      at->msg = &msg;
      at->current_t = time(NULL);
      analyze_message(at);
      at->msg = NULL;
      sleep(1);
    }
  }
  shutdown_timer_event(at);
  hs_free_heka_message(&msg);
  hs_log(g_module, 6, "exiting input_thread: %d", at->tid);
  pthread_exit(NULL);
}


void hs_init_analysis_plugins(hs_analysis_plugins* plugins,
                              hs_config* cfg,
                              hs_message_match_builder* mmb)
{
  hs_init_output(&plugins->output, cfg->output_path, hs_analysis_dir);

  plugins->thread_cnt = cfg->analysis_threads;
  plugins->cfg = cfg;
  plugins->stop = false;
  plugins->sample = false;
  plugins->mmb = mmb;

  plugins->list = malloc(sizeof(hs_analysis_thread) * cfg->analysis_threads);
  for (unsigned i = 0; i < cfg->analysis_threads; ++i) {
    init_analysis_thread(plugins, i);
  }
  plugins->threads = malloc(sizeof(pthread_t*) * (cfg->analysis_threads));
}