/* Update metric oldest and latest timestamps when removing old values */
void pg_cache_update_metric_times(struct pg_cache_page_index *page_index)
{
Pvoid_t *firstPValue, *lastPValue;
Word_t firstIndex, lastIndex;
struct rrdeng_page_cache_descr *descr;
usec_t oldest_time = INVALID_TIME;
usec_t latest_time = INVALID_TIME;
uv_rwlock_rdlock(&page_index->lock);
/* Find first page in range */
firstIndex = (Word_t)0;
firstPValue = JudyLFirst(page_index->JudyL_array, &firstIndex, PJE0);
if (likely(NULL != firstPValue)) {
descr = *firstPValue;
oldest_time = descr->start_time;
}
lastIndex = (Word_t)-1;
lastPValue = JudyLLast(page_index->JudyL_array, &lastIndex, PJE0);
if (likely(NULL != lastPValue)) {
descr = *lastPValue;
latest_time = descr->end_time;
}
uv_rwlock_rdunlock(&page_index->lock);
if (unlikely(NULL == firstPValue)) {
assert(NULL == lastPValue);
page_index->oldest_time = page_index->latest_time = INVALID_TIME;
return;
}
page_index->oldest_time = oldest_time;
page_index->latest_time = latest_time;
}
void RefMap::free()
{
unsigned long idx = 0;
Node** pp = (Node**) JudyLFirst(nodes, &idx, NULL);
while (pp != NULL)
{
free_node(*pp);
pp = (Node**) JudyLNext(nodes, &idx, NULL);
}
JudyLFreeArray(&nodes, NULL);
if (overflow != NULL) { free_node(overflow); overflow = NULL; }
}
int
JudyLDup(PPvoid_t PPDest, Pvoid_t PSource, JError_t * PJError)
{
Pvoid_t newJArray = 0; // new JudyL array to ppopulate
Word_t kindex; // Key/index
int Ins_rv = 0; // Insert return value
for (kindex = 0L, Ins_rv = JudyLFirst(PSource, &kindex, PJError);
Ins_rv == 1; Ins_rv = JudyLNext(PSource, &kindex, PJError))
{
Ins_rv = JudyLSet(&newJArray, kindex, PJError);
}
if (Ins_rv == JERR)
return Ins_rv;
*PPDest = newJArray;
return Ins_rv;
} /* JudyLDup */
unsigned long flx_collector_t::sweep()
{
if(debug)
fprintf(stderr,"Collector: Sweep, garbage bit value=%d\n",(int)parity);
unsigned long sweeped = 0;
void *current = NULL;
Word_t *pshape = (Word_t*)JudyLFirst(j_shape,(Word_t*)¤t,&je);
if(pshape==(Word_t*)PPJERR)judyerror("sweep");
while(pshape!=NULL)
{
if((*pshape & 1) == (parity & 1UL))
{
if(debug)
fprintf(stderr,"Garbage %p=%s\n",current,((gc_shape_t*)(*pshape & ~1UL))->cname);
++ sweeped;
//fprintf(stderr,"Unlinking ..\n");
unlink(current);
//fprintf(stderr,"Posting delete ..\n");
post_delete(current);
//fprintf(stderr,"Reaping done\n");
}
else
if(debug)
fprintf(stderr,"Reachable %p=%s\n",current,((gc_shape_t*)(*pshape & ~1UL))->cname);
//fprintf(stderr,"Calling Judy for next object\n");
pshape = (Word_t*)JudyLNext(j_shape,(Word_t*)(void*)¤t,&je);
//fprintf(stderr,"Judy got next object %p\n",pshape);
}
parity = !parity;
if(debug)
fprintf(stderr,"Sweeped %ld\n",sweeped);
return reap();
}
FUNCTION Word_t JUDY_EXTERN Judy1Count
#else
FUNCTION Word_t JUDY_EXTERN JudyLCount
#endif
(
Pcvoid_t PArray, // JRP to first branch/leaf in SM.
Word_t Index1, // starting Index.
Word_t Index2, // ending Index.
PJError_t PJError // optional, for returning error info.
)
{
jpm_t fakejpm; // local temporary for small arrays.
Pjpm_t Pjpm; // top JPM or local temporary for error info.
jp_t fakejp; // constructed for calling j__udy1LCountSM().
Pjp_t Pjp; // JP to pass to j__udy1LCountSM().
Word_t pop1; // total for the array.
Word_t pop1above1; // indexes at or above Index1, inclusive.
Word_t pop1above2; // indexes at or above Index2, exclusive.
int retcode; // from Judy*First() calls.
JUDYLCODE(PPvoid_t PPvalue); // from JudyLFirst() calls.
// CHECK FOR SHORTCUTS:
//
// As documented, return C_JERR if the Judy array is empty or Index1 > Index2.
if ((PArray == (Pvoid_t) NULL) || (Index1 > Index2))
{
JU_SET_ERRNO(PJError, JU_ERRNO_NONE);
return(C_JERR);
}
// If Index1 == Index2, simply check if the specified Index is set; pass
// through the return value from Judy1Test() or JudyLGet() with appropriate
// translations.
if (Index1 == Index2)
{
#ifdef JUDY1
retcode = Judy1Test(PArray, Index1, PJError);
if (retcode == JERRI) return(C_JERR); // pass through error.
if (retcode == 0)
{
JU_SET_ERRNO(PJError, JU_ERRNO_NONE);
return(C_JERR);
}
#else
PPvalue = JudyLGet(PArray, Index1, PJError);
if (PPvalue == PPJERR) return(C_JERR); // pass through error.
if (PPvalue == (PPvoid_t) NULL) // Index is not set.
{
JU_SET_ERRNO(PJError, JU_ERRNO_NONE);
return(C_JERR);
}
#endif
return(1); // single index is set.
}
// CHECK JRP TYPE:
//
// Use an if/then for speed rather than a switch, and put the most common cases
// first.
//
// Note: Since even cJU_LEAFW types require counting between two Indexes,
// prepare them here for common code below that calls j__udy1LCountSM(), rather
// than handling them even more specially here.
if (JU_LEAFW_POP0(PArray) < cJU_LEAFW_MAXPOP1) // must be a LEAFW
{
Pjlw_t Pjlw = P_JLW(PArray); // first word of leaf.
Pjpm = & fakejpm;
Pjp = & fakejp;
Pjp->jp_Addr = (Word_t) Pjlw;
Pjp->jp_Type = cJU_LEAFW;
Pjpm->jpm_Pop0 = Pjlw[0]; // from first word of leaf.
pop1 = Pjpm->jpm_Pop0 + 1;
}
else
{
Pjpm = P_JPM(PArray);
Pjp = &(Pjpm->jpm_JP);
pop1 = (Pjpm->jpm_Pop0) + 1; // note: can roll over to 0.
#if (defined(JUDY1) && (! defined(JU_64BIT)))
if (pop1 == 0) // rare special case of full array:
{
Word_t count = Index2 - Index1 + 1; // can roll over again.
if (count == 0)
{
JU_SET_ERRNO(PJError, JU_ERRNO_FULL);
return(C_JERR);
}
return(count);
}
#else
assert(pop1); // JudyL or 64-bit cannot create a full array!
#endif
}
// COUNT POP1 ABOVE INDEX1, INCLUSIVE:
assert(pop1); // just to be safe.
if (Index1 == 0) // shortcut, pop1above1 is entire population:
{
pop1above1 = pop1;
}
else // find first valid Index above Index1, if any:
{
#ifdef JUDY1
if ((retcode = Judy1First(PArray, & Index1, PJError)) == JERRI)
return(C_JERR); // pass through error.
#else
if ((PPvalue = JudyLFirst(PArray, & Index1, PJError)) == PPJERR)
return(C_JERR); // pass through error.
retcode = (PPvalue != (PPvoid_t) NULL); // found a next Index.
#endif
// If theres no Index at or above Index1, just return C_JERR (early exit):
if (retcode == 0)
{
JU_SET_ERRNO(PJError, JU_ERRNO_NONE);
return(C_JERR);
}
// If a first/next Index was found, call the counting motor starting with that
// known valid Index, meaning the return should be positive, not C_JERR except
// in case of a real error:
if ((pop1above1 = j__udy1LCountSM(Pjp, Index1, Pjpm)) == C_JERR)
{
JU_COPY_ERRNO(PJError, Pjpm); // pass through error.
return(C_JERR);
}
}
// COUNT POP1 ABOVE INDEX2, EXCLUSIVE, AND RETURN THE DIFFERENCE:
//
// In principle, calculate the ordinal of each Index and take the difference,
// with caution about off-by-one errors due to the specified Indexes being set
// or unset. In practice:
//
// - The ordinals computed here are inverse ordinals, that is, the populations
// ABOVE the specified Indexes (Index1 inclusive, Index2 exclusive), so
// subtract pop1above2 from pop1above1, rather than vice-versa.
//
// - Index1s result already includes a count for Index1 and/or Index2 if
// either is set, so calculate pop1above2 exclusive of Index2.
//
// TBD: If Index1 and Index2 fall in the same expanse in the top-state
// branch(es), would it be faster to walk the SM only once, to their divergence
// point, before calling j__udy1LCountSM() or equivalent? Possibly a non-issue
// if a top-state pop1 becomes stored with each Judy1 array. Also, consider
// whether the first call of j__udy1LCountSM() fills the cache, for common tree
// branches, for the second call.
//
// As for pop1above1, look for shortcuts for special cases when pop1above2 is
// zero. Otherwise call the counting "motor".
assert(pop1above1); // just to be safe.
if (Index2++ == cJU_ALLONES) return(pop1above1); // Index2 at limit.
#ifdef JUDY1
if ((retcode = Judy1First(PArray, & Index2, PJError)) == JERRI)
return(C_JERR);
#else
if ((PPvalue = JudyLFirst(PArray, & Index2, PJError)) == PPJERR)
return(C_JERR);
retcode = (PPvalue != (PPvoid_t) NULL); // found a next Index.
#endif
if (retcode == 0) return(pop1above1); // no Index above Index2.
// Just as for Index1, j__udy1LCountSM() cannot return 0 (locally == C_JERR)
// except in case of a real error:
if ((pop1above2 = j__udy1LCountSM(Pjp, Index2, Pjpm)) == C_JERR)
{
JU_COPY_ERRNO(PJError, Pjpm); // pass through error.
return(C_JERR);
}
if (pop1above1 == pop1above2)
{
JU_SET_ERRNO(PJError, JU_ERRNO_NONE);
return(C_JERR);
}
return(pop1above1 - pop1above2);
} // Judy1Count() / JudyLCount()
int ugh_subreq_gen(ugh_subreq_t *r, strp u_host)
{
ugh_client_t *c = r->c;
/* generate request line */
if (0 == r->u.uri.size)
{
aux_buffer_printf(&r->b_send, c->pool, "%s %.*s%s%.*s %s" CRLF
, ugh_method_string[r->method]
, (int) c->uri.size, c->uri.data
, c->args.size ? "?" : ""
, (int) c->args.size, c->args.data
, ugh_version_string[c->version]
);
}
else if (0 == r->u.args.size)
{
aux_buffer_printf(&r->b_send, c->pool, "%s %.*s%s%.*s %s" CRLF
, ugh_method_string[r->method]
, (int) r->u.uri.size, r->u.uri.data
, c->args.size ? "?" : ""
, (int) c->args.size, c->args.data
, ugh_version_string[c->version]
);
}
else
{
aux_buffer_printf(&r->b_send, c->pool, "%s %.*s?%.*s %s" CRLF
, ugh_method_string[r->method]
, (int) r->u.uri.size, r->u.uri.data
, (int) r->u.args.size, r->u.args.data
, ugh_version_string[c->version]
);
}
/* copy original request headers, change host header with new value */
Word_t idx = 0;
void **vptr;
for (vptr = JudyLFirst(c->headers_hash, &idx, PJE0); NULL != vptr;
vptr = JudyLNext (c->headers_hash, &idx, PJE0))
{
ugh_header_t *h = *vptr;
if (4 == h->key.size && aux_hash_key_lc_header("Host", 4) == aux_hash_key_lc_header(h->key.data, h->key.size))
{
aux_buffer_printf(&r->b_send, c->pool, "Host: %.*s" CRLF
, (int) u_host->size, u_host->data
);
}
else if (14 == h->key.size && aux_hash_key_lc_header("Content-Length", 14) == aux_hash_key_lc_header(h->key.data, h->key.size) && r->method == UGH_HTTP_GET)
{
continue; /* remove Content-Length header if it was in original request, but this request is GET */
}
else
{
aux_buffer_printf(&r->b_send, c->pool, "%.*s: %.*s" CRLF
, (int) h->key.size, h->key.data
, (int) h->value.size, h->value.data
);
}
}
if (NULL != r->request_body.data)
{
if (r->c->method != UGH_HTTP_POST) /* don't write new Content-Length if it was in original request */
{
aux_buffer_printf(&r->b_send, c->pool, "Content-Length: %"PRIuMAX CRLF
, (uintmax_t) r->request_body.size
);
}
/* TODO Content-Type */
aux_buffer_strcpy(&r->b_send, c->pool, CRLF);
aux_buffer_memcpy(&r->b_send, c->pool, r->request_body.data, r->request_body.size);
}
else
{
aux_buffer_strcpy(&r->b_send, c->pool, CRLF);
}
/* log_debug("ugh_subreq(%.*s)", (int) r->buf_send.size, r->buf_send.data); */
return 0;
}
/*
* Searches for a page and triggers disk I/O if necessary and possible.
* Does not get a reference.
* Returns page index pointer for given metric UUID.
*/
struct pg_cache_page_index *
pg_cache_preload(struct rrdengine_instance *ctx, uuid_t *id, usec_t start_time, usec_t end_time)
{
struct page_cache *pg_cache = &ctx->pg_cache;
struct rrdeng_page_cache_descr *descr = NULL, *preload_array[PAGE_CACHE_MAX_PRELOAD_PAGES];
int i, j, k, count, found;
unsigned long flags;
Pvoid_t *PValue;
struct pg_cache_page_index *page_index;
Word_t Index;
uint8_t failed_to_reserve;
uv_rwlock_rdlock(&pg_cache->metrics_index.lock);
PValue = JudyHSGet(pg_cache->metrics_index.JudyHS_array, id, sizeof(uuid_t));
if (likely(NULL != PValue)) {
page_index = *PValue;
}
uv_rwlock_rdunlock(&pg_cache->metrics_index.lock);
if (NULL == PValue) {
debug(D_RRDENGINE, "%s: No page was found to attempt preload.", __func__);
return NULL;
}
uv_rwlock_rdlock(&page_index->lock);
/* Find first page in range */
found = 0;
Index = (Word_t)(start_time / USEC_PER_SEC);
PValue = JudyLLast(page_index->JudyL_array, &Index, PJE0);
if (likely(NULL != PValue)) {
descr = *PValue;
if (is_page_in_time_range(descr, start_time, end_time)) {
found = 1;
}
}
if (!found) {
Index = (Word_t)(start_time / USEC_PER_SEC);
PValue = JudyLFirst(page_index->JudyL_array, &Index, PJE0);
if (likely(NULL != PValue)) {
descr = *PValue;
if (is_page_in_time_range(descr, start_time, end_time)) {
found = 1;
}
}
}
if (!found) {
uv_rwlock_rdunlock(&page_index->lock);
debug(D_RRDENGINE, "%s: No page was found to attempt preload.", __func__);
return page_index;
}
for (count = 0 ;
descr != NULL && is_page_in_time_range(descr, start_time, end_time);
PValue = JudyLNext(page_index->JudyL_array, &Index, PJE0),
descr = unlikely(NULL == PValue) ? NULL : *PValue) {
/* Iterate all pages in range */
if (unlikely(0 == descr->page_length))
continue;
uv_mutex_lock(&descr->mutex);
flags = descr->flags;
if (pg_cache_can_get_unsafe(descr, 0)) {
if (flags & RRD_PAGE_POPULATED) {
/* success */
uv_mutex_unlock(&descr->mutex);
debug(D_RRDENGINE, "%s: Page was found in memory.", __func__);
continue;
}
}
if (!(flags & RRD_PAGE_POPULATED) && pg_cache_try_get_unsafe(descr, 1)) {
preload_array[count++] = descr;
if (PAGE_CACHE_MAX_PRELOAD_PAGES == count) {
uv_mutex_unlock(&descr->mutex);
break;
}
}
uv_mutex_unlock(&descr->mutex);
};
uv_rwlock_rdunlock(&page_index->lock);
failed_to_reserve = 0;
for (i = 0 ; i < count && !failed_to_reserve ; ++i) {
struct rrdeng_cmd cmd;
struct rrdeng_page_cache_descr *next;
descr = preload_array[i];
if (NULL == descr) {
continue;
}
if (!pg_cache_try_reserve_pages(ctx, 1)) {
failed_to_reserve = 1;
break;
}
cmd.opcode = RRDENG_READ_EXTENT;
cmd.read_extent.page_cache_descr[0] = descr;
/* don't use this page again */
preload_array[i] = NULL;
for (j = 0, k = 1 ; j < count ; ++j) {
next = preload_array[j];
if (NULL == next) {
continue;
}
if (descr->extent == next->extent) {
/* same extent, consolidate */
if (!pg_cache_try_reserve_pages(ctx, 1)) {
failed_to_reserve = 1;
break;
}
cmd.read_extent.page_cache_descr[k++] = next;
/* don't use this page again */
preload_array[j] = NULL;
}
}
cmd.read_extent.page_count = k;
rrdeng_enq_cmd(&ctx->worker_config, &cmd);
}
if (failed_to_reserve) {
debug(D_RRDENGINE, "%s: Failed to reserve enough memory, canceling I/O.", __func__);
for (i = 0 ; i < count ; ++i) {
descr = preload_array[i];
if (NULL == descr) {
continue;
}
pg_cache_put(descr);
}
}
if (!count) {
/* no such page */
debug(D_RRDENGINE, "%s: No page was eligible to attempt preload.", __func__);
}
return page_index;
}
int ugh_client_send(ugh_client_t *c, int status)
{
#if 0
if (0 == c->content_type.size)
{
c->content_type.size = sizeof("text/plain") - 1;
c->content_type.data = "text/plain";
}
#endif
size_t i;
for (i = 0; i < ugh_module_handles_size; ++i)
{
c->bufs_sumlen += c->bufs[i].size;
}
c->buf_send.data = (char *) aux_pool_nalloc(c->pool, UGH_HEADER_BUF);
c->buf_send.size = snprintf(c->buf_send.data, UGH_HEADER_BUF,
"HTTP/1.1 %s" CRLF
"Server: ugh/"UGH_VERSION CRLF
"Content-Length: %"PRIuMAX CRLF
"Connection: close" CRLF
/* "Content-Type: %.*s" CRLF */
, ugh_status_header[status]
, (uintmax_t) c->bufs_sumlen
/* , (int) c->content_type.size, c->content_type.data */
);
void **vptr;
Word_t idx = 0;
for (vptr = JudyLFirst(c->headers_out_hash, &idx, PJE0); vptr; vptr = JudyLNext(c->headers_out_hash, &idx, PJE0))
{
ugh_header_t *h = *vptr;
c->buf_send.size += snprintf(c->buf_send.data + c->buf_send.size, UGH_HEADER_BUF - c->buf_send.size,
"%.*s: %.*s" CRLF, (int) h->key.size, h->key.data, (int) h->value.size, h->value.data);
}
#if 0
if (0 != c->location.size)
{
c->buf_send.size += snprintf(c->buf_send.data + c->buf_send.size, UGH_HEADER_BUF - c->buf_send.size,
"Location: %.*s" CRLF, (int) c->location.size, c->location.data);
}
#endif
int rc;
idx = 0;
for (rc = Judy1First(c->cookies_out_hash, &idx, PJE0); rc; rc = Judy1Next(c->cookies_out_hash, &idx, PJE0))
{
strp cookie = (strp) idx;
c->buf_send.size += snprintf(c->buf_send.data + c->buf_send.size, UGH_HEADER_BUF - c->buf_send.size,
"Set-Cookie: %.*s" CRLF, (int) cookie->size, cookie->data);
}
c->buf_send.size += snprintf(c->buf_send.data + c->buf_send.size, UGH_HEADER_BUF - c->buf_send.size, CRLF);
log_notice("access %s:%u '%.*s%s%.*s' %.*s %"PRIuMAX, inet_ntoa(c->addr.sin_addr), ntohs(c->addr.sin_port), (int) c->uri.size, c->uri.data,
c->args.size ? "?" : "", (int) c->args.size, c->args.data, 3, ugh_status_header[status],
(uintmax_t) c->bufs_sumlen);
ev_io_start(loop, &c->wev_send);
return 0;
}
