void CombinerDestReceiverFlush(DestReceiver *self) { CombinerState *c = (CombinerState *) self; int i; int ntups = 0; Size size = 0; microbatch_t *mb; if (CombinerFlushHook) CombinerFlushHook(); mb = microbatch_new(CombinerTuple, bms_make_singleton(c->cont_query->id), NULL); microbatch_add_acks(mb, c->cont_exec->batch->sync_acks); for (i = 0; i < continuous_query_num_combiners; i++) { List *tups = c->tups_per_combiner[i]; ListCell *lc; if (tups == NIL) continue; ntups += list_length(tups); foreach(lc, tups) { tagged_ref_t *ref = lfirst(lc); HeapTuple tup = (HeapTuple) ref->ptr; uint64 hash = ref->tag; if (!microbatch_add_tuple(mb, tup, hash)) { microbatch_send_to_combiner(mb, i); microbatch_add_tuple(mb, tup, hash); } size += HEAPTUPLESIZE + tup->t_len; } if (!microbatch_is_empty(mb)) { microbatch_send_to_combiner(mb, i); microbatch_reset(mb); } list_free_deep(tups); c->tups_per_combiner[i] = NIL; }
/* * CStoreEndWrite finishes a cstore data load operation. If we have an unflushed * stripe, we flush it. Then, we sync and close the cstore data file. Last, we * flush the footer to a temporary file, and atomically rename this temporary * file to the original footer file. */ void CStoreEndWrite(TableWriteState *writeState) { StringInfo tableFooterFilename = NULL; StringInfo tempTableFooterFileName = NULL; int renameResult = 0; StripeData *stripeData = writeState->stripeData; if (stripeData != NULL) { MemoryContext oldContext = MemoryContextSwitchTo(writeState->stripeWriteContext); StripeMetadata stripeMetadata = FlushStripe(writeState); MemoryContextReset(writeState->stripeWriteContext); MemoryContextSwitchTo(oldContext); AppendStripeMetadata(writeState->tableFooter, stripeMetadata); } SyncAndCloseFile(writeState->tableFile); tableFooterFilename = writeState->tableFooterFilename; tempTableFooterFileName = makeStringInfo(); appendStringInfo(tempTableFooterFileName, "%s%s", tableFooterFilename->data, CSTORE_TEMP_FILE_SUFFIX); CStoreWriteFooter(tempTableFooterFileName, writeState->tableFooter); renameResult = rename(tempTableFooterFileName->data, tableFooterFilename->data); if (renameResult != 0) { ereport(ERROR, (errcode_for_file_access(), errmsg("could not rename file \"%s\" to \"%s\": %m", tempTableFooterFileName->data, tableFooterFilename->data))); } pfree(tempTableFooterFileName->data); pfree(tempTableFooterFileName); MemoryContextDelete(writeState->stripeWriteContext); list_free_deep(writeState->tableFooter->stripeMetadataList); pfree(writeState->tableFooter); pfree(writeState->tableFooterFilename->data); pfree(writeState->tableFooterFilename); pfree(writeState->comparisonFunctionArray); pfree(writeState); }
/* * Destroy query context */ void pool_query_context_destroy(POOL_QUERY_CONTEXT *query_context) { POOL_SESSION_CONTEXT *session_context; if (query_context) { session_context = pool_get_session_context(); pool_unset_query_in_progress(); session_context->query_context = NULL; list_free(query_context->move_counts); list_free(query_context->move_counts_ids); list_free_deep(query_context->move_queries); pool_memory_delete(query_context->memory_context, 0); free(query_context); } }
void test__opexpr_to_pxffilter__twoVars(void **state) { PxfFilterDesc *filter = (PxfFilterDesc*) palloc0(sizeof(PxfFilterDesc)); Var *arg_var_left = build_var(INT4OID, 8); Var *arg_var_right = build_var(INT4OID, 9); OpExpr *expr = build_op_expr(arg_var_left, arg_var_right, 0 /* whatever */); /* run test */ assert_false(opexpr_to_pxffilter(expr, filter)); pxf_free_filter(filter); list_free_deep(expr->args); /* free all args */ pfree(expr); }
void test__opexpr_to_pxffilter__unsupportedTypeCircle(void **state) { PxfFilterDesc *filter = (PxfFilterDesc*) palloc0(sizeof(PxfFilterDesc)); Var *arg_var = build_var(CIRCLEOID, 8); Const *arg_const = build_const(CIRCLEOID, NULL); OpExpr *expr = build_op_expr(arg_const, arg_var, 0 /* whatever */); /* run test */ assert_false(opexpr_to_pxffilter(expr, filter)); pxf_free_filter(filter); list_free_deep(expr->args); /* free all args */ pfree(expr); }
void test__opexpr_to_pxffilter__unsupportedOpNot(void **state) { PxfFilterDesc *filter = (PxfFilterDesc*) palloc0(sizeof(PxfFilterDesc)); Var *arg_var = build_var(INT2OID, 3); char* const_value = strdup("not"); /* will be free'd by const_to_str */ Const *arg_const = build_const(INT2OID, const_value); OpExpr *expr = build_op_expr(arg_const, arg_var, 1877 /* int2not */); /* run test */ assert_false(opexpr_to_pxffilter(expr, filter)); pxf_free_filter(filter); list_free_deep(expr->args); /* free all args */ pfree(expr); }
/* * Handle table synchronization cooperation from the apply worker. * * Walk over all subscription tables that are individually tracked by the * apply process (currently, all that have state other than * SUBREL_STATE_READY) and manage synchronization for them. * * If there are tables that need synchronizing and are not being synchronized * yet, start sync workers for them (if there are free slots for sync * workers). To prevent starting the sync worker for the same relation at a * high frequency after a failure, we store its last start time with each sync * state info. We start the sync worker for the same relation after waiting * at least wal_retrieve_retry_interval. * * For tables that are being synchronized already, check if sync workers * either need action from the apply worker or have finished. This is the * SYNCWAIT to CATCHUP transition. * * If the synchronization position is reached (SYNCDONE), then the table can * be marked as READY and is no longer tracked. */ static void process_syncing_tables_for_apply(XLogRecPtr current_lsn) { struct tablesync_start_time_mapping { Oid relid; TimestampTz last_start_time; }; static List *table_states = NIL; static HTAB *last_start_times = NULL; ListCell *lc; bool started_tx = false; Assert(!IsTransactionState()); /* We need up-to-date sync state info for subscription tables here. */ if (!table_states_valid) { MemoryContext oldctx; List *rstates; ListCell *lc; SubscriptionRelState *rstate; /* Clean the old list. */ list_free_deep(table_states); table_states = NIL; StartTransactionCommand(); started_tx = true; /* Fetch all non-ready tables. */ rstates = GetSubscriptionNotReadyRelations(MySubscription->oid); /* Allocate the tracking info in a permanent memory context. */ oldctx = MemoryContextSwitchTo(CacheMemoryContext); foreach(lc, rstates) { rstate = palloc(sizeof(SubscriptionRelState)); memcpy(rstate, lfirst(lc), sizeof(SubscriptionRelState)); table_states = lappend(table_states, rstate); }
/* * Test for a query with different types. * Types pairing are not checked, it is covered by the * supported operations which are type specific. */ void test__opexpr_to_pxffilter__differentTypes(void **state) { PxfFilterDesc *filter = (PxfFilterDesc*) palloc0(sizeof(PxfFilterDesc)); Var *arg_var = build_var(INT2OID, 3); char* const_value = strdup("13"); /* will be free'd by const_to_str */ Const *arg_const = build_const(INT8OID, const_value); OpExpr *expr = build_op_expr(arg_const, arg_var, 1864 /* int28lt */); /* run test */ assert_true(opexpr_to_pxffilter(expr, filter)); PxfFilterDesc *expected = build_filter( PXF_CONST_CODE, 0, "13", PXF_ATTR_CODE, 3, NULL, PXFOP_LT); compare_filters(filter, expected); pxf_free_filter(filter); list_free_deep(expr->args); /* free all args */ pfree(expr); }
/* NOTE: this test is not a use case - when the query includes * 'is null' or 'is not null' the qualifier code is T_NullTest and not T_OpExpr */ void test__opexpr_to_pxffilter__attributeIsNull(void **state) { PxfFilterDesc *filter = (PxfFilterDesc*) palloc0(sizeof(PxfFilterDesc)); Var *arg_var = build_var(INT2OID, 1); Const* arg_const = build_const(INT2OID, NULL); OpExpr *expr = build_op_expr(arg_var, arg_const, 94 /* int2eq */); PxfFilterDesc* expected = build_filter( PXF_ATTR_CODE, 1, NULL, PXF_CONST_CODE, 0, "\"NULL\"", PXFOP_EQ); /* run test */ assert_true(opexpr_to_pxffilter(expr, filter)); compare_filters(filter, expected); pxf_free_filter(filter); pxf_free_filter(expected); list_free_deep(expr->args); /* free all args */ pfree(expr); }
void run__opexpr_to_pxffilter__positive(Oid dbop, PxfOperatorCode expectedPxfOp) { PxfFilterDesc *filter = (PxfFilterDesc*) palloc0(sizeof(PxfFilterDesc)); Var *arg_var = build_var(INT2OID, 1); char* const_value = strdup("1984"); /* will be free'd by const_to_str */ Const* arg_const = build_const(INT2OID, const_value); OpExpr *expr = build_op_expr(arg_var, arg_const, dbop); PxfFilterDesc* expected = build_filter( PXF_ATTR_CODE, 1, NULL, PXF_CONST_CODE, 0, "1984", expectedPxfOp); /* run test */ assert_true(opexpr_to_pxffilter(expr, filter)); compare_filters(filter, expected); pxf_free_filter(expected); pxf_free_filter(filter); list_free_deep(expr->args); /* free all args */ pfree(expr); }
/* * Insert tuples to a given page. * * This is analogous with gistinserttuples() in the regular insertion code. * * Returns the block number of the page where the (first) new or updated tuple * was inserted. Usually that's the original page, but might be a sibling page * if the original page was split. * * Caller should hold a lock on 'buffer' on entry. This function will unlock * and unpin it. */ static BlockNumber gistbufferinginserttuples(GISTBuildState *buildstate, Buffer buffer, int level, IndexTuple *itup, int ntup, OffsetNumber oldoffnum, BlockNumber parentblk, OffsetNumber downlinkoffnum) { GISTBuildBuffers *gfbb = buildstate->gfbb; List *splitinfo; bool is_split; BlockNumber placed_to_blk = InvalidBlockNumber; is_split = gistplacetopage(buildstate->indexrel, buildstate->freespace, buildstate->giststate, buffer, itup, ntup, oldoffnum, &placed_to_blk, InvalidBuffer, &splitinfo, false); /* * If this is a root split, update the root path item kept in memory. This * ensures that all path stacks are always complete, including all parent * nodes up to the root. That simplifies the algorithm to re-find correct * parent. */ if (is_split && BufferGetBlockNumber(buffer) == GIST_ROOT_BLKNO) { Page page = BufferGetPage(buffer); OffsetNumber off; OffsetNumber maxoff; Assert(level == gfbb->rootlevel); gfbb->rootlevel++; elog(DEBUG2, "splitting GiST root page, now %d levels deep", gfbb->rootlevel); /* * All the downlinks on the old root page are now on one of the child * pages. Visit all the new child pages to memorize the parents of the * grandchildren. */ if (gfbb->rootlevel > 1) { maxoff = PageGetMaxOffsetNumber(page); for (off = FirstOffsetNumber; off <= maxoff; off++) { ItemId iid = PageGetItemId(page, off); IndexTuple idxtuple = (IndexTuple) PageGetItem(page, iid); BlockNumber childblkno = ItemPointerGetBlockNumber(&(idxtuple->t_tid)); Buffer childbuf = ReadBuffer(buildstate->indexrel, childblkno); LockBuffer(childbuf, GIST_SHARE); gistMemorizeAllDownlinks(buildstate, childbuf); UnlockReleaseBuffer(childbuf); /* * Also remember that the parent of the new child page is the * root block. */ gistMemorizeParent(buildstate, childblkno, GIST_ROOT_BLKNO); } } } if (splitinfo) { /* * Insert the downlinks to the parent. This is analogous with * gistfinishsplit() in the regular insertion code, but the locking is * simpler, and we have to maintain the buffers on internal nodes and * the parent map. */ IndexTuple *downlinks; int ndownlinks, i; Buffer parentBuffer; ListCell *lc; /* Parent may have changed since we memorized this path. */ parentBuffer = gistBufferingFindCorrectParent(buildstate, BufferGetBlockNumber(buffer), level, &parentblk, &downlinkoffnum); /* * If there's a buffer associated with this page, that needs to be * split too. gistRelocateBuildBuffersOnSplit() will also adjust the * downlinks in 'splitinfo', to make sure they're consistent not only * with the tuples already on the pages, but also the tuples in the * buffers that will eventually be inserted to them. */ gistRelocateBuildBuffersOnSplit(gfbb, buildstate->giststate, buildstate->indexrel, level, buffer, splitinfo); /* Create an array of all the downlink tuples */ ndownlinks = list_length(splitinfo); downlinks = (IndexTuple *) palloc(sizeof(IndexTuple) * ndownlinks); i = 0; foreach(lc, splitinfo) { GISTPageSplitInfo *splitinfo = lfirst(lc); /* * Remember the parent of each new child page in our parent map. * This assumes that the downlinks fit on the parent page. If the * parent page is split, too, when we recurse up to insert the * downlinks, the recursive gistbufferinginserttuples() call will * update the map again. */ if (level > 0) gistMemorizeParent(buildstate, BufferGetBlockNumber(splitinfo->buf), BufferGetBlockNumber(parentBuffer)); /* * Also update the parent map for all the downlinks that got moved * to a different page. (actually this also loops through the * downlinks that stayed on the original page, but it does no * harm). */ if (level > 1) gistMemorizeAllDownlinks(buildstate, splitinfo->buf); /* * Since there's no concurrent access, we can release the lower * level buffers immediately. This includes the original page. */ UnlockReleaseBuffer(splitinfo->buf); downlinks[i++] = splitinfo->downlink; } /* Insert them into parent. */ gistbufferinginserttuples(buildstate, parentBuffer, level + 1, downlinks, ndownlinks, downlinkoffnum, InvalidBlockNumber, InvalidOffsetNumber); list_free_deep(splitinfo); /* we don't need this anymore */ }
/* * Insert tuples to a given page. * * This is analogous with gistinserttuples() in the regular insertion code. */ static void gistbufferinginserttuples(GISTBuildState *buildstate, Buffer buffer, IndexTuple *itup, int ntup, OffsetNumber oldoffnum, GISTBufferingInsertStack *path) { GISTBuildBuffers *gfbb = buildstate->gfbb; List *splitinfo; bool is_split; is_split = gistplacetopage(buildstate->indexrel, buildstate->freespace, buildstate->giststate, buffer, itup, ntup, oldoffnum, InvalidBuffer, &splitinfo, false); /* * If this is a root split, update the root path item kept in memory. This * ensures that all path stacks are always complete, including all parent * nodes up to the root. That simplifies the algorithm to re-find correct * parent. */ if (is_split && BufferGetBlockNumber(buffer) == GIST_ROOT_BLKNO) { GISTBufferingInsertStack *oldroot = gfbb->rootitem; Page page = BufferGetPage(buffer); ItemId iid; IndexTuple idxtuple; BlockNumber leftmostchild; gfbb->rootitem = (GISTBufferingInsertStack *) MemoryContextAlloc( gfbb->context, sizeof(GISTBufferingInsertStack)); gfbb->rootitem->parent = NULL; gfbb->rootitem->blkno = GIST_ROOT_BLKNO; gfbb->rootitem->downlinkoffnum = InvalidOffsetNumber; gfbb->rootitem->level = oldroot->level + 1; gfbb->rootitem->refCount = 1; /* * All the downlinks on the old root page are now on one of the child * pages. Change the block number of the old root entry in the stack * to point to the leftmost child. The other child pages will be * accessible from there by walking right. */ iid = PageGetItemId(page, FirstOffsetNumber); idxtuple = (IndexTuple) PageGetItem(page, iid); leftmostchild = ItemPointerGetBlockNumber(&(idxtuple->t_tid)); oldroot->parent = gfbb->rootitem; oldroot->blkno = leftmostchild; oldroot->downlinkoffnum = InvalidOffsetNumber; } if (splitinfo) { /* * Insert the downlinks to the parent. This is analogous with * gistfinishsplit() in the regular insertion code, but the locking is * simpler, and we have to maintain the buffers. */ IndexTuple *downlinks; int ndownlinks, i; Buffer parentBuffer; ListCell *lc; /* Parent may have changed since we memorized this path. */ gistBufferingFindCorrectParent(buildstate, path); /* * If there's a buffer associated with this page, that needs to be * split too. gistRelocateBuildBuffersOnSplit() will also adjust the * downlinks in 'splitinfo', to make sure they're consistent not only * with the tuples already on the pages, but also the tuples in the * buffers that will eventually be inserted to them. */ gistRelocateBuildBuffersOnSplit(gfbb, buildstate->giststate, buildstate->indexrel, path, buffer, splitinfo); /* Create an array of all the downlink tuples */ ndownlinks = list_length(splitinfo); downlinks = (IndexTuple *) palloc(sizeof(IndexTuple) * ndownlinks); i = 0; foreach(lc, splitinfo) { GISTPageSplitInfo *splitinfo = lfirst(lc); /* * Since there's no concurrent access, we can release the lower * level buffers immediately. Don't release the buffer for the * original page, though, because the caller will release that. */ if (splitinfo->buf != buffer) UnlockReleaseBuffer(splitinfo->buf); downlinks[i++] = splitinfo->downlink; } /* Insert them into parent. */ parentBuffer = ReadBuffer(buildstate->indexrel, path->parent->blkno); LockBuffer(parentBuffer, GIST_EXCLUSIVE); gistbufferinginserttuples(buildstate, parentBuffer, downlinks, ndownlinks, path->downlinkoffnum, path->parent); UnlockReleaseBuffer(parentBuffer); list_free_deep(splitinfo); /* we don't need this anymore */ }
region_table_t *parse_regions_from_gff_file(char *filename, const char *url, const char *species, const char *version) { gff_file_t *file = gff_open(filename); if (file == NULL) { return NULL; } region_table_t *regions_table = new_region_table_from_ws(url, species, version); int ret_code = 0; size_t max_batches = 20, batch_size = 2000; list_t *read_list = (list_t*) malloc (sizeof(list_t)); list_init("batches", 1, max_batches, read_list); #pragma omp parallel sections { // The producer reads the GFF file #pragma omp section { LOG_DEBUG_F("Thread %d reads the GFF file\n", omp_get_thread_num()); ret_code = gff_read_batches(read_list, batch_size, file); list_decr_writers(read_list); if (ret_code) { LOG_FATAL_F("Error while reading GFF file %s (%d)\n", filename, ret_code); } } // The consumer inserts regions in the structure #pragma omp section { list_item_t *item = NULL; gff_batch_t *batch; gff_record_t *record; region_t *regions_batch[REGIONS_CHUNKSIZE]; int avail_regions = 0; while ( item = list_remove_item(read_list) ) { batch = item->data_p; // For each record in the batch, generate a new region for (int i = 0; i < batch->records->size; i++) { record = batch->records->items[i]; region_t *region = region_new(strndup(record->sequence, record->sequence_len), record->start, record->end, record->strand ? strndup(&record->strand, 1) : NULL, record->feature ? strndup(record->feature, record->feature_len) : NULL); LOG_DEBUG_F("region '%s:%u-%u'\n", region->chromosome, region->start_position, region->end_position); regions_batch[avail_regions++] = region; // Save when the recommended size is reached if (avail_regions == REGIONS_CHUNKSIZE) { insert_regions(regions_batch, avail_regions, regions_table); for (int i = 0; i < avail_regions; i++) { free(regions_batch[i]); } avail_regions = 0; } } gff_batch_free(batch); list_item_free(item); } // Save the remaining regions that did not fill a batch if (avail_regions > 0) { insert_regions(regions_batch, avail_regions, regions_table); for (int i = 0; i < avail_regions; i++) { free(regions_batch[i]); } avail_regions = 0; } } } finish_region_table_loading(regions_table); list_free_deep(read_list, NULL); gff_close(file, 1); return regions_table; }
/* * parse_checkpoint */ bool parse_checkpoint(const char *message, const char *timestamp) { static CheckpointLog *ckpt = NULL; List *params; if ((params = capture(message, msg_checkpoint_starting, NUM_CHECKPOINT_STARTING)) != NIL) { /* log for checkpoint starting */ const char *type = (char *) list_nth(params, 0); const char *flags = (char *) list_nth(params, 1); CheckpointType ckpt_type; if (strcmp(type, "checkpoint") == 0) ckpt_type = CKPT_TYPE_CHECKPOINT; else if (strcmp(type, "restartpoint") == 0) ckpt_type = CKPT_TYPE_RESTARTPOINT; else { /* not a checkpoint log */ list_free_deep(params); return false; } /* ignore shutdown checkpoint */ if (strstr(flags, "shutdown")) { free(ckpt); ckpt = NULL; list_free_deep(params); return true; /* handled, but forget */ } if (ckpt == NULL) ckpt = pgut_new(CheckpointLog); /* copy type, flags and start timestamp */ ckpt->type = ckpt_type; strlcpy(ckpt->flags, flags, sizeof(ckpt->flags)); strlcpy(ckpt->start, timestamp, sizeof(ckpt->start)); list_free_deep(params); return true; } if ((params = capture(message, msg_checkpoint_complete, NUM_CHECKPOINT_COMPLETE)) != NIL || (params = capture(message, msg_restartpoint_complete, NUM_RESTARTPOINT_COMPLETE)) != NIL) { /* log for checkpoint complete */ /* ignore if we have not seen any checkpoint start */ if (ckpt == NULL) { list_free_deep(params); return true; /* handled, but forget */ } /* send checkpoint log to writer */ ckpt->params = params; ckpt->base.type = QUEUE_CHECKPOINT; ckpt->base.free = (QueueItemFree) Checkpoint_free; ckpt->base.exec = (QueueItemExec) Checkpoint_exec; writer_send((QueueItem *) ckpt); ckpt = NULL; return true; } /* not a checkpoint log */ return false; }
/* * purge_dropped_db_segments */ static void purge_dropped_db_segments(bool force) { static TimestampTz last_purge_time = 0; List *db_oids; List *dbs_to_remove = NIL; HASH_SEQ_STATUS status; broker_db_meta *db_meta; if (!force && !TimestampDifferenceExceeds(last_purge_time, GetCurrentTimestamp(), 10 * 1000)) /* 10s */ return; db_oids = get_database_oids(); LWLockAcquire(IPCMessageBrokerIndexLock, LW_SHARED); hash_seq_init(&status, broker_meta->db_meta_hash); while ((db_meta = (broker_db_meta *) hash_seq_search(&status)) != NULL) { bool found = false; ListCell *lc; foreach(lc, db_oids) { if (lfirst_oid(lc) == db_meta->dbid) { found = true; break; } } if (!found) dbs_to_remove = lappend_oid(dbs_to_remove, db_meta->dbid); } LWLockRelease(IPCMessageBrokerIndexLock); if (list_length(dbs_to_remove)) { ListCell *lc; LWLockAcquire(IPCMessageBrokerIndexLock, LW_EXCLUSIVE); foreach(lc, dbs_to_remove) { Oid dbid = lfirst_oid(lc); bool found; db_meta = hash_search(broker_meta->db_meta_hash, &dbid, HASH_FIND, &found); Assert(found); Assert(db_meta->handle > 0); /* detach from main db segment */ if (db_meta->segment) dsm_detach(db_meta->segment); if (db_meta->lqueues) { int i; for (i = 0; i < continuous_query_num_workers; i++) { local_queue *local_buf = &db_meta->lqueues[i]; if (local_buf->slots) list_free_deep(local_buf->slots); } pfree(db_meta->lqueues); } hash_search(broker_meta->db_meta_hash, &dbid, HASH_REMOVE, &found); Assert(found); } mark_unused_locks_as_free(db_oids); LWLockRelease(IPCMessageBrokerIndexLock); }