Ejemplo n.º 1
0
static void LockSegfilesOnMasterForSingleRel(Relation rel, int32 segno) {
  Insist(Gp_role == GP_ROLE_DISPATCH);

  /*
   * do not lock segfile with content id = -1
   */
  /*
   for (i = 1; i < rel->rd_segfile0_count; ++i)
   {
   if (RelationIsAoRows(rel) || RelationIsParquet(rel))
   {
   LockRelationAppendOnlySegmentFile(&rel->rd_node, segno,
   AccessExclusiveLock, false, i - 1);
   }
   }
   */
  {
    if (RelationIsAoRows(rel) || RelationIsParquet(rel)) {
      LockRelationAppendOnlySegmentFile(&rel->rd_node, segno,
      AccessExclusiveLock,
                                        false);
    }
  }

}
Ejemplo n.º 2
0
/*
 * Update the eof and filetupcount of a parquet table.
 */
void
UpdateParquetFileSegInfo(Relation parentrel,
				  AppendOnlyEntry *aoEntry,
				  int segno,
				  int64 eof,
				  int64 eof_uncompressed,
				  int64 tuples_added)
{

	LockAcquireResult acquireResult;

	Relation			pg_parquetseg_rel;
	TupleDesc			pg_parquetseg_dsc;
	ScanKeyData			key[1];
	SysScanDesc			parquetscan;
	HeapTuple			tuple, new_tuple;
	Datum				filetupcount;
	Datum				new_tuple_count;
	Datum			   *new_record;
	bool			   *new_record_nulls;
	bool			   *new_record_repl;
	bool				isNull;

	/* overflow sanity checks. don't check the same for tuples_added,
	 * it may be coming as a negative diff from gp_update_ao_master_stats */
	Assert(eof >= 0);

	Insist(Gp_role != GP_ROLE_EXECUTE);

	elog(DEBUG3, "UpdateParquetFileSegInfo called. segno = %d", segno);

	if (Gp_role != GP_ROLE_DISPATCH)
	{
		/*
		 * Verify we already have the write-lock!
		 */
		acquireResult = LockRelationAppendOnlySegmentFile(
													&parentrel->rd_node,
													segno,
													AccessExclusiveLock,
													/* dontWait */ false);
		if (acquireResult != LOCKACQUIRE_ALREADY_HELD)
		{
			elog(ERROR, "Should already have the (transaction-scope) write-lock on Parquet segment file #%d, "
				 "relation %s", segno, RelationGetRelationName(parentrel));
		}
	}

	/*
	 * Open the aoseg relation and its index.
	 */
	pg_parquetseg_rel = heap_open(aoEntry->segrelid, RowExclusiveLock);
	pg_parquetseg_dsc = pg_parquetseg_rel->rd_att;

	/*
	 * Setup a scan key to fetch from the index by segno.
	 */
	ScanKeyInit(&key[0], (AttrNumber) Anum_pg_parquetseg_segno,
			BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(segno));

	parquetscan = systable_beginscan(pg_parquetseg_rel, aoEntry->segidxid, TRUE,
			SnapshotNow, 1, &key[0]);

	tuple = systable_getnext(parquetscan);

	if (!HeapTupleIsValid(tuple))
		ereport(ERROR,
				(errcode(ERRCODE_UNDEFINED_OBJECT),
				errmsg("parquet table \"%s\" file segment \"%d\" entry "
						"does not exist", RelationGetRelationName(parentrel),
						segno)));

	new_record = palloc0(sizeof(Datum) * pg_parquetseg_dsc->natts);
	new_record_nulls = palloc0(sizeof(bool) * pg_parquetseg_dsc->natts);
	new_record_repl = palloc0(sizeof(bool) * pg_parquetseg_dsc->natts);

	/* get the current tuple count so we can add to it */
	filetupcount = fastgetattr(tuple,
								Anum_pg_parquetseg_tupcount,
								pg_parquetseg_dsc,
								&isNull);

	if(isNull)
		ereport(ERROR,
				(errcode(ERRCODE_UNDEFINED_OBJECT),
				errmsg("got invalid pg_aoseg filetupcount value: NULL")));

	/* calculate the new tuple count */
	new_tuple_count = DirectFunctionCall2(float8pl,
										  filetupcount,
										  Float8GetDatum((float8)tuples_added));

	/*
	 * Build a tuple to update
	 */
	new_record[Anum_pg_parquetseg_eof - 1] = Float8GetDatum((float8)eof);
	new_record_repl[Anum_pg_parquetseg_eof - 1] = true;

	new_record[Anum_pg_parquetseg_tupcount - 1] = new_tuple_count;
	new_record_repl[Anum_pg_parquetseg_tupcount - 1] = true;

	new_record[Anum_pg_parquetseg_eofuncompressed - 1] = Float8GetDatum((float8)eof_uncompressed);
	new_record_repl[Anum_pg_parquetseg_eofuncompressed - 1] = true;

	/*
	 * update the tuple in the pg_aoseg table
	 */
	new_tuple = heap_modify_tuple(tuple, pg_parquetseg_dsc, new_record,
								new_record_nulls, new_record_repl);

	simple_heap_update(pg_parquetseg_rel, &tuple->t_self, new_tuple);

	CatalogUpdateIndexes(pg_parquetseg_rel, new_tuple);

	heap_freetuple(new_tuple);

	/* Finish up scan */
	systable_endscan(parquetscan);
	heap_close(pg_parquetseg_rel, RowExclusiveLock);

	pfree(new_record);
	pfree(new_record_nulls);
	pfree(new_record_repl);
}
Ejemplo n.º 3
0
/*
 * Open the next file segment for write.
 */
static void SetCurrentFileSegForWrite(ParquetInsertDesc parquetInsertDesc, ResultRelSegFileInfo *segfileinfo) {
	ParquetFileSegInfo *fsinfo;
	int32 fileSegNo;

	/* Make the 'segment' file name */
	MakeAOSegmentFileName(parquetInsertDesc->parquet_rel,
			parquetInsertDesc->cur_segno, -1, &fileSegNo,
			parquetInsertDesc->parquetFilePathName);
	Assert(
			strlen(parquetInsertDesc->parquetFilePathName) + 1 <=
			parquetInsertDesc->parquetFilePathNameMaxLen);

	/*
	 * In order to append to this file segment entry we must first
	 * acquire the relation parquet segment file (transaction-scope) lock (tag
	 * LOCKTAG_RELATION_APPENDONLY_SEGMENT_FILE) in order to guarantee
	 * stability of the pg_aoseg information on this segment file and exclusive right
	 * to append data to the segment file.
	 *
	 * NOTE: This is a transaction scope lock that must be held until commit / abort.
	 */
	LockRelationAppendOnlySegmentFile(&parquetInsertDesc->parquet_rel->rd_node,
			parquetInsertDesc->cur_segno, AccessExclusiveLock,
			/* dontWait */false);

	/* Now, get the information for the file segment we are going to append to. */
	parquetInsertDesc->fsInfo = (ParquetFileSegInfo *) palloc0(sizeof(ParquetFileSegInfo));

	/*
	 * in hawq, we cannot insert a new catalog entry and then update,
	 * since we cannot get the tid of added tuple.
	 * we should add the new catalog entry on master and then dispatch it to segments for update.
	 */
	Assert(parquetInsertDesc->fsInfo != NULL);
	Assert(segfileinfo->numfiles == 1);
	fsinfo = parquetInsertDesc->fsInfo;
	fsinfo->segno = segfileinfo->segno;
	fsinfo->tupcount = segfileinfo->tupcount;
	fsinfo->eof = segfileinfo->eof[0];
	fsinfo->eof_uncompressed = segfileinfo->uncompressed_eof[0];

	parquetInsertDesc->fileLen = (int64)fsinfo->eof;
	parquetInsertDesc->fileLen_uncompressed = (int64)fsinfo->eof_uncompressed;
	parquetInsertDesc->rowCount = fsinfo->tupcount;

	/* Open the existing file for write.*/
	OpenSegmentFile(
			parquetInsertDesc->mirroredOpen,
			parquetInsertDesc->parquetFilePathName, fsinfo->eof,
			&parquetInsertDesc->parquet_rel->rd_node,
			parquetInsertDesc->cur_segno, parquetInsertDesc->relname,
			&parquetInsertDesc->parquet_file,
			&parquetInsertDesc->file_previousmetadata,
			&parquetInsertDesc->protocol_read,
			parquetInsertDesc->parquet_rel->rd_att,
			&parquetInsertDesc->parquetMetadata,
			&parquetInsertDesc->fileLen,
			&parquetInsertDesc->fileLen_uncompressed,
			&parquetInsertDesc->previous_rowgroupcnt);

	initSerializeFooter(&(parquetInsertDesc->footerProtocol), parquetInsertDesc->parquetFilePathName);

}
Ejemplo n.º 4
0
/*
 * Performs a compaction of an append-only relation in column-orientation.
 *
 * In non-utility mode, all compaction segment files should be
 * marked as in-use/in-compaction in the appendonlywriter.c code. If
 * set, the insert_segno should also be marked as in-use.
  * When the insert segno is negative, only truncate to eof operations
 * can be executed.
 *
 * The caller is required to hold either an AccessExclusiveLock (vacuum full)
 * or a ShareLock on the relation.
 */
void
AOCSCompact(Relation aorel,
			List *compaction_segno,
			int insert_segno,
			bool isFull)
{
	const char *relname;
	int			total_segfiles;
	AOCSFileSegInfo **segfile_array;
	AOCSInsertDesc insertDesc = NULL;
	int			i,
				segno;
	LockAcquireResult acquireResult;
	AOCSFileSegInfo *fsinfo;
	Snapshot	appendOnlyMetaDataSnapshot = RegisterSnapshot(GetCatalogSnapshot(InvalidOid));

	Assert(RelationIsAoCols(aorel));
	Assert(Gp_role == GP_ROLE_EXECUTE || Gp_role == GP_ROLE_UTILITY);
	Assert(insert_segno >= 0);

	relname = RelationGetRelationName(aorel);

	elogif(Debug_appendonly_print_compaction, LOG,
		   "Compact AO relation %s", relname);

	/* Get information about all the file segments we need to scan */
	segfile_array = GetAllAOCSFileSegInfo(aorel, appendOnlyMetaDataSnapshot, &total_segfiles);

	if (insert_segno >= 0)
	{
		insertDesc = aocs_insert_init(aorel, insert_segno, false);
	}

	for (i = 0; i < total_segfiles; i++)
	{
		segno = segfile_array[i]->segno;
		if (!list_member_int(compaction_segno, segno))
		{
			continue;
		}
		if (segno == insert_segno)
		{
			/* We cannot compact the segment file we are inserting to. */
			continue;
		}

		/*
		 * Try to get the transaction write-lock for the Append-Only segment
		 * file.
		 *
		 * NOTE: This is a transaction scope lock that must be held until
		 * commit / abort.
		 */
		acquireResult = LockRelationAppendOnlySegmentFile(
														  &aorel->rd_node,
														  segfile_array[i]->segno,
														  AccessExclusiveLock,
														   /* dontWait */ true);
		if (acquireResult == LOCKACQUIRE_NOT_AVAIL)
		{
			elog(DEBUG5, "compaction skips AOCS segfile %d, "
				 "relation %s", segfile_array[i]->segno, relname);
			continue;
		}

		/* Re-fetch under the write lock to get latest committed eof. */
		fsinfo = GetAOCSFileSegInfo(aorel, appendOnlyMetaDataSnapshot, segno);

		/*
		 * This should not occur since this segfile info was found by the
		 * "all" method, but better to catch for trouble shooting (possibly
		 * index corruption?)
		 */
		if (fsinfo == NULL)
			elog(ERROR, "file seginfo for AOCS relation %s %u/%u/%u (segno=%u) is missing",
				 relname,
				 aorel->rd_node.spcNode,
				 aorel->rd_node.dbNode,
				 aorel->rd_node.relNode,
				 segno);

		if (AppendOnlyCompaction_ShouldCompact(aorel,
											   fsinfo->segno, fsinfo->total_tupcount, isFull,
											   appendOnlyMetaDataSnapshot))
		{
			AOCSSegmentFileFullCompaction(aorel, insertDesc, fsinfo,
										  appendOnlyMetaDataSnapshot);
		}

		pfree(fsinfo);
	}

	if (insertDesc != NULL)
		aocs_insert_finish(insertDesc);

	if (segfile_array)
	{
		FreeAllAOCSSegFileInfo(segfile_array, total_segfiles);
		pfree(segfile_array);
	}

	UnregisterSnapshot(appendOnlyMetaDataSnapshot);
}
Ejemplo n.º 5
0
/*
 * Performs a compaction of an append-only AOCS relation.
 *
 * In non-utility mode, all compaction segment files should be
 * marked as in-use/in-compaction in the appendonlywriter.c code.
 *
 */
void
AOCSDrop(Relation aorel,
		 List *compaction_segno)
{
	const char *relname;
	int			total_segfiles;
	AOCSFileSegInfo **segfile_array;
	int			i,
				segno;
	LockAcquireResult acquireResult;
	AOCSFileSegInfo *fsinfo;
	Snapshot	appendOnlyMetaDataSnapshot = RegisterSnapshot(GetCatalogSnapshot(InvalidOid));

	Assert(Gp_role == GP_ROLE_EXECUTE || Gp_role == GP_ROLE_UTILITY);
	Assert(RelationIsAoCols(aorel));

	relname = RelationGetRelationName(aorel);

	elogif(Debug_appendonly_print_compaction, LOG,
		   "Drop AOCS relation %s", relname);

	/* Get information about all the file segments we need to scan */
	segfile_array = GetAllAOCSFileSegInfo(aorel,
										  appendOnlyMetaDataSnapshot, &total_segfiles);

	for (i = 0; i < total_segfiles; i++)
	{
		segno = segfile_array[i]->segno;
		if (!list_member_int(compaction_segno, segno))
		{
			continue;
		}

		/*
		 * Try to get the transaction write-lock for the Append-Only segment
		 * file.
		 *
		 * NOTE: This is a transaction scope lock that must be held until
		 * commit / abort.
		 */
		acquireResult = LockRelationAppendOnlySegmentFile(
														  &aorel->rd_node,
														  segfile_array[i]->segno,
														  AccessExclusiveLock,
														   /* dontWait */ true);
		if (acquireResult == LOCKACQUIRE_NOT_AVAIL)
		{
			elog(DEBUG5, "drop skips AOCS segfile %d, "
				 "relation %s", segfile_array[i]->segno, relname);
			continue;
		}

		/* Re-fetch under the write lock to get latest committed eof. */
		fsinfo = GetAOCSFileSegInfo(aorel, appendOnlyMetaDataSnapshot, segno);

		if (fsinfo->state == AOSEG_STATE_AWAITING_DROP)
		{
			Assert(HasLockForSegmentFileDrop(aorel));
			AOCSCompaction_DropSegmentFile(aorel, segno);
			ClearAOCSFileSegInfo(aorel, segno, AOSEG_STATE_DEFAULT);
		}
		pfree(fsinfo);
	}

	if (segfile_array)
	{
		FreeAllAOCSSegFileInfo(segfile_array, total_segfiles);
		pfree(segfile_array);
	}
	UnregisterSnapshot(appendOnlyMetaDataSnapshot);
}
Ejemplo n.º 6
0
/*
 * Truncates each segment file to the AOCS relation to its EOF.
 * If we cannot get a lock on the segment file (because e.g. a concurrent insert)
 * the segment file is skipped.
 */
void
AOCSTruncateToEOF(Relation aorel)
{
	const char *relname;
	int			total_segfiles;
	AOCSFileSegInfo **segfile_array;
	int			i,
				segno;
	LockAcquireResult acquireResult;
	AOCSFileSegInfo *fsinfo;
	Snapshot	appendOnlyMetaDataSnapshot = RegisterSnapshot(GetCatalogSnapshot(InvalidOid));

	Assert(RelationIsAoCols(aorel));

	relname = RelationGetRelationName(aorel);

	elogif(Debug_appendonly_print_compaction, LOG,
		   "Compact AO relation %s", relname);

	/* Get information about all the file segments we need to scan */
	segfile_array = GetAllAOCSFileSegInfo(aorel, appendOnlyMetaDataSnapshot, &total_segfiles);

	for (i = 0; i < total_segfiles; i++)
	{
		segno = segfile_array[i]->segno;

		/*
		 * Try to get the transaction write-lock for the Append-Only segment
		 * file.
		 *
		 * NOTE: This is a transaction scope lock that must be held until
		 * commit / abort.
		 */
		acquireResult = LockRelationAppendOnlySegmentFile(
														  &aorel->rd_node,
														  segfile_array[i]->segno,
														  AccessExclusiveLock,
														   /* dontWait */ true);
		if (acquireResult == LOCKACQUIRE_NOT_AVAIL)
		{
			elog(DEBUG5, "truncate skips AO segfile %d, "
				 "relation %s", segfile_array[i]->segno, relname);
			continue;
		}

		/* Re-fetch under the write lock to get latest committed eof. */
		fsinfo = GetAOCSFileSegInfo(aorel, appendOnlyMetaDataSnapshot, segno);

		/*
		 * This should not occur since this segfile info was found by the
		 * "all" method, but better to catch for trouble shooting (possibly
		 * index corruption?)
		 */
		if (fsinfo == NULL)
			elog(ERROR, "file seginfo for AOCS relation %s %u/%u/%u (segno=%u) is missing",
				 relname,
				 aorel->rd_node.spcNode,
				 aorel->rd_node.dbNode,
				 aorel->rd_node.relNode,
				 segno);

		AOCSSegmentFileTruncateToEOF(aorel, fsinfo);
		pfree(fsinfo);
	}

	if (segfile_array)
	{
		FreeAllAOCSSegFileInfo(segfile_array, total_segfiles);
		pfree(segfile_array);
	}
	UnregisterSnapshot(appendOnlyMetaDataSnapshot);
}
Ejemplo n.º 7
0
/*
 * Performs a compaction of an append-only relation.
 *
 * In non-utility mode, all compaction segment files should be
 * marked as in-use/in-compaction in the appendonlywriter.c code. If
 * set, the insert_segno should also be marked as in-use.
  * When the insert segno is negative, only truncate to eof operations
 * can be executed.
 *
 * The caller is required to hold either an AccessExclusiveLock (vacuum full) 
 * or a ShareLock on the relation.
 */ 
void
AppendOnlyCompact(Relation aorel, 
		List* compaction_segno, 
		int insert_segno,
		bool isFull)
{
	const char* relname;
	int total_segfiles;
	FileSegInfo** segfile_array;
	AppendOnlyInsertDesc insertDesc = NULL;
	int i, segno;
	FileSegInfo* fsinfo;

	Assert (Gp_role == GP_ROLE_EXECUTE || Gp_role == GP_ROLE_UTILITY);
	Assert(insert_segno >= 0);

	relname = RelationGetRelationName(aorel);
	AppendOnlyEntry *aoEntry = GetAppendOnlyEntry(RelationGetRelid(aorel), SnapshotNow);

	elogif (Debug_appendonly_print_compaction, LOG, 
			"Compact AO relation %s", relname);

	/* Get information about all the file segments we need to scan */
	segfile_array = GetAllFileSegInfo(aorel, aoEntry, SnapshotNow, &total_segfiles);

	insertDesc = appendonly_insert_init(aorel, SnapshotNow,
		insert_segno, false);

	for(i = 0 ; i < total_segfiles ; i++)
	{
		segno = segfile_array[i]->segno;
		if (list_find_int(compaction_segno, segno) < 0)
		{
			continue;
		}
		if (segno == insert_segno)
		{
			/* We cannot compact the segment file we are inserting to. */
			continue;
		}

		/*
		 * Try to get the transaction write-lock for the Append-Only segment file.
		 *
		 * NOTE: This is a transaction scope lock that must be held until commit / abort.
		 */
		LockRelationAppendOnlySegmentFile(
												&aorel->rd_node,
												segfile_array[i]->segno,
												AccessExclusiveLock,
												false);

		/* Re-fetch under the write lock to get latest committed eof. */
		fsinfo = GetFileSegInfo(aorel, aoEntry, SnapshotNow, segno);

		/*
		 * This should not occur since this segfile info was found by the
		 * "all" method, but better to catch for trouble shooting
		 * (possibly index corruption?)
		 */
		if (fsinfo == NULL)
			elog(ERROR, "file seginfo for AO relation %s %u/%u/%u (segno=%u) is missing",
				 relname,
				 aorel->rd_node.spcNode,
				 aorel->rd_node.dbNode,
				 aorel->rd_node.relNode,
				 segno);

		if (AppendOnlyCompaction_ShouldCompact(aorel, aoEntry,
				fsinfo->segno, fsinfo->total_tupcount, isFull))
		{
			AppendOnlySegmentFileFullCompaction(aorel, aoEntry, 
				insertDesc, 
				fsinfo);
		} 
		pfree(fsinfo);
	}

	appendonly_insert_finish(insertDesc);

	pfree(aoEntry);

	if (segfile_array)
	{
		FreeAllSegFileInfo(segfile_array, total_segfiles);
		pfree(segfile_array);
	}
}
Ejemplo n.º 8
0
/*
 * Performs a compaction of an append-only relation.
 *
 * In non-utility mode, all compaction segment files should be
 * marked as in-use/in-compaction in the appendonlywriter.c code.
 *
 */ 
void
AppendOnlyDrop(Relation aorel,
		List* compaction_segno)
{
	const char* relname;
	int total_segfiles;
	FileSegInfo** segfile_array;
	int i, segno;
	FileSegInfo* fsinfo;

	Assert (Gp_role == GP_ROLE_EXECUTE || Gp_role == GP_ROLE_UTILITY);
	Assert (RelationIsAoRows(aorel));

	relname = RelationGetRelationName(aorel);
	AppendOnlyEntry *aoEntry = GetAppendOnlyEntry(RelationGetRelid(aorel), SnapshotNow);

	elogif (Debug_appendonly_print_compaction, LOG, 
			"Drop AO relation %s", relname);

	/* Get information about all the file segments we need to scan */
	segfile_array = GetAllFileSegInfo(aorel, aoEntry, SnapshotNow, &total_segfiles);

	for(i = 0 ; i < total_segfiles ; i++)
	{
		segno = segfile_array[i]->segno;
		if (list_find_int(compaction_segno, segno) < 0)
		{
			continue;
		}

		/*
		 * Try to get the transaction write-lock for the Append-Only segment file.
		 *
		 * NOTE: This is a transaction scope lock that must be held until commit / abort.
		 */
		LockRelationAppendOnlySegmentFile(
												&aorel->rd_node,
												segfile_array[i]->segno,
												AccessExclusiveLock,
												false);

		/* Re-fetch under the write lock to get latest committed eof. */
		fsinfo = GetFileSegInfo(aorel, aoEntry, SnapshotNow, segno);

		if (fsinfo->state == AOSEG_STATE_AWAITING_DROP)
		{
			Assert(HasLockForSegmentFileDrop(aorel));
			Assert(!HasSerializableBackends(false));
			AppendOnlyCompaction_DropSegmentFile(aorel, segno);
			ClearFileSegInfo(aorel, aoEntry, segno,
					AOSEG_STATE_DEFAULT);
		}
		pfree(fsinfo);
	}

	pfree(aoEntry);

	if (segfile_array)
	{
		FreeAllSegFileInfo(segfile_array, total_segfiles);
		pfree(segfile_array);
	}
}