Ejemplo n.º 1
0
/*
 *  Estimate metadata cache shared memory size
 *      - Metadata cache hash size
 *      - Block info (3 types) hash size
 *      - Metadata cache shared structure size
 *      - Metadata hdfs block array size
 */
Size 
MetadataCache_ShmemSize(void)
{
    Size size;
    
    size = hash_estimate_size((Size)metadata_cache_max_hdfs_file_num, sizeof(MetadataCacheEntry));

    size = add_size(size, hash_estimate_size((Size)MAX_HDFS_HOST_NUM, sizeof(BlockInfoEntry)) * METADATA_BLOCK_INFO_TYPE_NUM);

    size = add_size(size, sizeof(MetadataCacheSharedData));

    size = add_size(size, metadata_cache_block_capacity* sizeof(MetadataHdfsBlockInfo));

    return size;
}
Ejemplo n.º 2
0
Size
Persistent_PostDTMRecv_ShmemSize(void)
{
	Size    size;

	size = hash_estimate_size((Size)PT_MAX_NUM_POSTDTMRECV_DB, sizeof(postDTMRecv_dbTblSpc_Hash_Entry));
	size = add_size(size, sizeof(PT_PostDTMRecv_Data));

	return size;
}
Ejemplo n.º 3
0
/*
 * IPCMessageBrokerShmemSize
 */
Size
IPCMessageBrokerShmemSize(void)
{
	Size		size;

	size = MAXALIGN(sizeof(BrokerMeta));
	size = add_size(size, max_locks * MAXALIGN(sizeof(lw_lock_slot)));
	size = add_size(size, hash_estimate_size(16, broker_db_meta_size));

	return size;
}
Ejemplo n.º 4
0
/*
 * Estimate shared memory space needed.
 */
static Size
pgss_memsize(void)
{
	Size		size;
	Size		entrysize;

	size = MAXALIGN(sizeof(pgssSharedState));
	entrysize = offsetof(pgssEntry, query) +pgstat_track_activity_query_size;
	size = add_size(size, hash_estimate_size(pgss_max, entrysize));

	return size;
}
Ejemplo n.º 5
0
/****************************************************************
 * FAULT INJECTOR routines
 ****************************************************************/
Size
FaultInjector_ShmemSize(void)
{
	Size	size;
	
	size = hash_estimate_size(
							  (Size)FAULTINJECTOR_MAX_SLOTS, 
							  sizeof(FaultInjectorEntry_s));
	
	size = add_size(size, sizeof(FaultInjectorShmem_s));
	
	return size;	
}
Ejemplo n.º 6
0
/*
 * Return the required shared-memory size for this module.
 */
Size PersistentFilespace_ShmemSize(void)
{
	Size		size;

	/* The hash table of persistent filespaces */
	size = hash_estimate_size((Size)gp_max_filespaces,
							  sizeof(FilespaceDirEntryData));

	/* The shared-memory structure. */
	size = add_size(size, PersistentFilespace_SharedDataSize());

	return size;
}
Ejemplo n.º 7
0
Size
FileRepAckPrimary_ShmemSize(void)
{
	Size	size;
	
	size = hash_estimate_size(
				(Size)FILEREP_MAX_OPEN_FILES, 
				sizeof(FileRepAckHashEntry_s));
	
	size = add_size(size, sizeof(FileRepAckHashShmem_s));
	
	return size;	
}
Ejemplo n.º 8
0
/* -----------------------------------------------------
 * BufferShmemSize
 *
 * compute the size of shared memory for the buffer pool including
 * data pages, buffer descriptors, hash tables, etc.
 * ----------------------------------------------------
 */
int
BufferShmemSize(void)
{
	int			size = 0;

	/* size of shmem index hash table */
	size += hash_estimate_size(SHMEM_INDEX_SIZE, sizeof(ShmemIndexEnt));

	/* size of buffer descriptors */
	size += MAXALIGN((NBuffers + 1) * sizeof(BufferDesc));

	/* size of data pages */
	size += NBuffers * MAXALIGN(BLCKSZ);

	/* size of buffer hash table */
	size += hash_estimate_size(NBuffers, sizeof(BufferLookupEnt));

#ifdef BMTRACE
	size += (BMT_LIMIT * sizeof(bmtrace)) + sizeof(long);
#endif

	return size;
}
Ejemplo n.º 9
0
/*
 * ResPortalIncrementShmemSize -- Estimate the size of the increment hash.
 *
 * Notes
 *	We allow one extra slot for unnamed portals, as it is simpler to not
 *	count them at all.
 */
Size 
ResPortalIncrementShmemSize(void)
{

	Size	size;
	long	max_table_size = (MaxResourcePortalsPerXact + 1) * MaxBackends;

	size = hash_estimate_size(max_table_size, sizeof(ResPortalIncrement));

	/* Add a safety margin */
	size = add_size(size, size / 10);

	return size;
}
Ejemplo n.º 10
0
/*
 * Computes shared memory size needed for this cache
 */
Size
Cache_SharedMemSize(uint32 nEntries, uint32 entryPayloadSize)
{
	Size size = 0;

	/* Size of anchor hashtable. It has the same number of entries as the cache */
	size = add_size(size, hash_estimate_size(nEntries, sizeof(CacheAnchor)));

	/* Size of cache shared control header */
	size = add_size(size, MAXALIGN(sizeof(CacheHdr)));

	Size entrySize = add_size(CACHE_ENTRY_HEADER_SIZE, MAXALIGN(entryPayloadSize));
	size = add_size(size, mul_size(nEntries, entrySize));

	return size;
}
Ejemplo n.º 11
0
/*
 * ResSchedulerShmemSize -- estimate size the schedular structures will need in
 *	shared memory.
 *
 */
Size
ResSchedulerShmemSize(void)
{
	Size		size;

	/* The hash of queues.*/
	size = hash_estimate_size(MaxResourceQueues, sizeof(ResQueueData));

	/* The scheduler structure. */
	size = add_size(size, sizeof(ResSchedulerData));

	/* Add a safety margin */
	size = add_size(size, size / 10);


	return size;
}
Ejemplo n.º 12
0
/*
 * Return the required shared-memory size for this module.
 */
Size PersistentRelation_ShmemSize(void)
{
	Size size;

	/* The hash table of persistent relations */
	size = hash_estimate_size((Size)gp_max_relations,
							sizeof(RelationDirEntryData));

	/* The shared-memory structure. */
	size = add_size(size, PersistentRelation_SharedDataSize());

	elog(LOG, "PersistentRelation_ShmemSize: %zu = "
			"gp_max_relations: %d "
			"* sizeof(RelationDirEntryData): %zu "
			"+ PersistentRelation_SharedDataSize(): %zu",
			size,
			gp_max_relations,
			sizeof(RelationDirEntryData),
			PersistentRelation_SharedDataSize());

	return size;
}
Ejemplo n.º 13
0
/*
 * CreateSharedMemoryAndSemaphores
 *		Creates and initializes shared memory and semaphores.
 *
 * This is called by the postmaster or by a standalone backend.
 * It is also called by a backend forked from the postmaster in the
 * EXEC_BACKEND case.  In the latter case, the shared memory segment
 * already exists and has been physically attached to, but we have to
 * initialize pointers in local memory that reference the shared structures,
 * because we didn't inherit the correct pointer values from the postmaster
 * as we do in the fork() scenario.  The easiest way to do that is to run
 * through the same code as before.  (Note that the called routines mostly
 * check IsUnderPostmaster, rather than EXEC_BACKEND, to detect this case.
 * This is a bit code-wasteful and could be cleaned up.)
 *
 * If "makePrivate" is true then we only need private memory, not shared
 * memory.	This is true for a standalone backend, false for a postmaster.
 */
void
CreateSharedMemoryAndSemaphores(bool makePrivate, int port)
{
	if (!IsUnderPostmaster)
	{
		PGShmemHeader *seghdr;
		Size		size;
		int			numSemas;

		/*
		 * Size of the Postgres shared-memory block is estimated via
		 * moderately-accurate estimates for the big hogs, plus 100K for the
		 * stuff that's too small to bother with estimating.
		 *
		 * We take some care during this phase to ensure that the total size
		 * request doesn't overflow size_t.  If this gets through, we don't
		 * need to be so careful during the actual allocation phase.
		 */
		size = 100000;
		size = add_size(size, SpinlockSemaSize());
		size = add_size(size, hash_estimate_size(SHMEM_INDEX_SIZE,
												 sizeof(ShmemIndexEnt)));
		size = add_size(size, BufferShmemSize());
		size = add_size(size, LockShmemSize());
		size = add_size(size, PredicateLockShmemSize());
		size = add_size(size, ProcGlobalShmemSize());
		size = add_size(size, XLOGShmemSize());
		size = add_size(size, CLOGShmemSize());
		size = add_size(size, SUBTRANSShmemSize());
		size = add_size(size, TwoPhaseShmemSize());
		size = add_size(size, BackgroundWorkerShmemSize());
		size = add_size(size, MultiXactShmemSize());
		size = add_size(size, LWLockShmemSize());
		size = add_size(size, ProcArrayShmemSize());
		size = add_size(size, BackendStatusShmemSize());
		size = add_size(size, SInvalShmemSize());
		size = add_size(size, PMSignalShmemSize());
		size = add_size(size, ProcSignalShmemSize());
		size = add_size(size, CheckpointerShmemSize());
		size = add_size(size, AutoVacuumShmemSize());
		size = add_size(size, ReplicationSlotsShmemSize());
		size = add_size(size, WalSndShmemSize());
		size = add_size(size, WalRcvShmemSize());
		size = add_size(size, BTreeShmemSize());
		size = add_size(size, SyncScanShmemSize());
		size = add_size(size, AsyncShmemSize());
#ifdef EXEC_BACKEND
		size = add_size(size, ShmemBackendArraySize());
#endif

		/* freeze the addin request size and include it */
		addin_request_allowed = false;
		size = add_size(size, total_addin_request);

		/* might as well round it off to a multiple of a typical page size */
		size = add_size(size, 8192 - (size % 8192));

		elog(DEBUG3, "invoking IpcMemoryCreate(size=%zu)", size);

		/*
		 * Create the shmem segment
		 */
		seghdr = PGSharedMemoryCreate(size, makePrivate, port);

		InitShmemAccess(seghdr);

		/*
		 * Create semaphores
		 */
		numSemas = ProcGlobalSemas();
		numSemas += SpinlockSemas();
		PGReserveSemaphores(numSemas, port);
	}
	else
	{
		/*
		 * We are reattaching to an existing shared memory segment. This
		 * should only be reached in the EXEC_BACKEND case, and even then only
		 * with makePrivate == false.
		 */
#ifdef EXEC_BACKEND
		Assert(!makePrivate);
#else
		elog(PANIC, "should be attached to shared memory already");
#endif
	}

	/*
	 * Set up shared memory allocation mechanism
	 */
	if (!IsUnderPostmaster)
		InitShmemAllocation();

	/*
	 * Now initialize LWLocks, which do shared memory allocation and are
	 * needed for InitShmemIndex.
	 */
	CreateLWLocks();

	/*
	 * Set up shmem.c index hashtable
	 */
	InitShmemIndex();

	/*
	 * Set up xlog, clog, and buffers
	 */
	XLOGShmemInit();
	CLOGShmemInit();
	SUBTRANSShmemInit();
	MultiXactShmemInit();
	InitBufferPool();

	/*
	 * Set up lock manager
	 */
	InitLocks();

	/*
	 * Set up predicate lock manager
	 */
	InitPredicateLocks();

	/*
	 * Set up process table
	 */
	if (!IsUnderPostmaster)
		InitProcGlobal();
	CreateSharedProcArray();
	CreateSharedBackendStatus();
	TwoPhaseShmemInit();
	BackgroundWorkerShmemInit();

	/*
	 * Set up shared-inval messaging
	 */
	CreateSharedInvalidationState();

	/*
	 * Set up interprocess signaling mechanisms
	 */
	PMSignalShmemInit();
	ProcSignalShmemInit();
	CheckpointerShmemInit();
	AutoVacuumShmemInit();
	ReplicationSlotsShmemInit();
	WalSndShmemInit();
	WalRcvShmemInit();

	/*
	 * Set up other modules that need some shared memory space
	 */
	BTreeShmemInit();
	SyncScanShmemInit();
	AsyncShmemInit();

#ifdef EXEC_BACKEND

	/*
	 * Alloc the win32 shared backend array
	 */
	if (!IsUnderPostmaster)
		ShmemBackendArrayAllocation();
#endif

	/* Initialize dynamic shared memory facilities. */
	if (!IsUnderPostmaster)
		dsm_postmaster_startup();

	/*
	 * Now give loadable modules a chance to set up their shmem allocations
	 */
	if (shmem_startup_hook)
		shmem_startup_hook();
}
Ejemplo n.º 14
0
Archivo: ipci.c Proyecto: LJoNe/gpdb
/*
 * CreateSharedMemoryAndSemaphores
 *		Creates and initializes shared memory and semaphores.
 *
 * This is called by the postmaster or by a standalone backend.
 * It is also called by a backend forked from the postmaster in the
 * EXEC_BACKEND case.  In the latter case, the shared memory segment
 * already exists and has been physically attached to, but we have to
 * initialize pointers in local memory that reference the shared structures,
 * because we didn't inherit the correct pointer values from the postmaster
 * as we do in the fork() scenario.  The easiest way to do that is to run
 * through the same code as before.  (Note that the called routines mostly
 * check IsUnderPostmaster, rather than EXEC_BACKEND, to detect this case.
 * This is a bit code-wasteful and could be cleaned up.)
 *
 * If "makePrivate" is true then we only need private memory, not shared
 * memory.	This is true for a standalone backend, false for a postmaster.
 */
void
CreateSharedMemoryAndSemaphores(bool makePrivate, int port)
{
	if (!IsUnderPostmaster)
	{
		PGShmemHeader *seghdr;
		Size		size;
		int			numSemas;

		/*
		 * Size of the Postgres shared-memory block is estimated via
		 * moderately-accurate estimates for the big hogs, plus 100K for the
		 * stuff that's too small to bother with estimating.
		 *
		 * We take some care during this phase to ensure that the total size
		 * request doesn't overflow size_t.  If this gets through, we don't
		 * need to be so careful during the actual allocation phase.
		 */
		size = 150000;
		size = add_size(size, hash_estimate_size(SHMEM_INDEX_SIZE,
												 sizeof(ShmemIndexEnt)));
		size = add_size(size, BufferShmemSize());
		size = add_size(size, LockShmemSize());
		size = add_size(size, workfile_mgr_shmem_size());
		if (Gp_role == GP_ROLE_DISPATCH)
		{
			size = add_size(size, AppendOnlyWriterShmemSize());
			
			if(ResourceScheduler)
			{
				size = add_size(size, ResSchedulerShmemSize());
				size = add_size(size, ResPortalIncrementShmemSize());				
			}
		}
		size = add_size(size, ProcGlobalShmemSize());
		size = add_size(size, XLOGShmemSize());
		size = add_size(size, DistributedLog_ShmemSize());
		size = add_size(size, CLOGShmemSize());
		size = add_size(size, ChangeTrackingShmemSize());
		size = add_size(size, SUBTRANSShmemSize());
		size = add_size(size, TwoPhaseShmemSize());
		size = add_size(size, MultiXactShmemSize());
		size = add_size(size, LWLockShmemSize());
		size = add_size(size, ProcArrayShmemSize());
		size = add_size(size, BackendStatusShmemSize());
		size = add_size(size, SharedSnapshotShmemSize());

		size = add_size(size, SInvalShmemSize());
		size = add_size(size, PMSignalShmemSize());
		size = add_size(size, ProcSignalShmemSize());
		size = add_size(size, primaryMirrorModeShmemSize());
		size = add_size(size, FreeSpaceShmemSize());
		//size = add_size(size, AutoVacuumShmemSize());
		size = add_size(size, FtsShmemSize());
		size = add_size(size, tmShmemSize());
		size = add_size(size, SeqServerShmemSize());
		size = add_size(size, PersistentFileSysObj_ShmemSize());
		size = add_size(size, PersistentFilespace_ShmemSize());
		size = add_size(size, PersistentTablespace_ShmemSize());
		size = add_size(size, PersistentDatabase_ShmemSize());
		size = add_size(size, PersistentRelation_ShmemSize());
		size = add_size(size, Pass2Recovery_ShmemSize());

		/*Add shared memory for PT verification checks*/
		if (Gp_role == GP_ROLE_DISPATCH && debug_persistent_ptcat_verification)
		{
			size = add_size(size, Persistent_PostDTMRecv_ShmemSize());
		}

		if (GPAreFileReplicationStructuresRequired()) {
			size = add_size(size, FileRep_SubProcShmemSize());
			size = add_size(size, FileRep_ShmemSize());
			size = add_size(size, FileRepAck_ShmemSize());
			size = add_size(size, FileRepAckPrimary_ShmemSize());
			size = add_size(size, FileRepResync_ShmemSize()); 
			size = add_size(size, FileRepIpc_ShmemSize());
			size = add_size(size, FileRepLog_ShmemSize());
			size = add_size(size, FileRepStats_ShmemSize());
		}
		
#ifdef FAULT_INJECTOR
		size = add_size(size, FaultInjector_ShmemSize());
#endif			
		
#ifdef EXEC_BACKEND
		size = add_size(size, ShmemBackendArraySize());
#endif

#ifdef USE_TEST_UTILS
		if (gp_simex_init)
		{
			// initialize SimEx
			simex_init();
			size = add_size(size, SyncBitVector_ShmemSize(simex_get_subclass_count()));
		}
#endif

		/* This elog happens before we know the name of the log file we are supposed to use */
		elog(DEBUG1, "Size not including the buffer pool %lu",
			 (unsigned long) size);

		size = add_size(size, BgWriterShmemSize());
		size = add_size(size, AutoVacuumShmemSize());
		size = add_size(size, BTreeShmemSize());
		size = add_size(size, SyncScanShmemSize());
		size = add_size(size, CheckpointShmemSize());

		size = add_size(size, WalSndShmemSize());
		size = add_size(size, WalRcvShmemSize());

		/* freeze the addin request size and include it */
		addin_request_allowed = false;
		size = add_size(size, total_addin_request);

		/* might as well round it off to a multiple of a typical page size */
		size = add_size(size, BLCKSZ - (size % BLCKSZ));

		/* Consider the size of the SessionState array */
		size = add_size(size, SessionState_ShmemSize());

		/*
		 * Create the shmem segment
		 */
		seghdr = PGSharedMemoryCreate(size, makePrivate, port);

		InitShmemAccess(seghdr);

		/*
		 * Create semaphores
		 */
		numSemas = ProcGlobalSemas();
		numSemas += SpinlockSemas();

		if (GPAreFileReplicationStructuresRequired()) 
		{
			numSemas += FileRepSemas();
		}
		
		elog(DEBUG3,"reserving %d semaphores",numSemas);
		PGReserveSemaphores(numSemas, port);
		
	}
	else
	{
		/*
		 * We are reattaching to an existing shared memory segment. This
		 * should only be reached in the EXEC_BACKEND case, and even then only
		 * with makePrivate == false.
		 */
#ifdef EXEC_BACKEND
		Assert(!makePrivate);
#else
		elog(PANIC, "should be attached to shared memory already");
#endif
	}

	/*
	 * Set up shared memory allocation mechanism
	 */
	if (!IsUnderPostmaster)
		InitShmemAllocation();

	/*
	 * Now initialize LWLocks, which do shared memory allocation and are
	 * needed for InitShmemIndex.
	 */
	if (!IsUnderPostmaster)
		CreateLWLocks();

	/*
	 * Set up shmem.c index hashtable
	 */
	InitShmemIndex();

	primaryMirrorModeShmemInit();

	/*
	 * Set up xlog, clog, and buffers
	 */
	XLOGShmemInit();
	CLOGShmemInit();
	ChangeTrackingShmemInit();
	DistributedLog_ShmemInit();
	SUBTRANSShmemInit();
	TwoPhaseShmemInit();
	MultiXactShmemInit();
    FtsShmemInit();
    tmShmemInit();
	InitBufferPool();

	/*
	 * Set up lock manager
	 */
	InitLocks();

	/*
	 * Set up append only writer
	 */
	if (Gp_role == GP_ROLE_DISPATCH)
		InitAppendOnlyWriter();

	PersistentFileSysObj_ShmemInit();
	PersistentFilespace_ShmemInit();
	PersistentTablespace_ShmemInit();
	PersistentDatabase_ShmemInit();
	PersistentRelation_ShmemInit();
	Pass2Recovery_ShmemInit();

	if (Gp_role == GP_ROLE_DISPATCH && debug_persistent_ptcat_verification)
		Persistent_PostDTMRecv_ShmemInit();

	/*
	 * Set up resource schedular
	 */
	if (Gp_role == GP_ROLE_DISPATCH && ResourceScheduler)
	{
		InitResScheduler();
		InitResPortalIncrementHash();
	}


	if (!IsUnderPostmaster)
	{
		/* Set up process table */
		InitProcGlobal(PostmasterGetMppLocalProcessCounter());
	}

	/* Initialize SessionState shared memory array */
	SessionState_ShmemInit();
	/* Initialize vmem protection */
	GPMemoryProtect_ShmemInit();

	CreateSharedProcArray();
	CreateSharedBackendStatus();
	
	/*
	 * Set up Shared snapshot slots
	 *
	 * TODO: only need to do this if we aren't the QD. for now we are just 
	 *		 doing it all the time and wasting shemem on the QD.  This is 
	 *		 because this happens at postmaster startup time when we don't
	 *		 know who we are.  
	 */
	CreateSharedSnapshotArray();

	/*
	 * Set up shared-inval messaging
	 */
	CreateSharedInvalidationState();

	/*
	 * Set up free-space map
	 */
	InitFreeSpaceMap();

	/*
	 * Set up interprocess signaling mechanisms
	 */
	PMSignalShmemInit();
	ProcSignalShmemInit();
	BgWriterShmemInit();
	CheckpointShmemInit();
	WalSndShmemInit();
	WalRcvShmemInit();
	//AutoVacuumShmemInit();
	SeqServerShmemInit();

	if (GPAreFileReplicationStructuresRequired()) {
	
		FileRep_SubProcShmemInit();
		FileRep_ShmemInit();
		FileRepAck_ShmemInit();
		FileRepAckPrimary_ShmemInit();
		FileRepResync_ShmemInit();
		FileRepIpc_ShmemInit();
		FileRepLog_ShmemInit();
		FileRepStats_ShmemInit();
	}
	
#ifdef FAULT_INJECTOR
	FaultInjector_ShmemInit();
#endif

#ifdef USE_TEST_UTILS
	if (gp_simex_init)
	{
		// initialize shmem segment for SimEx
		simex_set_sync_bitvector_container(
			SyncBitVector_ShmemInit("SimEx bit vector container", simex_get_subclass_count()));
	}
#endif /* USE_TEST_UTILS */

	/*
	 * Set up other modules that need some shared memory space
	 */
	BTreeShmemInit();
	SyncScanShmemInit();
	workfile_mgr_cache_init();

#ifdef EXEC_BACKEND

	/*
	 * Alloc the win32 shared backend array
	 */
	if (!IsUnderPostmaster)
		ShmemBackendArrayAllocation();
#endif

	BackoffStateInit();
	
	if (gp_resqueue_memory_policy != RESQUEUE_MEMORY_POLICY_NONE)
	{
		SPI_InitMemoryReservation();
	}
	
	/*
	 * Now give loadable modules a chance to set up their shmem allocations
	 */
	if (shmem_startup_hook)
		shmem_startup_hook();
}
Ejemplo n.º 15
0
/*
 * Estimate space needed for mapping hashtable
 *		size is the desired hash table size (possibly more than NBuffers)
 */
Size
BufTableShmemSize(int size)
{
	return hash_estimate_size(size, sizeof(BufferLookupEnt));
}
Ejemplo n.º 16
0
static Size ShmemIndexSize()
{
	return (Size)hash_estimate_size(SHMEM_INDEX_SIZE, sizeof(ShmemIndexEnt));
}