/*
* Sweeps through the cache and marks all entries as deleted
*
* Returns the number of elements it found and marked deleted.
*/
int32
Cache_Clear(Cache *cache)
{
Assert(NULL != cache);
int32 startIdx = cdb_randint(cache->cacheHdr->nEntries - 1, 0);
int32 entryIdx = startIdx;
int32 numClearedEntries = 0;
while (true)
{
entryIdx = (entryIdx + 1) % cache->cacheHdr->nEntries;
if (entryIdx == startIdx)
{
/* Completed one loop through the list of all entries. We're done */
break;
}
CacheEntry *crtEntry = Cache_GetEntryByIndex(cache->cacheHdr, entryIdx);
/* Lock entry so that nobody else changes its state until we're done with it */
Cache_LockEntry(cache, crtEntry);
if (crtEntry->state != CACHE_ENTRY_CACHED)
{
/* Not interested in free/acquired/deleted entries. Go back and look at next entry */
Cache_UnlockEntry(cache, crtEntry);
continue;
}
/* Found cached entry */
Cache_EntryAddRef(cache, crtEntry);
if (crtEntry->state == CACHE_ENTRY_FREE || crtEntry->state == CACHE_ENTRY_ACQUIRED)
{
/* Someone freed up the entry before we had a chance to Add-Ref it. Skip it. */
Assert(false);
Cache_EntryDecRef(cache, crtEntry);
Cache_UnlockEntry(cache, crtEntry);
continue;
}
Cache_RegisterCleanup(cache, crtEntry, true /* isCachedEntry */);
Cache_Remove(cache, crtEntry);
/* Done with changing the state. Unlock the entry */
Cache_UnlockEntry(cache, crtEntry);
Cache_Release(cache, crtEntry);
numClearedEntries++;
}
return numClearedEntries;
}
/*
* Create a CdbHash for this session.
*
* CdbHash maintains the following information about the hash.
* In here we set the variables that should not change in the scope of the newly created
* CdbHash, these are:
*
* 1 - number of segments in Greenplum Database.
* 2 - hashingalgorithm used.
* 3 - reduction method.
*
* The hash value itself will be initialized for every tuple in cdbhashinit()
*/
CdbHash *
makeCdbHash(int numsegs, CdbHashAlg algorithm)
{
CdbHash *h;
assert(numsegs > 0); /* verify number of segments is legal. */
assert(algorithm == HASH_FNV_1); /* make sure everybody uses same algorithm */
/* Create a pointer to a CdbHash that includes the hash properties */
h = palloc(sizeof(CdbHash));
/*
* set this hash session characteristics.
*/
h->hash = 0;
h->numsegs = numsegs;
h->hashalg = algorithm;
if (h->hashalg == HASH_FNV_1)
h->hashfn = &fnv1_32_buf;
else if (h->hashalg == HASH_FNV_1A)
h->hashfn = &fnv1a_32_buf;
/*
* set the reduction algorithm: If num_segs is power of 2 use bit mask,
* else use lazy mod (h mod n)
*/
if (ispowof2(numsegs))
{
h->reducealg = REDUCE_BITMASK;
}
else
{
h->reducealg = REDUCE_LAZYMOD;
}
/*
* if we distribute into a relation with an empty partitioning policy,
* we will round robin the tuples starting off from this index. Note that
* the random number is created one per makeCdbHash. This means that commands
* that create a cdbhash object only once for all tuples (like COPY,
* INSERT-INTO-SELECT) behave more like a round-robin distribution, while
* commands that create a cdbhash per row (like INSERT) behave more like a
* random distribution.
*/
h->rrindex = cdb_randint(0, UPPER_VAL);
ereport(DEBUG4,
(errmsg("CDBHASH started using algorithm %d into %d segment databases",
h->hashalg,
h->numsegs)));
return h;
}
/*
* Given total number of primary segment databases and a number of
* segments to "skip" - this routine creates a boolean map (array) the
* size of total number of segments and randomly selects several
* entries (total number of total_to_skip) to be marked as
* "skipped". This is used for external tables with the 'gpfdist'
* protocol where we want to get a number of *random* segdbs to
* connect to a gpfdist client.
*
* Caller of this function should pfree skip_map when done with it.
*/
bool *
makeRandomSegMap(int total_primaries, int total_to_skip)
{
int randint; /* some random int representing a seg */
int skipped = 0; /* num segs already marked to be skipped */
bool *skip_map;
skip_map = (bool *) palloc(total_primaries * sizeof(bool));
MemSet(skip_map, false, total_primaries * sizeof(bool));
while (total_to_skip != skipped)
{
/*
* create a random int between 0 and (total_primaries - 1).
*
* NOTE that the lower and upper limits in cdb_randint() are
* inclusive so we take them into account. In reality the
* chance of those limits to get selected by the random
* generator is extremely small, so we may want to find a
* better random generator some time (not critical though).
*/
randint = cdb_randint(0, total_primaries - 1);
/*
* mark this random index 'true' in the skip map (marked to be
* skipped) unless it was already marked.
*/
if (skip_map[randint] == false)
{
skip_map[randint] = true;
skipped++;
}
}
return skip_map;
}
void
add_filesystem_credential(const char * uri)
{
char *protocol;
char *host;
bool found = false;
struct FileSystemCredentialKey key;
HTAB * currentFilesystemCredentials;
MemoryContext currentFilesystemCredentialsMemoryContext;
get_current_credential_cache_and_memcxt(¤tFilesystemCredentials,
¤tFilesystemCredentialsMemoryContext);
Insist(NULL != currentFilesystemCredentials);
Insist(NULL != currentFilesystemCredentialsMemoryContext);
MemoryContext old = MemoryContextSwitchTo(currentFilesystemCredentialsMemoryContext);
memset(&key, 0, sizeof(key));
if (HdfsParsePath(uri, &protocol, &host, &key.port, NULL)
|| NULL == protocol || NULL == host)
elog(ERROR, "fail to parse uri: %s", uri);
StrNCpy(key.protocol, protocol, sizeof(key.protocol));
StrNCpy(key.host, host, sizeof(key.host));
hash_search(currentFilesystemCredentials, &key, HASH_FIND, &found);
if (!found)
{
int retry = 5;
bool success = false;
/*
* Canceling the query at this point will result in an exception.
* We have to make sure we clean the hash table from the new entry,
* because all credentials are being removed from HDFS when
* we end the transaction, and this entry doesn't have a valid
* credential yet.
*/
PG_TRY();
{
struct FileSystemCredential *entry =
(struct FileSystemCredential *) hash_search(
currentFilesystemCredentials, &key, HASH_ENTER, NULL);
Assert(NULL != entry);
while (true)
{
success = get_filesystem_credential_internal(entry);
if (success)
{
break;
}
if (--retry <= 0)
{
hash_search(currentFilesystemCredentials, &key, HASH_REMOVE, NULL);
break;
}
elog(DEBUG5, "failed to getting credentials for %s://%s:%d, retrying...",
key.protocol, key.host, key.port);
pg_usleep(cdb_randint(0, 5) * 1000000L);
}
}
PG_CATCH();
{
if (!success)
{
hash_search(currentFilesystemCredentials, &key, HASH_REMOVE, NULL);
}
PG_RE_THROW();
}
PG_END_TRY();
if (retry <= 0)
{
elog(ERROR, "fail to get filesystem credential for uri %s", uri);
}
}
MemoryContextSwitchTo(old);
}
