NABoolean HHDFSTableStats::validateAndRefresh(Int64 expirationJTimestamp, NABoolean refresh)
{
NABoolean result = TRUE;
// initial heap allocation size
Int32 initialSize = heap_->getAllocSize();
diags_.reset();
// check if the stats needs to be fetched within a specified time interval
// when not requested to refresh
if (! refresh && (expirationJTimestamp == -1 ||
(expirationJTimestamp > 0 &&
validationJTimestamp_ < expirationJTimestamp)))
return result; // consider the stats still valid
// if partitions get added or deleted, that gets
// caught in the Hive metadata, so no need to check for
// that here
for (int p=0; p<totalNumPartitions_ && result && diags_.isSuccess(); p++)
{
HHDFSListPartitionStats *partStats = listPartitionStatsList_[p];
NAString hdfsHost;
Int32 hdfsPort;
NAString partDir;
result = splitLocation(partStats->getDirName(),
hdfsHost,
hdfsPort,
partDir,
diags_,
hdfsPortOverride_);
if (! result)
break;
if (! connectHDFS(hdfsHost, hdfsPort))
return FALSE;
subtract(partStats);
result = partStats->validateAndRefresh(fs_, diags_, refresh);
if (result)
add(partStats);
}
disconnectHDFS();
validationJTimestamp_ = JULIANTIMESTAMP();
// account for the heap used by stats. Heap released during
// stats refresh will also be included
hiveStatsSize_ += (heap_->getAllocSize() - initialSize);
return result;
}
NABoolean HHDFSTableStats::validateAndRefresh(Int64 expirationJTimestamp, NABoolean refresh)
{
NABoolean result = TRUE;
// initial heap allocation size
Int32 initialSize = heap_->getAllocSize();
// check only once within a specified time interval
if (expirationJTimestamp == -1 ||
(expirationJTimestamp > 0 &&
validationJTimestamp_ < expirationJTimestamp))
return result; // consider the stats still valid
// if partitions get added or deleted, that gets
// caught in the Hive metadata, so no need to check for
// that here
for (int p=0; p<totalNumPartitions_ && result; p++)
{
HHDFSListPartitionStats *partStats = listPartitionStatsList_[p];
NAString hdfsHost;
Int32 hdfsPort;
NAString partDir;
splitLocation(partStats->getDirName(), hdfsHost, hdfsPort, partDir);
if (! connectHDFS(hdfsHost, hdfsPort))
CMPASSERT(fs_);
subtract(partStats);
result = partStats->validateAndRefresh(fs_, refresh);
add(partStats);
}
disconnectHDFS();
validationJTimestamp_ = JULIANTIMESTAMP();
// account for the heap used by stats. Heap released during
// stats refresh will also be included
hiveStatsSize_ += (heap_->getAllocSize() - initialSize);
return result;
}
