void CDownload::OnDownloaded()
{
ASSERT( m_bComplete == false );
theApp.Message( MSG_NOTICE, IDS_DOWNLOAD_COMPLETED, GetDisplayName() );
m_tCompleted = GetTickCount();
m_bDownloading = false;
StopSearch();
CloseTransfers();
//AppendMetadata();
if ( GetTaskType() == dtaskMergeFile || GetTaskType() == dtaskPreviewRequest )
AbortTask();
// LibraryBuilder.m_bBusy = true; // ?
m_pTask.Copy();
Statistics.Current.Downloads.Files++;
SetModified();
}
void CDownloadWithSources::RemoveOverlappingSources(QWORD nOffset, QWORD nLength)
{
CQuickLock pLock( Transfers.m_pSection );
for ( POSITION posSource = GetIterator() ; posSource ; )
{
CDownloadSource* pSource = GetNext( posSource );
if ( pSource->TouchedRange( nOffset, nLength ) )
{
if ( GetTaskType() == dtaskMergeFile )
{
// Merging process can produce corrupted blocks, retry connection after 30 seconds
pSource->m_nFailures = 0;
pSource->Close( 30 );
}
else
{
theApp.Message( MSG_ERROR, IDS_DOWNLOAD_VERIFY_DROP,
(LPCTSTR)CString( inet_ntoa( pSource->m_pAddress ) ), (LPCTSTR)pSource->m_sServer, (LPCTSTR)m_sName, nOffset, nOffset + nLength - 1 );
pSource->Remove( TRUE, FALSE );
}
}
}
}
void TCatboostOptions::SetMetricDefaults(const TLossDescription& lossFunction) {
if (GetTaskType() == ETaskType::CPU) {
auto& evalMetric = MetricOptions->EvalMetric;
if (evalMetric.IsSet()) {
return;
} else {
evalMetric = lossFunction;
}
}
}
const char* CTaskSA::GetTaskName()
{
DEBUG_TRACE("char * CTaskSA::GetTaskName()");
int iTaskType = GetTaskType();
if (iTaskType != NO_TASK_TYPE)
if (TaskNames[iTaskType].szName)
return TaskNames[iTaskType].szName;
else
return "";
return sNoTaskName;
}
void TCatboostOptions::ValidateCtr(const TCtrDescription& ctr, ELossFunction lossFunction, bool isTreeCtrs) const {
if (ctr.TargetBinarization->BorderCount > 1) {
CB_ENSURE(lossFunction == ELossFunction::RMSE || lossFunction == ELossFunction::Quantile ||
lossFunction == ELossFunction::LogLinQuantile || lossFunction == ELossFunction::Poisson ||
lossFunction == ELossFunction::MAPE || lossFunction == ELossFunction::MAE,
"target-border-cnt is not supported for loss function " << lossFunction);
}
CB_ENSURE(ctr.GetPriors().size(), "Provide at least one prior for CTR" << ToString(*this));
const ETaskType taskType = GetTaskType();
const ECtrType ctrType = ctr.Type;
if (taskType == ETaskType::GPU) {
CB_ENSURE(IsSupportedOnGpu(ctrType),
"Ctr type " << ctrType << " is not implemented on GPU yet");
CB_ENSURE(ctr.TargetBinarization.IsDefault(), "Error: GPU doesn't not support target binarization per CTR description currently. Please use target_borders option instead");
} else {
CB_ENSURE(taskType == ETaskType::CPU);
CB_ENSURE(IsSupportedOnCpu(ctrType),
"Ctr type " << ctrType << " is not implemented on CPU yet");
CB_ENSURE(ctr.PriorEstimation == EPriorEstimation::No, "Error: CPU doesn't not support prior estimation currently");
}
const EBorderSelectionType borderSelectionType = ctr.CtrBinarization->BorderSelectionType;
if (taskType == ETaskType::CPU) {
CB_ENSURE(borderSelectionType == EBorderSelectionType::Uniform,
"Error: custom ctr binarization is not supported on CPU yet");
} else {
CB_ENSURE(taskType == ETaskType::GPU);
if (isTreeCtrs) {
EBorderSelectionType borderType = borderSelectionType;
CB_ENSURE(borderType == EBorderSelectionType::Uniform || borderType == EBorderSelectionType::Median,
"Error: GPU supports Median and Uniform combinations-ctr binarization only");
CB_ENSURE(ctr.CtrBinarization->BorderCount <= GetMaxTreeCtrBinarizationForGpu(), "Error: max combinations-ctr binarization for GPU is " << GetMaxTreeCtrBinarizationForGpu());
CB_ENSURE(ctr.PriorEstimation == EPriorEstimation::No, "Error: prior estimation is not available for combinations-ctr");
} else {
switch (ctrType) {
case ECtrType::Borders: {
break;
}
default: {
CB_ENSURE(ctr.PriorEstimation == EPriorEstimation::No, "Error: prior estimation is not available for ctr type " << ctrType);
}
}
}
}
if ((ctrType == ECtrType::FeatureFreq) && borderSelectionType == EBorderSelectionType::Uniform) {
MATRIXNET_WARNING_LOG << "Uniform ctr binarization for featureFreq ctr is not good choice. Use MinEntropy for simpleCtrs and Median for combinations-ctrs instead" << Endl;
}
}
void TCatboostOptions::Validate() const {
SystemOptions.Get().Validate();
BoostingOptions.Get().Validate();
ObliviousTreeOptions.Get().Validate();
ELossFunction lossFunction = LossFunctionDescription->GetLossFunction();
{
const ui32 classesCount = DataProcessingOptions->ClassesCount;
if (classesCount != 0 ) {
CB_ENSURE(IsMultiClassError(lossFunction), "classes_count parameter takes effect only with MultiClass/MultiClassOneVsAll loss functions");
CB_ENSURE(classesCount > 1, "classes-count should be at least 2");
}
const auto& classWeights = DataProcessingOptions->ClassWeights.Get();
if (!classWeights.empty()) {
CB_ENSURE(lossFunction == ELossFunction::Logloss || IsMultiClassError(lossFunction),
"class weights takes effect only with Logloss, MultiClass and MultiClassOneVsAll loss functions");
CB_ENSURE(IsMultiClassError(lossFunction) || (classWeights.size() == 2),
"if loss-function is Logloss, then class weights should be given for 0 and 1 classes");
CB_ENSURE(classesCount == 0 || classesCount == classWeights.size(), "class weights should be specified for each class in range 0, ... , classes_count - 1");
}
}
ELeavesEstimation leavesEstimation = ObliviousTreeOptions->LeavesEstimationMethod;
if (lossFunction == ELossFunction::Quantile ||
lossFunction == ELossFunction::MAE ||
lossFunction == ELossFunction::LogLinQuantile ||
lossFunction == ELossFunction::MAPE)
{
CB_ENSURE(leavesEstimation != ELeavesEstimation::Newton,
"Newton leave estimation method is not supported for " << lossFunction << " loss function");
CB_ENSURE(ObliviousTreeOptions->LeavesEstimationIterations == 1U,
"gradient_iterations should equals 1 for this mode");
}
if (GetTaskType() == ETaskType::CPU) {
CB_ENSURE(!(IsQuerywiseError(lossFunction) && leavesEstimation == ELeavesEstimation::Newton),
"This leaf estimation method is not supported for querywise error for CPU learning");
CB_ENSURE(!(IsPairwiseError(lossFunction) && leavesEstimation == ELeavesEstimation::Newton),
"This leaf estimation method is not supported for pairwise error");
}
ValidateCtrs(CatFeatureParams->SimpleCtrs, lossFunction, false);
for (const auto& perFeatureCtr : CatFeatureParams->PerFeatureCtrs.Get()) {
ValidateCtrs(perFeatureCtr.second, lossFunction, false);
}
ValidateCtrs(CatFeatureParams->CombinationCtrs, lossFunction, true);
}
TCtrDescription TCatboostOptions::CreateDefaultCounter(EProjectionType projectionType) const {
if (GetTaskType() == ETaskType::CPU) {
return TCtrDescription(ECtrType::Counter, GetDefaultPriors(ECtrType::Counter));
} else {
CB_ENSURE(GetTaskType() == ETaskType::GPU);
EBorderSelectionType borderSelectionType;
switch (projectionType) {
case EProjectionType::TreeCtr: {
borderSelectionType = EBorderSelectionType::Median;
break;
}
case EProjectionType::SimpleCtr: {
borderSelectionType = EBorderSelectionType::MinEntropy;
break;
}
default: {
ythrow TCatboostException() << "Unknown projection type " << projectionType;
}
}
return TCtrDescription(ECtrType::FeatureFreq,
GetDefaultPriors(ECtrType::FeatureFreq),
TBinarizationOptions(borderSelectionType, 15));
}
}
void Config::Set(const std::unordered_map<std::string, std::string>& params) {
// generate seeds by seed.
if (GetInt(params, "seed", &seed)) {
Random rand(seed);
int int_max = std::numeric_limits<short>::max();
data_random_seed = static_cast<int>(rand.NextShort(0, int_max));
bagging_seed = static_cast<int>(rand.NextShort(0, int_max));
drop_seed = static_cast<int>(rand.NextShort(0, int_max));
feature_fraction_seed = static_cast<int>(rand.NextShort(0, int_max));
}
GetTaskType(params, &task);
GetBoostingType(params, &boosting);
GetMetricType(params, &metric);
GetObjectiveType(params, &objective);
GetDeviceType(params, &device_type);
GetTreeLearnerType(params, &tree_learner);
GetMembersFromString(params);
if (valid_data_initscores.size() == 0 && valid.size() > 0) {
valid_data_initscores = std::vector<std::string>(valid.size(), "");
}
CHECK(valid.size() == valid_data_initscores.size());
// check for conflicts
CheckParamConflict();
if (verbosity == 1) {
LightGBM::Log::ResetLogLevel(LightGBM::LogLevel::Info);
} else if (verbosity == 0) {
LightGBM::Log::ResetLogLevel(LightGBM::LogLevel::Warning);
} else if (verbosity >= 2) {
LightGBM::Log::ResetLogLevel(LightGBM::LogLevel::Debug);
} else {
LightGBM::Log::ResetLogLevel(LightGBM::LogLevel::Fatal);
}
}
BOOL CTaskPropMgr::ReadTaskFromData(CTaskLineProp* poTaskLineProp, const CHAR* pszData, const UINT32 dwDataLen)
{
UINT32 dwLastTaskIdx = 0;
SGDP::CSDXMLFile xmlFile;
if(FALSE == xmlFile.LoadData(pszData, dwDataLen))
{
// XML文件解析失败
return FALSE;
}
CSDXMLNode rootNode = xmlFile.GetRootNode();
if(rootNode == NULL)
{
return FALSE;
}
CSDXMLNode xmlNode = rootNode["Tasks"];
if(xmlNode == NULL)
{
return FALSE;
}
CHAR* pszNodeName = "Task";
CSDXMLNode xmlNodePropTask = xmlNode[pszNodeName];
while(xmlNodePropTask != NULL)
{
STaskProp* pstTaskProp = SDNew STaskProp;
pstTaskProp->byTaskLineIdx = poTaskLineProp->byTaskLineIdx;
pstTaskProp->dwTaskIdx = xmlNodePropTask("Idx").AsInteger();
//idx必须按大小顺序，但不一定连续
pstTaskProp->dwTaskIdx = xmlNodePropTask("Idx").AsInteger();
if((pstTaskProp->dwTaskIdx <= dwLastTaskIdx) || (poTaskLineProp->mapTaskProp.end() != poTaskLineProp->mapTaskProp.find(pstTaskProp->dwTaskIdx)))
{
SYS_CRITICAL(_SDT("[%s: %d]: ReadTaskAcceptCondition failed, Idx invalid! LineIdx[%d], TaskIdx[%d]!"), MSG_MARK, pstTaskProp->byTaskLineIdx, pstTaskProp->dwTaskIdx);
SDDelete pstTaskProp;
pstTaskProp = NULL;
return FALSE;
}
dwLastTaskIdx = pstTaskProp->dwTaskIdx;
if(0 == poTaskLineProp->dwFirstTaskIdx)
{
poTaskLineProp->dwFirstTaskIdx = dwLastTaskIdx;
}
string strTmp = xmlNodePropTask("Type").AsString();
pstTaskProp->eTaskType = GetTaskType(strTmp);
if(INVALID == pstTaskProp->eTaskType)
{
SYS_CRITICAL(_SDT("[%s: %d]: task type is invalid! LineIdx[%d], TaskIdx[%d]!"), MSG_MARK, pstTaskProp->byTaskLineIdx, pstTaskProp->dwTaskIdx);
SDDelete pstTaskProp;
pstTaskProp = NULL;
return FALSE;
}
//需下发utf8编码，配置为utf8
pstTaskProp->strDispName = xmlNodePropTask("DispName").AsString();
pstTaskProp->strDesc = xmlNodePropTask("Desc").AsString();
CSDXMLNode xmlNodeTmp = xmlNodePropTask["AcceptCondition"];
if(!ReadTaskAcceptCondition(xmlNodeTmp, pstTaskProp))
{
SYS_CRITICAL(_SDT("[%s: %d]: ReadTaskAcceptCondition failed! LineIdx[%d], TaskIdx[%d]!"), MSG_MARK, pstTaskProp->byTaskLineIdx, pstTaskProp->dwTaskIdx);
SDDelete pstTaskProp;
pstTaskProp = NULL;
return FALSE;
}
xmlNodeTmp = xmlNodePropTask["FinishCondition"];
if(!ReadTaskFinishCondition(xmlNodeTmp, pstTaskProp))
{
SYS_CRITICAL(_SDT("[%s: %d]: ReadTaskFinishCondition failed! LineIdx[%d], TaskIdx[%d]!"), MSG_MARK, pstTaskProp->byTaskLineIdx, pstTaskProp->dwTaskIdx);
SDDelete pstTaskProp;
pstTaskProp = NULL;
return FALSE;
}
xmlNodeTmp = xmlNodePropTask["Encourage"];
if(!ReadTaskEncourage(xmlNodeTmp, pstTaskProp))
{
SYS_CRITICAL(_SDT("[%s: %d]: ReadTaskEncourage failed! LineIdx[%d], TaskIdx[%d]!"), MSG_MARK, pstTaskProp->byTaskLineIdx, pstTaskProp->dwTaskIdx);
SDDelete pstTaskProp;
pstTaskProp = NULL;
return FALSE;
}
//只有主线才有解锁副本配置
if(MAIN_LINE_TASK == poTaskLineProp->byTaskLineIdx)
{
xmlNodeTmp = xmlNodePropTask["UnlockInstance"];
if(!ReadTaskUnlockInstance(xmlNodeTmp, pstTaskProp))
{
SYS_CRITICAL(_SDT("[%s: %d]: ReadTaskUnlockBuild failed! LineIdx[%d], TaskIdx[%d]!"), MSG_MARK, pstTaskProp->byTaskLineIdx, pstTaskProp->dwTaskIdx);
SDDelete pstTaskProp;
pstTaskProp = NULL;
return FALSE;
}
}
poTaskLineProp->mapTaskProp[pstTaskProp->dwTaskIdx] = pstTaskProp;
xmlNodePropTask = xmlNodePropTask.Sibling(pszNodeName);
}
return TRUE;
}
CString CDownload::GetDownloadStatus() const
{
CString strText;
if ( m_bClearing ) // Briefly marked for removal
{
LoadString( strText, IDS_STATUS_CLEARING );
}
else if ( IsPaused() )
{
if ( GetFileError() == ERROR_SUCCESS || IsSeeding() )
LoadString( strText, IDS_STATUS_PAUSED );
else
LoadString( strText, IsMoving() ? IDS_STATUS_CANTMOVE : IDS_STATUS_FILEERROR );
}
else if ( IsCompleted() )
{
if ( IsSeeding() )
LoadString( strText, m_bTorrentTrackerError ? IDS_STATUS_TRACKERDOWN : IDS_STATUS_SEEDING );
else
LoadString( strText, IDS_STATUS_COMPLETED );
}
else if ( IsMoving() )
{
LoadString( strText, IDS_STATUS_MOVING );
}
else if ( IsStarted() && GetProgress() == 100.0f )
{
LoadString( strText, IDS_STATUS_VERIFYING );
}
else if ( ! IsTrying() )
{
LoadString( strText, IDS_STATUS_QUEUED );
}
else if ( IsDownloading() )
{
const DWORD nTime = GetTimeRemaining();
if ( nTime == 0xFFFFFFFF )
LoadString( strText, IDS_STATUS_ACTIVE ); // IDS_STATUS_DOWNLOADING
else if ( nTime == 0 )
LoadString( strText, IDS_STATUS_DOWNLOADING );
else if ( nTime > 86400 )
strText.Format( L"%u:%.2u:%.2u:%.2u", nTime / 86400, ( nTime / 3600 ) % 24, ( nTime / 60 ) % 60, nTime % 60 );
else
strText.Format( L"%u:%.2u:%.2u", nTime / 3600, ( nTime / 60 ) % 60, nTime % 60 );
}
else if ( GetEffectiveSourceCount() > 0 )
{
LoadString( strText, IDS_STATUS_PENDING );
}
else if ( IsTorrent() )
{
if ( GetTaskType() == dtaskAllocate )
LoadString( strText, IDS_STATUS_CREATING );
else if ( m_bTorrentTrackerError )
LoadString( strText, IDS_STATUS_TRACKERDOWN );
else
LoadString( strText, IDS_STATUS_TORRENT );
}
else // Inactive
{
LoadString( strText, IDS_STATUS_QUEUED );
}
return strText;
}