void CDownload::OnDownloaded() { ASSERT( m_bComplete == false ); theApp.Message( MSG_NOTICE, IDS_DOWNLOAD_COMPLETED, GetDisplayName() ); m_tCompleted = GetTickCount(); m_bDownloading = false; StopSearch(); CloseTransfers(); //AppendMetadata(); if ( GetTaskType() == dtaskMergeFile || GetTaskType() == dtaskPreviewRequest ) AbortTask(); // LibraryBuilder.m_bBusy = true; // ? m_pTask.Copy(); Statistics.Current.Downloads.Files++; SetModified(); }
void CDownloadWithSources::RemoveOverlappingSources(QWORD nOffset, QWORD nLength) { CQuickLock pLock( Transfers.m_pSection ); for ( POSITION posSource = GetIterator() ; posSource ; ) { CDownloadSource* pSource = GetNext( posSource ); if ( pSource->TouchedRange( nOffset, nLength ) ) { if ( GetTaskType() == dtaskMergeFile ) { // Merging process can produce corrupted blocks, retry connection after 30 seconds pSource->m_nFailures = 0; pSource->Close( 30 ); } else { theApp.Message( MSG_ERROR, IDS_DOWNLOAD_VERIFY_DROP, (LPCTSTR)CString( inet_ntoa( pSource->m_pAddress ) ), (LPCTSTR)pSource->m_sServer, (LPCTSTR)m_sName, nOffset, nOffset + nLength - 1 ); pSource->Remove( TRUE, FALSE ); } } } }
void TCatboostOptions::SetMetricDefaults(const TLossDescription& lossFunction) { if (GetTaskType() == ETaskType::CPU) { auto& evalMetric = MetricOptions->EvalMetric; if (evalMetric.IsSet()) { return; } else { evalMetric = lossFunction; } } }
const char* CTaskSA::GetTaskName() { DEBUG_TRACE("char * CTaskSA::GetTaskName()"); int iTaskType = GetTaskType(); if (iTaskType != NO_TASK_TYPE) if (TaskNames[iTaskType].szName) return TaskNames[iTaskType].szName; else return ""; return sNoTaskName; }
void TCatboostOptions::ValidateCtr(const TCtrDescription& ctr, ELossFunction lossFunction, bool isTreeCtrs) const { if (ctr.TargetBinarization->BorderCount > 1) { CB_ENSURE(lossFunction == ELossFunction::RMSE || lossFunction == ELossFunction::Quantile || lossFunction == ELossFunction::LogLinQuantile || lossFunction == ELossFunction::Poisson || lossFunction == ELossFunction::MAPE || lossFunction == ELossFunction::MAE, "target-border-cnt is not supported for loss function " << lossFunction); } CB_ENSURE(ctr.GetPriors().size(), "Provide at least one prior for CTR" << ToString(*this)); const ETaskType taskType = GetTaskType(); const ECtrType ctrType = ctr.Type; if (taskType == ETaskType::GPU) { CB_ENSURE(IsSupportedOnGpu(ctrType), "Ctr type " << ctrType << " is not implemented on GPU yet"); CB_ENSURE(ctr.TargetBinarization.IsDefault(), "Error: GPU doesn't not support target binarization per CTR description currently. Please use target_borders option instead"); } else { CB_ENSURE(taskType == ETaskType::CPU); CB_ENSURE(IsSupportedOnCpu(ctrType), "Ctr type " << ctrType << " is not implemented on CPU yet"); CB_ENSURE(ctr.PriorEstimation == EPriorEstimation::No, "Error: CPU doesn't not support prior estimation currently"); } const EBorderSelectionType borderSelectionType = ctr.CtrBinarization->BorderSelectionType; if (taskType == ETaskType::CPU) { CB_ENSURE(borderSelectionType == EBorderSelectionType::Uniform, "Error: custom ctr binarization is not supported on CPU yet"); } else { CB_ENSURE(taskType == ETaskType::GPU); if (isTreeCtrs) { EBorderSelectionType borderType = borderSelectionType; CB_ENSURE(borderType == EBorderSelectionType::Uniform || borderType == EBorderSelectionType::Median, "Error: GPU supports Median and Uniform combinations-ctr binarization only"); CB_ENSURE(ctr.CtrBinarization->BorderCount <= GetMaxTreeCtrBinarizationForGpu(), "Error: max combinations-ctr binarization for GPU is " << GetMaxTreeCtrBinarizationForGpu()); CB_ENSURE(ctr.PriorEstimation == EPriorEstimation::No, "Error: prior estimation is not available for combinations-ctr"); } else { switch (ctrType) { case ECtrType::Borders: { break; } default: { CB_ENSURE(ctr.PriorEstimation == EPriorEstimation::No, "Error: prior estimation is not available for ctr type " << ctrType); } } } } if ((ctrType == ECtrType::FeatureFreq) && borderSelectionType == EBorderSelectionType::Uniform) { MATRIXNET_WARNING_LOG << "Uniform ctr binarization for featureFreq ctr is not good choice. Use MinEntropy for simpleCtrs and Median for combinations-ctrs instead" << Endl; } }
void TCatboostOptions::Validate() const { SystemOptions.Get().Validate(); BoostingOptions.Get().Validate(); ObliviousTreeOptions.Get().Validate(); ELossFunction lossFunction = LossFunctionDescription->GetLossFunction(); { const ui32 classesCount = DataProcessingOptions->ClassesCount; if (classesCount != 0 ) { CB_ENSURE(IsMultiClassError(lossFunction), "classes_count parameter takes effect only with MultiClass/MultiClassOneVsAll loss functions"); CB_ENSURE(classesCount > 1, "classes-count should be at least 2"); } const auto& classWeights = DataProcessingOptions->ClassWeights.Get(); if (!classWeights.empty()) { CB_ENSURE(lossFunction == ELossFunction::Logloss || IsMultiClassError(lossFunction), "class weights takes effect only with Logloss, MultiClass and MultiClassOneVsAll loss functions"); CB_ENSURE(IsMultiClassError(lossFunction) || (classWeights.size() == 2), "if loss-function is Logloss, then class weights should be given for 0 and 1 classes"); CB_ENSURE(classesCount == 0 || classesCount == classWeights.size(), "class weights should be specified for each class in range 0, ... , classes_count - 1"); } } ELeavesEstimation leavesEstimation = ObliviousTreeOptions->LeavesEstimationMethod; if (lossFunction == ELossFunction::Quantile || lossFunction == ELossFunction::MAE || lossFunction == ELossFunction::LogLinQuantile || lossFunction == ELossFunction::MAPE) { CB_ENSURE(leavesEstimation != ELeavesEstimation::Newton, "Newton leave estimation method is not supported for " << lossFunction << " loss function"); CB_ENSURE(ObliviousTreeOptions->LeavesEstimationIterations == 1U, "gradient_iterations should equals 1 for this mode"); } if (GetTaskType() == ETaskType::CPU) { CB_ENSURE(!(IsQuerywiseError(lossFunction) && leavesEstimation == ELeavesEstimation::Newton), "This leaf estimation method is not supported for querywise error for CPU learning"); CB_ENSURE(!(IsPairwiseError(lossFunction) && leavesEstimation == ELeavesEstimation::Newton), "This leaf estimation method is not supported for pairwise error"); } ValidateCtrs(CatFeatureParams->SimpleCtrs, lossFunction, false); for (const auto& perFeatureCtr : CatFeatureParams->PerFeatureCtrs.Get()) { ValidateCtrs(perFeatureCtr.second, lossFunction, false); } ValidateCtrs(CatFeatureParams->CombinationCtrs, lossFunction, true); }
TCtrDescription TCatboostOptions::CreateDefaultCounter(EProjectionType projectionType) const { if (GetTaskType() == ETaskType::CPU) { return TCtrDescription(ECtrType::Counter, GetDefaultPriors(ECtrType::Counter)); } else { CB_ENSURE(GetTaskType() == ETaskType::GPU); EBorderSelectionType borderSelectionType; switch (projectionType) { case EProjectionType::TreeCtr: { borderSelectionType = EBorderSelectionType::Median; break; } case EProjectionType::SimpleCtr: { borderSelectionType = EBorderSelectionType::MinEntropy; break; } default: { ythrow TCatboostException() << "Unknown projection type " << projectionType; } } return TCtrDescription(ECtrType::FeatureFreq, GetDefaultPriors(ECtrType::FeatureFreq), TBinarizationOptions(borderSelectionType, 15)); } }
void Config::Set(const std::unordered_map<std::string, std::string>& params) { // generate seeds by seed. if (GetInt(params, "seed", &seed)) { Random rand(seed); int int_max = std::numeric_limits<short>::max(); data_random_seed = static_cast<int>(rand.NextShort(0, int_max)); bagging_seed = static_cast<int>(rand.NextShort(0, int_max)); drop_seed = static_cast<int>(rand.NextShort(0, int_max)); feature_fraction_seed = static_cast<int>(rand.NextShort(0, int_max)); } GetTaskType(params, &task); GetBoostingType(params, &boosting); GetMetricType(params, &metric); GetObjectiveType(params, &objective); GetDeviceType(params, &device_type); GetTreeLearnerType(params, &tree_learner); GetMembersFromString(params); if (valid_data_initscores.size() == 0 && valid.size() > 0) { valid_data_initscores = std::vector<std::string>(valid.size(), ""); } CHECK(valid.size() == valid_data_initscores.size()); // check for conflicts CheckParamConflict(); if (verbosity == 1) { LightGBM::Log::ResetLogLevel(LightGBM::LogLevel::Info); } else if (verbosity == 0) { LightGBM::Log::ResetLogLevel(LightGBM::LogLevel::Warning); } else if (verbosity >= 2) { LightGBM::Log::ResetLogLevel(LightGBM::LogLevel::Debug); } else { LightGBM::Log::ResetLogLevel(LightGBM::LogLevel::Fatal); } }
BOOL CTaskPropMgr::ReadTaskFromData(CTaskLineProp* poTaskLineProp, const CHAR* pszData, const UINT32 dwDataLen) { UINT32 dwLastTaskIdx = 0; SGDP::CSDXMLFile xmlFile; if(FALSE == xmlFile.LoadData(pszData, dwDataLen)) { // XML文件解析失败 return FALSE; } CSDXMLNode rootNode = xmlFile.GetRootNode(); if(rootNode == NULL) { return FALSE; } CSDXMLNode xmlNode = rootNode["Tasks"]; if(xmlNode == NULL) { return FALSE; } CHAR* pszNodeName = "Task"; CSDXMLNode xmlNodePropTask = xmlNode[pszNodeName]; while(xmlNodePropTask != NULL) { STaskProp* pstTaskProp = SDNew STaskProp; pstTaskProp->byTaskLineIdx = poTaskLineProp->byTaskLineIdx; pstTaskProp->dwTaskIdx = xmlNodePropTask("Idx").AsInteger(); //idx必须按大小顺序,但不一定连续 pstTaskProp->dwTaskIdx = xmlNodePropTask("Idx").AsInteger(); if((pstTaskProp->dwTaskIdx <= dwLastTaskIdx) || (poTaskLineProp->mapTaskProp.end() != poTaskLineProp->mapTaskProp.find(pstTaskProp->dwTaskIdx))) { SYS_CRITICAL(_SDT("[%s: %d]: ReadTaskAcceptCondition failed, Idx invalid! LineIdx[%d], TaskIdx[%d]!"), MSG_MARK, pstTaskProp->byTaskLineIdx, pstTaskProp->dwTaskIdx); SDDelete pstTaskProp; pstTaskProp = NULL; return FALSE; } dwLastTaskIdx = pstTaskProp->dwTaskIdx; if(0 == poTaskLineProp->dwFirstTaskIdx) { poTaskLineProp->dwFirstTaskIdx = dwLastTaskIdx; } string strTmp = xmlNodePropTask("Type").AsString(); pstTaskProp->eTaskType = GetTaskType(strTmp); if(INVALID == pstTaskProp->eTaskType) { SYS_CRITICAL(_SDT("[%s: %d]: task type is invalid! LineIdx[%d], TaskIdx[%d]!"), MSG_MARK, pstTaskProp->byTaskLineIdx, pstTaskProp->dwTaskIdx); SDDelete pstTaskProp; pstTaskProp = NULL; return FALSE; } //需下发utf8编码,配置为utf8 pstTaskProp->strDispName = xmlNodePropTask("DispName").AsString(); pstTaskProp->strDesc = xmlNodePropTask("Desc").AsString(); CSDXMLNode xmlNodeTmp = xmlNodePropTask["AcceptCondition"]; if(!ReadTaskAcceptCondition(xmlNodeTmp, pstTaskProp)) { SYS_CRITICAL(_SDT("[%s: %d]: ReadTaskAcceptCondition failed! LineIdx[%d], TaskIdx[%d]!"), MSG_MARK, pstTaskProp->byTaskLineIdx, pstTaskProp->dwTaskIdx); SDDelete pstTaskProp; pstTaskProp = NULL; return FALSE; } xmlNodeTmp = xmlNodePropTask["FinishCondition"]; if(!ReadTaskFinishCondition(xmlNodeTmp, pstTaskProp)) { SYS_CRITICAL(_SDT("[%s: %d]: ReadTaskFinishCondition failed! LineIdx[%d], TaskIdx[%d]!"), MSG_MARK, pstTaskProp->byTaskLineIdx, pstTaskProp->dwTaskIdx); SDDelete pstTaskProp; pstTaskProp = NULL; return FALSE; } xmlNodeTmp = xmlNodePropTask["Encourage"]; if(!ReadTaskEncourage(xmlNodeTmp, pstTaskProp)) { SYS_CRITICAL(_SDT("[%s: %d]: ReadTaskEncourage failed! LineIdx[%d], TaskIdx[%d]!"), MSG_MARK, pstTaskProp->byTaskLineIdx, pstTaskProp->dwTaskIdx); SDDelete pstTaskProp; pstTaskProp = NULL; return FALSE; } //只有主线才有解锁副本配置 if(MAIN_LINE_TASK == poTaskLineProp->byTaskLineIdx) { xmlNodeTmp = xmlNodePropTask["UnlockInstance"]; if(!ReadTaskUnlockInstance(xmlNodeTmp, pstTaskProp)) { SYS_CRITICAL(_SDT("[%s: %d]: ReadTaskUnlockBuild failed! LineIdx[%d], TaskIdx[%d]!"), MSG_MARK, pstTaskProp->byTaskLineIdx, pstTaskProp->dwTaskIdx); SDDelete pstTaskProp; pstTaskProp = NULL; return FALSE; } } poTaskLineProp->mapTaskProp[pstTaskProp->dwTaskIdx] = pstTaskProp; xmlNodePropTask = xmlNodePropTask.Sibling(pszNodeName); } return TRUE; }
CString CDownload::GetDownloadStatus() const { CString strText; if ( m_bClearing ) // Briefly marked for removal { LoadString( strText, IDS_STATUS_CLEARING ); } else if ( IsPaused() ) { if ( GetFileError() == ERROR_SUCCESS || IsSeeding() ) LoadString( strText, IDS_STATUS_PAUSED ); else LoadString( strText, IsMoving() ? IDS_STATUS_CANTMOVE : IDS_STATUS_FILEERROR ); } else if ( IsCompleted() ) { if ( IsSeeding() ) LoadString( strText, m_bTorrentTrackerError ? IDS_STATUS_TRACKERDOWN : IDS_STATUS_SEEDING ); else LoadString( strText, IDS_STATUS_COMPLETED ); } else if ( IsMoving() ) { LoadString( strText, IDS_STATUS_MOVING ); } else if ( IsStarted() && GetProgress() == 100.0f ) { LoadString( strText, IDS_STATUS_VERIFYING ); } else if ( ! IsTrying() ) { LoadString( strText, IDS_STATUS_QUEUED ); } else if ( IsDownloading() ) { const DWORD nTime = GetTimeRemaining(); if ( nTime == 0xFFFFFFFF ) LoadString( strText, IDS_STATUS_ACTIVE ); // IDS_STATUS_DOWNLOADING else if ( nTime == 0 ) LoadString( strText, IDS_STATUS_DOWNLOADING ); else if ( nTime > 86400 ) strText.Format( L"%u:%.2u:%.2u:%.2u", nTime / 86400, ( nTime / 3600 ) % 24, ( nTime / 60 ) % 60, nTime % 60 ); else strText.Format( L"%u:%.2u:%.2u", nTime / 3600, ( nTime / 60 ) % 60, nTime % 60 ); } else if ( GetEffectiveSourceCount() > 0 ) { LoadString( strText, IDS_STATUS_PENDING ); } else if ( IsTorrent() ) { if ( GetTaskType() == dtaskAllocate ) LoadString( strText, IDS_STATUS_CREATING ); else if ( m_bTorrentTrackerError ) LoadString( strText, IDS_STATUS_TRACKERDOWN ); else LoadString( strText, IDS_STATUS_TORRENT ); } else // Inactive { LoadString( strText, IDS_STATUS_QUEUED ); } return strText; }