// Regression void RFTrainer::train(RFClassifier& model, RegressionDataset const& dataset) { model.clearModels(); // added by TG 23.02.2015 //TODO O.K.: i am just fixing these things for now so that they are working. //Store the number of input dimensions m_inputDimension = inputDimension(dataset); //Store the size of the labels m_labelDimension = labelDimension(dataset); model.setInputDimension(m_inputDimension); model.setLabelDimension(m_labelDimension); m_regressionLearner = true; setDefaults(); //we need direct element access sicne we need to generate elementwise subsets std::size_t subsetSize = static_cast<std::size_t>(dataset.numberOfElements()*m_OOBratio); DataView<RegressionDataset const> elements(dataset); //Generate m_B trees SHARK_PARALLEL_FOR(int i = 0; i < (int)m_B; ++i){ //For each tree generate a subset of the dataset //generate indices of the dataset (pick k out of n elements) std::vector<std::size_t> subsetIndices(dataset.numberOfElements()); boost::iota(subsetIndices,0); boost::random_shuffle(subsetIndices); // create oob indices std::vector<std::size_t>::iterator oobStart = subsetIndices.begin() + subsetSize; std::vector<std::size_t>::iterator oobEnd = subsetIndices.end(); //generate the dataset by copying (TODO: this is a quick fix! subsetIndices.erase(oobStart, oobEnd); RegressionDataset dataTrain = toDataset(subset(elements,subsetIndices)); AttributeTables tables; createAttributeTables(dataTrain.inputs(), tables); std::size_t dataTrainSize = dataTrain.numberOfElements(); std::vector<RealVector> labels; for(std::size_t i = 0; i < dataTrainSize; i++){ labels.push_back(dataTrain.element(i).label); } CARTClassifier<RealVector>::TreeType tree = buildTree(tables, dataTrain, labels, 0); CARTClassifier<RealVector> cart(tree, m_inputDimension); // if oob error or importances have to be computed, create an oob sample if(m_computeOOBerror || m_computeFeatureImportances){ std::vector<std::size_t> subsetIndicesOOB(oobStart, oobEnd); RegressionDataset dataOOB = toDataset(subset(elements, subsetIndicesOOB)); // if importances should be computed, oob errors are computed implicitly if(m_computeFeatureImportances){ cart.computeFeatureImportances(dataOOB); } // if importances should not be computed, only compute the oob errors else{ cart.computeOOBerror(dataOOB); } } SHARK_CRITICAL_REGION{ model.addModel(cart); } } if(m_computeOOBerror){ model.computeOOBerror(); } if(m_computeFeatureImportances){ model.computeFeatureImportances(); } }
//Build CART tree in the regression case CARTTrainer::TreeType CARTTrainer::buildTree(AttributeTables const& tables, RegressionDataset const& dataset, std::vector<RealVector> const& labels, std::size_t nodeId, std::size_t trainSize){ size_t nextId = 0; std::queue<BuildData> bd; bd.emplace(tables, labels, nextId++); TreeType tree; while(!bd.empty()) { BuildData current(std::move(bd.front())); bd.pop(); //Construct tree CARTClassifier<RealVector>::NodeInfo nodeInfo; nodeInfo.nodeId = current.nodeId; nodeInfo.label = mean(current.labels); nodeInfo.leftNodeId = 0; nodeInfo.rightNodeId = 0; //Store the Total Sum of Squares (TSS) RealVector labelSum = current.labels[0]; for(std::size_t i=1; i< current.labels.size(); i++){ labelSum += current.labels[0]; } nodeInfo.misclassProp = totalSumOfSquares(current.labels, 0, current.labels.size(), labelSum)*((double)dataset.numberOfElements()/trainSize); //n = Total number of cases in the dataset //n1 = Number of cases to the left child node //n2 = number of cases to the right child node std::size_t n, n1, n2; n = current.tables[0].size(); size_t splitcount = n/m_numSplits; splitcount = splitcount ? splitcount : 1; // Make sure splitcount is never 0 std::cout << labels.size() << " " << splitcount << " " << m_nodeSize << std::endl; if(n > m_nodeSize){ //label vectors std::vector<RealVector> bestLabels, tmpLabels; RealVector labelSumAbove(m_labelDimension), labelSumBelow(m_labelDimension); //Index of attributes std::size_t bestAttributeIndex = 0; std::size_t bestAttributeValIndex = m_nodeSize; //Attribute values double bestAttributeVal = current.tables[bestAttributeIndex][bestAttributeValIndex-1].value; double impurity, fullImpurity, bestImpurity = -1; bool doSplit = false; for (size_t attributeIndex = 0; attributeIndex< m_inputDimension; attributeIndex++){ labelSumBelow.clear(); labelSumAbove.clear(); tmpLabels.clear(); //Create a labels table, that corresponds to the sorted attribute for(std::size_t k=0; k<current.tables[attributeIndex].size(); k++){ tmpLabels.push_back(dataset.element(current.tables[attributeIndex][k].id).label); noalias(labelSumBelow) += dataset.element(current.tables[attributeIndex][k].id).label; } for(std::size_t i=splitcount; i<n; i += splitcount){ // cerr << "Trying split at att: " << attributeIndex << " and point: " << i << endl; for(std::size_t j = i-splitcount; j < i; j++) { noalias(labelSumAbove) += tmpLabels[j]; noalias(labelSumBelow) -= tmpLabels[j]; } if(current.tables[attributeIndex][i-splitcount].value!=current.tables[attributeIndex][i].value){ n1=i; n2 = n-n1; //Calculate the squared error of the split fullImpurity = totalSumOfSquares(tmpLabels,0,n,labelSumBelow + labelSumAbove); impurity = (n1*totalSumOfSquares(tmpLabels,0,n1,labelSumAbove)+n2*totalSumOfSquares(tmpLabels,n1,n2,labelSumBelow))/(double)(n); double improvement = (fullImpurity - impurity) / fullImpurity; if(improvement*100 >= m_splitImpurityGain && (impurity<bestImpurity || bestImpurity<0)){ //Found a more pure split, store the attribute index and value doSplit = true; bestImpurity = impurity; bestAttributeIndex = attributeIndex; bestAttributeValIndex = i; bestAttributeVal = current.tables[attributeIndex][bestAttributeValIndex-1].value; bestLabels = tmpLabels; } } } } if(doSplit){ BuildData leftNode; BuildData rightNode; //Split the attribute tables splitAttributeTables(current.tables, bestAttributeIndex, bestAttributeValIndex-1, leftNode.tables, rightNode.tables); //Split the labels for(std::size_t i = 0; i < bestAttributeValIndex; i++){ leftNode.labels.push_back(bestLabels[i]); } for(std::size_t i = bestAttributeValIndex; i < bestLabels.size(); i++){ rightNode.labels.push_back(bestLabels[i]); } //Continue recursively nodeInfo.attributeIndex = bestAttributeIndex; nodeInfo.attributeValue = bestAttributeVal; nodeInfo.leftNodeId = nextId++; nodeInfo.rightNodeId = nextId++; leftNode.nodeId = nodeInfo.leftNodeId; rightNode.nodeId = nodeInfo.rightNodeId; bd.push(std::move(leftNode)); bd.push(std::move(rightNode)); } } tree.push_back(nodeInfo); } std::cerr << "Tree size: " << tree.size() << std::endl; cerr << "Will return\n"; return tree; }
CARTClassifier<RealVector>::TreeType RFTrainer::buildTree(AttributeTables& tables, RegressionDataset const& dataset, std::vector<RealVector> const& labels, std::size_t nodeId ){ //Construct tree CARTClassifier<RealVector>::NodeInfo nodeInfo; nodeInfo.nodeId = nodeId; nodeInfo.attributeIndex = 0; nodeInfo.attributeValue = 0.0; nodeInfo.leftNodeId = 0; nodeInfo.rightNodeId = 0; nodeInfo.label = average(labels); nodeInfo.misclassProp = 0.0; nodeInfo.r = 0; nodeInfo.g = 0.0; CARTClassifier<RealVector>::TreeType tree, lTree, rTree; //n = Total number of cases in the dataset std::size_t n = tables[0].size(); bool isLeaf = false; if(n <= m_nodeSize){ isLeaf = true; }else{ //label vectors std::vector<RealVector> bestLabels, tmpLabels; RealVector labelSumAbove(m_labelDimension), labelSumBelow(m_labelDimension); //Randomly select the attributes to test for split set<std::size_t> tableIndicies; generateRandomTableIndicies(tableIndicies); //Index of attributes std::size_t attributeIndex, bestAttributeIndex, bestAttributeValIndex; //Attribute values double bestAttributeVal; double bestImpurity = -1; std::size_t prev; bool doSplit = false; for (set<std::size_t>::iterator it=tableIndicies.begin() ; it != tableIndicies.end(); it++ ){ attributeIndex = *it; labelSumBelow.clear(); labelSumAbove.clear(); tmpLabels.clear(); //Create a labels table, that corresponds to the sorted attribute for(std::size_t k=0; k<tables[attributeIndex].size(); k++){ tmpLabels.push_back(dataset.element(tables[attributeIndex][k].id).label); labelSumBelow += dataset.element(tables[attributeIndex][k].id).label; } labelSumAbove += tmpLabels[0]; labelSumBelow -= tmpLabels[0]; for(std::size_t i=1; i<n; i++){ prev = i-1; if(tables[attributeIndex][prev].value!=tables[attributeIndex][i].value){ std::size_t n1=i; std::size_t n2 = n-n1; //Calculate the squared error of the split double impurity = (n1*totalSumOfSquares(tmpLabels,0,n1,labelSumAbove)+n2*totalSumOfSquares(tmpLabels,n1,n2,labelSumBelow))/(double)(n); if(impurity<bestImpurity || bestImpurity<0){ //Found a more pure split, store the attribute index and value doSplit = true; bestImpurity = impurity; bestAttributeIndex = attributeIndex; bestAttributeValIndex = prev; bestAttributeVal = tables[attributeIndex][bestAttributeValIndex].value; bestLabels = tmpLabels; } } labelSumAbove += tmpLabels[i]; labelSumBelow -= tmpLabels[i]; } } if(doSplit){ //Split the attribute tables AttributeTables rTables, lTables; splitAttributeTables(tables, bestAttributeIndex, bestAttributeValIndex, lTables, rTables); tables.clear();//save memory //Split the labels std::vector<RealVector> lLabels, rLabels; for(std::size_t i = 0; i <= bestAttributeValIndex; i++){ lLabels.push_back(bestLabels[i]); } for(std::size_t i = bestAttributeValIndex+1; i < bestLabels.size(); i++){ rLabels.push_back(bestLabels[i]); } //Continue recursively nodeInfo.attributeIndex = bestAttributeIndex; nodeInfo.attributeValue = bestAttributeVal; nodeInfo.leftNodeId = 2*nodeId+1; nodeInfo.rightNodeId = 2*nodeId+2; lTree = buildTree(lTables, dataset, lLabels, nodeInfo.leftNodeId); rTree = buildTree(rTables, dataset, rLabels, nodeInfo.rightNodeId); }else{ //Leaf node isLeaf = true; } } if(isLeaf){ tree.push_back(nodeInfo); return tree; } tree.push_back(nodeInfo); tree.insert(tree.end(), lTree.begin(), lTree.end()); tree.insert(tree.end(), rTree.begin(), rTree.end()); //Store entry in the tree return tree; }
//Build CART tree in the regression case CARTTrainer::TreeType CARTTrainer::buildTree(AttributeTables const& tables, RegressionDataset const& dataset, std::vector<RealVector> const& labels, std::size_t nodeId, std::size_t trainSize){ //Construct tree CARTClassifier<RealVector>::NodeInfo nodeInfo; nodeInfo.nodeId = nodeId; nodeInfo.label = mean(labels); nodeInfo.leftNodeId = 0; nodeInfo.rightNodeId = 0; //Store the Total Sum of Squares (TSS) RealVector labelSum = labels[0]; for(std::size_t i=1; i< labels.size(); i++){ labelSum += labels[0]; } nodeInfo.misclassProp = totalSumOfSquares(labels, 0, labels.size(), labelSum)*((double)dataset.numberOfElements()/trainSize); TreeType tree, lTree, rTree; //n = Total number of cases in the dataset //n1 = Number of cases to the left child node //n2 = number of cases to the right child node std::size_t n, n1, n2; n = tables[0].size(); if(n > m_nodeSize){ //label vectors std::vector<RealVector> bestLabels, tmpLabels; RealVector labelSumAbove(m_labelDimension), labelSumBelow(m_labelDimension); //Index of attributes std::size_t attributeIndex, bestAttributeIndex, bestAttributeValIndex; //Attribute values double bestAttributeVal; double impurity, bestImpurity = -1; std::size_t prev; bool doSplit = false; for ( attributeIndex = 0; attributeIndex< m_inputDimension; attributeIndex++){ labelSumBelow.clear(); labelSumAbove.clear(); tmpLabels.clear(); //Create a labels table, that corresponds to the sorted attribute for(std::size_t k=0; k<tables[attributeIndex].size(); k++){ tmpLabels.push_back(dataset.element(tables[attributeIndex][k].id).label); labelSumBelow += dataset.element(tables[attributeIndex][k].id).label; } labelSumAbove += tmpLabels[0]; labelSumBelow -= tmpLabels[0]; for(std::size_t i=1; i<n; i++){ prev = i-1; if(tables[attributeIndex][prev].value!=tables[attributeIndex][i].value){ n1=i; n2 = n-n1; //Calculate the squared error of the split impurity = (n1*totalSumOfSquares(tmpLabels,0,n1,labelSumAbove)+n2*totalSumOfSquares(tmpLabels,n1,n2,labelSumBelow))/(double)(n); if(impurity<bestImpurity || bestImpurity<0){ //Found a more pure split, store the attribute index and value doSplit = true; bestImpurity = impurity; bestAttributeIndex = attributeIndex; bestAttributeValIndex = prev; bestAttributeVal = tables[attributeIndex][bestAttributeValIndex].value; bestLabels = tmpLabels; } } labelSumAbove += tmpLabels[i]; labelSumBelow -= tmpLabels[i]; } } if(doSplit){ //Split the attribute tables AttributeTables rTables, lTables; splitAttributeTables(tables, bestAttributeIndex, bestAttributeValIndex, lTables, rTables); //Split the labels std::vector<RealVector> lLabels, rLabels; for(std::size_t i = 0; i <= bestAttributeValIndex; i++){ lLabels.push_back(bestLabels[i]); } for(std::size_t i = bestAttributeValIndex+1; i < bestLabels.size(); i++){ rLabels.push_back(bestLabels[i]); } //Continue recursively nodeInfo.attributeIndex = bestAttributeIndex; nodeInfo.attributeValue = bestAttributeVal; nodeInfo.leftNodeId = nodeId+1; lTree = buildTree(lTables, dataset, lLabels, nodeInfo.leftNodeId, trainSize); nodeInfo.rightNodeId = nodeInfo.leftNodeId + lTree.size(); rTree = buildTree(rTables, dataset, rLabels, nodeInfo.rightNodeId, trainSize); } } tree.push_back(nodeInfo); tree.insert(tree.end(), lTree.begin(), lTree.end()); tree.insert(tree.end(), rTree.begin(), rTree.end()); //Store entry in the tree return tree; }