// [[Rcpp::export]] std::shared_ptr<arrow::Table> Table__from_RecordBatchFileReader( const std::shared_ptr<arrow::ipc::RecordBatchFileReader>& reader) { int num_batches = reader->num_record_batches(); std::vector<std::shared_ptr<arrow::RecordBatch>> batches(num_batches); for (int i = 0; i < num_batches; i++) { STOP_IF_NOT_OK(reader->ReadRecordBatch(i, &batches[i])); } std::shared_ptr<arrow::Table> table; STOP_IF_NOT_OK(arrow::Table::FromRecordBatches(std::move(batches), &table)); return table; }
std::vector< SmartPointer<Batch> > Batch::Open(std::string filename) { Clock t1; printf("Opening file %s\n",filename.c_str()); Archive ar; ar.Open(filename,false); ar.Push("batches"); int num=ar.ReadInt("num"); std::vector< SmartPointer<Batch> > batches(num); for (int i=0;i<num;i++) { ar.Push("batch"); batches[i]=ar.ReadSmartPointer<Batch>(); ar.Pop("batch"); } ar.Pop("batches"); ar.Close(); //printf("done in %d msec\n",t1.msec()); return batches; }
void CPatchRData::RenderBlends(const std::vector<CPatchRData*>& patches, const CShaderDefines& context, ShadowMap* shadow, bool isDummyShader, const CShaderProgramPtr& dummy) { Allocators::Arena<> arena(ARENA_SIZE); typedef std::vector<SBlendBatch, ProxyAllocator<SBlendBatch, Allocators::Arena<> > > BatchesStack; BatchesStack batches((BatchesStack::allocator_type(arena))); CShaderDefines contextBlend = context; contextBlend.Add("BLEND", "1"); PROFILE_START("compute batches"); // Reserve an arbitrary size that's probably big enough in most cases, // to avoid heavy reallocations batches.reserve(256); typedef std::vector<SBlendStackItem, ProxyAllocator<SBlendStackItem, Allocators::Arena<> > > BlendStacks; BlendStacks blendStacks((BlendStacks::allocator_type(arena))); blendStacks.reserve(patches.size()); // Extract all the blend splats from each patch for (size_t i = 0; i < patches.size(); ++i) { CPatchRData* patch = patches[i]; if (!patch->m_BlendSplats.empty()) { blendStacks.push_back(SBlendStackItem(patch->m_VBBlends, patch->m_VBBlendIndices, patch->m_BlendSplats, arena)); // Reverse the splats so the first to be rendered is at the back of the list std::reverse(blendStacks.back().splats.begin(), blendStacks.back().splats.end()); } } // Rearrange the collection of splats to be grouped by texture, preserving // order of splats within each patch: // (This is exactly the same algorithm used in CPatchRData::BuildBlends, // but applied to patch-sized splats rather than to tile-sized splats; // see that function for comments on the algorithm.) while (true) { if (!batches.empty()) { CTerrainTextureEntry* tex = batches.back().m_Texture; for (size_t k = 0; k < blendStacks.size(); ++k) { SBlendStackItem::SplatStack& splats = blendStacks[k].splats; if (!splats.empty() && splats.back().m_Texture == tex) { CVertexBuffer::VBChunk* vertices = blendStacks[k].vertices; CVertexBuffer::VBChunk* indices = blendStacks[k].indices; BatchElements& batch = PooledPairGet(PooledMapGet(batches.back().m_Batches, vertices->m_Owner, arena), indices->m_Owner, arena); batch.first.push_back(splats.back().m_IndexCount); u8* indexBase = indices->m_Owner->GetBindAddress(); batch.second.push_back(indexBase + sizeof(u16)*(indices->m_Index + splats.back().m_IndexStart)); splats.pop_back(); } } } CTerrainTextureEntry* bestTex = NULL; size_t bestStackSize = 0; for (size_t k = 0; k < blendStacks.size(); ++k) { SBlendStackItem::SplatStack& splats = blendStacks[k].splats; if (splats.size() > bestStackSize) { bestStackSize = splats.size(); bestTex = splats.back().m_Texture; } } if (bestStackSize == 0) break; SBlendBatch layer(arena); layer.m_Texture = bestTex; batches.push_back(layer); } PROFILE_END("compute batches"); CVertexBuffer* lastVB = NULL; for (BatchesStack::iterator itt = batches.begin(); itt != batches.end(); ++itt) { if (itt->m_Texture->GetMaterial().GetSamplers().size() == 0) continue; int numPasses = 1; CShaderTechniquePtr techBase; if (!isDummyShader) { techBase = g_Renderer.GetShaderManager().LoadEffect(itt->m_Texture->GetMaterial().GetShaderEffect(), contextBlend, itt->m_Texture->GetMaterial().GetShaderDefines()); numPasses = techBase->GetNumPasses(); } CShaderProgramPtr previousShader; for (int pass = 0; pass < numPasses; ++pass) { if (!isDummyShader) { techBase->BeginPass(pass); TerrainRenderer::PrepareShader(techBase->GetShader(), shadow); glEnable(GL_BLEND); glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA); } const CShaderProgramPtr& shader = isDummyShader ? dummy : techBase->GetShader(pass); if (itt->m_Texture) { CMaterial::SamplersVector samplers = itt->m_Texture->GetMaterial().GetSamplers(); size_t samplersNum = samplers.size(); for (size_t s = 0; s < samplersNum; ++s) { CMaterial::TextureSampler &samp = samplers[s]; shader->BindTexture(samp.Name.c_str(), samp.Sampler); } shader->BindTexture("blendTex", itt->m_Texture->m_TerrainAlpha->second.m_hCompositeAlphaMap); itt->m_Texture->GetMaterial().GetStaticUniforms().BindUniforms(shader); #if !CONFIG2_GLES if (isDummyShader) { pglClientActiveTextureARB(GL_TEXTURE0); glMatrixMode(GL_TEXTURE); glLoadMatrixf(itt->m_Texture->GetTextureMatrix()); glMatrixMode(GL_MODELVIEW); } else #endif { float c = itt->m_Texture->GetTextureMatrix()[0]; float ms = itt->m_Texture->GetTextureMatrix()[8]; shader->Uniform("textureTransform", c, ms, -ms, 0.f); } } else { shader->BindTexture("baseTex", g_Renderer.GetTextureManager().GetErrorTexture()); } for (VertexBufferBatches::iterator itv = itt->m_Batches.begin(); itv != itt->m_Batches.end(); ++itv) { // Rebind the VB only if it changed since the last batch if (itv->first != lastVB || shader != previousShader) { lastVB = itv->first; previousShader = shader; GLsizei stride = sizeof(SBlendVertex); SBlendVertex *base = (SBlendVertex *)itv->first->Bind(); shader->VertexPointer(3, GL_FLOAT, stride, &base->m_Position[0]); shader->ColorPointer(4, GL_UNSIGNED_BYTE, stride, &base->m_DiffuseColor); shader->NormalPointer(GL_FLOAT, stride, &base->m_Normal[0]); shader->TexCoordPointer(GL_TEXTURE0, 3, GL_FLOAT, stride, &base->m_Position[0]); shader->TexCoordPointer(GL_TEXTURE1, 2, GL_FLOAT, stride, &base->m_AlphaUVs[0]); } shader->AssertPointersBound(); for (IndexBufferBatches::iterator it = itv->second.begin(); it != itv->second.end(); ++it) { it->first->Bind(); BatchElements& batch = it->second; if (!g_Renderer.m_SkipSubmit) { for (size_t i = 0; i < batch.first.size(); ++i) glDrawElements(GL_TRIANGLES, batch.first[i], GL_UNSIGNED_SHORT, batch.second[i]); } g_Renderer.m_Stats.m_DrawCalls++; g_Renderer.m_Stats.m_BlendSplats++; g_Renderer.m_Stats.m_TerrainTris += std::accumulate(batch.first.begin(), batch.first.end(), 0) / 3; } } if (!isDummyShader) { glDisable(GL_BLEND); techBase->EndPass(); } } } #if !CONFIG2_GLES if (isDummyShader) { pglClientActiveTextureARB(GL_TEXTURE0); glMatrixMode(GL_TEXTURE); glLoadIdentity(); glMatrixMode(GL_MODELVIEW); } #endif CVertexBuffer::Unbind(); }
bool Model::loadGeometryFile(std::string filename) { // Open file core::FileSystem::Ptr fs = getManager()->getFileSystem(); core::File::Ptr file = fs->open(filename, core::FileAccess::Read); if (!file) { getManager()->getLog()->error("Could not open file \"%s\".", filename.c_str()); return false; } // Read header // TODO: Adapt all this to big-endian GeometryFile::Header header; if (file->read(sizeof(header), &header) != sizeof(header)) { getManager()->getLog()->error("%s: Could not read geometry header.", getName().c_str()); return false; } if (header.tag != GeometryFile::tag || header.version != GeometryFile::version) { getManager()->getLog()->error("%s: Invalid geometry file.", getName().c_str()); return false; } // Read vertex and index data void *vertexdata = malloc(header.vertexdatasize); if (file->read(header.vertexdatasize, vertexdata) != (int)header.vertexdatasize) { getManager()->getLog()->error("%s: Could not read vertex data.", getName().c_str()); free(vertexdata); return false; } void *indexdata = malloc(header.indexdatasize); if (file->read(header.indexdatasize, indexdata) != (int)header.indexdatasize) { getManager()->getLog()->error("%s: Could not read index data.", getName().c_str()); free(vertexdata); free(indexdata); return false; } // Construct vertex/index buffers res::ResourceManager *rmgr = getManager(); vertexbuffer = rmgr->createResource<VertexBuffer>("VertexBuffer"); indexbuffer = rmgr->createResource<IndexBuffer>("IndexBuffer"); if (!vertexbuffer || !indexbuffer) { getManager()->getLog()->error("%s: Could not create buffers.", getName().c_str()); free(vertexdata); free(indexdata); return false; } vertexbuffer->set(header.vertexdatasize, vertexdata, VertexBufferUsage::Static, false); indexbuffer->set(header.indexdatasize, indexdata, false); // Read batch info std::vector<GeometryFile::Batch> batchdata(header.batchcount, GeometryFile::Batch()); for (unsigned int i = 0; i < header.batchcount; i++) { GeometryFile::AttribInfo &attribs = batchdata[i].attribs; GeometryFile::GeometryInfo &geom = batchdata[i].geom; if (file->read(sizeof(attribs), &attribs) != sizeof(attribs) || file->read(sizeof(geom), &geom) != sizeof(geom)) { getManager()->getLog()->error("%s: Could not read batch.", getName().c_str()); return false; } // Joint matrices if (geom.jointcount == 0) continue; int jointsize = sizeof(float) * geom.jointcount * 16; batchdata[i].jointmatrices = std::vector<float>(geom.jointcount * 16); if (file->read(jointsize, &batchdata[i].jointmatrices[0]) != jointsize) { getManager()->getLog()->error("%s: Could not joint matrices.", getName().c_str()); return false; } } // Construct batch info std::vector<Batch> batches(header.batchcount); for (unsigned int i = 0; i < header.batchcount; i++) { batches[i].layout = createVertexLayout(batchdata[i].attribs); if (!batches[i].layout) { getManager()->getLog()->error("%s: Could not create vertex layout.", getName().c_str()); return false; } GeometryFile::AttribInfo &attribs = batchdata[i].attribs; GeometryFile::GeometryInfo &geom = batchdata[i].geom; batches[i].basevertex = geom.basevertex; batches[i].indextype = geom.indextype; batches[i].indexcount = geom.indexcount; batches[i].startindex = geom.indexoffset / geom.indextype; batches[i].vertexoffset = geom.vertexoffset; batches[i].vertexcount = geom.vertexsize / attribs.stride; // Create joints batches[i].joints = std::vector<Joint>(batchdata[i].geom.jointcount); for (unsigned int j = 0; j < batchdata[i].geom.jointcount; j++) { math::Matrix4 &jointmat = batches[i].joints[j].jointmat; memcpy(&jointmat.m[0], &batchdata[i].jointmatrices[j * 16], sizeof(float) * 16); } } this->batches = batches; return true; }
void ComputationGraph::accumulate_model_gradients(int first_node, std::vector<int> y_grad_indices, std::vector<Eigen::VectorXf, Eigen::aligned_allocator<Eigen::VectorXf>> y_grad_values) { std::vector<std::vector<int>> gradients; for (int node_index = first_node; node_index < nodes.size(); node_index++) { gradients.push_back(std::vector<int>(nodes[node_index].output.size(), 0)); } for (int i = 0; i < y_grad_indices.size(); i++) { auto &gradient = gradients[y_grad_indices[i] - first_node]; auto &y_grad_value = y_grad_values[i]; for (int j = 0; j < y_grad_value.size(); j++) { gradient[j] = *((int *)&y_grad_value(j)); } } std::vector<Batch> batches(this->batches); while (batches.size() > 1 && first_node < batches.back().nodes_end) { std::vector<std::vector<int>> transformations_node_indices; for (int transformation_index = 0; transformation_index < transformations.size(); transformation_index++) { transformations_node_indices.push_back({}); } for (int node_index = batches[batches.size() - 2].nodes_end; node_index < batches[batches.size() - 1].nodes_end; node_index++) { auto &node = nodes[node_index]; if (node.transformation != -1) { auto &transformation_node_indices = transformations_node_indices[node.transformation]; transformation_node_indices.push_back(node_index); } } // create feed dict std::vector<std::pair<std::string, tensorflow::Tensor>> feed_dict; for (int transformation_index = 0; transformation_index < transformations.size(); transformation_index++) { auto &transformation = transformations[transformation_index]; auto &transformation_node_indices = transformations_node_indices[transformation_index]; if (transformation_node_indices.size() > 0) { for (int input_index = 0; input_index < transformation.inputs.size(); input_index++) { auto &input_name = transformation.inputs[input_index]; auto &input_shape = transformation.input_shapes[input_index]; auto &input_type = transformation.input_types[input_index]; auto &input_size = transformation.input_sizes[input_index]; tensorflow::TensorShape tensor_shape; tensor_shape.AddDim(transformation_node_indices.size()); for (int dim : input_shape) tensor_shape.AddDim(dim); tensorflow::Tensor tensor(input_type, tensor_shape); for (int i = 0; i < transformation_node_indices.size(); i++) { auto &node_index = transformation_node_indices[i]; auto &node = nodes[node_index]; if (input_type == tensorflow::DataType::DT_FLOAT) { copy_input_into_tensor<float>(i, input_size, node.inputs[input_index], tensor); } else if (input_type == tensorflow::DataType::DT_INT32) { copy_input_into_tensor<int>(i, input_size, node.inputs[input_index], tensor); } else { std::cout << "unsupported type " << input_type << std::endl; exit(123); } } feed_dict.push_back(std::make_pair(input_name, tensor)); } { tensorflow::TensorShape tensor_shape; tensor_shape.AddDim(transformation_node_indices.size()); for (int dim : transformation.output_shape) tensor_shape.AddDim(dim); tensorflow::Tensor tensor(transformation.output_type, tensor_shape); for (int i = 0; i < transformation_node_indices.size(); i++) { auto &node_index = transformation_node_indices[i]; copy_vector_into_tensor<float>(i, transformation.output_size, gradients[node_index - first_node], tensor); } feed_dict.push_back(std::make_pair(transformation.output_gradient, tensor)); } } } for (int transformation_index = 0; transformation_index < transformations.size(); transformation_index++) { auto &transformation = transformations[transformation_index]; auto &input_name = transformation.seed; auto &seed = batches.back().seeds[transformation_index]; tensorflow::Tensor tensor(tensorflow::DataType::DT_INT64, tensorflow::TensorShape({seed.size()})); for (int i = 0; i < seed.size(); i++) { tensor.flat<long long>()(i) = seed[i]; } feed_dict.push_back(std::make_pair(input_name, tensor)); } feed_dict.push_back(std::make_pair(training_name, training_tensor)); // create fetch outputs std::vector<std::string> fetch_outputs; std::vector<std::vector<int>> transformations_input_index_to_output; for (int transformation_index = 0; transformation_index < transformations.size(); transformation_index++) { auto &transformation = transformations[transformation_index]; auto &transformation_node_indices = transformations_node_indices[transformation_index]; transformations_input_index_to_output.push_back({}); if (transformation_node_indices.size() > 0) { for (auto &input_gradient : transformation.input_gradients) { transformations_input_index_to_output.back().push_back(fetch_outputs.size()); if (input_gradient != "") { fetch_outputs.push_back(input_gradient); } } } } // create run_outputs std::vector<std::string> run_outputs; for (int transformation_index = 0; transformation_index < transformations.size(); transformation_index++) { auto &transformation = transformations[transformation_index]; auto &transformation_node_indices = transformations_node_indices[transformation_index]; if (transformation_node_indices.size() > 0) { run_outputs.push_back(transformation.update_model_gradient_accumulators); } } // run std::vector<tensorflow::Tensor> outputs; TF_CHECK_OK(session->Run(feed_dict, fetch_outputs, run_outputs, &outputs)); // std::cout << "outputs " << batches[batches.size() - 2].nodes_end << std::endl; // for (int i = 0; i < outputs.size(); i++) { // std::cout << fetch_outputs[i] << std::endl; // auto output = outputs[i].flat<float>(); // for (int i = 0; i < output.size(); i++) { // std::cout << output(i) << " "; // } // std::cout << std::endl; // } // add gradients for (int transformation_index = 0; transformation_index < transformations.size(); transformation_index++) { auto &transformation = transformations[transformation_index]; auto &transformation_node_indices = transformations_node_indices[transformation_index]; auto &transformation_input_index_to_output = transformations_input_index_to_output[transformation_index]; for (int i = 0; i < transformation_node_indices.size(); i++) { auto node_index = transformation_node_indices[i]; auto &node = nodes[node_index]; for (int input_index = 0; input_index < transformation.inputs.size(); input_index++) { auto &input_gradient = transformation.input_gradients[input_index]; if (input_gradient != "") { auto output_index = transformation_input_index_to_output[input_index]; auto &output = outputs[output_index]; auto &node_input = node.inputs[input_index]; auto output_view = output.matrix<float>(); int output_view_iterator = 0; for (int partial_input_node_index : node_input) { if (partial_input_node_index - first_node >= 0) { auto &gradient = gradients[partial_input_node_index - first_node]; for (auto &x : gradient) { (*((float*)&x)) += output_view(i, output_view_iterator++); } } } } } } } batches.pop_back(); } }