// [[Rcpp::export]]
std::shared_ptr<arrow::Table> Table__from_RecordBatchFileReader(
const std::shared_ptr<arrow::ipc::RecordBatchFileReader>& reader) {
int num_batches = reader->num_record_batches();
std::vector<std::shared_ptr<arrow::RecordBatch>> batches(num_batches);
for (int i = 0; i < num_batches; i++) {
STOP_IF_NOT_OK(reader->ReadRecordBatch(i, &batches[i]));
}
std::shared_ptr<arrow::Table> table;
STOP_IF_NOT_OK(arrow::Table::FromRecordBatches(std::move(batches), &table));
return table;
}
std::vector< SmartPointer<Batch> > Batch::Open(std::string filename)
{
Clock t1;
printf("Opening file %s\n",filename.c_str());
Archive ar;
ar.Open(filename,false);
ar.Push("batches");
int num=ar.ReadInt("num");
std::vector< SmartPointer<Batch> > batches(num);
for (int i=0;i<num;i++)
{
ar.Push("batch");
batches[i]=ar.ReadSmartPointer<Batch>();
ar.Pop("batch");
}
ar.Pop("batches");
ar.Close();
//printf("done in %d msec\n",t1.msec());
return batches;
}
void CPatchRData::RenderBlends(const std::vector<CPatchRData*>& patches, const CShaderDefines& context,
ShadowMap* shadow, bool isDummyShader, const CShaderProgramPtr& dummy)
{
Allocators::Arena<> arena(ARENA_SIZE);
typedef std::vector<SBlendBatch, ProxyAllocator<SBlendBatch, Allocators::Arena<> > > BatchesStack;
BatchesStack batches((BatchesStack::allocator_type(arena)));
CShaderDefines contextBlend = context;
contextBlend.Add("BLEND", "1");
PROFILE_START("compute batches");
// Reserve an arbitrary size that's probably big enough in most cases,
// to avoid heavy reallocations
batches.reserve(256);
typedef std::vector<SBlendStackItem, ProxyAllocator<SBlendStackItem, Allocators::Arena<> > > BlendStacks;
BlendStacks blendStacks((BlendStacks::allocator_type(arena)));
blendStacks.reserve(patches.size());
// Extract all the blend splats from each patch
for (size_t i = 0; i < patches.size(); ++i)
{
CPatchRData* patch = patches[i];
if (!patch->m_BlendSplats.empty())
{
blendStacks.push_back(SBlendStackItem(patch->m_VBBlends, patch->m_VBBlendIndices, patch->m_BlendSplats, arena));
// Reverse the splats so the first to be rendered is at the back of the list
std::reverse(blendStacks.back().splats.begin(), blendStacks.back().splats.end());
}
}
// Rearrange the collection of splats to be grouped by texture, preserving
// order of splats within each patch:
// (This is exactly the same algorithm used in CPatchRData::BuildBlends,
// but applied to patch-sized splats rather than to tile-sized splats;
// see that function for comments on the algorithm.)
while (true)
{
if (!batches.empty())
{
CTerrainTextureEntry* tex = batches.back().m_Texture;
for (size_t k = 0; k < blendStacks.size(); ++k)
{
SBlendStackItem::SplatStack& splats = blendStacks[k].splats;
if (!splats.empty() && splats.back().m_Texture == tex)
{
CVertexBuffer::VBChunk* vertices = blendStacks[k].vertices;
CVertexBuffer::VBChunk* indices = blendStacks[k].indices;
BatchElements& batch = PooledPairGet(PooledMapGet(batches.back().m_Batches, vertices->m_Owner, arena), indices->m_Owner, arena);
batch.first.push_back(splats.back().m_IndexCount);
u8* indexBase = indices->m_Owner->GetBindAddress();
batch.second.push_back(indexBase + sizeof(u16)*(indices->m_Index + splats.back().m_IndexStart));
splats.pop_back();
}
}
}
CTerrainTextureEntry* bestTex = NULL;
size_t bestStackSize = 0;
for (size_t k = 0; k < blendStacks.size(); ++k)
{
SBlendStackItem::SplatStack& splats = blendStacks[k].splats;
if (splats.size() > bestStackSize)
{
bestStackSize = splats.size();
bestTex = splats.back().m_Texture;
}
}
if (bestStackSize == 0)
break;
SBlendBatch layer(arena);
layer.m_Texture = bestTex;
batches.push_back(layer);
}
PROFILE_END("compute batches");
CVertexBuffer* lastVB = NULL;
for (BatchesStack::iterator itt = batches.begin(); itt != batches.end(); ++itt)
{
if (itt->m_Texture->GetMaterial().GetSamplers().size() == 0)
continue;
int numPasses = 1;
CShaderTechniquePtr techBase;
if (!isDummyShader)
{
techBase = g_Renderer.GetShaderManager().LoadEffect(itt->m_Texture->GetMaterial().GetShaderEffect(), contextBlend, itt->m_Texture->GetMaterial().GetShaderDefines());
numPasses = techBase->GetNumPasses();
}
CShaderProgramPtr previousShader;
for (int pass = 0; pass < numPasses; ++pass)
{
if (!isDummyShader)
{
techBase->BeginPass(pass);
TerrainRenderer::PrepareShader(techBase->GetShader(), shadow);
glEnable(GL_BLEND);
glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
}
const CShaderProgramPtr& shader = isDummyShader ? dummy : techBase->GetShader(pass);
if (itt->m_Texture)
{
CMaterial::SamplersVector samplers = itt->m_Texture->GetMaterial().GetSamplers();
size_t samplersNum = samplers.size();
for (size_t s = 0; s < samplersNum; ++s)
{
CMaterial::TextureSampler &samp = samplers[s];
shader->BindTexture(samp.Name.c_str(), samp.Sampler);
}
shader->BindTexture("blendTex", itt->m_Texture->m_TerrainAlpha->second.m_hCompositeAlphaMap);
itt->m_Texture->GetMaterial().GetStaticUniforms().BindUniforms(shader);
#if !CONFIG2_GLES
if (isDummyShader)
{
pglClientActiveTextureARB(GL_TEXTURE0);
glMatrixMode(GL_TEXTURE);
glLoadMatrixf(itt->m_Texture->GetTextureMatrix());
glMatrixMode(GL_MODELVIEW);
}
else
#endif
{
float c = itt->m_Texture->GetTextureMatrix()[0];
float ms = itt->m_Texture->GetTextureMatrix()[8];
shader->Uniform("textureTransform", c, ms, -ms, 0.f);
}
}
else
{
shader->BindTexture("baseTex", g_Renderer.GetTextureManager().GetErrorTexture());
}
for (VertexBufferBatches::iterator itv = itt->m_Batches.begin(); itv != itt->m_Batches.end(); ++itv)
{
// Rebind the VB only if it changed since the last batch
if (itv->first != lastVB || shader != previousShader)
{
lastVB = itv->first;
previousShader = shader;
GLsizei stride = sizeof(SBlendVertex);
SBlendVertex *base = (SBlendVertex *)itv->first->Bind();
shader->VertexPointer(3, GL_FLOAT, stride, &base->m_Position[0]);
shader->ColorPointer(4, GL_UNSIGNED_BYTE, stride, &base->m_DiffuseColor);
shader->NormalPointer(GL_FLOAT, stride, &base->m_Normal[0]);
shader->TexCoordPointer(GL_TEXTURE0, 3, GL_FLOAT, stride, &base->m_Position[0]);
shader->TexCoordPointer(GL_TEXTURE1, 2, GL_FLOAT, stride, &base->m_AlphaUVs[0]);
}
shader->AssertPointersBound();
for (IndexBufferBatches::iterator it = itv->second.begin(); it != itv->second.end(); ++it)
{
it->first->Bind();
BatchElements& batch = it->second;
if (!g_Renderer.m_SkipSubmit)
{
for (size_t i = 0; i < batch.first.size(); ++i)
glDrawElements(GL_TRIANGLES, batch.first[i], GL_UNSIGNED_SHORT, batch.second[i]);
}
g_Renderer.m_Stats.m_DrawCalls++;
g_Renderer.m_Stats.m_BlendSplats++;
g_Renderer.m_Stats.m_TerrainTris += std::accumulate(batch.first.begin(), batch.first.end(), 0) / 3;
}
}
if (!isDummyShader)
{
glDisable(GL_BLEND);
techBase->EndPass();
}
}
}
#if !CONFIG2_GLES
if (isDummyShader)
{
pglClientActiveTextureARB(GL_TEXTURE0);
glMatrixMode(GL_TEXTURE);
glLoadIdentity();
glMatrixMode(GL_MODELVIEW);
}
#endif
CVertexBuffer::Unbind();
}
bool Model::loadGeometryFile(std::string filename)
{
// Open file
core::FileSystem::Ptr fs = getManager()->getFileSystem();
core::File::Ptr file = fs->open(filename, core::FileAccess::Read);
if (!file)
{
getManager()->getLog()->error("Could not open file \"%s\".",
filename.c_str());
return false;
}
// Read header
// TODO: Adapt all this to big-endian
GeometryFile::Header header;
if (file->read(sizeof(header), &header) != sizeof(header))
{
getManager()->getLog()->error("%s: Could not read geometry header.",
getName().c_str());
return false;
}
if (header.tag != GeometryFile::tag
|| header.version != GeometryFile::version)
{
getManager()->getLog()->error("%s: Invalid geometry file.",
getName().c_str());
return false;
}
// Read vertex and index data
void *vertexdata = malloc(header.vertexdatasize);
if (file->read(header.vertexdatasize, vertexdata) != (int)header.vertexdatasize)
{
getManager()->getLog()->error("%s: Could not read vertex data.",
getName().c_str());
free(vertexdata);
return false;
}
void *indexdata = malloc(header.indexdatasize);
if (file->read(header.indexdatasize, indexdata) != (int)header.indexdatasize)
{
getManager()->getLog()->error("%s: Could not read index data.",
getName().c_str());
free(vertexdata);
free(indexdata);
return false;
}
// Construct vertex/index buffers
res::ResourceManager *rmgr = getManager();
vertexbuffer = rmgr->createResource<VertexBuffer>("VertexBuffer");
indexbuffer = rmgr->createResource<IndexBuffer>("IndexBuffer");
if (!vertexbuffer || !indexbuffer)
{
getManager()->getLog()->error("%s: Could not create buffers.",
getName().c_str());
free(vertexdata);
free(indexdata);
return false;
}
vertexbuffer->set(header.vertexdatasize,
vertexdata,
VertexBufferUsage::Static,
false);
indexbuffer->set(header.indexdatasize, indexdata, false);
// Read batch info
std::vector<GeometryFile::Batch> batchdata(header.batchcount, GeometryFile::Batch());
for (unsigned int i = 0; i < header.batchcount; i++)
{
GeometryFile::AttribInfo &attribs = batchdata[i].attribs;
GeometryFile::GeometryInfo &geom = batchdata[i].geom;
if (file->read(sizeof(attribs), &attribs) != sizeof(attribs)
|| file->read(sizeof(geom), &geom) != sizeof(geom))
{
getManager()->getLog()->error("%s: Could not read batch.",
getName().c_str());
return false;
}
// Joint matrices
if (geom.jointcount == 0)
continue;
int jointsize = sizeof(float) * geom.jointcount * 16;
batchdata[i].jointmatrices = std::vector<float>(geom.jointcount * 16);
if (file->read(jointsize, &batchdata[i].jointmatrices[0]) != jointsize)
{
getManager()->getLog()->error("%s: Could not joint matrices.",
getName().c_str());
return false;
}
}
// Construct batch info
std::vector<Batch> batches(header.batchcount);
for (unsigned int i = 0; i < header.batchcount; i++)
{
batches[i].layout = createVertexLayout(batchdata[i].attribs);
if (!batches[i].layout)
{
getManager()->getLog()->error("%s: Could not create vertex layout.",
getName().c_str());
return false;
}
GeometryFile::AttribInfo &attribs = batchdata[i].attribs;
GeometryFile::GeometryInfo &geom = batchdata[i].geom;
batches[i].basevertex = geom.basevertex;
batches[i].indextype = geom.indextype;
batches[i].indexcount = geom.indexcount;
batches[i].startindex = geom.indexoffset / geom.indextype;
batches[i].vertexoffset = geom.vertexoffset;
batches[i].vertexcount = geom.vertexsize / attribs.stride;
// Create joints
batches[i].joints = std::vector<Joint>(batchdata[i].geom.jointcount);
for (unsigned int j = 0; j < batchdata[i].geom.jointcount; j++)
{
math::Matrix4 &jointmat = batches[i].joints[j].jointmat;
memcpy(&jointmat.m[0],
&batchdata[i].jointmatrices[j * 16],
sizeof(float) * 16);
}
}
this->batches = batches;
return true;
}
void ComputationGraph::accumulate_model_gradients(int first_node, std::vector<int> y_grad_indices, std::vector<Eigen::VectorXf, Eigen::aligned_allocator<Eigen::VectorXf>> y_grad_values) {
std::vector<std::vector<int>> gradients;
for (int node_index = first_node; node_index < nodes.size(); node_index++) {
gradients.push_back(std::vector<int>(nodes[node_index].output.size(), 0));
}
for (int i = 0; i < y_grad_indices.size(); i++) {
auto &gradient = gradients[y_grad_indices[i] - first_node];
auto &y_grad_value = y_grad_values[i];
for (int j = 0; j < y_grad_value.size(); j++) {
gradient[j] = *((int *)&y_grad_value(j));
}
}
std::vector<Batch> batches(this->batches);
while (batches.size() > 1 && first_node < batches.back().nodes_end) {
std::vector<std::vector<int>> transformations_node_indices;
for (int transformation_index = 0; transformation_index < transformations.size(); transformation_index++) {
transformations_node_indices.push_back({});
}
for (int node_index = batches[batches.size() - 2].nodes_end; node_index < batches[batches.size() - 1].nodes_end; node_index++) {
auto &node = nodes[node_index];
if (node.transformation != -1) {
auto &transformation_node_indices = transformations_node_indices[node.transformation];
transformation_node_indices.push_back(node_index);
}
}
// create feed dict
std::vector<std::pair<std::string, tensorflow::Tensor>> feed_dict;
for (int transformation_index = 0; transformation_index < transformations.size(); transformation_index++) {
auto &transformation = transformations[transformation_index];
auto &transformation_node_indices = transformations_node_indices[transformation_index];
if (transformation_node_indices.size() > 0) {
for (int input_index = 0; input_index < transformation.inputs.size(); input_index++) {
auto &input_name = transformation.inputs[input_index];
auto &input_shape = transformation.input_shapes[input_index];
auto &input_type = transformation.input_types[input_index];
auto &input_size = transformation.input_sizes[input_index];
tensorflow::TensorShape tensor_shape;
tensor_shape.AddDim(transformation_node_indices.size());
for (int dim : input_shape) tensor_shape.AddDim(dim);
tensorflow::Tensor tensor(input_type, tensor_shape);
for (int i = 0; i < transformation_node_indices.size(); i++) {
auto &node_index = transformation_node_indices[i];
auto &node = nodes[node_index];
if (input_type == tensorflow::DataType::DT_FLOAT) {
copy_input_into_tensor<float>(i, input_size, node.inputs[input_index], tensor);
} else if (input_type == tensorflow::DataType::DT_INT32) {
copy_input_into_tensor<int>(i, input_size, node.inputs[input_index], tensor);
} else {
std::cout << "unsupported type " << input_type << std::endl;
exit(123);
}
}
feed_dict.push_back(std::make_pair(input_name, tensor));
}
{
tensorflow::TensorShape tensor_shape;
tensor_shape.AddDim(transformation_node_indices.size());
for (int dim : transformation.output_shape) tensor_shape.AddDim(dim);
tensorflow::Tensor tensor(transformation.output_type, tensor_shape);
for (int i = 0; i < transformation_node_indices.size(); i++) {
auto &node_index = transformation_node_indices[i];
copy_vector_into_tensor<float>(i, transformation.output_size, gradients[node_index - first_node], tensor);
}
feed_dict.push_back(std::make_pair(transformation.output_gradient, tensor));
}
}
}
for (int transformation_index = 0; transformation_index < transformations.size(); transformation_index++) {
auto &transformation = transformations[transformation_index];
auto &input_name = transformation.seed;
auto &seed = batches.back().seeds[transformation_index];
tensorflow::Tensor tensor(tensorflow::DataType::DT_INT64, tensorflow::TensorShape({seed.size()}));
for (int i = 0; i < seed.size(); i++) {
tensor.flat<long long>()(i) = seed[i];
}
feed_dict.push_back(std::make_pair(input_name, tensor));
}
feed_dict.push_back(std::make_pair(training_name, training_tensor));
// create fetch outputs
std::vector<std::string> fetch_outputs;
std::vector<std::vector<int>> transformations_input_index_to_output;
for (int transformation_index = 0; transformation_index < transformations.size(); transformation_index++) {
auto &transformation = transformations[transformation_index];
auto &transformation_node_indices = transformations_node_indices[transformation_index];
transformations_input_index_to_output.push_back({});
if (transformation_node_indices.size() > 0) {
for (auto &input_gradient : transformation.input_gradients) {
transformations_input_index_to_output.back().push_back(fetch_outputs.size());
if (input_gradient != "") {
fetch_outputs.push_back(input_gradient);
}
}
}
}
// create run_outputs
std::vector<std::string> run_outputs;
for (int transformation_index = 0; transformation_index < transformations.size(); transformation_index++) {
auto &transformation = transformations[transformation_index];
auto &transformation_node_indices = transformations_node_indices[transformation_index];
if (transformation_node_indices.size() > 0) {
run_outputs.push_back(transformation.update_model_gradient_accumulators);
}
}
// run
std::vector<tensorflow::Tensor> outputs;
TF_CHECK_OK(session->Run(feed_dict, fetch_outputs, run_outputs, &outputs));
// std::cout << "outputs " << batches[batches.size() - 2].nodes_end << std::endl;
// for (int i = 0; i < outputs.size(); i++) {
// std::cout << fetch_outputs[i] << std::endl;
// auto output = outputs[i].flat<float>();
// for (int i = 0; i < output.size(); i++) {
// std::cout << output(i) << " ";
// }
// std::cout << std::endl;
// }
// add gradients
for (int transformation_index = 0; transformation_index < transformations.size(); transformation_index++) {
auto &transformation = transformations[transformation_index];
auto &transformation_node_indices = transformations_node_indices[transformation_index];
auto &transformation_input_index_to_output = transformations_input_index_to_output[transformation_index];
for (int i = 0; i < transformation_node_indices.size(); i++) {
auto node_index = transformation_node_indices[i];
auto &node = nodes[node_index];
for (int input_index = 0; input_index < transformation.inputs.size(); input_index++) {
auto &input_gradient = transformation.input_gradients[input_index];
if (input_gradient != "") {
auto output_index = transformation_input_index_to_output[input_index];
auto &output = outputs[output_index];
auto &node_input = node.inputs[input_index];
auto output_view = output.matrix<float>();
int output_view_iterator = 0;
for (int partial_input_node_index : node_input) {
if (partial_input_node_index - first_node >= 0) {
auto &gradient = gradients[partial_input_node_index - first_node];
for (auto &x : gradient) {
(*((float*)&x)) += output_view(i, output_view_iterator++);
}
}
}
}
}
}
}
batches.pop_back();
}
}
