void write_supertensor_index(const string& name, const vector<Ref<const supertensor_reader_t>>& readers) {
// Check consistency
GEODE_ASSERT(readers.size());
const uint32_t slice = readers[0]->header.stones;
const auto sections = descendent_sections(section_t(),slice).at(slice);
GEODE_ASSERT(sections->slice==int(slice));
Hashtable<section_t,Ref<const supertensor_reader_t>> section_reader;
for (const auto reader : readers) {
GEODE_ASSERT(int(reader->header.filter)==filter);
section_reader.set(reader->header.section,reader);
}
for (const auto section : sections->sections)
GEODE_ASSERT(section_reader.contains(section));
// Write index
FILE* file = fopen(name.c_str(),"wb");
if (!file)
throw IOError(format("write_supertensor_index: can't open '%s' for writing",name));
fwrite("pentago index \n",1,20,file);
fwrite(&slice,sizeof(uint32_t),1,file);
GEODE_ASSERT(ftell(file)==24);
for (const auto section : sections->sections) {
const auto reader = section_reader.get(section);
Array<compact_blob_t> blobs(reader->offset.flat.size(),uninit);
for (const int i : range(blobs.size())) {
const uint64_t offset = reader->offset.flat[i];
blobs[i].set_offset(offset);
blobs[i].size = reader->compressed_size_.flat[i];
}
fwrite(blobs.data(),sizeof(compact_blob_t),blobs.size(),file);
}
const auto index = new_<supertensor_index_t>(sections);
GEODE_ASSERT(uint64_t(ftell(file))==index->section_offset.back());
fclose(file);
}
//
// Run a single type test
//
void runtest(char type)
{
try {
debug("SStest", "Start test <%c>", type);
switch (type) {
case '.': // default
integrity();
break;
case '-':
adhoc();
break;
case 'a':
acls();
break;
case 'A':
authAcls();
break;
case 'b':
blobs();
break;
case 'c':
codeSigning();
break;
case 'd':
databases();
break;
case 'e':
desEncryption();
break;
case 'k':
keychainAcls();
break;
case 'K':
keyBlobs();
break;
case 's':
signWithRSA();
break;
case 't':
authorizations();
break;
case 'T':
timeouts();
break;
default:
error("Invalid test selection (%c)", type);
}
printf("** Test step complete.\n");
debug("SStest", "End test <%c>", type);
} catch (CssmCommonError &err) {
error(err, "Unexpected exception");
} catch (...) {
error("Unexpected system exception");
}
}
void blob_service_test_base_with_objects_to_delete::check_blob_list(const std::vector<azure::storage::cloud_blob>& list)
{
auto blob_list_sorted = std::is_sorted(list.cbegin(), list.cend(), [](const azure::storage::cloud_blob& a, const azure::storage::cloud_blob& b)
{
return a.name() < b.name();
});
CHECK(blob_list_sorted);
std::vector<azure::storage::cloud_blob> blobs(m_blobs_to_delete);
for (auto list_iter = list.begin(); list_iter != list.end(); ++list_iter)
{
for (auto iter = blobs.begin(); iter != blobs.end(); ++iter)
{
if (iter->name() == list_iter->name())
{
blobs.erase(iter);
break;
}
}
}
CHECK(blobs.empty());
}
Waifu2x::eWaifu2xError cNet::LoadParameterFromJson(const boost::filesystem::path &model_path, const boost::filesystem::path ¶m_path
, const boost::filesystem::path &modelbin_path, const boost::filesystem::path &caffemodel_path, const std::string &process)
{
Waifu2x::eWaifu2xError ret;
caffe::NetParameter param;
ret = readProtoText(model_path, ¶m);
if (ret != Waifu2x::eWaifu2xError_OK)
return ret;
ret = writeProtoBinary(param, modelbin_path);
if (ret != Waifu2x::eWaifu2xError_OK)
return ret;
ret = SetParameter(param, process);
if (ret != Waifu2x::eWaifu2xError_OK)
return ret;
mNet = boost::shared_ptr<caffe::Net<float>>(new caffe::Net<float>(param));
rapidjson::Document d;
std::vector<char> jsonBuf;
try
{
boost::iostreams::stream<boost::iostreams::file_descriptor_source> is;
try
{
is.open(param_path, std::ios_base::in | std::ios_base::binary);
}
catch (...)
{
return Waifu2x::eWaifu2xError_FailedOpenModelFile;
}
if (!is)
return Waifu2x::eWaifu2xError_FailedOpenModelFile;
const size_t size = is.seekg(0, std::ios::end).tellg();
is.seekg(0, std::ios::beg);
jsonBuf.resize(size + 1);
is.read(jsonBuf.data(), jsonBuf.size());
jsonBuf[jsonBuf.size() - 1] = '\0';
d.Parse(jsonBuf.data());
}
catch (...)
{
return Waifu2x::eWaifu2xError_FailedParseModelFile;
}
if (d.Size() != 7)
return Waifu2x::eWaifu2xError_FailedParseModelFile;
int inputPlane = 0;
int outputPlane = 0;
try
{
inputPlane = d[0]["nInputPlane"].GetInt();
outputPlane = d[d.Size() - 1]["nOutputPlane"].GetInt();
}
catch (...)
{
return Waifu2x::eWaifu2xError_FailedParseModelFile;
}
if (inputPlane == 0 || outputPlane == 0)
return Waifu2x::eWaifu2xError_FailedParseModelFile;
if (inputPlane != outputPlane)
return Waifu2x::eWaifu2xError_FailedParseModelFile;
//if (param.layer_size() < 17)
// return Waifu2x::eWaifu2xError_FailedParseModelFile;
std::vector<boost::shared_ptr<caffe::Layer<float>>> list;
auto &v = mNet->layers();
for (auto &l : v)
{
auto lk = l->type();
auto &bv = l->blobs();
if (bv.size() > 0)
list.push_back(l);
}
try
{
std::vector<float> weightList;
std::vector<float> biasList;
int count = 0;
for (auto it = d.Begin(); it != d.End(); ++it)
{
const auto &weight = (*it)["weight"];
const auto nInputPlane = (*it)["nInputPlane"].GetInt();
const auto nOutputPlane = (*it)["nOutputPlane"].GetInt();
const auto kW = (*it)["kW"].GetInt();
const auto &bias = (*it)["bias"];
auto leyer = list[count];
auto &b0 = leyer->blobs()[0];
auto &b1 = leyer->blobs()[1];
float *b0Ptr = nullptr;
float *b1Ptr = nullptr;
if (caffe::Caffe::mode() == caffe::Caffe::CPU)
{
b0Ptr = b0->mutable_cpu_data();
b1Ptr = b1->mutable_cpu_data();
}
else
{
b0Ptr = b0->mutable_gpu_data();
b1Ptr = b1->mutable_gpu_data();
}
const auto WeightSize1 = weight.Size();
const auto WeightSize2 = weight[0].Size();
const auto KernelHeight = weight[0][0].Size();
const auto KernelWidth = weight[0][0][0].Size();
if (!(b0->count() == WeightSize1 * WeightSize2 * KernelHeight * KernelWidth))
return Waifu2x::eWaifu2xError_FailedConstructModel;
if (!(b1->count() == bias.Size()))
return Waifu2x::eWaifu2xError_FailedConstructModel;
weightList.resize(0);
biasList.resize(0);
size_t weightCount = 0;
for (auto it2 = weight.Begin(); it2 != weight.End(); ++it2)
{
for (auto it3 = (*it2).Begin(); it3 != (*it2).End(); ++it3)
{
for (auto it4 = (*it3).Begin(); it4 != (*it3).End(); ++it4)
{
for (auto it5 = (*it4).Begin(); it5 != (*it4).End(); ++it5)
weightList.push_back((float)it5->GetDouble());
}
}
}
caffe::caffe_copy(b0->count(), weightList.data(), b0Ptr);
for (auto it2 = bias.Begin(); it2 != bias.End(); ++it2)
biasList.push_back((float)it2->GetDouble());
caffe::caffe_copy(b1->count(), biasList.data(), b1Ptr);
count++;
}
mNet->ToProto(¶m);
ret = writeProtoBinary(param, caffemodel_path);
if (ret != Waifu2x::eWaifu2xError_OK)
return ret;
}
catch (...)
{
return Waifu2x::eWaifu2xError_FailedConstructModel;
}
return Waifu2x::eWaifu2xError_OK;
}
std::vector<Blob> BlobDetection::Invoke(ImageLib::ImageRGB & image, int minBlobSize) {
//Get the image height
int height = image.height();
//Get the image width
int width = image.width();
//Remember on which lines we found a (part) blob, so we don't have to check every line again.
bool blobFoundOnRow = false;
//Save all the lines on which we found a (part) blob
std::vector<int> blobRows;
//Map used to map every white pixels, r|g|b > 0, to a label
std::vector< std::vector<int> > labelMap(height);
//Make sure the default values are 0.
for (int i = 0; i < height; i++) {
labelMap[i] = std::vector<int>(width);
}
//Maximum amount of labels an image can containt. this is a estimation.
int maxLabels = height * width / 2;
//If to labels meet somewhere, we link the highest labels to the lowest label. So we can link all labels together
std::vector<int> labelTable(maxLabels);
//We start labeling at 1. 0 means no label. (black(background) pixel)
int labelIndex = 1;
//used to find the labels of the surrounding pixels
int labelA, labelB, labelC, labelD;
//smallest neighboor label found. We use a huge value so we know for sure our if works
int smallestLabel = maxLabels;
//We don't want to check the edge(1px), since not all the surrounding pixels are there.
int tmpHeight = height - 1;
int tmpWidth = width - 1;
//Check every pixel for color, if color is white check surrounding pixels if they already have a label. If a neighbour (or more) has a label, assign the lowest label
// to the current pixel
for (int y = 1; y < tmpHeight; y++)
{
for (int x = 1; x < tmpWidth; x++)
{
//Only check blue. Saves time and if blue = 0 it's black, if blue is 255 it's white. Simple as that
if (image.at(x, y).red == 255)
{
if (blobFoundOnRow == false) blobFoundOnRow = true;
labelA = labelMap[y][x - 1];
labelB = labelMap[y - 1][x - 1];
labelC = labelMap[y - 1][x];
labelD = labelMap[y - 1][x + 1];
smallestLabel = 25000000;
if (labelA != 0 && labelA < smallestLabel) smallestLabel = labelA;
if (labelB != 0 && labelB < smallestLabel) smallestLabel = labelB;
if (labelC != 0 && labelC < smallestLabel) smallestLabel = labelC;
if (labelD != 0 && labelD < smallestLabel) smallestLabel = labelD;
if (smallestLabel == 25000000)
{
//No neighbours found
labelMap[y][x] = labelIndex;
labelTable[labelIndex] = labelIndex;
labelIndex++;
}
else
{
//Found atleast one neighbour label
labelMap[y][x] = smallestLabel;
if (labelA > smallestLabel) labelTable[labelA] = smallestLabel;
if (labelB > smallestLabel) labelTable[labelB] = smallestLabel;
if (labelC > smallestLabel) labelTable[labelC] = smallestLabel;
if (labelD > smallestLabel) labelTable[labelD] = smallestLabel;
}
}
}
//found blob on row
if (blobFoundOnRow == true) {
blobFoundOnRow = false;
blobRows.insert(blobRows.end(), y);
}
}
//Vector for assinging which labels belong to which blob
std::vector<int> labelToBlob(labelIndex);
int blobCount = 0;
int tmp1;
for(int i = 0; i < labelIndex; i++) {
if (labelTable[i] != i) {
tmp1 = labelTable[i];
while (labelTable[tmp1] != tmp1) {
tmp1 = labelTable[tmp1];
}
labelToBlob[i] = labelToBlob[tmp1];
labelTable[i] = tmp1;
}
else {
labelToBlob[i] = blobCount++;
}
}
//Change al labels to blob id
//Map used to map every white pixels, r|g|b > 0, to a label
std::vector< std::vector<int> > blobMap(height);
//Make sure the default values are 0.
for (int i = 0; i < height; i++) {
blobMap[i] = std::vector<int>(width);
}
for (int y = 0; y < height; y++) {
for (int x = 0; x < width; x++) {
blobMap[y][x] = labelToBlob[labelMap[y][x]];
}
}
std::vector< std::vector<int> > blobs(blobCount);
for (int i = 0; i < blobCount; i++) {
blobs[i] = std::vector<int>(6);
blobs[i][0] = 0; //Blob ID
blobs[i][1] = 0; //Mass (amount of pixels)
blobs[i][2] = 20000; //Min Y
blobs[i][3] = 0; //Max Y
blobs[i][4] = 20000; //Min X
blobs[i][5] = 0; //Max X
}
int originalLabel = 0;
int blobId = 0;
for (std::vector<int>::iterator it = blobRows.begin(); it != blobRows.end(); it++) {
for (int x = 0; x < width; x++) {
originalLabel = labelMap[*it][x];
if(originalLabel > 0) {
blobId = labelToBlob[originalLabel];
blobs[blobId][0] = blobId;
blobs[blobId][1]++;
if (x < blobs[blobId][4]) blobs[blobId][4] = x;
if (x > blobs[blobId][5]) blobs[blobId][5] = x;
if (*it < blobs[blobId][2]) blobs[blobId][2] = *it;
if (*it > blobs[blobId][3]) blobs[blobId][3] = *it;
}
}
}
std::vector<Blob> definiteBlobs;
for (int i = 0; i < blobCount; i++) {
if (blobs[i][1] > minBlobSize) {
definiteBlobs.insert(definiteBlobs.end(), Blob(blobs[i][0], blobs[i][1], blobs[i][2], blobs[i][3], blobs[i][4], blobs[i][5]));
}
}
BlobCheck bc;
return bc.CheckIfBlobIsLicensePlate(definiteBlobs, blobMap);
}
