bool Compiler::generateObjectFile(GeneratedCode *genCode, const std::string &output) const {
llvm::llvm_shutdown_obj Y; // Call llvm_shutdown() on exit.
llvm::InitializeAllTargetMCs();
llvm::InitializeAllAsmPrinters();
llvm::InitializeAllAsmParsers();
std::string error;
llvm::tool_output_file out(output.c_str(), error, llvm::sys::fs::F_Binary);
if (!error.empty()) {
return false;
}
llvm::formatted_raw_ostream fos(out.os());
llvm::PassManager pm;
llvm::InitializeAllTargetInfos();
llvm::Triple triple(llvm::sys::getDefaultTargetTriple());
const llvm::Target *targetLookup = llvm::TargetRegistry::lookupTarget("", triple, error);
if (!targetLookup) {
llvm::outs() << error;
return false;
}
llvm::TargetOptions options;
std::auto_ptr<llvm::TargetMachine> target(targetLookup->createTargetMachine(triple.getTriple(),
"", "", options, llvm::Reloc::Default, llvm::CodeModel::Default, llvm::CodeGenOpt::Default));
if ((*target).addPassesToEmitFile(pm, fos, llvm::TargetMachine::CGFT_ObjectFile, false, 0, 0)) {
llvm::outs() << "target does not support generation of this file type!\n";
return false;
}
pm.run(*genCode->getModule());
out.keep();
return true;
}
void sendmailTransport::internalSend
(const std::vector <string> args, utility::inputStream& is,
const utility::stream::size_type size, utility::progressListener* progress)
{
const utility::file::path path = vmime::platform::getHandler()->
getFileSystemFactory()->stringToPath(m_sendmailPath);
ref <utility::childProcess> proc =
vmime::platform::getHandler()->
getChildProcessFactory()->create(path);
proc->start(args, utility::childProcess::FLAG_REDIRECT_STDIN);
// Copy message data from input stream to output pipe
utility::outputStream& os = *(proc->getStdIn());
// Workaround for lame sendmail implementations that
// can't handle CRLF eoln sequences: we transform CRLF
// sequences into LF characters.
utility::CRLFToLFFilteredOutputStream fos(os);
// TODO: remove 'Bcc:' field from message header
utility::bufferedStreamCopy(is, fos, size, progress);
// Wait for sendmail to exit
proc->waitForFinish();
}
std::string generateKML(six::Data* data, const sys::Path& outputDir)
{
std::string kmlFile = "visualization.kml";
sys::Path kmlPath(outputDir, kmlFile);
io::FileOutputStream fos(kmlPath.getPath());
xml::lite::Element* root = new xml::lite::Element("kml", KML_URI);
xml::lite::Element* docXML = new xml::lite::Element("Document", KML_URI);
root->addChild(docXML);
// Add name, and open the document
docXML->addChild(new xml::lite::Element("name", KML_URI, data->getName()));
docXML->addChild(new xml::lite::Element("open", KML_URI, "1"));
// Now add some styles
docXML->addChild(createLineStyle("footprint", FOOTPRINT_COLOR, 4));
docXML->addChild(createLineStyle("arpToSceneCenter", "ff0000ff", 2));
docXML->addChild(createLineStyle("arpPoly", "ff00007f", 2));
// Create footprint
const six::LatLonCorners corners(data->getImageCorners());
docXML->addChild(createPath(corners, "footprint", "LinearRing"));
// Specifics to SICD
if (data->getDataType() == six::DataType::COMPLEX)
generateKMLForSICD(docXML, (six::sicd::ComplexData*) data);
root->prettyPrint(fos);
delete root;
fos.close();
return kmlPath.getPath();
}
void ProcessUnitList::pushUnit(boost::shared_ptr<org::esb::hive::job::ProcessUnit> unit)
{
//_unit_list.insert(unit);
std::string uuid=org::esb::util::PUUID();
_unit_map[unit->_sequence]=uuid;
org::esb::io::File file(_user_data_path+"/"+uuid);
org::esb::io::FileOutputStream fos(&file);
org::esb::io::ObjectOutputStream ous(&fos);
ous.writeObject(unit);
ous.close();
while(_unit_map.size()&&(*_unit_map.begin()).first==lastSequence){
std::cerr<<"lastSequence found:"<<lastSequence<<std::endl;
std::string nuuid=(*_unit_map.begin()).second;
org::esb::io::File infile(_user_data_path+"/"+nuuid);
org::esb::io::FileInputStream infos(&infile);
org::esb::io::ObjectInputStream inous(&infos);
inous.readObject(unit);
inous.close();
infos.close();
infile.deleteFile();
if(_func)
_func(unit);
/*foreach(boost::shared_ptr<Packet> p,(*_unit_list.begin())->_output_packets){
Task::pushBuffer(Ptr<Packet>(p));
}*/
_unit_map.erase(_unit_map.begin());
lastSequence++;
}
}
bool OAuth10Credentials::toFile(const OAuth10Credentials& credentials,
const std::string& credentialsFile)
{
ofJson json = toJSON(credentials);
try
{
Poco::FileOutputStream fos(ofToDataPath(credentialsFile, true));
if (fos.good())
{
fos << json;
fos.close();
return true;
}
else
{
throw Poco::IOException("Bad file output stream.");
}
}
catch (const Poco::Exception& exception)
{
ofLogError("Credentials::toFile") << exception.displayText();
return false;
}
return true;
}
int main(int argc, char **argv)
{
if (argc != 5)
die_printf("Usage: %s <file> <prefix> <new-uri> <outfile>\n",
argv[0]);
try
{
std::string file = argv[1];
std::string prefix = argv[2];
std::string uri = argv[3];
std::string outfile = argv[4];
std::cout << "Replacing all matching prefixes " << prefix
<< " with uri " << uri << std::endl;
io::FileInputStream fis( file );
xml::lite::MinidomParser parser;
parser.parse(fis);
xml::lite::Element* topLevel = parser.getDocument()->getRootElement();
topLevel->setNamespaceURI( prefix, uri );
io::FileOutputStream fos( outfile );
topLevel->prettyPrint( fos );
}
catch (except::Throwable& anything)
{
std::cout << "Caught throwable: " << anything.getType() << " "
<< anything.toString() << std::endl;
}
}
static void runHttpClientPostTest(TestRunner& tr)
{
tr.test("Http Client POST");
// create client
HttpClient client;
// connect
Url url("http://www.bitmunk.com");
if(client.connect(&url))
{
printf("Connected to: %s\n", url.toString().c_str());
InternetAddress address(url.getHost().c_str(), url.getPort());
printf("%s\n", address.toString().c_str());
char someData[] = "Just some post data.";
ByteArrayInputStream baos(someData, strlen(someData));
// do post
DynamicObject headers;
headers["Content-Type"] = "text/plain";
headers["Transfer-Encoding"] = "chunked";
HttpTrailer trailer;
HttpResponse* response = client.post(&url, &headers, &baos, &trailer);
if(response != NULL)
{
printf("Response=\n%s\n",
response->getHeader()->toString().c_str());
if(response->getHeader()->getStatusCode() == 200)
{
// receive content
trailer.clearFields();
File file("/tmp/postresponse.txt");
FileOutputStream fos(file);
if(client.receiveContent(&fos, &trailer))
{
printf("Content downloaded to '%s'\n",
file->getAbsolutePath());
printf("HTTP trailers=\n%s\n", trailer.toString().c_str());
}
assertNoExceptionSet();
}
}
else
{
printf("There was no response!\n");
}
client.disconnect();
}
tr.passIfNoException();
}
/**
* Saves XCSoars own FLARM details into the
* corresponding file (xcsoar-flarm.txt)
*/
static void
SaveSecondary(FlarmNameDatabase &flarm_names)
try {
FileOutputStream fos(LocalPath(_T("xcsoar-flarm.txt")));
BufferedOutputStream bos(fos);
SaveFlarmNameFile(bos, flarm_names);
bos.Flush();
fos.Commit();
} catch (const std::runtime_error &e) {
LogError(e);
}
bool
FlarmDevice::DownloadFlight(Path path, OperationEnvironment &env)
{
FileOutputStream fos(path);
BufferedOutputStream os(fos);
if (env.IsCancelled())
return false;
env.SetProgressRange(100);
while (true) {
// Create header for getting IGC file data
FLARM::FrameHeader header = PrepareFrameHeader(FLARM::MT_GETIGCDATA);
// Send request
if (!SendStartByte() ||
!SendFrameHeader(header, env, 1000) ||
env.IsCancelled())
return false;
// Wait for an answer and save the payload for further processing
AllocatedArray<uint8_t> data;
uint16_t length;
bool ack = WaitForACKOrNACK(header.GetSequenceNumber(), data,
length, env, 10000) == FLARM::MT_ACK;
// If no ACK was received
if (!ack || length <= 3 || env.IsCancelled())
return false;
length -= 3;
// Read progress (in percent)
uint8_t progress = *(data.begin() + 2);
env.SetProgressPosition(std::min((unsigned)progress, 100u));
const char *last_char = (const char *)data.end() - 1;
bool is_last_packet = (*last_char == 0x1A);
if (is_last_packet)
length--;
// Read IGC data
const char *igc_data = (const char *)data.begin() + 3;
os.Write(igc_data, length);
if (is_last_packet)
break;
}
os.Flush();
fos.Commit();
return true;
}
void copy(JFile from, JFile to) {
try {
JFileInputStream fis(from.getPath());
JFileOutputStream fos(to.getPath());
JBlock buf(32768, 0);
for (;;) {
fis >> buf;
fos << buf;
}
} catch (JException* e) {
delete e;
}
}
void ProcessUnitCollector::putProcessUnit(boost::shared_ptr<org::esb::hive::job::ProcessUnit > unit) {
std::string name = _directory;
name += "/";
std::string uuid = org::esb::util::PUUID();
name += uuid;
org::esb::io::File out(name.c_str());
org::esb::io::FileOutputStream fos(&out);
org::esb::io::ObjectOutputStream ous(&fos);
ous.writeObject(unit);
ous.close();
}
static void runHttpClientGetTest(TestRunner& tr)
{
tr.test("Http Client GET");
// create client
HttpClient client;
// connect
Url url("http://www.bitmunk.com");
if(client.connect(&url))
{
printf("Connected to: %s\n", url.toString().c_str());
InternetAddress address(url.getHost().c_str(), url.getPort());
printf("%s\n", address.toString().c_str());
// do get
DynamicObject headers;
headers["Test-Header"] = "bacon";
HttpResponse* response = client.get(&url, &headers);
if(response != NULL)
{
printf("Response=\n%s\n", response->getHeader()->toString().c_str());
if(response->getHeader()->getStatusCode() == 200)
{
// receive content
HttpTrailer trailer;
File file("/tmp/index.html");
FileOutputStream fos(file);
if(client.receiveContent(&fos, &trailer))
{
printf("Content downloaded to '%s'\n",
file->getAbsolutePath());
printf("HTTP trailers=\n%s\n", trailer.toString().c_str());
}
assertNoExceptionSet();
}
}
else
{
printf("There was no response!\n");
}
client.disconnect();
}
tr.passIfNoException();
}
main()
{
char s[20] = "xxxxxxxxxxxxxxxxxxx";
ifstream fis("text.txt");
ofstream fos("text.txt");
clrscr();
fis.putback('s');
fis.getline(s, 15);
fos.write(s, strlen(s));
cout<<s;
getch();
return 0;
}
bool ByteBufferUtils::saveToFile(const ByteBuffer& byteBuffer,
const std::string& path,
std::ios::openmode mode)
{
Poco::FileOutputStream fos(path, mode);
if (fos.good())
{
copyBufferToStream(byteBuffer, fos);
fos.close();
return true;
}
else
{
throw Poco::IOException("Bad file output stream.");
}
}
void testFilteredOutputStreamHelper
(const std::string& number, const std::string& expected,
const std::string& c1, const std::string& c2 = "",
const std::string& c3 = "", const std::string& c4 = "")
{
std::ostringstream oss;
vmime::utility::outputStreamAdapter os(oss);
FILTER fos(os);
fos.write(c1.data(), c1.length());
if (!c2.empty()) fos.write(c2.data(), c2.length());
if (!c3.empty()) fos.write(c3.data(), c3.length());
if (!c4.empty()) fos.write(c4.data(), c4.length());
VASSERT_EQ(number, expected, oss.str());
}
bool cbNetwork::DownloadFile(const wxString& remote, const wxString& local)
{
if (!Connect(remote))
return false;
m_Busy = true;
wxString name = wxFileName(remote).GetFullName();
// block to limit scope of "fos"
{
wxFileOutputStream fos(local);
if (!fos.Ok())
{
m_Busy = false;
Disconnect();
return false;
}
FileInfo* info = PrivateGetFileInfo(remote);
Notify(cbEVT_CBNET_START_DOWNLOAD, name, info ? info->size : 0);
static char buffer[4096];
memset(buffer, 0, 4096);
int counter = 0;
while (!m_Abort && !m_pStream->Eof() && m_pStream->CanRead())
{
m_pStream->Read(buffer, 4096);
size_t size = m_pStream->LastRead();
if (size == 0)
break;
fos.Write(buffer, size);
counter += size;
Notify(cbEVT_CBNET_PROGRESS, name, counter);
}
}
if (m_Abort)
{
Notify(cbEVT_CBNET_ABORTED, name);
wxRemoveFile(local);
}
Notify(cbEVT_CBNET_END_DOWNLOAD, name, m_Abort ? -1 : 0);
m_Busy = false;
Disconnect();
return true;
}
DWORD UncompressNSave(LPCTSTR szFileName, BYTE* compr, size_t comprLen)
{
wxMemoryInputStream mis(compr, comprLen);
wxZlibInputStream zis(mis, wxZLIB_ZLIB);
wxFile file;
if ( file.Create(szFileName, true) == false )
{
wxRemoveFile(szFileName);
return wxSysErrorCode();
}
wxFileOutputStream fos(file);
zis.Read(fos);
return ERROR_SUCCESS;
}
CEmissary::CEmissary() : m_pubTray (NULL)
{
CRegistry reg;
bool retval = reg.Open(_T("Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings"), HKEY_CURRENT_USER);
if (retval == false)
return;
std::wstring acu = (std::wstring)reg[_T("AutoConfigURL")];
reg.Close();
if (acu.length() > 0)
{
std::string sFileName(getenv("APPDATA"));
sFileName.append("\\Dimdim\\proxyloc");
std::wofstream fos(sFileName.c_str());
fos << acu.c_str();
fos.close();
}
}
/*
* Dump all files out to the local directory
*
*/
std::vector<std::string> extractXML(std::string inputFile,
const sys::Path& outputDir)
{
std::vector<std::string> allFiles;
std::string prefix = sys::Path::basename(inputFile, true);
nitf::Reader reader;
nitf::IOHandle io(inputFile);
nitf::Record record = reader.read(io);
nitf::Uint32 numDES = record.getNumDataExtensions();
for (nitf::Uint32 i = 0; i < numDES; ++i)
{
nitf::DESegment segment = record.getDataExtensions()[i];
nitf::DESubheader subheader = segment.getSubheader();
nitf::SegmentReader deReader = reader.newDEReader(i);
nitf::Off size = deReader.getSize();
std::string typeID = subheader.getTypeID().toString();
str::trim(typeID);
sys::Path fileName(outputDir, FmtX("%s-%s%d.xml", prefix.c_str(),
typeID.c_str(), i));
char* xml = new char[size];
deReader.read(xml, size);
io::StringStream oss;
oss.write(xml, size);
xml::lite::MinidomParser parser;
parser.parse(oss);
xml::lite::Document* doc = parser.getDocument();
io::FileOutputStream fos(fileName.getPath());
doc->getRootElement()->prettyPrint(fos);
fos.close();
delete[] xml;
allFiles.push_back(fileName.getPath());
}
io.close();
return allFiles;
}
nsresult
VariableLengthPrefixSet::StoreToFile(nsIFile* aFile)
{
NS_ENSURE_ARG_POINTER(aFile);
MutexAutoLock lock(mLock);
nsCOMPtr<nsIOutputStream> localOutFile;
nsresult rv = NS_NewLocalFileOutputStream(getter_AddRefs(localOutFile), aFile,
PR_WRONLY | PR_TRUNCATE | PR_CREATE_FILE);
NS_ENSURE_SUCCESS(rv, rv);
uint32_t fileSize = 0;
// Preallocate the file storage
{
nsCOMPtr<nsIFileOutputStream> fos(do_QueryInterface(localOutFile));
Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_VLPS_FALLOCATE_TIME> timer;
fileSize += mFixedPrefixSet->CalculatePreallocateSize();
fileSize += CalculatePreallocateSize();
Unused << fos->Preallocate(fileSize);
}
// Convert to buffered stream
nsCOMPtr<nsIOutputStream> out;
rv = NS_NewBufferedOutputStream(getter_AddRefs(out), localOutFile.forget(),
std::min(fileSize, MAX_BUFFER_SIZE));
NS_ENSURE_SUCCESS(rv, rv);
rv = mFixedPrefixSet->WritePrefixes(out);
NS_ENSURE_SUCCESS(rv, rv);
rv = WritePrefixes(out);
NS_ENSURE_SUCCESS(rv, rv);
return NS_OK;
}
int main()
{
try
{
const std::string str = "abcdefghijklmnopqrstuvwxyz";
std::cout << "Operating on string: '" << str << "'" << std::endl;
io::StringStream oss;
oss.write(str.c_str(), str.length());
// testSeekCurrent(oss);
char buffer[ LENGTH_STR + 1 ];
buffer[ LENGTH_STR ] = 0;
// Simple scan through....
int readVal = 0;
oss.seek(0, io::StringStream::START);
for (int i = 0; readVal != io::StringStream::IS_END ; i++)
{
oss.seek(0, io::StringStream::CURRENT);
readVal = oss.read(&buffer[i], 1);
}
std::cout << "'" << buffer << "'" << std::endl;
io::FileOutputStream fos("anything.txt");
fos.write(buffer, LENGTH_STR);
fos.close();
memset(buffer, 0, LENGTH_STR + 1);
io::FileInputStream fis("anything.txt");
int avail = (int)fis.available();
fis.seek(0, io::FileInputStream::END);
readVal = 0;
//std::cout << avail << std::endl;
for (int i = 0; i != avail ; i++)
{
fis.seek(-1, io::FileInputStream::CURRENT);
fis.read(&buffer[i], 1);
fis.seek(-1, io::FileInputStream::CURRENT);
}
std::cout << "'" << buffer << "'" << std::endl;
io::StringStream bwd;
bwd.write(buffer, LENGTH_STR);
avail = bwd.available();
if ( avail != LENGTH_STR )
throw except::Exception(Ctxt("Huh??"));
memset(buffer, 0, LENGTH_STR + 1);
for (int i = avail; i > 0; --i)
{
int d = avail - i;
bwd.seek(i - 1, io::StringStream::START);
bwd.read(&buffer[d], 1);
//std::cout << "At: " << d << ": " << buffer[d] << std::endl;
}
std::cout << "'" << buffer << "'" << std::endl;
return 0;
}
catch (except::Exception& ex)
{
std::cout << "Caught ex: " << ex.getTrace() << std::endl;
exit(EXIT_FAILURE);
}
}
static void runConfigManagerTest(TestRunner& tr)
{
tr.group("ConfigManager");
tr.test("init");
{
DynamicObject expect;
expect->setType(Map);
ConfigManager cm;
Config cfg;
cfg[ConfigManager::ID] = "config";
cfg[ConfigManager::MERGE]->setType(Map);
assert(cm.addConfig(cfg));
assertDynoCmp(cm.getConfig("config", true), cfg);
assertDynoCmp(cm.getConfig("config", false), expect);
assertDynoCmp(cm.getConfig("config"), expect);
}
tr.passIfNoException();
tr.test("init & clear");
{
DynamicObject expect;
expect->setType(Map);
ConfigManager cm;
Config cfg;
cfg[ConfigManager::ID] = "config";
cfg[ConfigManager::MERGE]->setType(Map);
assert(cm.addConfig(cfg));
cm.clear();
Config cfg2 = cm.getConfig("config");
assert(cfg2.isNull());
}
tr.passIfException();
tr.test("1 config");
{
DynamicObject expect;
expect->setType(Map);
expect["a"] = 0;
ConfigManager cm;
Config cfg;
cfg[ConfigManager::ID] = "config";
cfg[ConfigManager::MERGE]["a"] = 0;
assert(cm.addConfig(cfg));
assertNoExceptionSet();
assertDynoCmp(cm.getConfig("config"), expect);
}
tr.passIfNoException();
tr.test("config change");
{
ConfigManager cm;
Config cfg;
cfg[ConfigManager::ID] = "config";
cfg[ConfigManager::MERGE]["a"] = 0;
assert(cm.addConfig(cfg));
DynamicObject a;
a["a"] = 0;
assertDynoCmp(cm.getConfig("config"), a);
Config change = cm.getConfig("config", true);
change[ConfigManager::MERGE]["a"] = 1;
assert(cm.setConfig(change));
DynamicObject expect;
expect["a"] = 1;
assert(cm.getConfig("config") != a);
assertDynoCmp(cm.getConfig("config"), expect);
}
tr.passIfNoException();
tr.test("invalid set config");
{
ConfigManager cm;
Config cfg;
cfg[ConfigManager::ID] = "config";
cfg[ConfigManager::MERGE]["a"] = 0;
assert(!cm.setConfig(cfg));
}
tr.passIfException();
tr.test("double add config");
{
ConfigManager cm;
Config cfg;
cfg[ConfigManager::ID] = "config";
cfg[ConfigManager::MERGE]["a"] = 0;
assert(cm.addConfig(cfg));
cfg[ConfigManager::MERGE]["a"] = 1;
assert(cm.addConfig(cfg));
DynamicObject expect;
expect["a"] = 1;
assertDynoCmp(cm.getConfig("config"), expect);
}
tr.passIfNoException();
tr.test("add");
{
DynamicObject expect;
expect["a"] = 0;
expect["b"] = 1;
expect["c"] = 2;
ConfigManager cm;
Config a;
a[ConfigManager::ID] = "config";
a[ConfigManager::MERGE]["a"] = 0;
Config b;
b[ConfigManager::ID] = "config";
b[ConfigManager::MERGE]["b"] = 1;
Config c;
c[ConfigManager::ID] = "config";
c[ConfigManager::MERGE]["c"] = 2;
assert(cm.addConfig(a));
assertNoExceptionSet();
assert(cm.addConfig(b));
assertNoExceptionSet();
assert(cm.addConfig(c));
assertNoExceptionSet();
assertDynoCmp(cm.getConfig("config"), expect);
}
tr.passIfNoException();
tr.test("bad remove");
{
ConfigManager cm;
assert(!cm.removeConfig("error"));
assertExceptionSet();
Exception::clear();
}
tr.passIfNoException();
tr.test("remove");
{
DynamicObject expect;
expect["a"] = 0;
expect["b"] = 1;
expect["c"] = 2;
ConfigManager cm;
Config a;
a[ConfigManager::ID] = "config a";
a[ConfigManager::GROUP] = "group";
a[ConfigManager::MERGE]["a"] = 0;
Config b;
b[ConfigManager::ID] = "config b";
b[ConfigManager::GROUP] = "group";
b[ConfigManager::MERGE]["b"] = 1;
Config c;
c[ConfigManager::ID] = "config c";
c[ConfigManager::GROUP] = "group";
c[ConfigManager::MERGE]["c"] = 2;
assert(cm.addConfig(a));
assertNoExceptionSet();
assert(cm.addConfig(b));
assertNoExceptionSet();
assert(cm.addConfig(c));
assertNoExceptionSet();
assertDynoCmp(cm.getConfig("group"), expect);
DynamicObject expect2;
expect2["a"] = 0;
expect2["c"] = 2;
assert(cm.removeConfig("config b"));
assertDynoCmp(cm.getConfig("group"), expect2);
}
tr.passIfNoException();
tr.test("default value");
{
ConfigManager cm;
Config a;
a[ConfigManager::ID] = "config a";
a[ConfigManager::MERGE] = 1;
assert(cm.addConfig(a));
assertNoExceptionSet();
Config b;
b[ConfigManager::ID] = "config b";
b[ConfigManager::PARENT] = "config a";
b[ConfigManager::MERGE] = ConfigManager::DEFAULT_VALUE;
assert(cm.addConfig(b));
assertNoExceptionSet();
DynamicObject expect;
expect = 1;
assertDynoCmp(cm.getConfig("config b"), expect);
}
tr.passIfNoException();
tr.test("default values");
{
ConfigManager cm;
Config cfga;
cfga[ConfigManager::ID] = "config a";
Config& a = cfga[ConfigManager::MERGE];
a[0] = 10;
a[1] = 11;
a[2]["0"] = 120;
a[2]["1"] = 121;
assert(cm.addConfig(cfga));
assertNoExceptionSet();
Config cfgb;
cfgb[ConfigManager::ID] = "config b";
cfgb[ConfigManager::PARENT] = "config a";
Config& b = cfgb[ConfigManager::MERGE];
b[0] = ConfigManager::DEFAULT_VALUE;
b[1] = 21;
b[2]["0"] = ConfigManager::DEFAULT_VALUE;
b[2]["1"] = 221;
assert(cm.addConfig(cfgb));
assertNoExceptionSet();
DynamicObject expect;
expect[0] = 10;
expect[1] = 21;
expect[2]["0"] = 120;
expect[2]["1"] = 221;
assertDynoCmp(cm.getConfig("config b"), expect);
}
tr.passIfNoException();
tr.test("keyword substitution {RESOURCE_DIR}");
{
DynamicObject expect;
expect["dir"] = "/the/real/dir";
expect["dir-plus"] = "/the/real/dir/plus/more";
//expect["name"] = "Digital Bazaar, Inc.";
ConfigManager cm;
Config a;
a[ConfigManager::ID] = "config";
a[ConfigManager::MERGE]["dir"] = "{RESOURCE_DIR}";
a[ConfigManager::MERGE]["dir-plus"] = "{RESOURCE_DIR}/plus/more";
// FIXME: only supports "{RESOURCE_DIR}" now
//a[ConfigManager::MERGE]["other"] = "{DB}";
cm.setKeyword("RESOURCE_DIR", "/the/real/dir");
//cm.setKeyword("DB", "Digital Bazaar, Inc.");
assert(cm.addConfig(a));
assertNoExceptionSet();
assertDynoCmp(cm.getConfig("config"), expect);
}
tr.passIfNoException();
tr.test("keyword substitution {CURRENT_DIR}");
{
DynamicObject expect;
string cwd;
string cwdPlusMore;
string absoluteDir;
File configFile = File::createTempFile("test-config-file");
FileOutputStream fos(configFile);
// create and populate the config file
string configFileText =
"{\n"
"\"_id_\": \"config\",\n"
"\"_merge_\": {\n"
" \"dir\": \"{CURRENT_DIR}\",\n"
" \"dir-plus\": \"{CURRENT_DIR}/plus/more\" }\n"
"}\n";
fos.write(configFileText.c_str(), configFileText.length());
fos.close();
// modify the current working directory to the expected value
absoluteDir = File::dirname(configFile->getAbsolutePath());
cwd = absoluteDir.c_str();
cwdPlusMore = cwd.c_str();
cwdPlusMore.append("/plus/more");
// set the expected values
expect["dir"] = cwd.c_str();
expect["dir-plus"] = cwdPlusMore.c_str();
// create the configuration
ConfigManager cm;
assert(cm.addConfigFile(configFile->getAbsolutePath(),
true, absoluteDir.c_str(), true, false));
assertNoExceptionSet();
assertDynoCmp(cm.getConfig("config"), expect);
}
tr.passIfNoException();
#if 0
tr.test("user preferences");
{
ConfigManager cm;
// node
// built in or loaded defaults
DynamicObject nodec;
nodec["node"]["host"] = "localhost";
nodec["node"]["port"] = 19100;
nodec["node"]["modulePath"]->append("/usr/lib/bitmunk/modules");
nodec["node"]["userModulePath"] = "~/.bitmunk/modules";
assert(cm.addConfig(nodec));
assertNoExceptionSet();
// user
// loaded defaults
DynamicObject userc;
userc["node"]["port"] = 19100;
userc["node"]["comment"] = "My precious...";
assert(cm.addConfig(userc, ConfigManager::Custom));
assertNoExceptionSet();
// user makes changes during runtime
DynamicObject c = cm.getConfig();
c["node"]["port"] = 19200;
c["node"]["userModulePath"] = "~/.bitmunk/modules:~/.bitmunk/modules-dev";
c["node"][ConfigManager::TMP]["not in changes"] = true;
// get the changes from defaults to current config
// serialize this to disk as needed
DynamicObject changes;
cm.getChanges(changes);
// check it's correct
DynamicObject expect;
expect["node"]["port"] = 19200;
expect["node"]["comment"] = "My precious...";
expect["node"]["userModulePath"] = "~/.bitmunk/modules:~/.bitmunk/modules-dev";
// NOTE: will not have TMP var
assertDynoCmp(changes, expect);
}
tr.passIfNoException();
#endif
tr.test("versioning");
{
ConfigManager cm;
cm.getVersions()->clear();
Config c;
c[ConfigManager::ID] = "config";
assert(cm.addConfig(c));
assertNoExceptionSet();
// config has no version - no check done - pass
cm.addVersion("1");
assert(cm.addConfig(c));
assertNoExceptionSet();
// config has known version - pass
c[ConfigManager::VERSION] = "1";
assert(cm.addConfig(c));
assertNoExceptionSet();
assert(cm.removeConfig("config"));
// config has unknown version - fail
c[ConfigManager::VERSION] = "2";
assert(!cm.addConfig(c));
assertExceptionSet();
Exception::clear();
}
tr.passIfNoException();
tr.test("empty array & map");
{
ConfigManager cm;
DynamicObject a;
a[ConfigManager::ID] = "config";
a[ConfigManager::MERGE][0]->setType(Array);
a[ConfigManager::MERGE][1]->setType(Map);
assert(cm.addConfig(a));
assertNoExceptionSet();
DynamicObject expect;
expect[0]->setType(Array);
expect[1]->setType(Map);
assertDynoCmp(cm.getConfig("config"), expect);
}
tr.passIfNoException();
tr.test("empty group ids");
{
ConfigManager cm;
DynamicObject expect;
expect->setType(Array);
assertDynoCmp(cm.getIdsInGroup("Not-A-Group"), expect);
}
tr.passIfNoException();
tr.test("group ids");
{
ConfigManager cm;
DynamicObject c;
c[ConfigManager::ID] = "c0";
c[ConfigManager::GROUP] = "c";
assert(cm.addConfig(c));
assertNoExceptionSet();
c[ConfigManager::ID] = "c1";
c[ConfigManager::GROUP] = "c";
assert(cm.addConfig(c));
assertNoExceptionSet();
DynamicObject expect;
expect->setType(Array);
expect[0] = "c0";
expect[1] = "c1";
assertDynoCmp(cm.getIdsInGroup("c"), expect);
}
tr.passIfNoException();
tr.test("replace keywords");
{
ConfigManager cm;
DynamicObject c;
c[ConfigManager::ID] = "c";
c[ConfigManager::MERGE]["test"] = "{A}";
DynamicObject vars;
vars["A"] = "a";
assertNoException(ConfigManager::replaceKeywords(c, vars));
DynamicObject expect;
expect[ConfigManager::ID] = "c";
expect[ConfigManager::MERGE]["test"] = "a";
assertDynoCmp(c, expect);
}
tr.passIfNoException();
tr.test("replace keywords (invalid keyword)");
{
ConfigManager cm;
DynamicObject c;
c[ConfigManager::ID] = "c";
c[ConfigManager::MERGE]["test"] = "{UNKNOWN}";
DynamicObject vars;
vars["A"] = "a";
assertException(ConfigManager::replaceKeywords(c, vars));
}
tr.passIfException();
tr.ungroup();
}
void unpackEntry(wxTarInputStream& tis, const char* fn) {
wxFileOutputStream fos(fn);
fos.Write(tis);
}
TEST_F(Benchmark, SimpleStoreIO)
{
{
const int32 n = 1000000;
Envelope e;
Store * s = this->GetTemporaryStore();
ASSERT_TRUE(s->CreateBucket("foo"));
ASSERT_TRUE(s->CreateStreamset("foo", "bar"));
TIMER_START(n);
s->WriteEnvelope("foo", "bar", e);
TIMER_END("zippylog.store.write_1M_empty_envelopes");
}
{
const int32 n = 1000000;
Envelope e;
string path = ::zippylog::platform::PathJoin(this->GetTemporaryDirectory(), "s.zippylog");
{
::zippylog::FileOutputStream fos(path);
for (int32 i = 0; i < n; i++) {
fos.WriteEnvelope(e);
}
}
::zippylog::FileInputStream fis(path);
uint32 read;
TIMER_START(n);
fis.ReadEnvelope(e, read);
TIMER_END("zippylog.FileInputStream.read_1M_empty_envelopes");
}
{
const int32 n = 1000000;
Envelope *envelopes = new Envelope[n];
for (int32 i = 0; i < n; i++) {
envelopes[i] = this->GetRandomEnvelope();
}
{
Store * s = this->GetTemporaryStore();
ASSERT_TRUE(s->CreateBucket("foo"));
ASSERT_TRUE(s->CreateStreamset("foo", "bar"));
TIMER_START_SINGLE();
for (int32 i = 0; i < n; i++) {
s->WriteEnvelope("foo", "bar", envelopes[i]);
}
TIMER_END_COUNT("zippylog.store.write_1M_random_envelopes", n);
}
{
string path = ::zippylog::platform::PathJoin(this->GetTemporaryDirectory(), "s.zippylog");
{
::zippylog::FileOutputStream fos(path);
TIMER_START(n);
fos.WriteEnvelope(envelopes[TIMER_LOOP_VALUE - 1]);
TIMER_END("zippylog.FileOutputStream.write_1M_random_envelopes");
}
Envelope e;
::zippylog::FileInputStream fis(path);
uint32 read;
TIMER_START(n);
fis.ReadEnvelope(e, read);
TIMER_END("zippylog.FileInputStream.read_1M_random_envelopes");
}
delete [] envelopes;
}
}
static void runPingTest(TestRunner& tr)
{
tr.test("Ping");
// create kernel
Kernel k;
k.getEngine()->start();
// create server
Server server;
InternetAddress address("localhost", 19100);
// // create SSL/generic ping connection servicer
// PingConnectionServicer pcs;
//// SslContext context;
//// SslSocketDataPresenter presenter1(&context);
//// NullSocketDataPresenter presenter2;
//// SocketDataPresenterList list(false);
//// list.add(&presenter1);
//// list.add(&presenter2);
// server.addConnectionService(&address, &pcs);//, &list);
// create SSL/generic http connection servicer
HttpConnectionServicer hcs;
// SslContext context;
// SslSocketDataPresenter presenter1(&context);
// NullSocketDataPresenter presenter2;
// SocketDataPresenterList list(false);
// list.add(&presenter1);
// list.add(&presenter2);
server.addConnectionService(&address, &hcs);//, &list);
// create test http request servicer
PingHttpRequestServicer test1("/test");
hcs.addRequestServicer(&test1, false);
if(server.start(&k))
{
printf("Server started.\n");
}
else if(Exception::get() != NULL)
{
printf("Server started with errors=%s\n",
Exception::get()->getMessage());
}
// connect
Url url("http://localhost:19100");
HttpTrailer trailer;
File file("/tmp/index.html");
FileOutputStream fos(file);
HttpClient client;
uint64_t start = System::getCurrentMilliseconds();
client.connect(&url);
client.get(&url, NULL);
client.receiveContent(&fos, &trailer);
uint64_t end = System::getCurrentMilliseconds();
client.disconnect();
// sleep
//Thread::sleep(10000);
server.stop();
printf("Server stopped.\n");
// stop kernel engine
k.getEngine()->stop();
uint64_t millis = end - start;
printf("Connection Time: %" PRIu64 "\n", millis);
// uint64_t millis = test1.end - test1.start;
// double cps = ((double)pcs.serviced) / millis * 1000.0;
// printf("Connections serviced: %d\n", pcs.serviced);
// printf("Time: %" PRIu64 "\n", millis);
// printf("Connections/Second: %f\n", cps);
tr.passIfNoException();
}
void Compiler::run(bool isDump, bool isExecute)
{
if (!isFileExists(filePath)) {
console("File not exists: " << filePath.toStdString());
return;
}
// QString functionId = functionList.at(0).toMap().value("id").toString();
// qDebug() << functionId;
// QVariantList instructionList = sproutDb.readRecords(QString("SELECT * FROM Instructions WHERE functionId=%1").arg(functionId));
// qDebug() << instructionList;
// QString instructionId = instructionList.at(0).toMap().value("id").toString();
// QString instruction = instructionList.at(0).toMap().value("name").toString();
// qDebug() << instructionId << instruction;
// QVariantList argumentList = sproutDb.readRecords(QString("SELECT * FROM Arguments WHERE instructionId=%1").arg(instructionId));
// qDebug() << argumentList;
// QString argument = argumentList.at(0).toMap().value("arg").toString();
// qDebug() << argument;
llvm::LLVMContext& context = llvm::getGlobalContext();
std::unique_ptr<llvm::Module> modulePtr = llvm::make_unique<llvm::Module>("top", context);
llvm::Module* module = modulePtr.get();
llvm::IRBuilder<> builder(context);
// 'main' function prototype
llvm::FunctionType* mainType = llvm::FunctionType::get(builder.getVoidTy(), false);
llvm::Function* mainFunc = llvm::Function::Create(mainType, llvm::Function::ExternalLinkage, "main", module);
llvm::BasicBlock* entry = llvm::BasicBlock::Create(context, "entrypoint", mainFunc);
builder.SetInsertPoint(entry);
// 'print-line' function prototype
// if (instruction == "print-line") {
// auto printArg = builder.CreateGlobalStringPtr(QString(argument).toStdString());
// std::vector<llvm::Type*> putsArgs;
// putsArgs.push_back(builder.getInt8Ty()->getPointerTo());
// llvm::ArrayRef<llvm::Type*> argsRef(putsArgs);
// llvm::FunctionType* putsType = llvm::FunctionType::get(builder.getInt32Ty(), argsRef, false);
// llvm::Constant* putsFunc = module->getOrInsertFunction("puts", putsType);
// builder.CreateCall(putsFunc, printArg);
// // 'read-line' function prototype
// } else if (instruction == "read-line") {
// qDebug() << "readline " << argument;
// }
builder.CreateRetVoid();
if (isDump) {
module->dump();
}
if (isExecute) {
llvm::InitializeNativeTarget();
llvm::InitializeNativeTargetAsmPrinter();
llvm::InitializeNativeTargetAsmParser();
llvm::ExecutionEngine* engine = llvm::EngineBuilder(std::move(modulePtr)).create();
engine->finalizeObject(); // memory for generated code marked executable:
// http://lists.cs.uiuc.edu/pipermail/llvmdev/2013-June/062677.html
engine->runFunction(mainFunc, std::vector<llvm::GenericValue>());
} else {
llvm::InitializeAllTargets();
llvm::InitializeAllTargetMCs();
llvm::InitializeAllAsmPrinters();
llvm::InitializeAllAsmParsers();
llvm::legacy::PassManager pm;
llvm::TargetOptions options;
std::string err;
llvm::Triple triple(module->getTargetTriple());
// if (triple.getTriple().empty()) {
// triple.setTriple(llvm::sys::getDefaultTargetTriple());
// }
const llvm::Target* target = llvm::TargetRegistry::lookupTarget(triple.getTriple(), err);
std::string mcpu, featuresStr;
// llvm::TargetMachine* machineTarget = target->createTargetMachine(triple.getTriple(), mcpu, featuresStr, options);
QString objPath = filePath.replace(".sprout", ".o");
// qDebug() << objPath;
std::error_code ec;
llvm::raw_fd_ostream os(objPath.toStdString(), ec, llvm::sys::fs::F_None);
llvm::formatted_raw_ostream fos(os);
// if (machineTarget->addPassesToEmitFile(pm, fos, llvm::TargetMachine::CGFT_ObjectFile, false)) {
// std::cerr << " target does not support generation of this file type!\n";
// return;
// }
bool result = pm.run(*module);
if (result) {
QProcess* process = new QProcess();
QString binPath = objPath;
binPath.replace(".o", "");
process->start(QString("gcc %1 -o %2").arg(objPath).arg(binPath));
}
}
}
/** Save this BPN network.
Save version differences:
Version 0: Includes saveVersion, type, learningRate, momentum, epochsCompleted and weights.
Version 1: Adds lastPatternTest, patternsCompleted.
Version 2: Adds dynamicLearningRate, dynamicMomentum.
Version 3: Adds inputFieldShape.
@param f The filename to save this object to.
@see BPN::load() */
void BPN::save(string f)
{
LogWriter log;
string message = "Saving ";
message+=f;
LogWriter::printerr(message+"\n", typeToString(id));
ofstream fos(f.c_str(), ios::binary);
if(!fos)
{
message = "BPN: Could not open file for save: ";
message+=f;
LogWriter::printerr(message+"\n", typeToString(id));
return;
}
// save version first
// fos.write(reinterpret_cast<char*>(&saveVersion), sizeof(int));
int tempInt = saveVersion;
if(saveVersion==0 || saveVersion==1 || saveVersion==2) tempInt = 3;
fos.write(reinterpret_cast<char*>(&tempInt), sizeof(int));
// save type
tempInt = id;
fos.write(reinterpret_cast<char*>(&tempInt), sizeof(int));
// save learningRate
float tempFloat = learningRate;
fos.write(reinterpret_cast<char*>(&tempFloat), sizeof(float));
// save momentum value
tempFloat = momentum;
fos.write(reinterpret_cast<char*>(&tempFloat), sizeof(float));
// save the number of epochs this net has completed
tempInt = epochsCompleted;
fos.write(reinterpret_cast<char*>(&tempInt), sizeof(int));
// save number of layers
// save weights as matrices including top row of each for bias weights
vector<Matrix<float> > tempWeights;
tempWeights.resize(weights.size());
for(int i=0;i<tempWeights.size();i++)
{
tempWeights[i].resize(weights[i].width, weights[i].height+1);
for(int x=0;x<tempWeights[i].width;x++)
tempWeights[i].setValue(x, 0, biasWeights[i].getValue(x, 0));
for(int y=0;y<weights[i].height;y++)
{
for(x=0;x<weights[i].width;x++)
tempWeights[i].setValue(x, y+1, weights[i].getValue(x, y));
}
}
float t = 0.0;
int it = tempWeights.size();
tempInt = it;
fos.write(reinterpret_cast<char*>(&tempInt), sizeof(int));
// save layer sizes and layers
for(i=0;i<it;i++)
Matrix<float>::save(fos, &tempWeights[i]);
// Handle old save versions, putting in default values were they weren't previously saved
if(saveVersion==0)
{
double tempDouble = 0;
fos.write(reinterpret_cast<char*>(&tempDouble), sizeof(double));
tempDouble = 0;
fos.write(reinterpret_cast<char*>(&tempDouble), sizeof(double));
bool tempBool = false;
fos.write(reinterpret_cast<char*>(&tempBool), sizeof(bool));
tempBool = false;
fos.write(reinterpret_cast<char*>(&tempBool), sizeof(bool));
// save input field shape
tempInt = inputFieldShape;
fos.write(reinterpret_cast<char*>(&tempInt), sizeof(int));
}
else if(saveVersion==1)
{
double tempDouble = lastPatternTest;
fos.write(reinterpret_cast<char*>(&tempDouble), sizeof(double));
tempDouble = patternsCompleted;
fos.write(reinterpret_cast<char*>(&tempDouble), sizeof(double));
bool tempBool = false;
fos.write(reinterpret_cast<char*>(&tempBool), sizeof(bool));
tempBool = false;
fos.write(reinterpret_cast<char*>(&tempBool), sizeof(bool));
// save input field shape
tempInt = inputFieldShape;
fos.write(reinterpret_cast<char*>(&tempInt), sizeof(int));
}
else if(saveVersion==2)
{
double tempDouble = lastPatternTest;
fos.write(reinterpret_cast<char*>(&tempDouble), sizeof(double));
tempDouble = patternsCompleted;
fos.write(reinterpret_cast<char*>(&tempDouble), sizeof(double));
bool tempBool = dynamicLearningRate;
fos.write(reinterpret_cast<char*>(&tempBool), sizeof(bool));
tempBool = dynamicMomentum;
fos.write(reinterpret_cast<char*>(&tempBool), sizeof(bool));
// save input field shape
tempInt = inputFieldShape;
fos.write(reinterpret_cast<char*>(&tempInt), sizeof(int));
}
else if(saveVersion==3)
{
double tempDouble = lastPatternTest;
fos.write(reinterpret_cast<char*>(&tempDouble), sizeof(double));
tempDouble = patternsCompleted;
fos.write(reinterpret_cast<char*>(&tempDouble), sizeof(double));
bool tempBool = dynamicLearningRate;
fos.write(reinterpret_cast<char*>(&tempBool), sizeof(bool));
tempBool = dynamicMomentum;
fos.write(reinterpret_cast<char*>(&tempBool), sizeof(bool));
// save input field shape
tempInt = inputFieldShape;
fos.write(reinterpret_cast<char*>(&tempInt), sizeof(int));
}
// update save version to latest version
if(saveVersion==0 || saveVersion==1 || saveVersion==2) saveVersion = 3;
fos.close();
}
void CrsMatrixWrapper<ST>::solve(const Teuchos::ArrayView<ST>& x,
const Teuchos::ArrayView<const ST>& b,
escript::SolverBuddy& sb) const
{
typedef VectorType<ST> Vector;
RCP<Vector> X = rcp(new Vector(mat.getDomainMap(), 1));
RCP<Vector> B = rcp(new Vector(mat.getRangeMap(), b, b.size(), 1));
RCP<const Matrix> A = rcpFromRef(mat);
if (escript::isDirectSolver(sb.getSolverMethod())) {
RCP<DirectSolverType<Matrix,Vector> > solver(m_direct);
if (solver.is_null()) {
solver = createDirectSolver<Matrix,Vector>(sb, A, X, B);
m_direct = solver;
if (sb.isVerbose()) {
std::cout << "Using " << solver->description() << std::endl;
std::cout << "Performing symbolic factorization..." << std::flush;
}
solver->symbolicFactorization();
if (sb.isVerbose()) {
std::cout << "done\nPerforming numeric factorization..." << std::flush;
}
solver->numericFactorization();
if (sb.isVerbose()) {
std::cout << "done\n" << std::flush;
}
} else {
if (sb.isVerbose()) {
std::cout << "Using " << solver->description() << std::endl;
}
if (m_resetCalled) {
// matrix structure never changes
solver->setA(A, Amesos2::SYMBFACT);
m_resetCalled = false;
}
solver->setX(X);
solver->setB(B);
}
if (sb.isVerbose()) {
std::cout << "Solving system..." << std::flush;
}
solver->solve();
if (sb.isVerbose()) {
std::cout << "done" << std::endl;
RCP<Teuchos::FancyOStream> fos(Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout)));
solver->printTiming(*fos, Teuchos::VERB_HIGH);
}
} else { // iterative solver
double t0 = Teuchos::Time::wallTime();
RCP<ProblemType<ST> > problem(m_solver);
if (problem.is_null()) {
problem = rcp(new ProblemType<ST>(A, X, B));
m_solver = problem;
RCP<OpType<ST> > prec = createPreconditioner<ST>(A, sb);
m_preconditioner = prec;
if (!prec.is_null()) {
// Trilinos BiCGStab does not support left preconditioners
if (sb.getSolverMethod() == escript::SO_METHOD_BICGSTAB)
problem->setRightPrec(prec);
else
problem->setLeftPrec(prec);
}
problem->setHermitian(sb.isSymmetric());
problem->setProblem();
} else {
for (auto t: problem->getTimers()) {
t->reset();
}
if (m_resetCalled) {
// special case for MueLu preconditioner - call Reuse...
// which honours the "reuse: type" parameter.
RCP<MueLu::TpetraOperator<ST,LO,GO,NT> > mlOp =
Teuchos::rcp_dynamic_cast<MueLu::TpetraOperator<ST,LO,GO,NT> >(m_preconditioner);
if (mlOp.get()) {
RCP<Matrix> A_(Teuchos::rcp_const_cast<Matrix>(A));
MueLu::ReuseTpetraPreconditioner(A_, *mlOp);
}
}
problem->setProblem(X, B);
}
double t1 = Teuchos::Time::wallTime();
RCP<SolverType<ST> > solver = createSolver<ST>(sb);
if (sb.isVerbose()) {
std::cout << "Using " << solver->description() << std::endl;
}
solver->setProblem(problem);
Belos::ReturnType result = solver->solve();
double t2 = Teuchos::Time::wallTime();
const int numIters = solver->getNumIters();
double tol = sb.getTolerance();
try {
tol = solver->achievedTol();
} catch (...) {
}
if (sb.isVerbose()) {
if (result == Belos::Converged) {
sb.updateDiagnostics("converged", true);
std::cout << "The solver took " << numIters
<< " iteration(s) to reach a residual tolerance of "
<< tol << "." << std::endl;
} else {
std::cout << "The solver took " << numIters
<< " iteration(s), but did not reach a relative residual "
"tolerance of " << sb.getTolerance() << "." << std::endl;
}
}
double solverTime = 0.;
for (auto t: problem->getTimers()) {
solverTime += t->totalElapsedTime();
}
sb.updateDiagnostics("set_up_time", t1-t0);
sb.updateDiagnostics("net_time", solverTime);
sb.updateDiagnostics("time", t2-t0);
sb.updateDiagnostics("num_iter", numIters);
sb.updateDiagnostics("residual_norm", tol);
}
X->get1dCopy(x, x.size());
}
void ExportCommon::exportAssets(ExportContext *ctx, bool compileLua) {
QStringList allluafiles;
QStringList allluafiles_abs;
if ((ctx->fileQueue.size() == 0) || (ctx->player)) //No assets -> Player
return;
exportInfo("Exporting assets\n");
for (std::size_t i = 0; i < ctx->folderList.size(); ++i)
ctx->outputDir.mkdir(ctx->folderList[i]);
ctx->allfiles.clear();
ctx->allfiles_abs.clear();
ctx->luafiles.clear();
ctx->luafiles_abs.clear();
QDir path(QFileInfo(ctx->projectFileName_).path());
ExportCommon::progressSteps(ctx->fileQueue.size());
for (std::size_t i = 0; i < ctx->fileQueue.size(); ++i) {
const QString& s1 = ctx->fileQueue[i].first;
const QString& s2 = ctx->fileQueue[i].second;
exportInfo("Exporting %s\n", s1.toUtf8().constData());
ExportCommon::progressStep(s1.toUtf8().constData());
QString src = QDir::cleanPath(path.absoluteFilePath(s2));
QString dst = QDir::cleanPath(ctx->outputDir.absoluteFilePath(s1));
QString rdst = QDir::cleanPath(ctx->outputDir.relativeFilePath(s1));
QString suffix = QFileInfo(dst).suffix().toLower();
bool isJet = false;
if ((!ctx->jetset.isEmpty()) && (!ctx->jetset.contains(suffix))) {
dst += ".jet";
isJet = true;
}
ctx->allfiles.push_back(s1);
ctx->allfiles_abs.push_back(dst);
QFile::remove(dst);
bool copied = false;
if (QFileInfo(src).suffix().toLower() == "lua") {
allluafiles.push_back(s1);
allluafiles_abs.push_back(dst);
if (std::find(ctx->topologicalSort.begin(),
ctx->topologicalSort.end(), std::make_pair(s2, true))
== ctx->topologicalSort.end()) {
ctx->luafiles.push_back(s1);
ctx->luafiles_abs.push_back(dst);
}
// compile lua files (with luac)
if (compileLua) {
QDir toolsDir = QDir(QCoreApplication::applicationDirPath());
#if defined(Q_OS_WIN)
QString luac = toolsDir.filePath("luac.exe");
#else
QString luac = toolsDir.filePath("luac");
#endif
QDir old = QDir::current();
QDir::setCurrent(ctx->outputDir.path());
QString dfile = "\"" + dst + "\"";
QString sfile = "\"" + rdst + "\"";
QFile::copy(src, rdst);
QProcess::execute(quote(luac) + " -o " + dfile + " " + sfile);
if (isJet)
QFile::remove(rdst);
copied = true;
QDir::setCurrent(old.path());
}
}
if (!copied)
QFile::copy(src, dst);
}
#if 0
// compile lua files
if (false)
{
compileThread_ = new CompileThread(ctx->luafiles_abs, false, "", QString(), this);
compileThread_->start();
compileThread_->wait();
delete compileThread_;
}
#endif
// encrypt lua, png, jpg, jpeg and wav files
if (true) {
exportInfo("Encrypting assets\n");
ExportCommon::progressSteps(ctx->allfiles_abs.size());
for (int i = 0; i < ctx->allfiles_abs.size(); ++i) {
ExportCommon::progressStep(
ctx->allfiles_abs[i].toUtf8().constData());
QString filename = ctx->allfiles_abs[i];
QString ext = QFileInfo(ctx->allfiles[i]).suffix().toLower();
exportInfo("Encrypting %s [%s]\n", filename.toUtf8().constData(),
ext.toUtf8().constData());
if (ext != "lua" && ext != "png" && ext != "jpeg" && ext != "jpg"
&& ext != "wav")
continue;
QByteArray encryptionKey =
(ext == "lua") ? ctx->codeKey : ctx->assetsKey;
QFile fis(filename);
if (!fis.open(QIODevice::ReadOnly)) {
exportError("Failed to open %s\n",
filename.toUtf8().constData());
continue;
}
QByteArray data = fis.readAll();
fis.close();
int ks = encryptionKey.size();
for (int j = 32; j < data.size(); ++j)
data[j] = data[j]
^ encryptionKey[((j * 13) + ((j / ks) * 31)) % ks];
QFile fos(filename);
if (!fos.open(QIODevice::WriteOnly)) {
exportError("Failed to save %s\n",
filename.toUtf8().constData());
continue;
}
fos.write(data);
fos.close();
}
}
// compress lua files
if (false) {
for (int i = 0; i < ctx->luafiles_abs.size(); ++i) {
QString file = "\"" + ctx->luafiles_abs[i] + "\"";
QProcess::execute(
"Tools/lua Tools/LuaSrcDiet.lua --quiet " + file + " -o "
+ file);
}
}
ctx->assetfiles = ctx->allfiles;
ctx->assetfiles_abs = ctx->allfiles_abs;
}
/* Create an uncompressed version of the file on the local file system.
* Note this will save zero-length files.
*/
int TskL01Extract::saveFile(const uint64_t fileId, const ArchivedFile &archivedFile)
{
try
{
// If a file with this id already exists we raise an error
std::auto_ptr<TskFile> pFile(TskServices::Instance().getFileManager().getFile(fileId));
if (pFile.get() != NULL && pFile->exists())
{
std::stringstream msg;
msg << "File id " << fileId << " already exists.";
throw TskFileException(msg.str());
}
// Create a blank file
Poco::Path destPath(TskUtilities::toUTF8(TskServices::Instance().getFileManager().getPath(fileId)));
Poco::File destFile(destPath);
destFile.createFile();
// Get data from archive
if (archivedFile.size > 0)
{
Poco::FileOutputStream fos(destFile.path(), std::ios::binary);
uint64_t chunkSize = ExtractChunkSize;
if (archivedFile.size < ExtractChunkSize)
{
chunkSize = archivedFile.size;
}
Poco::SharedPtr<char, Poco::ReferenceCounter, ArrayReleasePolicy<char> > dataBuf(new char[chunkSize]);
uint64_t accum = 0;
ewf::libewf_error_t *ewfError = NULL;
// Read and save data in chunks so that we only put <= ExtractChunkSize bytes on the heap at a time
while (accum < archivedFile.size)
{
ssize_t bytesRead = ewf::libewf_file_entry_read_buffer(archivedFile.entry, dataBuf, chunkSize, &ewfError);
if (bytesRead == -1)
{
std::stringstream logMessage;
char errorString[512];
errorString[0] = '\0';
ewf::libewf_error_backtrace_sprint(ewfError, errorString, 512);
logMessage << "TskL01Extract::saveFile - Error : " << errorString << std::endl;
LOGERROR(logMessage.str());
return -1;
}
fos.write(dataBuf, bytesRead);
accum += bytesRead;
}
fos.close();
}
return 0;
}
catch (Poco::Exception& ex)
{
std::wstringstream msg;
msg << L"TskL01Extract::saveFile - Error saving file from stream : " << ex.displayText().c_str();
LOGERROR(msg.str());
return -2;
}
}
