// checkpoint related
void counter_service_impl::recover()
{
service::zauto_lock l(_lock);
_counters.clear();
decree max_ver = 0;
std::string name;
std::vector<std::string> sub_list;
auto& path = data_dir();
if (!dsn::utils::filesystem::get_subfiles(path, sub_list, false))
{
dassert(false, "Fail to get subfiles in %s.", path.c_str());
}
for (auto& fpath : sub_list)
{
auto&& s = dsn::utils::filesystem::get_file_name(fpath);
if (s.substr(0, strlen("checkpoint.")) != std::string("checkpoint."))
continue;
decree version = atol(s.substr(strlen("checkpoint.")).c_str());
if (version > max_ver)
{
max_ver = version;
name = data_dir() + "/" + s;
}
}
if (max_ver > 0)
{
recover(name, max_ver);
}
}
// checkpoint related
void simple_kv_service_impl::recover()
{
zauto_lock l(_lock);
_store.clear();
decree maxVersion = 0;
std::string name;
boost::filesystem::directory_iterator endtr;
for (boost::filesystem::directory_iterator it(data_dir());
it != endtr;
++it)
{
auto s = it->path().filename().string();
if (s.substr(0, strlen("checkpoint.")) != std::string("checkpoint."))
continue;
decree version = atol(s.substr(strlen("checkpoint.")).c_str());
if (version > maxVersion)
{
maxVersion = version;
name = data_dir() + "/" + s;
}
}
if (maxVersion > 0)
{
recover(name, maxVersion);
}
}
int counter_service_impl::close(bool clear_state)
{
service::zauto_lock l(_lock);
if (clear_state)
{
if (!dsn::utils::filesystem::remove_path(data_dir()))
{
dassert(false, "Fail to delete directory %s.", data_dir().c_str());
}
}
return 0;
}
int simple_kv_service_impl::open(bool create_new)
{
zauto_lock l(_lock);
if (create_new)
{
boost::filesystem::remove_all(data_dir());
boost::filesystem::create_directory(data_dir());
}
else
{
recover();
}
return 0;
}
int counter_service_impl::apply_checkpoint(learn_state& state, chkpt_apply_mode mode)
{
if (mode == CHKPT_LEARN)
{
recover(state.files[0], state.to_decree_included);
ddebug("counter_service_impl learn checkpoint succeed, last_committed_decree = %" PRId64 "", last_committed_decree());
return ERR_OK;
}
else
{
dassert(CHKPT_COPY == mode, "invalid mode %d", (int)mode);
dassert(state.to_decree_included > _last_durable_decree, "checkpoint's decree is smaller than current");
char name[256];
sprintf(name, "%s/checkpoint.%" PRId64,
data_dir().c_str(),
state.to_decree_included
);
std::string lname(name);
if (!utils::filesystem::rename_path(state.files[0], lname))
{
derror("counter_service_impl copy checkpoint failed, rename path failed");
return ERR_CHECKPOINT_FAILED;
}
else
{
_last_durable_decree = state.to_decree_included;
ddebug("counter_service_impl copy checkpoint succeed, last_durable_decree = %" PRId64 "", _last_durable_decree.load());
return ERR_OK;
}
}
}
int counter_service_impl::checkpoint()
{
service::zauto_lock l(_lock);
if (last_committed_decree() == last_durable_decree())
{
ddebug("counter_service_impl create checkpoint succeed, checkpoint already the latest, last_durable_decree = %" PRId64 "", _last_durable_decree.load());
return 0;
}
// TODO: should use async write instead
char name[256];
sprintf(name, "%s/checkpoint.%" PRId64, data_dir().c_str(), last_committed_decree());
std::ofstream os(name);
uint32_t count = (uint32_t)_counters.size();
os.write((const char*)&count, (uint32_t)sizeof(count));
for (auto it = _counters.begin(); it != _counters.end(); it++)
{
const std::string& k = it->first;
uint32_t sz = (uint32_t)k.length();
os.write((const char*)&sz, (uint32_t)sizeof(sz));
os.write((const char*)&k[0], sz);
os.write((const char*)&it->second, sizeof(int32_t));
}
_last_durable_decree = last_committed_decree();
ddebug("counter_service_impl create checkpoint succeed, last_durable_decree = %" PRId64 "", _last_durable_decree.load());
return 0;
}
// helper routines to accelerate learning
int counter_service_impl::get_checkpoint(decree start, const blob& learn_req, /*out*/ learn_state& state)
{
if (_last_durable_decree.load() == 0 && is_delta_state_learning_supported())
{
checkpoint();
}
if (_last_durable_decree.load() > 0)
{
state.from_decree_excluded = 0;
state.to_decree_included = _last_durable_decree;
char name[256];
sprintf(name, "%s/checkpoint.%" PRId64,
data_dir().c_str(),
_last_durable_decree.load()
);
state.files.push_back(name);
ddebug("counter_service_impl get checkpoint succeed, last_durable_decree = %" PRId64 "", _last_durable_decree.load());
return ERR_OK;
}
else
{
state.from_decree_excluded = 0;
state.to_decree_included = 0;
derror("counter_service_impl get checkpoint failed, no checkpoint found");
return ERR_OBJECT_NOT_FOUND;
}
}
::dsn::error_code simple_kv_service_impl::apply_checkpoint(int64_t commit, const dsn_app_learn_state& state, dsn_chkpt_apply_mode mode)
{
if (mode == DSN_CHKPT_LEARN)
{
recover(state.files[0], state.to_decree_included);
return ERR_OK;
}
else
{
dassert(DSN_CHKPT_COPY == mode, "invalid mode %d", (int)mode);
dassert(state.to_decree_included > last_durable_decree(), "checkpoint's decree is smaller than current");
char name[256];
sprintf(name, "%s/checkpoint.%" PRId64,
data_dir(),
state.to_decree_included
);
std::string lname(name);
if (!utils::filesystem::rename_path(state.files[0], lname))
return ERR_CHECKPOINT_FAILED;
else
{
set_last_durable_decree(state.to_decree_included);
return ERR_OK;
}
}
}
int counter_service_impl::flush(bool force)
{
zauto_lock l(_lock);
if (last_committed_decree() == last_durable_decree())
{
return ERR_OK;
}
// TODO: should use async write instead
char name[256];
sprintf(name, "%s/checkpoint.%lld", data_dir().c_str(),
static_cast<long long int>(last_committed_decree()));
std::ofstream os(name);
uint32_t count = (uint32_t)_counters.size();
os.write((const char*)&count, (uint32_t)sizeof(count));
for (auto it = _counters.begin(); it != _counters.end(); it++)
{
const std::string& k = it->first;
uint32_t sz = (uint32_t)k.length();
os.write((const char*)&sz, (uint32_t)sizeof(sz));
os.write((const char*)&k[0], sz);
os.write((const char*)&it->second, sizeof(int32_t));
}
_last_durable_decree = last_committed_decree();
return ERR_OK;
}
// helper routines to accelerate learning
::dsn::error_code simple_kv_service_impl::get_checkpoint(
int64_t start,
int64_t commit,
void* learn_request,
int learn_request_size,
/* inout */ app_learn_state& state
)
{
if (last_durable_decree() > 0)
{
char name[256];
sprintf(name, "%s/checkpoint.%" PRId64,
data_dir(),
last_durable_decree()
);
state.from_decree_excluded = 0;
state.to_decree_included = last_durable_decree();
state.files.push_back(std::string(name));
return ERR_OK;
}
else
{
state.from_decree_excluded = 0;
state.to_decree_included = 0;
return ERR_OBJECT_NOT_FOUND;
}
}
static void console_command_compile(ConsoleServer& cs, TCPSocket client, const char* json)
{
TempAllocator4096 ta;
JsonObject obj(ta);
DynamicString id(ta);
DynamicString data_dir(ta);
DynamicString platform(ta);
sjson::parse(json, obj);
sjson::parse_string(obj["id"], id);
sjson::parse_string(obj["data_dir"], data_dir);
sjson::parse_string(obj["platform"], platform);
{
TempAllocator512 ta;
StringStream ss(ta);
ss << "{\"type\":\"compile\",\"id\":\"" << id.c_str() << "\",\"start\":true}";
cs.send(client, string_stream::c_str(ss));
}
logi("Compiling '%s'", id.c_str());
bool succ = device()->data_compiler()->compile(data_dir.c_str(), platform.c_str());
if (succ)
logi("Compiled '%s'", id.c_str());
else
loge("Error while compiling '%s'", id.c_str());
{
TempAllocator512 ta;
StringStream ss(ta);
ss << "{\"type\":\"compile\",\"id\":\"" << id.c_str() << "\",\"success\":" << (succ ? "true" : "false") << "}";
cs.send(client, string_stream::c_str(ss));
}
}
void JE_loadPic(SDL_Surface *screen, uint8_t PCXnumber, bool storepal )
{
PCXnumber--;
FILE *f = dir_fopen_die(data_dir(), "tyrian.pic", "rb");
static bool first = true;
if (first)
{
first = false;
uint16_t temp;
efread(&temp, sizeof(uint16_t), 1, f);
for (int i = 0; i < PCX_NUM; i++)
{
efread(&pcxpos[i], sizeof(int32_t), 1, f);
}
pcxpos[PCX_NUM] = ftell_eof(f);
}
uint32_t size = pcxpos[PCXnumber + 1] - pcxpos[PCXnumber];
uint8_t *buffer = malloc(size);
fseek(f, pcxpos[PCXnumber], SEEK_SET);
efread(buffer, sizeof(uint8_t), size, f);
fclose(f);
uint8_t *p = buffer;
uint8_t *s; /* screen pointer, 8-bit specific */
s = (uint8_t *)screen->pixels;
for (int i = 0; i < 320 * 200; )
{
if ((*p & 0xc0) == 0xc0)
{
i += (*p & 0x3f);
memset(s, *(p + 1), (*p & 0x3f));
s += (*p & 0x3f); p += 2;
} else {
i++;
*s = *p;
s++; p++;
}
if (i && (i % 320 == 0))
{
s += screen->pitch - 320;
}
}
free(buffer);
memcpy(colors, palettes[pcxpal[PCXnumber]], sizeof(colors));
if (storepal)
set_palette(colors, 0, 255);
}
void JE_loadMainShapeTables( const char *shpfile )
{
#ifdef TYRIAN2000
const int SHP_NUM = 13;
#else
const int SHP_NUM = 12;
#endif
FILE *f = dir_fopen_die(data_dir(), shpfile, "rb");
JE_word shpNumb;
JE_longint shpPos[SHP_NUM + 1]; // +1 for storing file length
efread(&shpNumb, sizeof(JE_word), 1, f);
assert(shpNumb + 1u == COUNTOF(shpPos));
for (unsigned int i = 0; i < shpNumb; ++i)
efread(&shpPos[i], sizeof(JE_longint), 1, f);
fseek(f, 0, SEEK_END);
for (unsigned int i = shpNumb; i < COUNTOF(shpPos); ++i)
shpPos[i] = ftell(f);
int i;
// fonts, interface, option sprites
for (i = 0; i < 7; i++)
{
fseek(f, shpPos[i], SEEK_SET);
load_sprites(i, f);
}
// player shot sprites
shapesC1.size = shpPos[i + 1] - shpPos[i];
JE_loadCompShapesB(&shapesC1, f);
i++;
// player ship sprites
shapes9.size = shpPos[i + 1] - shpPos[i];
JE_loadCompShapesB(&shapes9 , f);
i++;
// power-up sprites
eShapes[5].size = shpPos[i + 1] - shpPos[i];
JE_loadCompShapesB(&eShapes[5], f);
i++;
// coins, datacubes, etc sprites
eShapes[4].size = shpPos[i + 1] - shpPos[i];
JE_loadCompShapesB(&eShapes[4], f);
i++;
// more player shot sprites
shapesW2.size = shpPos[i + 1] - shpPos[i];
JE_loadCompShapesB(&shapesW2, f);
fclose(f);
}
int simple_kv_service_impl::close(bool clear_state)
{
zauto_lock l(_lock);
if (clear_state)
{
boost::filesystem::remove_all(data_dir());
}
return 0;
}
void Path::FillPlatformSpecificPaths() {
agi::fs::path home = home_dir();
SetToken("?user", home/".aegisub");
SetToken("?local", home/".aegisub");
SetToken("?data", data_dir());
SetToken("?temp", boost::filesystem::temp_directory_path());
SetToken("?dictionary", "/usr/share/hunspell");
SetToken("?docs", P_DOC);
}
ProcessUnitList::ProcessUnitList()
{
lastSequence=0;
_user_data_path=org::esb::config::Config::get("hive.data_path")+"/unitlist";
org::esb::io::File data_dir(_user_data_path+"/");
if(!data_dir.exists()){
data_dir.mkdirs();
}
}
void load_sprites_file( unsigned int table, const char *filename )
{
free_sprites(table);
FILE *f = dir_fopen_die(data_dir(), filename, "rb");
load_sprites(table, f);
fclose(f);
}
::dsn::error_code simple_kv_service_impl::stop(bool clear_state)
{
close_service(gpid());
{
zauto_lock l(_lock);
if (clear_state)
{
dsn_get_app_data_dir(gpid());
if (!dsn::utils::filesystem::remove_path(data_dir()))
{
dassert(false, "Fail to delete directory %s.", data_dir());
}
}
}
return ERR_OK;
}
void JE_loadMainShapeTables( const char *shpfile )
{
const int SHP_NUM = 12;
FILE *f = dir_fopen_die(data_dir(), shpfile, "rb");
JE_word shpNumb;
JE_longint shpPos[SHP_NUM + 1]; // +1 for storing file length
efread(&shpNumb, sizeof(JE_word), 1, f);
assert(shpNumb + 1 <= COUNTOF(shpPos));
for (int i = 0; i < shpNumb; i++)
{
efread(&shpPos[i], sizeof(JE_longint), 1, f);
}
fseek(f, 0, SEEK_END);
shpPos[shpNumb] = ftell(f);
int i;
// fonts, interface, option sprites
for (i = 0; i < 7; i++)
{
fseek(f, shpPos[i], SEEK_SET);
load_sprites(i, f);
}
// player shot sprites
shapesC1.size = shpPos[i + 1] - shpPos[i];
JE_loadCompShapesB(&shapesC1, f);
i++;
// player ship sprites
shapes9.size = shpPos[i + 1] - shpPos[i];
JE_loadCompShapesB(&shapes9 , f);
i++;
// power-up sprites
eShapes6.size = shpPos[i + 1] - shpPos[i];
JE_loadCompShapesB(&eShapes6, f);
i++;
// coins, datacubes, etc sprites
eShapes5.size = shpPos[i + 1] - shpPos[i];
JE_loadCompShapesB(&eShapes5, f);
i++;
// more player shot sprites
shapesW2.size = shpPos[i + 1] - shpPos[i];
JE_loadCompShapesB(&shapesW2, f);
fclose(f);
}
void JE_analyzeLevel( void )
{
FILE *f = dir_fopen_die(data_dir(), levelFile, "rb");
efread(&lvlNum, sizeof(JE_word), 1, f);
for (int x = 0; x < lvlNum; x++)
efread(&lvlPos[x], sizeof(JE_longint), 1, f);
lvlPos[lvlNum] = ftell_eof(f);
fclose(f);
}
void JE_loadCompShapes( Sprite2_array *sprite2s, char s )
{
char buffer[20];
snprintf(buffer, sizeof(buffer), "newsh%c.shp", tolower((unsigned char)s));
FILE *f = dir_fopen_die(data_dir(), buffer, "rb");
sprite2s->size = ftell_eof(f);
JE_loadCompShapesB(sprite2s, f);
fclose(f);
}
error_code replication_app_base::write_internal(mutation_ptr& mu)
{
dassert (mu->data.header.decree == last_committed_decree() + 1, "");
dassert(mu->client_requests.size() == mu->data.updates.size()
&& mu->client_requests.size() > 0,
"data inconsistency in mutation");
int count = static_cast<int>(mu->client_requests.size());
_batch_state = (count == 1 ? BS_NOT_BATCH : BS_BATCH);
for (int i = 0; i < count; i++)
{
if (_batch_state == BS_BATCH && i + 1 == count)
{
_batch_state = BS_BATCH_LAST;
}
auto& r = mu->client_requests[i];
if (r.code != RPC_REPLICATION_WRITE_EMPTY)
{
dinfo("%s: mutation %s dispatch rpc call: %s",
_replica->name(), mu->name(), dsn_task_code_to_string(r.code));
binary_reader reader(mu->data.updates[i]);
dsn_message_t resp = (r.req ? dsn_msg_create_response(r.req) : nullptr);
uint64_t now = dsn_now_ns();
dispatch_rpc_call(r.code, reader, resp);
now = dsn_now_ns() - now;
_app_commit_latency.set(now);
}
else
{
// empty mutation write
}
if (_physical_error != 0)
{
derror("%s: physical error %d occurs in replication local app %s",
_replica->name(), _physical_error, data_dir().c_str());
return ERR_LOCAL_APP_FAILURE;
}
}
++_last_committed_decree;
_replica->update_commit_statistics(count);
_app_commit_throughput.add((uint64_t)count);
_app_commit_decree.increment();
return ERR_OK;
}
void load_music( void )
{
if (music_file == NULL)
{
music_file = dir_fopen_die(data_dir(), "music.mus", "rb");
efread(&song_count, sizeof(song_count), 1, music_file);
song_offset = malloc((song_count + 1) * sizeof(song_offset));
efread(song_offset, 4, song_count, music_file);
song_offset[song_count] = ftell_eof(music_file);
}
}
int counter_service_impl::open(bool create_new)
{
service::zauto_lock l(_lock);
if (create_new)
{
auto& dir = data_dir();
dsn::utils::filesystem::remove_path(dir);
dsn::utils::filesystem::create_directory(dir);
}
else
{
recover();
}
return 0;
}
// checkpoint related
void simple_kv_service_impl::recover()
{
zauto_lock l(_lock);
_store.clear();
int64_t maxVersion = 0;
std::string name;
std::vector<std::string> sub_list;
std::string path = data_dir();
if (!dsn::utils::filesystem::get_subfiles(path, sub_list, false))
{
dassert(false, "Fail to get subfiles in %s.", path.c_str());
}
for (auto& fpath : sub_list)
{
auto&& s = dsn::utils::filesystem::get_file_name(fpath);
if (s.substr(0, strlen("checkpoint.")) != std::string("checkpoint."))
continue;
int64_t version = static_cast<int64_t>(atoll(s.substr(strlen("checkpoint.")).c_str()));
if (version > maxVersion)
{
maxVersion = version;
name = std::string(data_dir()) + "/" + s;
}
}
sub_list.clear();
if (maxVersion > 0)
{
recover(name, maxVersion);
set_last_durable_decree(maxVersion);
}
}
::dsn::error_code simple_kv_service_impl::checkpoint(int64_t version)
{
char name[256];
sprintf(name, "%s/checkpoint.%" PRId64, data_dir(),
version
);
zauto_lock l(_lock);
if (version == last_durable_decree())
{
dassert(utils::filesystem::file_exists(name),
"checkpoint file %s is missing!",
name
);
return ERR_OK;
}
// TODO: should use async write instead
std::ofstream os(name, std::ios::binary);
uint64_t count = (uint64_t)_store.size();
int magic = 0xdeadbeef;
os.write((const char*)&count, (uint32_t)sizeof(count));
os.write((const char*)&magic, (uint32_t)sizeof(magic));
for (auto it = _store.begin(); it != _store.end(); ++it)
{
const std::string& k = it->first;
uint32_t sz = (uint32_t)k.length();
os.write((const char*)&sz, (uint32_t)sizeof(sz));
os.write((const char*)&k[0], sz);
const std::string& v = it->second;
sz = (uint32_t)v.length();
os.write((const char*)&sz, (uint32_t)sizeof(sz));
os.write((const char*)&v[0], sz);
}
os.close();
// TODO: gc checkpoints
set_last_durable_decree(version);
return ERR_OK;
}
void replication_app_base::dispatch_rpc_call(int code, binary_reader& reader, dsn_message_t response)
{
auto it = _handlers.find(code);
if (it != _handlers.end())
{
if (response)
{
int err = 0; // replication layer error
::marshall(response, err);
}
it->second(reader, response);
}
else
{
dassert(false, "cannot find handler for rpc code %d in %s", code, data_dir().c_str());
}
}
TKeyhoteeApplication::TKeyhoteeApplication(int& argc, char** argv)
:QApplication(argc, argv),
_loaded_profile_name(DEF_PROFILE_NAME),
_main_window(nullptr),
_profile_wizard(nullptr),
_exit_status(TExitStatus::SUCCESS)
{
assert(s_Instance == nullptr && "Only one instance allowed at time");
s_Instance = this;
_backend_app = bts::application::instance();
/// \warning use stdwstring to avoid problems related to paths containing native chars.
auto str_data_dir = QStandardPaths::writableLocation(QStandardPaths::DataLocation).toStdWString();
boost::filesystem::path data_dir(str_data_dir);
fc::path profile_dir( data_dir / "profiles" );
_backend_app->set_profile_directory( profile_dir );
}
// helper routines to accelerate learning
int simple_kv_service_impl::get_learn_state(decree start, const blob& learn_req, __out_param learn_state& state)
{
::dsn::binary_writer writer;
zauto_lock l(_lock);
int magic = 0xdeadbeef;
writer.write(magic);
writer.write(_last_committed_decree.load());
dassert(_last_committed_decree >= 0, "");
uint64_t count = static_cast<uint64_t>(_store.size());
writer.write(count);
for (auto it = _store.begin(); it != _store.end(); it++)
{
writer.write(it->first);
writer.write(it->second);
}
auto bb = writer.get_buffer();
auto buf = bb.buffer();
state.meta.push_back(blob(buf, static_cast<int>(bb.data() - bb.buffer().get()), bb.length()));
// Test Sample
if (_test_file_learning)
{
std::stringstream ss;
ss << env::random32(0, 10000);
auto learn_test_file = data_dir() + "/test_learning_" + ss.str() + ".txt";
state.files.push_back(learn_test_file);
std::ofstream fout(learn_test_file.c_str());
fout << "DEADBEEF" << std::endl;
fout.close();
}
return 0;
}
void JE_loadPCX( const char *file ) // this is only meant to load tshp2.pcx
{
Uint8 *s = (Uint8 *)VGAScreen->pixels; /* 8-bit specific */
SDL_RWops *f = dir_fopen_die(data_dir(), file, "rb");
efseek(f, -769, SEEK_END);
if (efgetc(f) == 12)
{
for (int i = 0; i < 256; i++)
{
efread(f, &colors[i].r, 1, 1);
efread(f, &colors[i].g, 1, 1);
efread(f, &colors[i].b, 1, 1);
}
}
efseek(f, 128, SEEK_SET);
for (int i = 0; i < 320 * 200; )
{
Uint8 p = efgetc(f);
if ((p & 0xc0) == 0xc0)
{
i += (p & 0x3f);
memset(s, efgetc(f), (p & 0x3f));
s += (p & 0x3f);
} else {
i++;
*s = p;
s++;
}
if (i && (i % 320 == 0))
{
s += VGAScreen->pitch - 320;
}
}
efclose(f);
}
