static void skip_empty_file(Bitcask* bc)
{
int i, last=0;
char opath[255], npath[255];
const char* base = mgr_base(bc->mgr);
for (i=0; i<MAX_BUCKET_COUNT; i++)
{
if (file_exists(gen_path(opath, base, DATA_FILE, i)))
{
if (i != last)
{
mgr_rename(opath, gen_path(npath, base, DATA_FILE, last));
if (file_exists(gen_path(opath, base, HINT_FILE, i)))
{
mgr_rename(opath, gen_path(npath, base, HINT_FILE, last));
}
mgr_unlink(gen_path(opath, base, HTREE_FILE, i));
}
last ++;
}
}
}
vector<vector<string>> findLadders(string beginWord, string endWord, unordered_set<string> &wordList){
unordered_set<string> current, next;
unordered_set<string> visited;
/*** the key-part-structure : record-the-father-relation-ship-to-reconstruct-all-possible-path ***/
unordered_map<string, vector<string>> father;
vector<vector<string>> result;
current.insert(beginWord);
while(!current.empty()){
for(const auto& state : current)
visited.insert(state);
for(const auto& state : current){
if(state_is_target(state, endWord)){
vector<string> path;
gen_path(father, path, beginWord, state, result);
}
const auto new_states = state_extend(state, endWord, wordList, visited);
for(const auto& new_state : new_states){
next.insert(new_state);
father[new_state].push_back(state);
}
}
current.clear();
swap(current, next);
}
return result;
}
void gen_path(unordered_map<string, vector<string> > &father,
const string &start, const string &state, vector<string> &path,
vector<vector<string> > &result) {
path.push_back(state);
if (state == start) {
if (!result.empty()) {
if (path.size() < result[0].size()) {
result.clear();
}
else if (path.size() == result[0].size()) {
// do nothing
}
else { // not possible
throw std::logic_error("not possible to get here ");
}
}
result.push_back(path);
reverse(result.back().begin(), result.back().end());
}
else {
for (const auto& f : father[state]) {
gen_path(father, start, f, path, result);
}
}
path.pop_back();
}
vector<vector<string>> findLadders(string start, string end, unordered_set<string> &dict) {
vector<vector<string>> res;
if (start.size() != end.size()) return res;
unordered_set<string> visited;
unordered_map<string, vector<string> > parents;
unordered_set<string> current, next;
current.insert(start);
visited.insert(start);
bool flag = false;
while (!current.empty() && !flag) {
for (auto word: current) visited.insert(word);
for (auto word: current) {
for (int i = 0; i < word.size(); ++i) {
string origin = word;
for (char c = 'a'; c <= 'z'; ++c) {
if (c == origin[i]) continue;
word[i] = c;
if (word == end) flag = true;
if (dict.count(word) && visited.count(word) == 0) {
next.insert(word);
parents[word].push_back(origin);
}
}
word[i] = origin[i];
}
}
current.clear();
current.swap(next);
}
vector<string> path;
if (flag) gen_path(res, parents, path, start, end);
return res;
}
// A*探索
std::list<Action> a_star(const Field& _field, const Pos& _start, const Pos& _goal)
const
{
if(_goal == _start)
{
return std::list<Action>();
}
//std::cout << "AStar((" << _start.x << ", " << _start.y << ")->(" << _goal.x << ", " << _goal.y <<")" << std::endl;
std::vector< std::vector<AStar_dat> > data(_field.width(), std::vector<AStar_dat>(_field.height(), AStar_dat()));
std::vector<AStar_dat*> open_nodes; //Open状態のノードリスト
std::list<AStar_dat*> close_nodes; //Close状態のノードリスト
//初期ノード生成
data.at(_start.x).at(_start.y).open(_start, _goal);
open_nodes.push_back(&data.at(_start.x).at(_start.y));
Pos buf_pos, buf_move;
AStar_dat* ref_buf_data;
//探索
int open_num = 0;
while(true)
{
// スコア最小のノードを導出
int buf_index = 0, buf_score = open_nodes.front()->get_score();
for(size_t s = 1, cond_s = open_nodes.size(); s < cond_s; ++s)
{
if(buf_score > open_nodes.at(s)->get_score())
{
buf_index = s;
buf_score = open_nodes.at(s)->get_score();
}
}
buf_pos = open_nodes.at(buf_index)->m_pos;
// スコア最小のノードの上下左右を探索する(OPENにする)
for(int i = 0; i < 4; ++i)
{
buf_move = buf_pos.move((Action)i);
ref_buf_data = &data.at(buf_move.x).at(buf_move.y);
if(ref_buf_data->m_state == AStar_dat::NONE)
{
if(ref_buf_data->open(buf_move, _goal, open_nodes.at(buf_index), _field.isWall(buf_move)))
{
open_nodes.push_back(ref_buf_data);
}
++open_num;
if(ref_buf_data->m_pos == _goal)
{
return gen_path(data, _goal);
}
}
}
open_nodes.at(buf_index)->m_state = AStar_dat::CLOSE;
open_nodes.erase(std::next(open_nodes.begin(), buf_index));
}
}
uint64_t data_file_size(Bitcask *bc, int bucket)
{
struct stat st;
char path[255];
gen_path(path, mgr_base(bc->mgr), DATA_FILE, bucket);
if (stat(path, &st) != 0) return 0;
return st.st_size;
}
static inline char *new_path(char *dst, Mgr *mgr, const char *fmt, int i)
{
char *path = gen_path(dst, mgr_base(mgr), fmt, i);
if (!file_exists(dst))
{
char name[16];
sprintf(name, fmt, i);
sprintf(path, "%s/%s", mgr_alloc(mgr, name), name);
}
return path;
}
/* generate download paths ---------------------------------------------------*/
static int gen_paths(gtime_t time, gtime_t time_p, int seqnos, int seqnoe,
const url_t *url, char **stas, int nsta, const char *dir,
paths_t *paths)
{
int i;
if (strstr(url->path,"%s")||strstr(url->path,"%S")) {
for (i=0;i<nsta;i++) {
if (!gen_path(time,time_p,seqnos,seqnoe,url,stas[i],dir,paths)) {
return 0;
}
}
}
else {
if (!gen_path(time,time_p,seqnos,seqnoe,url,"",dir,paths)) {
return 0;
}
}
return 1;
}
void gen_path(vector<vector<string>> &res, unordered_map<string, vector<string> > &parents,
vector<string> &path, string &start, string &word) {
path.push_back(word);
if (start == word) {
res.push_back(path);
reverse(res.back().begin(), res.back().end());
} else {
for (auto s: parents[word]) {
gen_path(res, parents, path, start, s);
}
}
path.pop_back();
}
/* create a temporary filename in directory */
const char *
create_temp_file (const char *directory, const char *prefix, struct gc_arena *gc)
{
static unsigned int counter;
struct buffer fname = alloc_buf_gc (256, gc);
int fd;
const char *retfname = NULL;
unsigned int attempts = 0;
do
{
uint8_t rndbytes[16];
const char *rndstr;
++attempts;
++counter;
prng_bytes (rndbytes, sizeof rndbytes);
rndstr = format_hex_ex (rndbytes, sizeof rndbytes, 40, 0, NULL, gc);
buf_printf (&fname, PACKAGE "_%s_%s.tmp", prefix, rndstr);
retfname = gen_path (directory, BSTR (&fname), gc);
if (!retfname)
{
msg (M_FATAL, "Failed to create temporary filename and path");
return NULL;
}
/* Atomically create the file. Errors out if the file already
exists. */
fd = platform_open (retfname, O_CREAT | O_EXCL | O_WRONLY, S_IRUSR | S_IWUSR);
if (fd != -1)
{
close (fd);
return retfname;
}
else if (fd == -1 && errno != EEXIST)
{
/* Something else went wrong, no need to retry. */
struct gc_arena gcerr = gc_new ();
msg (M_FATAL, "Could not create temporary file '%s': %s",
retfname, strerror_ts (errno, &gcerr));
gc_free (&gcerr);
return NULL;
}
}
while (attempts < 6);
msg (M_FATAL, "Failed to create temporary file after %i attempts", attempts);
return NULL;
}
/*** construct the word-change-path-inversely ***/
void gen_path(unordered_map<string, vector<string>> &father,
vector<string> &path, const string& start, const string& word,
vector<vector<string>> &result){
path.push_back(word);
if(word==start){
result.push_back(path);
/*** why not reverse path directly ? ***/
reverse(result.back().begin(), result.back().end());
} else {
for(const auto &f : father[word]){
gen_path(father, path, start, f, result);
}
}
path.pop_back();
}
/*
* bc_close() is not thread safe, should stop other threads before call it.
* */
void bc_close(Bitcask *bc)
{
char datapath[255], hintpath[255];
if (bc->optimize_flag > 0)
{
bc->optimize_flag = 2;
while (bc->optimize_flag > 0)
{
sleep(1);
}
}
pthread_mutex_lock(&bc->write_lock);
bc_flush(bc, 0, 0);
if (NULL != bc->curr_tree)
{
if (bc->curr_bytes > 0)
{
build_hint(bc->curr_tree, new_path(hintpath, bc->mgr, HINT_FILE, bc->curr));
}
else
{
ht_destroy(bc->curr_tree);
}
bc->curr_tree = NULL;
}
if (bc->curr_bytes == 0) bc->curr --;
if (bc->curr - bc->last_snapshot >= SAVE_HTREE_LIMIT)
{
if (ht_save(bc->tree, new_path(datapath, bc->mgr, HTREE_FILE, bc->curr)) == 0)
{
mgr_unlink(gen_path(datapath, mgr_base(bc->mgr), HTREE_FILE, bc->last_snapshot));
}
else
{
fprintf(stderr, "save HTree to %s failed\n", datapath);
}
}
ht_destroy(bc->tree);
mgr_destroy(bc->mgr);
free(bc->write_buffer);
free(bc);
}
// DFS 遍历树，生成路径
void gen_path(const string &s, const vector<vector<bool> > &prev,
int cur, vector<string> &path, vector<string> &result) {
if (cur == 0) {
string tmp;
for (auto iter = path.crbegin(); iter != path.crend(); ++iter)
tmp += *iter + " ";
tmp.erase(tmp.end() - 1);
result.push_back(tmp);
}
for (size_t i = 0; i < s.size(); ++i) {
if (prev[cur][i]) {
path.push_back(s.substr(i, cur - i));
gen_path(s, prev, i, path, result);
path.pop_back();
}
}
}
// DFS 遍历树，生成路径
void gen_path(const string &s, const vector<vector<bool> > &prev,
int cur, vector<string> &path, vector<string> &result) {
if (cur == 0) {
string tmp;
//push的时候是逆序的，现在再逆过来
for (auto iter = path.crbegin(); iter != path.crend(); ++iter){
tmp += *iter + " ";
}
tmp.erase(tmp.end() - 1);
result.push_back(tmp);
}
for (size_t i = 0; i < s.size(); ++i) {
if (prev[i][cur]) { //如果从s[cur, i)是一个dict中的单词
path.push_back(s.substr(i, cur - i)); //把该单词入栈
gen_path(s, prev, i, path, result); //对字符串剩余部分递归
path.pop_back(); //还原现场
}
}
}
vector<string> wordBreak(string s, unordered_set<string> &dict) {
// 长度为n 的字符串有n+1 个隔板
vector<bool> f(s.length() + 1, false);
// path[i][j] 为true，表示s[j, i) 是一个合法单词，可以从j 处切开
// 第一行未用
vector<vector<bool> > prev(s.length() + 1, vector<bool>(s.length()));
f[0] = true;
for (size_t i = 1; i <= s.length(); ++i) {
for (int j = i - 1; j >= 0; --j) {
if (f[j] && dict.find(s.substr(j, i - j)) != dict.end()) {
f[i] = true;
prev[i][j] = true;
}
}
}
vector<string> result;
vector<string> path;
gen_path(s, prev, s.length(), path, result);
return result;
}
vector<string> wordBreak3(string s, unordered_set<string> &dict)
{
vector<bool> f(s.length() + 1, false);
vector<vector<bool> > prev(s.length() + 1, vector<bool>(s.length()));
f[0] = true;
for (size_t i = 1; i <= s.length(); ++i)
{
for (int j = i - 1; j >= 0; --j)
{
if (f[j] && dict.find(s.substr(j, i - j)) != dict.end())
{
f[i] = true;
prev[i][j] = true;
}
}
}
vector<string> result;
vector<string> path;
gen_path(s, prev, s.length(), path, result);
return result;
}
//跟word break区别：用prev记录下哪些位置可以分词出来
vector<string> wordBreak(string s, unordered_set<string> &dict) {
// 长度为n 的字符串有n+1 个隔板
vector<bool> f(s.length() + 1, false);
// prev[i][j] 为true，表示s[j, i) 是一个合法单词，可以从j 处切开
// 第一行未用
vector<vector<bool> > prev(s.length() + 1, vector<bool>(s.length()));
/*DP, 设状态为f(i)，表示字符串s从第0个到第i个字符之前那个位置是否可以分词，
状态转移方程为：
f(i) = any_of(f(j)&&s[j, i) 在 dict 中); 0 <= j < i
*/
f[0] = true; // 空字符串
for (size_t i = 1; i <= s.length(); ++i) { //从第一个分隔到尾后元素
for (int j = i - 1; j >= 0; --j) {
if (f[j] && dict.find(s.substr(j, i - j)) != dict.end()) {
f[i] = true; //发现第i个位置前能分词仍然要试探其他的位置
prev[i][j] = true; //从j到i前的位置是一个dict中的单词
}
}
}
vector<string> result;
vector<string> path;
gen_path(s, prev, s.length(), path, result);
return result;
}
void bc_scan(Bitcask* bc)
{
char datapath[255], hintpath[255];
int i=0;
struct stat st, hst;
skip_empty_file(bc);
const char* base = mgr_base(bc->mgr);
// load snapshot of htree
for (i=MAX_BUCKET_COUNT-1; i>=0; i--)
{
if (stat(gen_path(datapath, base, HTREE_FILE, i), &st) == 0
&& stat(gen_path(hintpath, base, HINT_FILE, i), &hst) == 0
&& st.st_mtime >= hst.st_mtime
&& (bc->before == 0 || st.st_mtime < bc->before))
{
bc->tree = ht_open(bc->depth, bc->pos, datapath);
if (bc->tree != NULL)
{
bc->last_snapshot = i;
break;
}
else
{
fprintf(stderr, "open HTree from %s failed\n", datapath);
mgr_unlink(datapath);
}
}
}
if (bc->tree == NULL)
{
bc->tree = ht_new(bc->depth, bc->pos);
}
for (i=0; i<MAX_BUCKET_COUNT; i++)
{
if (stat(gen_path(datapath, base, DATA_FILE, i), &st) != 0)
{
break;
}
bc->bytes += st.st_size;
if (i <= bc->last_snapshot) continue;
gen_path(hintpath, base, HINT_FILE, i);
if (bc->before == 0)
{
if (0 == stat(hintpath, &st))
{
scanHintFile(bc->tree, i, hintpath, NULL);
}
else
{
scanDataFile(bc->tree, i, datapath,
new_path(hintpath, bc->mgr, HINT_FILE, i));
}
}
else
{
if (0 == stat(hintpath, &st) &&
(st.st_mtime < bc->before || 0 == stat(datapath, &st) && st.st_mtime < bc->before))
{
scanHintFile(bc->tree, i, hintpath, NULL);
}
else
{
scanDataFileBefore(bc->tree, i, datapath, bc->before);
}
}
}
if (i - bc->last_snapshot > SAVE_HTREE_LIMIT)
{
if (ht_save(bc->tree, new_path(datapath, bc->mgr, HTREE_FILE, i-1)) == 0)
{
mgr_unlink(gen_path(NULL, base, HTREE_FILE, bc->last_snapshot));
bc->last_snapshot = i-1;
}
else
{
fprintf(stderr, "save HTree to %s failed\n", datapath);
}
}
bc->curr = i;
}
vector<vector<string> > findLadders(const string& start,
const string& end, const unordered_set<string> &dict) {
queue<string> q;
unordered_map<string, int> visited; // 判重
unordered_map<string, vector<string> > father; // DAG
// only used by state_extend()
const unordered_map<string, unordered_set<string> >& g = build_graph(dict);
auto state_is_valid = [&](const string& s) {
return dict.find(s) != dict.end() || s == end;
};
auto state_is_target = [&](const string &s) {return s == end; };
auto state_extend = [&](const string &s) {
vector<string> result;
const int new_depth = visited[s] + 1;
auto iter = g.find(s);
if (iter == g.end()) return result;
const auto& list = iter->second;
for (const auto& new_state : list) {
if (state_is_valid(new_state)) {
auto visited_iter = visited.find(new_state);
if (visited_iter != visited.end()) {
const int depth = visited_iter->second;
if (depth < new_depth) {
// do nothing
}
else if (depth == new_depth) {
result.push_back(new_state);
} else { // not possible
throw std::logic_error("not possible to get here");
}
}
else {
result.push_back(new_state);
}
}
}
return result;
};
vector<vector<string>> result;
q.push(start);
visited[start] = 0;
while (!q.empty()) {
// 千万不能用 const auto&，pop() 会删除元素，
// 引用就变成了悬空引用
const auto state = q.front();
q.pop();
// 如果当前路径长度已经超过当前最短路径长度，
// 可以中止对该路径的处理，因为我们要找的是最短路径
if (!result.empty() && visited[state] + 1 > result[0].size()) break;
if (state_is_target(state)) {
vector<string> path;
gen_path(father, start, state, path, result);
continue;
}
// 必须挪到下面，比如同一层A和B两个节点均指向了目标节点，
// 那么目标节点就会在q中出现两次，输出路径就会翻倍
// visited.insert(state);
// 扩展节点
const auto& new_states = state_extend(state);
for (const auto& new_state : new_states) {
if (visited.find(new_state) == visited.end()) {
q.push(new_state);
visited[new_state] = visited[state] + 1;
}
father[new_state].push_back(state);
}
}
return result;
}
void bc_optimize(Bitcask *bc, int limit)
{
int i, total, last = -1;
bc->optimize_flag = 1;
const char *base = mgr_base(bc->mgr);
// remove htree
for (i=0; i < bc->curr; i++)
{
mgr_unlink(gen_path(NULL, base, HTREE_FILE, i));
}
bc->last_snapshot = -1;
time_t limit_time = 0;
if (limit > 3600 * 24 * 365 * 10) // more than 10 years
{
limit_time = limit; // absolute time
}
else
{
limit_time = time(NULL) - limit; // relative time
}
struct stat st;
bool skipped = false;
for (i=0; i < bc->curr && bc->optimize_flag == 1; i++)
{
char datapath[255], hintpath[255];
gen_path(datapath, base, DATA_FILE, i);
gen_path(hintpath, base, HINT_FILE, i);
if (stat(datapath, &st) != 0)
{
continue; // skip empty file
}
// skip recent modified file
if (st.st_mtime > limit_time)
{
skipped = true;
last ++;
if (last != i) // rotate data file
{
char npath[255];
gen_path(npath, base, DATA_FILE, last);
if (symlink(datapath, npath) != 0)
{
fprintf(stderr, "symlink failed: %s -> %s\n", datapath, npath);
last = i;
continue;
}
// update HTree to use new index
if (stat(hintpath, &st) != 0)
{
fprintf(stderr, "no hint file: %s, skip it\n", hintpath);
last = i;
continue;
}
HTree *tree = ht_new(bc->depth, bc->pos);
scanHintFile(tree, i, hintpath, NULL);
struct update_args args;
args.tree = bc->tree;
args.index = last;
ht_visit(tree, update_item_pos, &args);
ht_destroy(tree);
unlink(npath);
mgr_rename(datapath, npath);
mgr_rename(hintpath, gen_path(npath, base, HINT_FILE, last));
}
continue;
}
int deleted = count_deleted_record(bc->tree, i, hintpath, &total);
uint64_t curr_size = data_file_size(bc, i) * (total - deleted/2) / (total+1); // guess
uint64_t last_size = last >= 0 ? data_file_size(bc, last) : -1;
// last data file size
uint32_t recoverd = 0;
if (last == -1 || last_size + curr_size > MAX_BUCKET_SIZE)
{
last ++;
}
while (last < i)
{
char ldpath[255], lhpath[255];
new_path(ldpath, bc->mgr, DATA_FILE, last);
new_path(lhpath, bc->mgr, HINT_FILE, last);
recoverd = optimizeDataFile(bc->tree, i, datapath, hintpath,
skipped, MAX_BUCKET_SIZE, last, ldpath, lhpath);
if (recoverd == 0)
{
last ++;
}
else
{
break;
}
}
if (recoverd == 0)
{
// last == i
recoverd = optimizeDataFile(bc->tree, i, datapath, hintpath,
skipped, MAX_BUCKET_SIZE, last, NULL, NULL);
}
if (recoverd < 0) break; // failed
pthread_mutex_lock(&bc->buffer_lock);
bc->bytes -= recoverd;
pthread_mutex_unlock(&bc->buffer_lock);
}
// update pos of items in curr_tree
pthread_mutex_lock(&bc->write_lock);
pthread_mutex_lock(&bc->flush_lock);
if (i == bc->curr && ++last < bc->curr)
{
char opath[255], npath[255];
gen_path(opath, base, DATA_FILE, bc->curr);
if (file_exists(opath))
{
gen_path(npath, base, DATA_FILE, last);
if (symlink(opath, npath) != 0)
fprintf(stderr, "symlink failed: %s -> %s\n", opath, npath);
struct update_args args;
args.tree = bc->tree;
args.index = last;
ht_visit(bc->curr_tree, update_item_pos, &args);
unlink(npath);
mgr_rename(opath, npath);
}
bc->curr = last;
}
pthread_mutex_unlock(&bc->flush_lock);
pthread_mutex_unlock(&bc->write_lock);
bc->optimize_flag = 0;
}
