void error(int errnum, const char *format, ...)
{
va_list ap;
va_start(ap,format);
fprintf(stderr,"%s",progname);
if ( format!=NULL ) {
fprintf(stderr,": ");
vfprintf(stderr,format,ap);
}
if ( errnum!=0 ) {
fprintf(stderr,": %s",strerror(errnum));
}
if ( format==NULL && errnum==0 ) {
fprintf(stderr,": unknown error");
}
fprintf(stderr,"\nTry `%s --help' for more information.\n",progname);
va_end(ap);
write_meta("internal-error","%s","runguard error");
exit(exit_failure);
}
void relation(const osmium::Relation& relation) {
if (m_write_change_ops) {
open_close_op_tag(relation.visible() ? (relation.version() == 1 ? operation::op_create : operation::op_modify) : operation::op_delete);
}
write_prefix();
m_out += "<relation";
write_meta(relation);
if (relation.tags().empty() && relation.members().empty()) {
m_out += "/>\n";
return;
}
m_out += ">\n";
for (const auto& member : relation.members()) {
write_prefix();
m_out += " <member type=\"";
m_out += item_type_to_name(member.type());
oprintf(m_out, "\" ref=\"%" PRId64 "\" role=\"", member.ref());
xml_string(m_out, member.role());
m_out += "\"/>\n";
}
write_tags(relation.tags());
write_prefix();
m_out += "</relation>\n";
}
void way(const osmium::Way& way) {
if (m_write_change_ops) {
open_close_op_tag(way.visible() ? (way.version() == 1 ? operation::op_create : operation::op_modify) : operation::op_delete);
}
write_prefix();
m_out += "<way";
write_meta(way);
if (way.tags().empty() && way.nodes().empty()) {
m_out += "/>\n";
return;
}
m_out += ">\n";
for (const auto& node_ref : way.nodes()) {
write_prefix();
oprintf(m_out, " <nd ref=\"%" PRId64 "\"/>\n", node_ref.ref());
}
write_tags(way.tags());
write_prefix();
m_out += "</way>\n";
}
void node(const osmium::Node& node) {
if (m_write_change_ops) {
open_close_op_tag(node.visible() ? (node.version() == 1 ? operation::op_create : operation::op_modify) : operation::op_delete);
}
write_prefix();
m_out += "<node";
write_meta(node);
if (node.location()) {
m_out += " lat=\"";
osmium::util::double2string(std::back_inserter(m_out), node.location().lat_without_check(), 7);
m_out += "\" lon=\"";
osmium::util::double2string(std::back_inserter(m_out), node.location().lon_without_check(), 7);
m_out += "\"";
}
if (node.tags().empty()) {
m_out += "/>\n";
return;
}
m_out += ">\n";
write_tags(node.tags());
write_prefix();
m_out += "</node>\n";
}
static void process(int ibuf, int nbuf){
if (VERBOSE){
fprintf(stderr, "%02d\n", ibuf);
}
char buf[80];
sprintf(buf, "%s/%06d/", OUTROOT, CYCLE);
mkdir(buf, 0777);
sprintf(buf, "%s/%06d/%d.%02d", OUTROOT, CYCLE, dev->getDevnum(), ibuf);
if (NO_OVERWRITE){
struct stat stat_buf;
int rc = stat(buf, &stat_buf);
DIAG("stat:rc %d errno %d\n", rc, errno);
if (rc == 0){
err("OVERRUN: NO_OVERWRITE SET and \"%s\" exists",
buf);
exit(1);
}else if (errno != ENOENT){
err("OVERRUN: NO_OVERWRITE SET and \"%s\" exists",
buf);
perror("buf");
exit(errno);
}else{
; /* ENOENT - that's good! */
}
}
int outfp = open(buf, O_MODE, PERM);
if (outfp == -1){
perror(buf);
_exit(errno);
}
write(outfp, dev->getHostBufferMapping(ibuf), maxlen);
strcat(buf, ".id");
int out_meta = open(buf, O_MODE, PERM);
write_meta(out_meta, ibuf, nbuf);
close(out_meta);
close(outfp); /* close data last - we monitor this one */
if (PUT_DATA){
char *cp = strstr(buf, ".id");
if (cp){
*cp = '\0';
}
}
puts(buf);
if (USLEEP){
usleep(USLEEP);
}
}
static int create_meta(TEE_Result *errno, struct sql_fs_fd *fdp)
{
TEE_Result res;
memset(&fdp->meta, 0, sizeof(fdp->meta));
res = tee_fs_generate_fek(fdp->encrypted_fek, TEE_FS_KM_FEK_SIZE);
if (res != TEE_SUCCESS)
return -1;
return write_meta(errno, fdp);
}
static void process(int ibuf, int nbuf){
if (VERBOSE){
fprintf(stderr, "%02d\n", ibuf);
}
static char data_fname[80];
static char old_fname[80];
char buf[80];
if (icat == 0){
sprintf(buf, "%s/%06d/", OUTROOT, CYCLE);
mkdir(buf, 0777);
sprintf(data_fname, "%s/%06d/%d.%02d", OUTROOT, CYCLE, dev->getDevnum(), ibuf);
outfp = open(data_fname, O_MODE, PERM);
if (outfp == -1){
perror(buf);
_exit(errno);
}
strcpy(buf, data_fname);
strcat(buf, ".id");
int out_meta = open(buf, O_MODE, PERM);
write_meta(out_meta, ibuf, nbuf);
close(out_meta);
}
if (NO_OVERWRITE){
fail_if_exists(data_fname);
}
write(outfp, dev->getHostBufferMapping(ibuf), maxlen);
if (++icat > CONCAT){
close(outfp); /* close data last - we monitor this one */
if (PUT_DATA){
if (strlen(old_fname)){
puts(old_fname);
}
strcpy(old_fname, data_fname);
}else{
puts(buf);
}
icat = 0;
}
if (USLEEP){
usleep(USLEEP);
}
}
void way(const osmium::Way& way) {
*m_out += 'w';
write_meta(way);
*m_out += " N";
bool first = true;
for (const auto& node_ref : way.nodes()) {
if (first) {
first = false;
} else {
*m_out += ',';
}
output_formatted("n%" PRId64, node_ref.ref());
}
*m_out += '\n';
}
void relation(const osmium::Relation& relation) {
*m_out += 'r';
write_meta(relation);
*m_out += " M";
bool first = true;
for (const auto& member : relation.members()) {
if (first) {
first = false;
} else {
*m_out += ',';
}
*m_out += item_type_to_char(member.type());
output_formatted("%" PRId64 "@", member.ref());
*m_out += member.role();
}
*m_out += '\n';
}
void relation(const osmium::Relation& relation) {
m_out << "r";
write_meta(relation);
m_out << " M";
int n=0;
for (const auto& member : relation.members()) {
if (n++ != 0) {
m_out << ",";
}
m_out << item_type_to_char(member.type())
<< member.ref()
<< "!"
<< member.role();
}
write_tags(relation.tags());
m_out << "\n";
::write(this->fd(), m_out.str().c_str(), m_out.str().size());
m_out.str("");
}
void node(const osmium::Node& node) {
*m_out += 'n';
write_meta(node);
write_location(node.location(), 'x', 'y');
*m_out += '\n';
}
static int sql_fs_write(TEE_Result *errno, int fd, const void *buf, size_t len)
{
struct sql_fs_fd *fdp;
size_t remain_bytes = len;
const uint8_t *data_ptr = buf;
int start_block_num;
int end_block_num;
int res = -1;
int ret;
DMSG("(fd: %d, buf: %p, len: %zu)...", fd, (void *)buf, len);
mutex_lock(&sql_fs_mutex);
*errno = TEE_ERROR_GENERIC;
fdp = handle_lookup(&fs_db, fd);
if (!fdp) {
*errno = TEE_ERROR_BAD_PARAMETERS;
goto exit_ret;
}
if (!len) {
res = 0;
goto exit_ret;
}
if (!buf) {
*errno = TEE_ERROR_BAD_PARAMETERS;
goto exit_ret;
}
if (fdp->flags & TEE_FS_O_RDONLY) {
*errno = TEE_ERROR_ACCESS_CONFLICT;
goto exit_ret;
}
sql_fs_begin_transaction_rpc();
if (fdp->meta.length < (size_t)fdp->pos) {
/* Fill hole */
res = sql_fs_ftruncate_internal(errno, fd, fdp->pos);
if (res < 0)
goto exit;
}
start_block_num = block_num(fdp->pos);
end_block_num = block_num(fdp->pos + len - 1);
while (start_block_num <= end_block_num) {
tee_fs_off_t offset = fdp->pos % BLOCK_SIZE;
size_t size_to_write = MIN(remain_bytes, (size_t)BLOCK_SIZE);
if (size_to_write + offset > BLOCK_SIZE)
size_to_write = BLOCK_SIZE - offset;
res = write_block_partial(errno, fdp, start_block_num,
data_ptr, size_to_write, offset);
if (res < 0)
goto exit;
data_ptr += size_to_write;
remain_bytes -= size_to_write;
fdp->pos += size_to_write;
start_block_num++;
}
fdp->meta.length = fdp->pos;
res = write_meta(errno, fdp);
exit:
sql_fs_end_transaction_rpc(res < 0);
exit_ret:
mutex_unlock(&sql_fs_mutex);
ret = (res < 0) ? res : (int)len;
DMSG("...%d", ret);
return ret;
}
static int sql_fs_ftruncate_internal(TEE_Result *errno, int fd,
tee_fs_off_t new_length)
{
struct sql_fs_fd *fdp;
tee_fs_off_t old_length;
int rc = -1;
DMSG("(fd: %d, new_length: %" PRId64 ")...", fd, new_length);
*errno = TEE_ERROR_GENERIC;
fdp = handle_lookup(&fs_db, fd);
if (!fdp) {
*errno = TEE_ERROR_BAD_PARAMETERS;
goto exit_ret;
}
old_length = (tee_fs_off_t)fdp->meta.length;
if (new_length == old_length) {
rc = 0;
goto exit_ret;
}
sql_fs_begin_transaction_rpc();
if (new_length < old_length) {
/* Trim unused blocks */
int old_last_block = block_num(old_length);
int last_block = block_num(new_length);
tee_fs_off_t off;
if (last_block < old_last_block) {
off = block_pos_raw(last_block);
rc = tee_fs_rpc_ftruncate(OPTEE_MSG_RPC_CMD_SQL_FS,
fd, off);
if (rc < 0)
goto exit;
}
} else {
/* Extend file with zeroes */
tee_fs_off_t off = old_length % BLOCK_SIZE;
size_t bnum = block_num(old_length);
size_t end_bnum = block_num(new_length);
while (bnum <= end_bnum) {
size_t len = (size_t)BLOCK_SIZE - (size_t)off;
rc = write_block_partial(errno, fdp, bnum, NULL, len,
off);
if (rc < 0)
goto exit;
off = 0;
bnum++;
}
}
fdp->meta.length = new_length;
rc = write_meta(errno, fdp);
exit:
sql_fs_end_transaction_rpc(rc < 0);
exit_ret:
DMSG("...%d", rc);
return rc;
}
void add_dir(uint parent, struct direct *sp)
{
uint size, dsize, dseek, nlink = 2;
int f, n, i;
struct direct *de, *p;
DIR *d;
struct dirent *dp;
struct stat st;
static uchar zeros[4096];
// build directory
de = sp;
d = opendir(".");
sp->d_ino = bn; xstrncpy(sp->d_name, ".", DIRSIZ); sp++;
sp->d_ino = parent; xstrncpy(sp->d_name, "..", DIRSIZ); sp++;
while (dp = readdir(d)) {
if (!strcmp(dp->d_name, ".") || !strcmp(dp->d_name, "..") || strlen(dp->d_name) > DIRSIZ) continue;
if (stat(dp->d_name, &st)) { dprintf(2, "stat(%s) failed\n", dp->d_name); exit(-1); }
if ((st.st_mode & S_IFMT) == S_IFREG) sp->d_ino = st.st_size;
else if ((st.st_mode & S_IFMT) == S_IFDIR) { sp->d_ino = -1; nlink++; }
else continue;
xstrncpy(sp->d_name, dp->d_name, DIRSIZ);
sp++;
}
closedir(d);
parent = bn;
// write inode
write_meta(dsize = (uint)sp - (uint)de, S_IFDIR, nlink);
dseek = (bn - ((dsize + 4095) / 4096)) * 4096;
// write directory
write_disk(de, dsize);
if (dsize & 4095) write_disk(zeros, 4096 - (dsize & 4095));
// add directory contents
for (p = de + 2; p < sp; p++) {
size = p->d_ino; p->d_ino = bn;
if (size == -1) { // subdirectory
chdir(p->d_name);
add_dir(parent, sp);
chdir("..");
} else { // file
write_meta(size, S_IFREG, 1);
if (size) {
if ((f = open(p->d_name, O_RDONLY)) < 0) { dprintf(2, "open(%s) failed\n", p->d_name); exit(-1); }
for (n = size; n; n -= i) {
if ((i = read(f, buf, (n > BUFSZ) ? BUFSZ : n)) < 0) { dprintf(2, "read(%s) failed\n", p->d_name); exit(-1); }
write_disk(buf, i);
}
close(f);
if (size & 4095) write_disk(zeros, 4096 - (size & 4095));
}
}
}
// update directory
lseek(disk, dseek, SEEK_SET);
write_disk(de, dsize);
lseek(disk, 0, SEEK_END);
}
int main(int argc, char **argv)
{
struct sigaction sigact;
sigset_t sigmask;
pid_t pid;
int status;
int exitcode, myexitcode;
int opt;
char *arg;
int i, r, fd_out, newcmd, argsize = 0;
int pipe_fd[MAX_CMDS][2];
int progout_pipe_fd[MAX_CMDS][2];
progname = argv[0];
/* Parse command-line options */
be_verbose = show_help = show_version = 0;
opterr = 0;
while ( (opt = getopt_long(argc,argv,"+o:M:v",long_opts,(int *) 0))!=-1 ) {
switch ( opt ) {
case 0: /* long-only option */
break;
case 'v': /* verbose option */
be_verbose = 1;
verb("verbose mode enabled");
break;
case 'o': /* outprog option */
write_progout = 1;
progoutfilename = strdup(optarg);
verb("writing program #2 output to '%s'", progoutfilename);
break;
case 'M': /* outmeta option */
outputmeta = 1;
metafilename = strdup(optarg);
verb("writing metadata to '%s'", metafilename);
break;
case ':': /* getopt error */
case '?':
error(0,"unknown option or missing argument `%c'",optopt);
break;
default:
error(0,"getopt returned character code `%c' ??",(char)opt);
}
}
if ( show_help ) usage();
if ( show_version ) version(PROGRAM,VERSION);
if ( argc<=optind ) error(0,"no command specified");
/* Parse commands to be executed */
ncmds = 0; /* Zero-based index to current command in loop,
contains #commands specified after loop */
newcmd = 1; /* Is current command newly started? */
for(i=optind; i<argc; i++) {
/* Check for commands separator */
if ( strcmp(argv[i],"=")==0 ) {
ncmds++;
if ( newcmd ) error(0,"empty command #%d specified", ncmds);
newcmd = 1;
if ( ncmds+1>MAX_CMDS ) {
error(0,"too many commands specified: %d > %d", ncmds+1, MAX_CMDS);
}
continue;
}
/* Un-escape multiple = at start of argument */
arg = argv[i];
if ( strncmp(arg,"==",2)==0 ) arg++;
if ( newcmd ) {
newcmd = 0;
cmd_name[ncmds] = arg;
cmd_nargs[ncmds] = 0;
argsize = 5;
cmd_args[ncmds] = malloc(argsize*sizeof(void *));
if ( cmd_args[ncmds]==NULL ) error(0,"cannot allocate memory");
} else {
if ( cmd_nargs[ncmds]+1>argsize ) {
argsize += 10;
cmd_args[ncmds] = realloc(cmd_args[ncmds],argsize*sizeof(void *));
if ( cmd_args[ncmds]==NULL ) error(0,"cannot allocate memory");
}
cmd_args[ncmds][cmd_nargs[ncmds]++] = arg;
}
}
ncmds++;
if ( newcmd ) error(0,"empty command #%d specified", ncmds);
if ( ncmds!=2 ) {
error(0,"%d commands specified, 2 required", ncmds);
}
/* Install TERM signal handler */
if ( sigemptyset(&sigmask)!=0 ) error(errno,"creating signal mask");
if ( sigprocmask(SIG_SETMASK, &sigmask, NULL)!=0 ) {
error(errno,"unmasking signals");
}
if ( sigaddset(&sigmask,SIGTERM)!=0 ) error(errno,"setting signal mask");
sigact.sa_handler = terminate;
sigact.sa_flags = SA_RESETHAND | SA_RESTART;
sigact.sa_mask = sigmask;
if ( sigaction(SIGTERM,&sigact,NULL)!=0 ) {
error(errno,"installing signal handler");
}
validator_exited_first = 0;
submission_still_alive = 1;
/* Create pipes and by default close all file descriptors when
executing a forked subcommand, required ones are reset below. */
for(i=0; i<ncmds; i++) {
if ( pipe(pipe_fd[i])!=0 ) error(errno,"creating pipes");
verb("command #%d: read fd=%d, write fd=%d", i, pipe_fd[i][0], pipe_fd[i][1]);
set_fd_close_exec(pipe_fd[i][0], 1);
set_fd_close_exec(pipe_fd[i][1], 1);
#ifdef PROC_MAX_PIPE_SIZE
resize_pipe(pipe_fd[i][1]);
#endif
}
/* Setup file and extra pipe for writing program output. */
if ( write_progout ) {
if ( (progoutfile = fopen(progoutfilename,"w"))==NULL ) {
error(errno,"cannot open `%s'",progoutfilename);
}
for(i=0; i<ncmds; i++) {
if ( pipe(progout_pipe_fd[i])!=0 ) error(errno,"creating pipes");
set_fd_close_exec(progout_pipe_fd[i][0], 1);
set_fd_close_exec(progout_pipe_fd[i][1], 1);
#ifdef PROC_MAX_PIPE_SIZE
resize_pipe(progout_pipe_fd[i][1]);
#endif
verb("writing program #%d output via pipe %d -> %d",
i, progout_pipe_fd[i][1], progout_pipe_fd[i][0]);
}
}
/* Execute commands as subprocesses and connect pipes as required. */
for(i=0; i<ncmds; i++) {
fd_out = write_progout ? progout_pipe_fd[i][1] : pipe_fd[1-i][1];
cmd_fds[i][0] = pipe_fd[i][0];
cmd_fds[i][1] = fd_out;
cmd_fds[i][2] = FDREDIR_NONE;
verb("pipes for command #%d are %d and %d", i, cmd_fds[i][0], cmd_fds[i][1]);
set_fd_close_exec(pipe_fd[i][0], 0);
set_fd_close_exec(fd_out, 0);
cmd_exit[i] = -1;
cmd_pid[i] = execute(cmd_name[i], (const char **)cmd_args[i],
cmd_nargs[i], cmd_fds[i], 0);
if ( cmd_pid[i]==-1 ) error(errno,"failed to execute command #%d",i+1);
verb("started #%d, pid %d: %s",i+1,cmd_pid[i],cmd_name[i]);
set_fd_close_exec(pipe_fd[i][0], 1);
set_fd_close_exec(fd_out, 1);
}
gettimeofday(&start_time, NULL);
if ( write_progout ) {
if ( close(pipe_fd[1][0])!=0 ) error(errno,"closing pipe read end");
if ( close(pipe_fd[0][0])!=0 ) error(errno,"closing pipe read end");
if ( close(progout_pipe_fd[0][1])!=0 ) error(errno,"closing pipe write end");
if ( close(progout_pipe_fd[1][1])!=0 ) error(errno,"closing pipe write end");
} else {
for(i=0; i<ncmds; i++) {
if ( close(pipe_fd[i][0])!=0 ) error(errno,"closing pipe read end");
if ( close(pipe_fd[i][1])!=0 ) error(errno,"closing pipe write end");
}
}
/* Wait for running child commands and check exit status. */
while ( 1 ) {
if ( write_progout ) {
for(i=0; i<ncmds; i++) {
pump_pipes(&progout_pipe_fd[i][0], &pipe_fd[1-i][1], 1-i);
}
pid = 0;
for(i=0; i<ncmds; i++) {
if ( cmd_exit[i]==-1 ) {
pid = waitpid(cmd_pid[i], &status, WNOHANG);
if ( pid != 0 ) break;
}
}
if ( pid==0 ) continue;
} else {
pid = waitpid(-1, &status, 0);
}
if ( pid<0 ) {
/* No more child processes, we're done. */
if ( errno==ECHILD ) break;
error(errno,"waiting for children");
}
/* Pump pipes one more time to improve detection which program exited first. */
if ( write_progout ) {
for(i=0; i<ncmds; i++) {
pump_pipes(&progout_pipe_fd[i][0], &pipe_fd[1-i][1], 1-i);
}
}
for(i=0; i<ncmds; i++) if ( cmd_pid[i]==pid ) {
if (i == 1) {
submission_still_alive = 0;
}
warning(0, "command #%d, pid %d has exited (with status %d)",i+1,pid,status);
break;
}
if ( i>=ncmds ) error(0, "waited for unknown child");
cmd_exit[i] = status;
verb("command #%d, pid %d has exited (with status %d)",i+1,pid,status);
if (cmd_exit[0] != -1 && cmd_exit[1] != -1) {
/* Both child processes are done. */
if (validator_exited_first && WIFEXITED(cmd_exit[0])) {
exitcode = WEXITSTATUS(cmd_exit[0]);
if ( outputmeta && (metafile = fopen(metafilename,"w"))==NULL ) {
error(errno,"cannot open `%s'",metafilename);
}
write_meta("exitcode","%d", exitcode);
/* TODO: add more meta data like run time. */
if ( outputmeta && fclose(metafile)!=0 ) {
error(errno,"closing file `%s'",metafilename);
}
}
break;
}
};
/* Reset pipe filedescriptors to use blocking I/O. */
for(i=0; i<ncmds; i++) {
if ( write_progout && progout_pipe_fd[i][0]>=0 ) {
r = fcntl(progout_pipe_fd[i][0], F_GETFL);
if (r == -1) error(errno, "fcntl, getting flags");
r = fcntl(progout_pipe_fd[i][0], F_SETFL, r ^ O_NONBLOCK);
if (r == -1) error(errno, "fcntl, setting flags");
do {
pump_pipes(&progout_pipe_fd[i][0], &pipe_fd[1-i][1], 1-i);
} while ( progout_pipe_fd[i][0]>=0 );
}
}
/* Check exit status of commands and report back the exit code of the first. */
myexitcode = exitcode = 0;
for(i=0; i<ncmds; i++) {
if ( cmd_exit[i]!=0 ) {
status = cmd_exit[i];
/* Test whether command has finished abnormally */
if ( ! WIFEXITED(status) ) {
if ( WIFSIGNALED(status) ) {
warning(0,"command #%d terminated with signal %d",i+1,WTERMSIG(status));
exitcode = 128+WTERMSIG(status);
} else {
error(0,"command #%d exit status unknown: %d",i+1,status);
}
} else {
/* Log the exitstatus of the failed commands */
exitcode = WEXITSTATUS(status);
if ( exitcode!=0 ) {
warning(0,"command #%d exited with exitcode %d",i+1,exitcode);
}
}
/* Only report it for the first command. */
if ( i==0 ) myexitcode = exitcode;
}
}
return myexitcode;
}
