static int serialize_params(sstream_t *ss, param_t **params)
{
if (is_input_sstream(ss)) { /* read */
int i = 0;
param_t *p, *last = NULL;
*params = NULL;
if (serialize_int(ss, &i) != 0) return -1; /* can't read "terminator" */
while (i) {
p = (param_t *)cds_malloc(sizeof(param_t));
if (!p) {
ERROR_LOG("serialize_params(): can't allocate memory\n");
return -1;
}
p->next = NULL;
if (last) last->next = p;
else *params = p;
last = p;
if (serialize_param(ss, p) != 0) return -1;
if (serialize_int(ss, &i) != 0) return -1; /* can't read "terminator" */
}
}
else { /* store */
param_t *p = *params;
int i = 1;
while (p) {
if (serialize_int(ss, &i) != 0) return -1; /* params terminator ! */
if (serialize_param(ss, p) != 0) return -1;
p = p->next;
}
i = 0;
if (serialize_int(ss, &i) != 0) return -1; /* params terminator ! */
}
return 0;
}
/* {{{ MongoCursor->doQuery
*/
PHP_METHOD(MongoCursor, doQuery) {
int sent;
mongo_msg_header header;
mongo_cursor *cursor;
CREATE_BUF(buf, INITIAL_BUF_SIZE);
cursor = (mongo_cursor*)zend_object_store_get_object(getThis() TSRMLS_CC);
MONGO_CHECK_INITIALIZED(cursor->link, MongoCursor);
CREATE_HEADER_WITH_OPTS(buf, cursor->ns, OP_QUERY, cursor->opts);
serialize_int(&buf, cursor->skip);
serialize_int(&buf, cursor->limit);
zval_to_bson(&buf, HASH_P(cursor->query), NO_PREP TSRMLS_CC);
if (cursor->fields && zend_hash_num_elements(HASH_P(cursor->fields)) > 0) {
zval_to_bson(&buf, HASH_P(cursor->fields), NO_PREP TSRMLS_CC);
}
serialize_size(buf.start, &buf);
// sends
sent = mongo_say(cursor->link, &buf TSRMLS_CC);
efree(buf.start);
if (sent == FAILURE) {
zend_throw_exception(mongo_ce_CursorException, "couldn't send query.", 0 TSRMLS_CC);
return;
}
get_reply(cursor TSRMLS_CC);
}
static int serialize_dlg_state(sstream_t *ss, dlg_state_t *state)
{
int i = -1;
if (is_input_sstream(ss)) { /* read state */
if (serialize_int(ss, &i) != 0) return -1;
switch (i) {
case 0: *state = DLG_NEW; break;
case 1: *state = DLG_EARLY; break;
case 2: *state = DLG_CONFIRMED; break;
case 3: *state = DLG_DESTROYED; break;
default:
ERROR_LOG("deserializing unknow dialog state (%d)!\n", i);
return -1; /* unknown dialog state */
}
}
else { /* store state */
switch (*state) {
case DLG_NEW: i = 0; break;
case DLG_EARLY: i = 1; break;
case DLG_CONFIRMED: i = 2; break;
case DLG_DESTROYED: i = 3; break;
}
if (i == -1) {
WARN_LOG("serializing unknow dialog state (probably unloadable!)\n");
}
serialize_int(ss, &i);
}
return 0;
}
static int serialize_route(sstream_t *ss, rr_t **_r)
{
int do_it = 0;
int res = 0;
if (is_input_sstream(ss)) { /* read route */
if (serialize_int(ss, &do_it) != 0) return -1;
*_r = NULL;
}
else { /* store route */
if (*_r) do_it = 1;
else do_it = 0;
if (serialize_int(ss, &do_it) != 0) return -1;
}
if (do_it) {
str s;
if (*_r) {
s.s = (*_r)->nameaddr.name.s;
s.len = (*_r)->len;
}
res = serialize_str_ex(ss, &s) | res;
if (is_input_sstream(ss)) {
rr_t *pkg_rr = NULL;
parse_rr_body(s.s, s.len, &pkg_rr);
rr_dup(_r, pkg_rr);
}
}
return res;
}
void LZ78::encoding(Stream &stream, Alphabet &alpha, File &file) {
LzNode *curr = root;
char c;
// Para cada caractere de entrada
while(stream.hasNext()) {
c = stream.next();
// Se já existe um nó com o caractere c'
if(curr->child.count(c)) {
curr = curr->child[c];
// Não existe a substring procurada, então ..
} else {
// printf(" (%2d, %c)", curr->id, c);
serialize_int(file, curr->id);
serialize_int(file, alpha.getIndex(c));
// Criar um novo nó
LzNode *new_node = new LzNode(nextId++);
curr->child.insert(std::make_pair(c, new_node));
curr = root;
}
}
// Terminal
// printf(" (%2d, 0)\n", curr->id);
serialize_int(file, curr->id);
serialize_int(file, alpha.getIndex('\0'));
}
void page_write_checksum(char* page, unsigned int page_num) {
if (!checksum_write)
return;
else {
int checksum = adler32(0, (unsigned char*) page + PAGE_DATA_OFFSET, PAGE_DATA_BYTES);
serialize_int(page + PAGE_CHECKSUM_OFFSET, checksum);
serialize_int(page + PAGE_NUM_OFFSET, page_num);
}
}
void serialize_t_metadata_program(t_metadata_program *x, char **output) {
*output = (char*) calloc (1, sizeof(t_metadata_program));
serialize_t_puntero_instruccion(x->instruccion_inicio, output);
serlize_t_size(x->instrucciones_size, output);
serialize_t_intructions(x->instrucciones_serializado,x->instrucciones_size, output);
serlize_t_size(x->etiquetas_size, output);
serialize_etiquetas(x->etiquetas, x->etiquetas_size, output);
serialize_int(x->cantidad_de_funciones, output);
serialize_int(x->cantidad_de_etiquetas, output);
}
// SERIALIZACION INDICE DE CODIGO INSTRUCCION_DE_INICIO-INS|23-45-11-11-233-44
void serialize_t_intructions(int cantidad_instrucciones , t_intructions* instrucciones_serializado, char** buffer) {
int i=0;
(*buffer)[(strlen(*buffer)-1)] = '|';
for(i=0;i<cantidad_instrucciones;i++){
serialize_int(instrucciones_serializado->start,buffer);
serialize_int(instrucciones_serializado->offset,buffer);
instrucciones_serializado++;
}
(*buffer)[(strlen(*buffer)-1)] = '|';
}
static int serialize_ptype(sstream_t *ss, ptype_t *type)
{
int i = -1;
if (is_input_sstream(ss)) { /* read ptype */
if (serialize_int(ss, &i) != 0) return -1;
switch (i) {
case 0: *type = P_OTHER; break;
case 1: *type = P_Q; break;
case 2: *type = P_EXPIRES; break;
case 3: *type = P_METHOD; break;
case 4: *type = P_RECEIVED; break;
case 5: *type = P_TRANSPORT; break;
case 6: *type = P_LR; break;
case 7: *type = P_R2; break;
case 8: *type = P_MADDR; break;
case 9: *type = P_TTL; break;
case 10: *type = P_DSTIP; break;
case 11: *type = P_DSTPORT; break;
case 12: *type = P_INSTANCE; break;
default:
ERROR_LOG("deserializing unknow ptype (%d)!\n", i);
return -1;
}
}
else { /* store ptype */
switch (*type) {
case P_OTHER: i = 0; break;
case P_Q: i = 1; break;
case P_EXPIRES: i = 2; break;
case P_METHOD: i = 3; break;
case P_RECEIVED: i = 4; break;
case P_TRANSPORT: i = 5; break;
case P_LR: i = 6; break;
case P_R2: i = 7; break;
case P_MADDR: i = 8; break;
case P_TTL: i = 9; break;
case P_DSTIP: i = 10; break;
case P_DSTPORT: i = 11; break;
case P_INSTANCE: i = 12; break;
}
if (i == -1) {
WARN_LOG("serializing unknow ptype (probably unloadable!)\n");
}
serialize_int(ss, &i);
}
return 0;
}
static void storage_serialize(at **pp, int code)
{
storage_t *st;
int type, kind;
size_t size;
if (code != SRZ_READ) {
st = Mptr(*pp);
type = (int)st->type;
kind = (int)st->kind;
size = st->size;
}
// Read/write basic info
serialize_int(&type, code);
serialize_int(&kind, code);
serialize_size(&size, code);
// Create storage if needed
if (code == SRZ_READ) {
st = new_storage_managed((storage_type_t)type, size, NIL);
*pp = st->backptr;
}
// Read/write storage data
st = Mptr(*pp);
if (type == ST_AT) {
at **data = st->data;
for (int i=0; i<size; i++)
serialize_atstar( &data[i], code);
} else {
FILE *f = serialization_file_descriptor(code);
if (code == SRZ_WRITE) {
extern int in_bwrite;
in_bwrite += sizeof(int) + size * storage_sizeof[type];
write4(f, STORAGE_NORMAL);
storage_save(st, f);
} else if (code == SRZ_READ) {
int magic = read4(f);
storage_load(st, f);
if (magic == STORAGE_SWAPPED)
swap_buffer(st->data, size, storage_sizeof[type]);
else if (magic != STORAGE_NORMAL)
RAISEF("Corrupted binary file",NIL);
}
}
}
int serialize_to_mpi(void *_pvCtx, int *_piAddr, int **_piBuffer, int *_piBufferSize)
{
int iType = 0;
SciErr sciErr = getVarType(_pvCtx, _piAddr, &iType);
if (sciErr.iErr)
{
printError(&sciErr, 0);
return 0;
}
switch (iType)
{
case sci_matrix:
return serialize_double(_pvCtx, _piAddr, _piBuffer, _piBufferSize);
break;
case sci_strings:
return serialize_string(_pvCtx, _piAddr, _piBuffer, _piBufferSize);
break;
case sci_boolean:
return serialize_boolean(_pvCtx, _piAddr, _piBuffer, _piBufferSize);
break;
case sci_sparse:
return serialize_sparse(_pvCtx, _piAddr, _piBuffer, _piBufferSize, TRUE);
break;
case sci_boolean_sparse:
return serialize_sparse(_pvCtx, _piAddr, _piBuffer, _piBufferSize, FALSE);
break;
case sci_ints:
return serialize_int(_pvCtx, _piAddr, _piBuffer, _piBufferSize);
break;
default:
return -1;
break;
}
}
unsigned char* serialize_request(request_data req) {
unsigned char* c = new unsigned char [R_D_LENGTH];
unsigned char* c2 = serialize_int(static_cast<unsigned int> (req.time));
for (int i = 0; i < 4; ++i) c[i] = c2[i];
delete c2;
c[4] = req.method;
c2 = serialize_int(req.receiver_ip.s_addr);
for (int i = 0; i < 4; ++i) c[i+5] = c2[i];
delete c2;
for (int i = 0; i < 4; ++i) c[i+9] = req.response[i];
return c;
}
void serialize_string(FILE *out, const char *s, long long *fpos, const char *fname) {
int len = strlen(s);
serialize_int(out, len + 1, fpos, fname);
fwrite_check(s, sizeof(char), len, out, fname);
fwrite_check("", sizeof(char), 1, out, fname);
*fpos += len + 1;
}
unsigned char* serialize_command(command comm) {
unsigned char* c = new unsigned char [5];
c[0] = comm.type;
unsigned char* c2 = serialize_int(static_cast<unsigned int> (comm.time));
for (int i = 0; i < 4; ++i) c[1+i] = c2[i];
delete c2;
return c;
}
static int serialize_route_ex(sstream_t *ss, rr_t **_r)
{
int do_it = 0;
int res = 0;
if (is_input_sstream(ss)) { /* read route */
if (serialize_int(ss, &do_it) != 0) return -1;
if (!do_it) *_r = NULL;
else {
*_r = (rr_t*)cds_malloc(sizeof(rr_t));
if (!*_r) {
ERROR_LOG("serialize_route(): can't allocate memory\n");
return -1;
}
(*_r)->r2 = NULL;
(*_r)->params = NULL;
(*_r)->next = NULL;
}
}
else { /* store route */
if (*_r) do_it = 1;
else do_it = 0;
if (serialize_int(ss, &do_it) != 0) return -1;
}
if (do_it) {
rr_t *r = *_r;
str s = { r->nameaddr.name.s, r->len };
int delta = r->nameaddr.uri.s - r->nameaddr.name.s;
res = serialize_str(ss, &s) | res;
res = serialize_int(ss, &r->nameaddr.name.len) | res;
res = serialize_int(ss, &r->nameaddr.uri.len) | res;
res = serialize_int(ss, &r->nameaddr.len) | res;
res = serialize_int(ss, &delta) | res;
if (is_input_sstream(ss)) {
r->nameaddr.name.s = s.s;
r->nameaddr.uri.s = s.s + delta;
}
/* !!! optimalized strings - use carefuly !!! */
/*res = serialize_str(ss, &r->nameaddr.name) | res;
res = serialize_str(ss, &r->nameaddr.uri) | res;
res = serialize_int(ss, &r->nameaddr.len) | res;*/
/* ??? res = serialize_r2(ss, &r->nameaddr.params) | res; ??? */
res = serialize_params(ss, &r->params) | res;
res = serialize_int(ss, &r->len) | res;
TRACE_LOG("ROUTE: rlen=%d name=%.*s, uri=%.*s\n len=%d WHOLE=%.*s\n",
r->len,
FMT_STR(r->nameaddr.name),
FMT_STR(r->nameaddr.uri),
r->nameaddr.len,
r->nameaddr.len, r->nameaddr.name.s
);
}
return res;
}
static int serialize_param(sstream_t *ss, param_t *p)
{
int res = 0;
res = serialize_ptype(ss, &p->type) | res;
res = serialize_str(ss, &p->name) | res;
res = serialize_str(ss, &p->body) | res;
res = serialize_int(ss, &p->len) | res;
return res;
}
template <typename Stream> bool ConnectionPacket::Serialize( Stream & stream )
{
ConnectionContext * context = (ConnectionContext*) stream.GetContext();
assert( context );
assert( context->magic == ConnectionContextMagic );
assert( context->messageFactory );
assert( context->connectionConfig );
// ack system
bool perfect_acks = Stream::IsWriting ? ( ack_bits == 0xFFFFFFFF ) : 0;
serialize_bool( stream, perfect_acks );
if ( !perfect_acks )
serialize_bits( stream, ack_bits, 32 );
else
ack_bits = 0xFFFFFFFF;
serialize_bits( stream, sequence, 16 );
serialize_ack_relative( stream, sequence, ack );
// channel entries
const int numChannels = context->connectionConfig->numChannels;
serialize_int( stream, numChannelEntries, 0, context->connectionConfig->numChannels );
#if YOJIMBO_VALIDATE_PACKET_BUDGET
assert( stream.GetBitsProcessed() <= ConservativeConnectionPacketHeaderEstimate );
#endif // #if YOJIMBO_VALIDATE_PACKET_BUDGET
if ( numChannelEntries > 0 )
{
if ( Stream::IsReading )
{
if ( !AllocateChannelData( *context->messageFactory, numChannelEntries ) )
return false;
}
for ( int i = 0; i < numChannelEntries; ++i )
{
if ( !channelEntry[i].SerializeInternal( stream, *m_messageFactory, context->connectionConfig->channelConfig, numChannels ) )
return false;
}
}
return true;
}
void Test_serialize_int(CuTest * tc){
printf("Int serialization tests and experiments \n");
printf("maximum int = %d\n",INT_MAX);
printf("size of int in bytes= %zu\n",sizeof(int));
int a = 18505;
int value;
unsigned char buffer[sizeof(int)];
serialize_int(buffer,a);
printf(" serialized is %s\n",buffer);
deserialize_int(buffer,&value);
printf(" deserialized is %d\n",value);
CuAssertIntEquals(tc,value,a);
a = INT_MAX;
serialize_int(buffer,a);
deserialize_int(buffer,&value);
CuAssertIntEquals(tc,value,a);
a = INT_MIN;
serialize_int(buffer,a);
deserialize_int(buffer,&value);
CuAssertIntEquals(tc,value,a);
}
void
serialize_message( struct message *msg, struct buffer *b )
{
//buffer = malloc( sizeof(struct message) + 13 );
//printf("%d \n", msg->m_length);
//msg->m_length = htonl( msg->m_length );
//memmove( buffer, msg, sizeof(struct message) );
//memcpy( buffer + sizeof(struct message), msg.m_param, msg.m_length );
serialize_int( msg->m_length, b );
serialize_short( msg->m_code, b );
serialize_string( msg->m_param, b );
}
int serialize_str_ex(sstream_t *ss, str_t *s)
{
int res = 0;
if (!s) return -1;
if (serialize_int(ss, &s->len) != 0) return -1;
if (is_input_sstream(ss)) {
if (s->len == 0) s->s = NULL;
else res = sstream_get_str_ex(ss, s->len, s); /* doesn't duplicate read string */
}
else res = sstream_put(ss, s->s, s->len);
return res;
}
/* {{{ MongoCursor::hasNext
*/
PHP_METHOD(MongoCursor, hasNext) {
mongo_msg_header header;
buffer buf;
int size;
mongo_cursor *cursor = (mongo_cursor*)zend_object_store_get_object(getThis() TSRMLS_CC);
MONGO_CHECK_INITIALIZED(cursor->link, MongoCursor);
if (!cursor->started_iterating) {
MONGO_METHOD(MongoCursor, doQuery)(INTERNAL_FUNCTION_PARAM_PASSTHRU);
cursor->started_iterating = 1;
}
if ((cursor->limit > 0 && cursor->at >= cursor->limit) ||
cursor->num == 0) {
RETURN_FALSE;
}
if (cursor->at < cursor->num) {
RETURN_TRUE;
}
else if (cursor->cursor_id == 0) {
RETURN_FALSE;
}
// we have to go and check with the db
size = 34+strlen(cursor->ns);
buf.start = (unsigned char*)emalloc(size);
buf.pos = buf.start;
buf.end = buf.start + size;
CREATE_RESPONSE_HEADER(buf, cursor->ns, strlen(cursor->ns), cursor->header.request_id, OP_GET_MORE);
serialize_int(&buf, cursor->limit);
serialize_long(&buf, cursor->cursor_id);
serialize_size(buf.start, &buf);
// fails if we're out of elems
if(mongo_say(cursor->link, &buf TSRMLS_CC) == FAILURE) {
efree(buf.start);
RETURN_FALSE;
}
efree(buf.start);
// if we have cursor->at == cursor->num && recv fails,
// we're probably just out of results
RETURN_BOOL(get_reply(cursor TSRMLS_CC) == SUCCESS);
}
int ObSqlGetParam::serialize_rowkeys(char* buf, const int64_t buf_len,
int64_t& pos, const RowkeyListType & rowkey_list) const
{
int ret = OB_SUCCESS;
int64_t i = 0;
if (NULL == buf || buf_len <= 0 || pos > buf_len)
{
TBSYS_LOG(WARN, "invalid param, buf=%p, buf_len=%ld, pos=%ld",
buf, buf_len, pos);
ret = OB_INVALID_ARGUMENT;
}
if (OB_SUCCESS == ret)
{
//serialize row keys
if (OB_SUCCESS != (ret = serialize_flag(buf, buf_len, pos, ObActionFlag::FORMED_ROW_KEY_FIELD)))
{
TBSYS_LOG(WARN, "fail to serialize flag. ret=%d", ret);
}
else if (OB_SUCCESS != (ret = serialize_int(buf, buf_len, pos, rowkey_list.count())))
{
TBSYS_LOG(WARN, "fail to serialize rowkey size. ret=%d", ret);
}
else
{
if (OB_SUCCESS == ret)
{
for (i = 0; OB_SUCCESS == ret && i < rowkey_list.count(); i++)
{
ret = rowkey_list.at(i).serialize(buf, buf_len, pos);
// TBSYS_LOG(DEBUG, "serialize rowkey[%ld]: %s. ret=%d", i, to_cstring(rowkey_list.at(i)), ret);
}
}
}
}
if (OB_SUCCESS != ret)
{
TBSYS_LOG(WARN, "fail to serialize rowkey list. buf=%p, buf_len=%ld, pos=%ld, rowkey list size=%ld",
buf, buf_len, pos, rowkey_list.count());
}
return ret;
}
int read_json(int argc, char **argv, char *fname, const char *layername, int maxzoom, int minzoom, sqlite3 *outdb, struct pool *exclude, struct pool *include, int exclude_all, double droprate, int buffer, const char *tmpdir, double gamma, char *prevent) {
int ret = EXIT_SUCCESS;
char metaname[strlen(tmpdir) + strlen("/meta.XXXXXXXX") + 1];
char geomname[strlen(tmpdir) + strlen("/geom.XXXXXXXX") + 1];
char indexname[strlen(tmpdir) + strlen("/index.XXXXXXXX") + 1];
sprintf(metaname, "%s%s", tmpdir, "/meta.XXXXXXXX");
sprintf(geomname, "%s%s", tmpdir, "/geom.XXXXXXXX");
sprintf(indexname, "%s%s", tmpdir, "/index.XXXXXXXX");
int metafd = mkstemp(metaname);
if (metafd < 0) {
perror(metaname);
exit(EXIT_FAILURE);
}
int geomfd = mkstemp(geomname);
if (geomfd < 0) {
perror(geomname);
exit(EXIT_FAILURE);
}
int indexfd = mkstemp(indexname);
if (indexfd < 0) {
perror(indexname);
exit(EXIT_FAILURE);
}
FILE *metafile = fopen(metaname, "wb");
if (metafile == NULL) {
perror(metaname);
exit(EXIT_FAILURE);
}
FILE *geomfile = fopen(geomname, "wb");
if (geomfile == NULL) {
perror(geomname);
exit(EXIT_FAILURE);
}
FILE *indexfile = fopen(indexname, "wb");
if (indexfile == NULL) {
perror(indexname);
exit(EXIT_FAILURE);
}
long long metapos = 0;
long long geompos = 0;
long long indexpos = 0;
unlink(metaname);
unlink(geomname);
unlink(indexname);
unsigned file_bbox[] = { UINT_MAX, UINT_MAX, 0, 0 };
unsigned midx = 0, midy = 0;
long long seq = 0;
int nlayers = argc;
if (nlayers == 0) {
nlayers = 1;
}
int n;
for (n = 0; n < nlayers; n++) {
json_pull *jp;
const char *reading;
FILE *fp;
long long found_hashes = 0;
long long found_features = 0;
if (n >= argc) {
reading = "standard input";
fp = stdin;
} else {
reading = argv[n];
fp = fopen(argv[n], "r");
if (fp == NULL) {
perror(argv[n]);
continue;
}
}
jp = json_begin_file(fp);
while (1) {
json_object *j = json_read(jp);
if (j == NULL) {
if (jp->error != NULL) {
fprintf(stderr, "%s:%d: %s\n", reading, jp->line, jp->error);
}
json_free(jp->root);
break;
}
if (j->type == JSON_HASH) {
found_hashes++;
if (found_hashes == 50 && found_features == 0) {
fprintf(stderr, "%s:%d: Not finding any GeoJSON features in input. Is your file just bare geometries?\n", reading, jp->line);
break;
}
}
json_object *type = json_hash_get(j, "type");
if (type == NULL || type->type != JSON_STRING || strcmp(type->string, "Feature") != 0) {
continue;
}
found_features++;
json_object *geometry = json_hash_get(j, "geometry");
if (geometry == NULL) {
fprintf(stderr, "%s:%d: feature with no geometry\n", reading, jp->line);
json_free(j);
continue;
}
json_object *geometry_type = json_hash_get(geometry, "type");
if (geometry_type == NULL) {
static int warned = 0;
if (!warned) {
fprintf(stderr, "%s:%d: null geometry (additional not reported)\n", reading, jp->line);
warned = 1;
}
json_free(j);
continue;
}
if (geometry_type->type != JSON_STRING) {
fprintf(stderr, "%s:%d: geometry without type\n", reading, jp->line);
json_free(j);
continue;
}
json_object *properties = json_hash_get(j, "properties");
if (properties == NULL || (properties->type != JSON_HASH && properties->type != JSON_NULL)) {
fprintf(stderr, "%s:%d: feature without properties hash\n", reading, jp->line);
json_free(j);
continue;
}
json_object *coordinates = json_hash_get(geometry, "coordinates");
if (coordinates == NULL || coordinates->type != JSON_ARRAY) {
fprintf(stderr, "%s:%d: feature without coordinates array\n", reading, jp->line);
json_free(j);
continue;
}
int t;
for (t = 0; t < GEOM_TYPES; t++) {
if (strcmp(geometry_type->string, geometry_names[t]) == 0) {
break;
}
}
if (t >= GEOM_TYPES) {
fprintf(stderr, "%s:%d: Can't handle geometry type %s\n", reading, jp->line, geometry_type->string);
json_free(j);
continue;
}
{
unsigned bbox[] = { UINT_MAX, UINT_MAX, 0, 0 };
int nprop = 0;
if (properties->type == JSON_HASH) {
nprop = properties->length;
}
long long metastart = metapos;
char *metakey[nprop];
char *metaval[nprop];
int metatype[nprop];
int m = 0;
int i;
for (i = 0; i < nprop; i++) {
if (properties->keys[i]->type == JSON_STRING) {
if (exclude_all) {
if (!is_pooled(include, properties->keys[i]->string, VT_STRING)) {
continue;
}
} else if (is_pooled(exclude, properties->keys[i]->string, VT_STRING)) {
continue;
}
metakey[m] = properties->keys[i]->string;
if (properties->values[i] != NULL && properties->values[i]->type == JSON_STRING) {
metatype[m] = VT_STRING;
metaval[m] = properties->values[i]->string;
m++;
} else if (properties->values[i] != NULL && properties->values[i]->type == JSON_NUMBER) {
metatype[m] = VT_NUMBER;
metaval[m] = properties->values[i]->string;
m++;
} else if (properties->values[i] != NULL && (properties->values[i]->type == JSON_TRUE || properties->values[i]->type == JSON_FALSE)) {
metatype[m] = VT_BOOLEAN;
metaval[m] = properties->values[i]->type == JSON_TRUE ? "true" : "false";
m++;
} else if (properties->values[i] != NULL && (properties->values[i]->type == JSON_NULL)) {
;
} else {
fprintf(stderr, "%s:%d: Unsupported property type for %s\n", reading, jp->line, properties->keys[i]->string);
json_free(j);
continue;
}
}
}
serialize_int(metafile, m, &metapos, fname);
for (i = 0; i < m; i++) {
serialize_int(metafile, metatype[i], &metapos, fname);
serialize_string(metafile, metakey[i], &metapos, fname);
serialize_string(metafile, metaval[i], &metapos, fname);
}
long long geomstart = geompos;
serialize_byte(geomfile, mb_geometry[t], &geompos, fname);
serialize_byte(geomfile, n, &geompos, fname);
serialize_long_long(geomfile, metastart, &geompos, fname);
parse_geometry(t, coordinates, bbox, &geompos, geomfile, VT_MOVETO, fname, jp);
serialize_byte(geomfile, VT_END, &geompos, fname);
/*
* Note that minzoom for lines is the dimension
* of the geometry in world coordinates, but
* for points is the lowest zoom level (in tiles,
* not in pixels) at which it should be drawn.
*
* So a line that is too small for, say, z8
* will have minzoom of 18 (if tile detail is 10),
* not 8.
*/
int minzoom = 0;
if (mb_geometry[t] == VT_LINE) {
for (minzoom = 0; minzoom < 31; minzoom++) {
unsigned mask = 1 << (32 - (minzoom + 1));
if (((bbox[0] & mask) != (bbox[2] & mask)) ||
((bbox[1] & mask) != (bbox[3] & mask))) {
break;
}
}
} else if (mb_geometry[t] == VT_POINT) {
double r = ((double) rand()) / RAND_MAX;
if (r == 0) {
r = .00000001;
}
minzoom = maxzoom - floor(log(r) / - log(droprate));
}
serialize_byte(geomfile, minzoom, &geompos, fname);
struct index index;
index.start = geomstart;
index.end = geompos;
index.index = encode(bbox[0] / 2 + bbox[2] / 2, bbox[1] / 2 + bbox[3] / 2);
fwrite_check(&index, sizeof(struct index), 1, indexfile, fname);
indexpos += sizeof(struct index);
for (i = 0; i < 2; i++) {
if (bbox[i] < file_bbox[i]) {
file_bbox[i] = bbox[i];
}
}
for (i = 2; i < 4; i++) {
if (bbox[i] > file_bbox[i]) {
file_bbox[i] = bbox[i];
}
}
if (seq % 10000 == 0) {
fprintf(stderr, "Read %.2f million features\r", seq / 1000000.0);
}
seq++;
}
json_free(j);
/* XXX check for any non-features in the outer object */
}
json_end(jp);
fclose(fp);
}
fclose(metafile);
fclose(geomfile);
fclose(indexfile);
struct stat geomst;
struct stat metast;
if (fstat(geomfd, &geomst) != 0) {
perror("stat geom\n");
exit(EXIT_FAILURE);
}
if (fstat(metafd, &metast) != 0) {
perror("stat meta\n");
exit(EXIT_FAILURE);
}
if (geomst.st_size == 0 || metast.st_size == 0) {
fprintf(stderr, "did not read any valid geometries\n");
exit(EXIT_FAILURE);
}
char *meta = (char *) mmap(NULL, metast.st_size, PROT_READ, MAP_PRIVATE, metafd, 0);
if (meta == MAP_FAILED) {
perror("mmap meta");
exit(EXIT_FAILURE);
}
struct pool file_keys1[nlayers];
struct pool *file_keys[nlayers];
int i;
for (i = 0; i < nlayers; i++) {
pool_init(&file_keys1[i], 0);
file_keys[i] = &file_keys1[i];
}
char *layernames[nlayers];
for (i = 0; i < nlayers; i++) {
if (argc <= 1 && layername != NULL) {
layernames[i] = strdup(layername);
} else {
char *src = argv[i];
if (argc < 1) {
src = fname;
}
char *trunc = layernames[i] = malloc(strlen(src) + 1);
const char *ocp, *use = src;
for (ocp = src; *ocp; ocp++) {
if (*ocp == '/' && ocp[1] != '\0') {
use = ocp + 1;
}
}
strcpy(trunc, use);
char *cp = strstr(trunc, ".json");
if (cp != NULL) {
*cp = '\0';
}
cp = strstr(trunc, ".mbtiles");
if (cp != NULL) {
*cp = '\0';
}
layername = trunc;
char *out = trunc;
for (cp = trunc; *cp; cp++) {
if (isalpha(*cp) || isdigit(*cp) || *cp == '_') {
*out++ = *cp;
}
}
*out = '\0';
printf("using layer %d name %s\n", i, trunc);
}
}
/* Sort the index by geometry */
{
int bytes = sizeof(struct index);
fprintf(stderr, "Sorting %lld features\n", (long long) indexpos / bytes);
int page = sysconf(_SC_PAGESIZE);
long long unit = (50 * 1024 * 1024 / bytes) * bytes;
while (unit % page != 0) {
unit += bytes;
}
int nmerges = (indexpos + unit - 1) / unit;
struct merge merges[nmerges];
long long start;
for (start = 0; start < indexpos; start += unit) {
long long end = start + unit;
if (end > indexpos) {
end = indexpos;
}
if (nmerges != 1) {
fprintf(stderr, "Sorting part %lld of %d\r", start / unit + 1, nmerges);
}
merges[start / unit].start = start;
merges[start / unit].end = end;
merges[start / unit].next = NULL;
void *map = mmap(NULL, end - start, PROT_READ | PROT_WRITE, MAP_PRIVATE, indexfd, start);
if (map == MAP_FAILED) {
perror("mmap");
exit(EXIT_FAILURE);
}
qsort(map, (end - start) / bytes, bytes, indexcmp);
// Sorting and then copying avoids the need to
// write out intermediate stages of the sort.
void *map2 = mmap(NULL, end - start, PROT_READ | PROT_WRITE, MAP_SHARED, indexfd, start);
if (map2 == MAP_FAILED) {
perror("mmap (write)");
exit(EXIT_FAILURE);
}
memcpy(map2, map, end - start);
munmap(map, end - start);
munmap(map2, end - start);
}
if (nmerges != 1) {
fprintf(stderr, "\n");
}
void *map = mmap(NULL, indexpos, PROT_READ, MAP_PRIVATE, indexfd, 0);
if (map == MAP_FAILED) {
perror("mmap");
exit(EXIT_FAILURE);
}
FILE *f = fopen(indexname, "w");
if (f == NULL) {
perror(indexname);
exit(EXIT_FAILURE);
}
merge(merges, nmerges, (unsigned char *) map, f, bytes, indexpos / bytes);
munmap(map, indexpos);
fclose(f);
close(indexfd);
}
/* Copy geometries to a new file in index order */
indexfd = open(indexname, O_RDONLY);
if (indexfd < 0) {
perror("reopen sorted index");
exit(EXIT_FAILURE);
}
struct index *index_map = mmap(NULL, indexpos, PROT_READ, MAP_PRIVATE, indexfd, 0);
if (index_map == MAP_FAILED) {
perror("mmap index");
exit(EXIT_FAILURE);
}
unlink(indexname);
char *geom_map = mmap(NULL, geomst.st_size, PROT_READ, MAP_PRIVATE, geomfd, 0);
if (geom_map == MAP_FAILED) {
perror("mmap unsorted geometry");
exit(EXIT_FAILURE);
}
if (close(geomfd) != 0) {
perror("close unsorted geometry");
}
sprintf(geomname, "%s%s", tmpdir, "/geom.XXXXXXXX");
geomfd = mkstemp(geomname);
if (geomfd < 0) {
perror(geomname);
exit(EXIT_FAILURE);
}
geomfile = fopen(geomname, "wb");
if (geomfile == NULL) {
perror(geomname);
exit(EXIT_FAILURE);
}
{
geompos = 0;
/* initial tile is 0/0/0 */
serialize_int(geomfile, 0, &geompos, fname);
serialize_uint(geomfile, 0, &geompos, fname);
serialize_uint(geomfile, 0, &geompos, fname);
long long i;
long long sum = 0;
long long progress = 0;
for (i = 0; i < indexpos / sizeof(struct index); i++) {
fwrite_check(geom_map + index_map[i].start, sizeof(char), index_map[i].end - index_map[i].start, geomfile, fname);
sum += index_map[i].end - index_map[i].start;
long long p = 1000 * i / (indexpos / sizeof(struct index));
if (p != progress) {
fprintf(stderr, "Reordering geometry: %3.1f%%\r", p / 10.0);
progress = p;
}
}
/* end of tile */
serialize_byte(geomfile, -2, &geompos, fname);
fclose(geomfile);
}
if (munmap(index_map, indexpos) != 0) {
perror("unmap sorted index");
}
if (munmap(geom_map, geomst.st_size) != 0) {
perror("unmap unsorted geometry");
}
if (close(indexfd) != 0) {
perror("close sorted index");
}
/* Traverse and split the geometries for each zoom level */
geomfd = open(geomname, O_RDONLY);
if (geomfd < 0) {
perror("reopen sorted geometry");
exit(EXIT_FAILURE);
}
unlink(geomname);
if (fstat(geomfd, &geomst) != 0) {
perror("stat sorted geom\n");
exit(EXIT_FAILURE);
}
int fd[4];
off_t size[4];
fd[0] = geomfd;
size[0] = geomst.st_size;
int j;
for (j = 1; j < 4; j++) {
fd[j] = -1;
size[j] = 0;
}
fprintf(stderr, "%lld features, %lld bytes of geometry, %lld bytes of metadata\n", seq, (long long) geomst.st_size, (long long) metast.st_size);
int written = traverse_zooms(fd, size, meta, file_bbox, file_keys, &midx, &midy, layernames, maxzoom, minzoom, outdb, droprate, buffer, fname, tmpdir, gamma, nlayers, prevent);
if (maxzoom != written) {
fprintf(stderr, "\n\n\n*** NOTE TILES ONLY COMPLETE THROUGH ZOOM %d ***\n\n\n", written);
maxzoom = written;
ret = EXIT_FAILURE;
}
if (munmap(meta, metast.st_size) != 0) {
perror("munmap meta");
}
if (close(metafd) < 0) {
perror("close meta");
}
double minlat = 0, minlon = 0, maxlat = 0, maxlon = 0, midlat = 0, midlon = 0;
tile2latlon(midx, midy, maxzoom, &maxlat, &minlon);
tile2latlon(midx + 1, midy + 1, maxzoom, &minlat, &maxlon);
midlat = (maxlat + minlat) / 2;
midlon = (maxlon + minlon) / 2;
tile2latlon(file_bbox[0], file_bbox[1], 32, &maxlat, &minlon);
tile2latlon(file_bbox[2], file_bbox[3], 32, &minlat, &maxlon);
if (midlat < minlat) {
midlat = minlat;
}
if (midlat > maxlat) {
midlat = maxlat;
}
if (midlon < minlon) {
midlon = minlon;
}
if (midlon > maxlon) {
midlon = maxlon;
}
mbtiles_write_metadata(outdb, fname, layernames, minzoom, maxzoom, minlat, minlon, maxlat, maxlon, midlat, midlon, file_keys, nlayers); // XXX layers
for (i = 0; i < nlayers; i++) {
pool_free_strings(&file_keys1[i]);
free(layernames[i]);
}
return ret;
}
