Esempio n. 1
0
File: read_all.c Progetto: kraj/ltp
static void run(void)
{
	spawn_workers();
	visit_dir(root_dir);
	stop_workers();

	tst_reap_children();
	tst_res(TPASS, "Finished reading files");
}
Esempio n. 2
0
int main(int argc, char **argv)
{
    char *spath = strdup(RENDER_SOCKET);
    const char *mapname_default = XMLCONFIG_DEFAULT;
    const char *mapname = mapname_default;
    const char *tile_dir = tile_dir_default;
    int minX=-1, maxX=-1, minY=-1, maxY=-1;
    int x, y, z;
    char name[PATH_MAX];
    struct timeval start, end;
    int num_render = 0, num_all = 0;
    int c;
    int all=0;
    int numThreads = 1;
    int force=0;
    int bboxset=0;
    struct storage_backend * store;
    struct stat_info s;

    while (1) {
        int option_index = 0;
        static struct option long_options[] = {
            {"min-zoom", 1, 0, 'z'},
            {"max-zoom", 1, 0, 'Z'},
            {"min-x", 1, 0, 'x'},
            {"max-x", 1, 0, 'X'},
            {"min-y", 1, 0, 'y'},
            {"max-y", 1, 0, 'Y'},
            {"socket", 1, 0, 's'},
            {"num-threads", 1, 0, 'n'},
            {"max-load", 1, 0, 'l'},
            {"tile-dir", 1, 0, 't'},
            {"map", 1, 0, 'm'},
            {"verbose", 0, 0, 'v'},
            {"force", 0, 0, 'f'},
            {"all", 0, 0, 'a'},
            {"bbox", 1, 0, 'b'},
            {"help", 0, 0, 'h'},
            {0, 0, 0, 0}
        };

        //c = getopt_long(argc, argv, "hvaz:Z:x:X:y:Y:s:m:t:n:l:f", long_options, &option_index);
        c = getopt_long(argc, argv, "hvab:z:Z:x:X:y:Y:s:m:t:n:l:f", long_options, &option_index);
        if (c == -1)
            break;

        switch (c) {
            case 'a':   /* -a, --all */
                all=1;
                break;
            case 'b':   /* -b, --bbox */
                if (!handle_bbox(optarg))
                {
                    fprintf(stderr, "invalid bbox argument - must be of the form east,south,west,north\n");
                    return -1;
                }
                bboxset = 1;
                break;
            case 's':   /* -s, --socket */
                free(spath);
                spath = strdup(optarg);
                break;
            case 't':   /* -t, --tile-dir */
                tile_dir=strdup(optarg);
                break;
            case 'm':   /* -m, --map */
                mapname=strdup(optarg);
                break;
            case 'l':   /* -l, --max-load */
                maxLoad = atoi(optarg);
                break;
            case 'n':   /* -n, --num-threads */
                numThreads=atoi(optarg);
                if (numThreads <= 0) {
                    fprintf(stderr, "Invalid number of threads, must be at least 1\n");
                    return 1;
                }
                break;
            case 'x':   /* -x, --min-x */
                minX=atoi(optarg);
                break;
            case 'X':   /* -X, --max-x */
                maxX=atoi(optarg);
                break;
            case 'y':   /* -y, --min-y */
                minY=atoi(optarg);
                break;
            case 'Y':   /* -Y, --max-y */
                maxY=atoi(optarg);
                break;
            case 'z':   /* -z, --min-zoom */
                minZoom=atoi(optarg);
                if (minZoom < 0 || minZoom > MAX_ZOOM) {
                    fprintf(stderr, "Invalid minimum zoom selected, must be between 0 and %d\n", MAX_ZOOM);
                    return 1;
                }
                break;
            case 'Z':   /* -Z, --max-zoom */
                maxZoom=atoi(optarg);
                if (maxZoom < 0 || maxZoom > MAX_ZOOM) {
                    fprintf(stderr, "Invalid maximum zoom selected, must be between 0 and %d\n", MAX_ZOOM);
                    return 1;
                }
                break;
            case 'f':   /* -f, --force */
                force=1;
                break;
            case 'v':   /* -v, --verbose */
                verbose=1;
                break;
            case 'h':   /* -h, --help */
                fprintf(stderr, "Usage: render_list [OPTION] ...\n");
                fprintf(stderr, "  -a, --all            render all tiles in given zoom level range instead of reading from STDIN\n");
                fprintf(stderr, "  -b, --bbox           render all tiles in given bbox (min Longitude, min Latitude, max Longitude, max Latitude) range instead of reading from STDIN\n");
                fprintf(stderr, "  -f, --force          render tiles even if they seem current\n");
                fprintf(stderr, "  -m, --map=MAP        render tiles in this map (defaults to '" XMLCONFIG_DEFAULT "')\n");
                fprintf(stderr, "  -l, --max-load=LOAD  sleep if load is this high (defaults to %d)\n", MAX_LOAD_OLD);
                fprintf(stderr, "  -s, --socket=SOCKET  unix domain socket name for contacting renderd\n");
                fprintf(stderr, "  -n, --num-threads=N the number of parallel request threads (default 1)\n");
                fprintf(stderr, "  -t, --tile-dir       tile cache directory (defaults to '" HASH_PATH "')\n");
                fprintf(stderr, "  -z, --min-zoom=ZOOM  filter input to only render tiles greater or equal to this zoom level (default is 0)\n");
                fprintf(stderr, "  -Z, --max-zoom=ZOOM  filter input to only render tiles less than or equal to this zoom level (default is %d)\n", MAX_ZOOM);
                fprintf(stderr, "If you are using --all, you can restrict the tile range by adding these options:\n");
                fprintf(stderr, "  -x, --min-x=X        minimum X tile coordinate\n");
                fprintf(stderr, "  -X, --max-x=X        maximum X tile coordinate\n");
                fprintf(stderr, "  -y, --min-y=Y        minimum Y tile coordinate\n");
                fprintf(stderr, "  -Y, --max-y=Y        maximum Y tile coordinate\n");
                fprintf(stderr, "Without --all, send a list of tiles to be rendered from STDIN in the format:\n");
                fprintf(stderr, "  X Y Z\n");
                fprintf(stderr, "e.g.\n");
                fprintf(stderr, "  0 0 1\n");
                fprintf(stderr, "  0 1 1\n");
                fprintf(stderr, "  1 0 1\n");
                fprintf(stderr, "  1 1 1\n");
                fprintf(stderr, "The above would cause all 4 tiles at zoom 1 to be rendered\n");
                return -1;
            default:
                fprintf(stderr, "unhandled char '%c'\n", c);
                break;
        }
    }

    if (maxZoom < minZoom) {
        fprintf(stderr, "Invalid zoom range, max zoom must be greater or equal to minimum zoom\n");
        return 1;
    }

    store = init_storage_backend(tile_dir);
    if (store == NULL) {
        fprintf(stderr, "Failed to initialise storage backend %s\n", tile_dir);
        return 1;
    }

    if (all) {
        if ((minX != -1 || minY != -1 || maxX != -1 || maxY != -1) && minZoom != maxZoom) {
            fprintf(stderr, "min-zoom must be equal to max-zoom when using min-x, max-x, min-y, or max-y options\n");
            return 1;
        }

        if (minX == -1) { minX = 0; }
        if (minY == -1) { minY = 0; }

        int lz = (1 << minZoom) - 1;

        if (minZoom == maxZoom) {
            if (maxX == -1) { maxX = lz; }
            if (maxY == -1) { maxY = lz; }
            if (minX > lz || minY > lz || maxX > lz || maxY > lz) {
                fprintf(stderr, "Invalid range, x and y values must be <= %d (2^zoom-1)\n", lz);
                return 1;
            }
        }

        if (minX < 0 || minY < 0 || maxX < -1 || maxY < -1) {
            fprintf(stderr, "Invalid range, x and y values must be >= 0\n");
            return 1;
        }

    }

    fprintf(stderr, "Rendering client\n");

    gettimeofday(&start, NULL);

    spawn_workers(numThreads, spath, maxLoad);

    if (all) {
        int x, y, z;
        printf("Rendering all tiles from zoom %d to zoom %d\n", minZoom, maxZoom);
        for (z=minZoom; z <= maxZoom; z++) {
            int current_maxX = (maxX == -1) ? (1 << z)-1 : maxX;
            int current_maxY = (maxY == -1) ? (1 << z)-1 : maxY;
            printf("Rendering all tiles for zoom %d from (%d, %d) to (%d, %d)\n", z, minX, minY, current_maxX, current_maxY);
            for (x=minX; x <= current_maxX; x+=METATILE) {
                for (y=minY; y <= current_maxY; y+=METATILE) {
                    if (!force) s = store->tile_stat(store, mapname, "", x, y, z);
                    if (force || (s.size < 0) || (s.expired)) {
                        enqueue(mapname, x, y, z);
                        num_render++;
                    }
                    num_all++;

                }
            }
        }
      } else if (bboxset) {
        int x, y, z, mask = METATILE - 1;
        printf("Rendering bbox (%.6f, %.6f) to (%.6f, %.6f) tiles from zoom %d to zoom %d\n", bbox[0], bbox[1], bbox[2], bbox[3], minZoom, maxZoom);
        for (z=minZoom; z <= maxZoom; z++) {
            minX = long2tilex(bbox[0], z) & ~mask;
            minY = lat2tiley(bbox[1], z) & ~mask;
            maxX = long2tilex(bbox[2], z) & ~mask;
            maxY = lat2tiley(bbox[3], z) & ~mask;
            if (minX < 0 || minY < 0 || maxX < -1 || maxY < -1) {
                fprintf(stderr, "Invalid range, x and y values must be >= 0\n");
                fprintf(stderr, "Zoom %d from (%d, %d) to (%d, %d)\n", z, minX, minY, maxX, maxY);
                fprintf(stderr, "Exit from zoom %d\n", z);
                continue;
            }
            if (minX > maxX) {
                // swap minX and maxX
                minX = minX ^ maxX;
                maxX = minX ^ maxX;
                minX = minX ^ maxX;
            }
            if (minY > maxY) {
                // swap minY and maxY
                minY = minY ^ maxY;
                maxY = minY ^ maxY;
                minY = minY ^ maxY;
            }
            printf("Rendering bbox tiles for zoom %d from (%d, %d) to (%d, %d)\n", z, minX, minY, maxX, maxY);
            for (x=minX; x <= maxX; x+=METATILE) {
                for (y=minY; y <= maxY; y+=METATILE) {
                    if (!force) s = store->tile_stat(store, mapname, "", x, y, z);
                    if (force || (s.size < 0) || (s.expired)) {
                        enqueue(mapname, x, y, z);
                        num_render++;
                    }
                    num_all++;
                }
            }
         }
    } else {
        while(!feof(stdin)) {
            int n = fscanf(stdin, "%d %d %d", &x, &y, &z);

            if (verbose)


            if (n != 3) {
                // Discard input line
                char tmp[1024];
                char *r = fgets(tmp, sizeof(tmp), stdin);
                if (!r)
                    continue;
                fprintf(stderr, "bad line %d: %s", num_all, tmp);
                continue;
            }

            if (verbose)
                printf("got: x(%d) y(%d) z(%d)\n", x, y, z);

            if (z < minZoom || z > maxZoom) {
                printf("Ignoring tile, zoom %d outside valid range (%d..%d)\n", z, minZoom, maxZoom);
                continue;
            }

            num_all++;

            if (!force) s = store->tile_stat(store, mapname, "", x, y, z);
            if (force || (s.size < 0) || (s.expired)) {
                // missing or old, render it
                //ret = process_loop(fd, mapname, x, y, z);
                enqueue(mapname, x, y, z);
                num_render++;
                // Attempts to adjust the stats for the QMAX tiles which are likely in the queue
                if (!(num_render % 10)) {
                    gettimeofday(&end, NULL);
                    printf("\n");
                    printf("Meta tiles rendered: ");
                    display_rate(start, end, num_render);
                    printf("Total tiles rendered: ");
                    display_rate(start, end, (num_render) * METATILE * METATILE);
                    printf("Total tiles handled from input: ");
                    display_rate(start, end, num_all);
                }
            } else {
                if (verbose)
                    printf("Tile %s is clean, ignoring\n", store->tile_storage_id(store, mapname, "", x, y, z, name));
            }
        }
    }

    store->close_storage(store);
    if (store != NULL) {
        free(store);
    }
    finish_workers();

    free(spath);
    if (mapname != mapname_default) {
        free((void *)mapname);
    }
    if (tile_dir != tile_dir_default) {
        free((void *)tile_dir);
    }


    gettimeofday(&end, NULL);
    printf("\n*****************************************************\n");
    printf("*****************************************************\n");
    printf("Total for all tiles rendered\n");
    printf("Meta tiles rendered: ");
    display_rate(start, end, num_render);
    printf("Total tiles rendered: ");
    display_rate(start, end, num_render * METATILE * METATILE);
    printf("Total tiles handled: ");
    display_rate(start, end, num_all);
    print_statistics();

    return 0;
}
Esempio n. 3
0
int32_t main(int32_t argc, char** argv) {
    // init common parameters
    int32_t i, j, k;

    // TODO parse command line options
    int32_t parse_cmd_opt_res = parse_cmd_opt(argc, argv);
    CHECK_RES(parse_cmd_opt_res, FATAL, true, -1,
            "parse_cmd_opt failed, res=[%d]", parse_cmd_opt_res);

    // TODO init encryption/decryption method
    cryption_t* crpt = init_cryption(g_conf.cryption_method);
    CHECK_RES(crpt == NULL, FATAL, true, -2,
            "init_cryption failed, res=[%d]", init_cryption_res);

    // prepare to fork children
    g_pids = (pid_t*)malloc(g_conf.worker_num * sizeof(pid_t));
    CHECK_RES((g_pids == NULL), FATAL, true, -3,
            "malloc for g_pids failed");
    for (i = 0; i < worker_num; ++i) {
        g_pids[i] = -1;
    }

    parent_pid = getpid();
    cur_pid    = getpid();

    loop = uv_default_loop();

    uv_tcp_t server;
    uv_tcp_init(loop, &server);

    struct sockaddr_in addr;
    uv_ip4_addr(g_conf.listen_host, g_conf.listen_port, &addr);
    uv_tcp_bind(&server, (const struct sockaddr*)&addr, 0);

    int32_t uv_listen_res = uv_listen(
        (uv_stream_t*)&server, 
        g_conf.max_connection_num,
        on_new_downstream_connection
    );

    //TODO(xuruiqi) write spawn_workers
    int32_t spawn_workers_res = spawn_workers(g_conf.worker_num);
    CHECK_RES(spawn_workers_res, FATAL, true, -4,
            "spawn_workers failed, res=[%d]", spawn_workers_res);
    if (cur_pid == parent_pid) {
        // close master's listening event
        uv_close((uv_handle_t*)&server, NULL);

        int32_t join_res = 0;
        for (i = 0; i < g_conf.worker_num; ++i) {
            if (g_pids[i] <= 0) {
                continue;
            }
            join_res = waitpid(g_pids[i], NULL, 0);
            if (join_res == -1) {
                LOG_ERROR("Wait for pid %d failed\n", g_pids[i]);
            } else {
                LOG_INFO("Child process %d finished\n", g_pids[i]);
            }
        }
    } else {
        return uv_run(loop, UV_RUN_DEFAULT);
    }
    
    return 0;
}
Esempio n. 4
0
int main(int argc, char **argv)
{
    char spath[PATH_MAX] = RENDER_SOCKET;
    char *config_file = RENDERD_CONFIG;
    char *map = NULL;
    int c;
    int numThreads = 1;
    int dd, mm, yy;
    struct tm tm;

    while (1) {
        int option_index = 0;
        static struct option long_options[] = {
            {"config",1,0,'c'},
            {"min-zoom", 1, 0, 'z'},
            {"max-zoom", 1, 0, 'Z'},
            {"max-load", 1, 0, 'l'},
            {"socket", 1, 0, 's'},
            {"num-threads", 1, 0, 'n'},
            {"tile-dir", 1, 0, 't'},
            {"timestamp", 1, 0, 'T'},
            {"map", 1, 0, 'm'},
            {"verbose", 0, 0, 'v'},
            {"help", 0, 0, 'h'},
            {0, 0, 0, 0}
        };

        c = getopt_long(argc, argv, "hvz:Z:s:t:n:c:l:T:m:", long_options, &option_index);
        if (c == -1)
            break;

        switch (c) {
            case 's':   /* -s, --socket */
                strncpy(spath, optarg, PATH_MAX-1);
                spath[PATH_MAX-1] = 0;
                break;
            case 't':   /* -t, --tile-dir */
                tile_dir=strdup(optarg);
                break;

            case 'c':   /* -c, --config */
                config_file=strdup(optarg);
                break;
            case 'm':   /* -m, --map */
                map=strdup(optarg);
                break;
            case 'n':   /* -n, --num-threads */
                numThreads=atoi(optarg);
                if (numThreads <= 0) {
                    fprintf(stderr, "Invalid number of threads, must be at least 1\n");
                    return 1;
                }
                break;
           case 'z':   /* -z, --min-zoom */
                        minZoom=atoi(optarg);
                if (minZoom < 0 || minZoom > MAX_ZOOM) {
                    fprintf(stderr, "Invalid minimum zoom selected, must be between 0 and %d\n", MAX_ZOOM);
                    return 1;
                }
                break;
            case 'Z':   /* -Z, --max-zoom */
                maxZoom=atoi(optarg);
                if (maxZoom < 0 || maxZoom > MAX_ZOOM) { 
                    fprintf(stderr, "Invalid maximum zoom selected, must be between 0 and %d\n", MAX_ZOOM);
                    return 1;
                }
                break;
             case 'l':
                 max_load = atoi(optarg);
                 if (max_load < 0) {
                     fprintf(stderr, "Invalid maximum load specified, must be greater than 0\n");
                     return 1;
                 }
                 break;
            case 'T':
                
                if (sscanf(optarg,"%d/%d/%d", &dd, &mm, &yy) < 3) {
                    fprintf(stderr, "Invalid planet time stamp, must be in the format dd/mm/yyyy\n");
                    return 1;
                }
                
                if (yy > 100) yy -= 1900;
                if (yy < 70) yy += 100;
                
                tm.tm_sec = 0; tm.tm_min = 0; tm.tm_hour = 0; tm.tm_mday = dd; tm.tm_mon = mm - 1; tm.tm_year =  yy;
                planetTime = mktime(&tm);

            case 'v':   /* -v, --verbose */
                verbose=1;
                break;
            case 'h':   /* -h, --help */
                fprintf(stderr, "Usage: render_old [OPTION] ...\n");
                fprintf(stderr, "Search the rendered tiles and re-render tiles which are older then the last planet import\n");
                fprintf(stderr, "  -c, --config=CONFIG  specify the renderd config file\n");
                fprintf(stderr, "  -n, --num-threads=N  the number of parallel request threads (default 1)\n");
                fprintf(stderr, "  -t, --tile-dir       tile cache directory (defaults to '" HASH_PATH "')\n");
                fprintf(stderr, "  -z, --min-zoom=ZOOM  filter input to only render tiles greater or equal to this zoom level (default 0)\n");
                fprintf(stderr, "  -Z, --max-zoom=ZOOM  filter input to only render tiles less than or equal to this zoom level (default %d)\n", MAX_ZOOM);
                fprintf(stderr, "  -s, --socket=SOCKET  unix domain socket name for contacting renderd\n");
                fprintf(stderr, "  -l, --max-load=LOAD  maximum system load with which requests are submitted\n");
                fprintf(stderr, "  -T, --timestamp=DD/MM/YY  Overwrite the assumed data of the planet import\n");
                fprintf(stderr, "  -m, --map=STYLE      Instead of going through all styls of CONFIG, only use a specific map-style\n");
                return -1;
            default:
                fprintf(stderr, "unhandled char '%c'\n", c);
                break;
        }
    }

    if (maxZoom < minZoom) {
        fprintf(stderr, "Invalid zoom range, max zoom must be greater or equal to minimum zoom\n");
        return 1;
    }

    fprintf(stderr, "Rendering old tiles\n");

    if (planetTime != 0) {
        printf("Overwriting planet file update to %s", ctime(&planetTime));
    }

    gettimeofday(&start, NULL);

    FILE * hini ;
    char line[INILINE_MAX];
    char value[INILINE_MAX];

    // Load the config
    if ((hini=fopen(config_file, "r"))==NULL) {
        fprintf(stderr, "Config: cannot open %s\n", config_file);
        exit(7);
    }

    spawn_workers(numThreads, spath);

    if (map) {
        render_layer(map);
    } else {
        while (fgets(line, INILINE_MAX, hini)!=NULL) {
            if (line[0] == '[') {
                if (strlen(line) >= XMLCONFIG_MAX){
                    fprintf(stderr, "XML name too long: %s\n", line);
                    exit(7);
                }
                sscanf(line, "[%[^]]", value);
                // Skip mapnik & renderd sections which are config, not tile layers
                if (strcmp(value,"mapnik") && strncmp(value, "renderd", 7))
                    render_layer(value);
            }
        }
    }
    fclose(hini);

    finish_workers(numThreads);

    gettimeofday(&end, NULL);
    printf("\nTotal for all tiles rendered\n");
    printf("Meta tiles rendered: ");
    display_rate(start, end, num_render);
    printf("Total tiles rendered: ");
    display_rate(start, end, num_render * METATILE * METATILE);
    printf("Total tiles handled: ");
    display_rate(start, end, num_all);

    return 0;
}
Esempio n. 5
0
File: cws.c Progetto: jtsymon/cws
int main (int argc, char **argv) {

    srand (time (NULL));
    int port = 8080;
    int work = 4;

    sighandleall (&signal_handler, SA_RESTART);

    argv0 = *argv;
    setname ("cws[master]");

    int selfpiperead;
    {
        int pipefd[2];
        if (pipe (pipefd) == -1) {
            die("pipe");
        }
        selfpipe = pipefd[1];
        selfpiperead = pipefd[0];
    }

    worker_init (port, "file");
    spawn_workers (work);

    tcgetattr (STDIN_FILENO, &old_term);
    atexit (&cleanup);
    setterm();

    int epollfd = epoll();
    epoll_add (epollfd, STDIN_FILENO);
    epoll_add (epollfd, selfpiperead);

    struct epoll_event events[MAX_QUEUE];

    int i, n;
    char cmd;
    while (1) {
        n = epoll_wait (epollfd, events, MAX_QUEUE, -1);
        for (i = 0; i < n; i++) {
            if ((events[i].events & EPOLLERR) ||
                    (events[i].events & EPOLLHUP) ||
                    (!(events[i].events & EPOLLIN))) {
                exit(1);
            } else if (STDIN_FILENO == events[i].data.fd) {
                // user input event
                cmd = getchar();
                switch (cmd) {
                    case 'k':
                        printf ("Killing workers\n");
                        finish(1);
                        break;
                    case 'q':
                        printf ("Telling workers to exit\n");
                        finish(0);
                        break;
                    case '+':
                        spawn_workers (worker_count + 1);
                        break;
                    case '-':
                        spawn_workers (worker_count - 1);
                        break;
                }
            } else {
                // pipe event
                if (read (selfpiperead, &cmd, 1) <= 0) {
                    die ("read");
                }
                switch (cmd) {
                    case CMD_KILL:
                        printf ("Killing workers\n");
                        finish(0);
                        break;
                    case CMD_TERM:
                        printf ("Telling workers to exit\n");
                        finish(1);
                        break;
                    case CMD_INCR:
                        spawn_workers (worker_count + 1);
                        break;
                    case CMD_DECR:
                        spawn_workers (worker_count - 1);
                        break;
                    case CMD_CHLD:
                        check_workers();
                        break;
                }
            }
        }
    }
    return 0; 
}