void Instruction::debug() const { std::cout << "infix: "; print_tokens(tokens_); std::cout << "postfix: "; print_tokens(postfix_); }
static void dump_name (bt_name * name) { printf ("total number of tokens = %d\n", name->tokens->num_items); print_tokens ("first", name->parts[BTN_FIRST], name->part_len[BTN_FIRST]); print_tokens ("von", name->parts[BTN_VON], name->part_len[BTN_VON]); print_tokens ("last", name->parts[BTN_LAST], name->part_len[BTN_LAST]); print_tokens ("jr", name->parts[BTN_JR], name->part_len[BTN_JR]); }
void verbose(void) { register int i; if (!vflag) return; null_rules = (Yshort *) NEW2(nrules, null_rules[0]); if (null_rules == 0) no_space(); BtYacc_puts("\f\n", verbose_file); for (i = 0; i < nstates; ++i) print_state(i); FREE(null_rules); if (nunused) log_unused(); if (SRtotal || RRtotal) log_conflicts(); print_tokens(); BtYacc_printf(verbose_file, "\n\n%d terminals, %d nonterminals\n%d grammar rules, %d states\n", ntokens, nvars, nrules - 2, nstates); }
int main() { std::string s; std::getline(std::cin, s); std::vector<std::pair<int, int> > tokens; str_tokenize(s, tokens); print_tokens(s, tokens); return 0; }
void yyerror (YYSTYPE lvalp, YYLTYPE const *llocp, parse_context const *context, char const *s) { L printf ("%s", s); if (lvalp != NULL) { L printf ("offending token: "); print_tokens (lvalp); } L printf ("deparsed correct subtree: "); print_tokens (parse_context_unit (context)); L printf ("current subtree:"); print_members_recursive (llocp, parse_context_unit (context), stdout, 0); fputc ('\n', stdout); exit (EXIT_FAILURE); }
void formula_lexer::tokenize() { #if IXION_DEBUG_LEXER __IXION_DEBUG_OUT__ << "formula string: '" << std::string(mp_first, m_size) << "'" << endl; #endif tokenizer tkr(m_tokens, mp_first, m_size); tkr.run(); #if IXION_DEBUG_LEXER __IXION_DEBUG_OUT__ << print_tokens(m_tokens, true) << endl; #endif }
bool test_expandreadmacro() { list *l = tokenize(" abc 123 "); char *s; assert(list_length(l) == 2); delete_tokens(l); l = tokenize(" (( 1 2 3) (hello world) )"); l = expand_readmacro(l); print_tokens(l); delete_tokens(l); l = tokenize("'x"); l = expand_readmacro(l); print_tokens(l); delete_tokens(l); l = tokenize("`x"); l = expand_readmacro(l); print_tokens(l); s = (char*)car(l); delete_tokens(l); l = tokenize(",x"); l = expand_readmacro(l); print_tokens(l); delete_tokens(l); l = tokenize(",@x"); l = expand_readmacro(l); print_tokens(l); delete_tokens(l); l = tokenize("`(xxx ,y ,@a )"); l = expand_readmacro(l); print_tokens(l); delete_tokens(l); l = tokenize("`(xxx (,y (,@a) . z ))"); l = expand_readmacro(l); print_tokens(l); delete_tokens(l); l = tokenize("'(1 . 2)"); l = expand_readmacro(l); print_tokens(l); delete_tokens(l); return true; }
int main(int argc, char *argv[]) { const int buf_size = 4096; char buffer[buf_size + 1]; buffer[0] = '\0'; buffer[buf_size] = '\0'; while (fgets(buffer, buf_size, stdin) != NULL) { print_tokens(buffer); } return 0; }
int main() { char start, stop; int dir; while(get_tokens(&dir, &start, &stop) == 1) { print_tokens(&dir, &start, &stop); putchar('\n'); } return 0; }
// 计算query (短文本) 与 document (长文本) 的相似度 // 可选的指标包括: // 1. document主题分布生成query的likelihood, 值越大相似度越高 // 2. 基于TWE模型的相似度计算 void cal_query_doc_similarity(const string& query, const string& document) { // 分词 vector<string> q_tokens, d_tokens; _tokenizer->tokenize(query, q_tokens); _tokenizer->tokenize(document, d_tokens); print_tokens("Query Tokens", q_tokens); print_tokens("Doc Tokens", d_tokens); // 对长文本进行主题推断,获取主题分布 LDADoc doc; _engine.infer(d_tokens, doc); vector<Topic> doc_topic_dist; doc.sparse_topic_dist(doc_topic_dist); float lda_sim = SemanticMatching::likelihood_based_similarity(q_tokens, doc_topic_dist, _engine.get_model()); float twe_sim = SemanticMatching::twe_based_similarity(q_tokens, doc_topic_dist, _twe); cout << "LDA Similarity = " << lda_sim << endl << "TWE Similarity = " << twe_sim << endl; }
std::vector<Token> lexical_analysis(const std::string& src) { std::vector<Token> tokens; print_source(src); tokens = scanner(src); print_tokens(tokens, TokenType::WHITESPACE); tokens = evaluate_scanned_tokens(tokens); std::vector<Token> stripped_tokens = strip_unnecessary(tokens); // remove all comments and whitespace tokens. return stripped_tokens; }
int main() { print_tokens("0:4:1:335577090:1:1:352452736::", ':', false); print_tokens("0:4:1:335577090:1:1:352452736::", ':', true); print_tokens("0:4:1:335577090:1:1:352452736:p", ':', false); print_tokens("0:4:1:335577090:1:1:352452736:p", ':', true); print_tokens("0:4:1:335577090:1:1:352452736:p:", ':', false); print_tokens("0:4:1:335577090:1:1:352452736:p:", ':', true); }
static void print_tokens (pt_node const *node) { char const *const *members; if (strcmp (pt_node_type_name (node), "token") == 0 && pt_token_token (node) != EOF) printf ("%s ", pt_token_text (node)); else for (members = pt_node_members (node); *members != NULL; members++) { pt_node const *next = pt_node_member (node, *members); if (next != NULL) print_tokens (next); } }
void parse_data(char *data, char *end) { char *token_alloc; // allocated memory for tokens killscript = false; // dont kill the script straight away // allocate space for the tokens token_alloc = new char[current_script->len + T_MAXTOKENS]; prev_section = NULL; // clear it while(*rover) // go through the script executing each statement { // past end of script? if(rover > end) break; // reset the tokens before getting the next line tokens[0] = token_alloc; prev_section = current_section; // store from prev. statement // get the line and tokens get_tokens(rover); if(killscript) break; if(!num_tokens) { if(current_section) // no tokens but a brace { // possible } at end of loop: // refer to spec.c spec_brace(); } continue; // continue to next statement } if(script_debug) print_tokens(); // debug run_statement(); // run the statement } delete token_alloc; }
static void setup_transition(struct state_machine *sm, int state) { char *token; int target_state; printf("\n\nTransitions for the state 'E%d'\n", state); printf("Available tokens: "); print_tokens(sm->tokens); for (;;) { token = read_token(sm); if (token == NULL) break; target_state = read_target_state(sm); sm_add_transition(sm, sm_make_transition(token, state, target_state)); } }
/** * @brief Test execution thread function. * * @param[in] p pointer to a @p BaseChannel object for test output * @return A failure boolean value. */ msg_t TestThread(void *p) { int i, j; chp = p; test_println(""); test_println("*** ChibiOS/RT test suite"); test_println("***"); test_print("*** Kernel: "); test_println(CH_KERNEL_VERSION); test_print("*** Compiled: "); test_println(__DATE__ " - " __TIME__); #ifdef PORT_COMPILER_NAME test_print("*** Compiler: "); test_println(PORT_COMPILER_NAME); #endif test_print("*** Architecture: "); test_println(PORT_ARCHITECTURE_NAME); #ifdef PORT_CORE_VARIANT_NAME test_print("*** Core Variant: "); test_println(PORT_CORE_VARIANT_NAME); #endif #ifdef PORT_INFO test_print("*** Port Info: "); test_println(PORT_INFO); #endif #ifdef PLATFORM_NAME test_print("*** Platform: "); test_println(PLATFORM_NAME); #endif #ifdef BOARD_NAME test_print("*** Test Board: "); test_println(BOARD_NAME); #endif test_println(""); global_fail = FALSE; i = 0; while (patterns[i]) { j = 0; while (patterns[i][j]) { print_line(); test_print("--- Test Case "); test_printn(i + 1); test_print("."); test_printn(j + 1); test_print(" ("); test_print(patterns[i][j]->name); test_println(")"); #if DELAY_BETWEEN_TESTS > 0 chThdSleepMilliseconds(DELAY_BETWEEN_TESTS); #endif execute_test(patterns[i][j]); if (local_fail) { test_print("--- Result: FAILURE (#"); test_printn(failpoint); test_print(" ["); print_tokens(); test_println("])"); } else test_println("--- Result: SUCCESS"); j++; } i++; } print_line(); test_println(""); test_print("Final result: "); if (global_fail) test_println("FAILURE"); else test_println("SUCCESS"); return (msg_t)global_fail; }
int main(int argc, char **argv) { int ch; int i; FILE *fp; while ((ch = getopt(argc, argv, "d:lnprsx")) != -1) { switch(ch) { case 'd': del = optarg; break; case 'l': oneline = 1; break; case 'n': oflags |= AU_OFLAG_NORESOLVE; break; case 'p': partial = 1; break; case 'r': if (oflags & AU_OFLAG_SHORT) usage(); /* Exclusive from shortfrm. */ oflags |= AU_OFLAG_RAW; break; case 's': if (oflags & AU_OFLAG_RAW) usage(); /* Exclusive from raw. */ oflags |= AU_OFLAG_SHORT; break; case 'x': oflags |= AU_OFLAG_XML; break; case '?': default: usage(); } } if (oflags & AU_OFLAG_XML) au_print_xml_header(stdout); /* For each of the files passed as arguments dump the contents. */ if (optind == argc) { print_tokens(stdin); return (1); } for (i = optind; i < argc; i++) { fp = fopen(argv[i], "r"); if ((fp == NULL) || (print_tokens(fp) == -1)) perror(argv[i]); if (fp != NULL) fclose(fp); } if (oflags & AU_OFLAG_XML) au_print_xml_footer(stdout); return (1); }
int main(int argc, char** argv) { int agentx_subagent = 1; /* change this if you want to be a SNMP master agent */ /* Defs for arg-handling code: handles setting of policy-related variables */ int ch; extern char* optarg; int dont_fork = 0, use_syslog = 0; char const* agentx_socket = NULL; char const* app_name = "moninor-agent"; char const* directory = "/home/"; while ((ch = getopt(argc, argv, "a:p:hdD:fHLMx:")) != EOF) { switch (ch) { case 'a': app_name = optarg; break; case 'p': directory = optarg; break; case 'h': print_usage(); exit(0); case 'd': print_tokens(); exit(0); break; case 'D': debug_register_tokens(optarg); snmp_set_do_debugging(1); break; case 'f': dont_fork = 1; break; case 'M': agentx_subagent = 0; break; case 'L': use_syslog = 0; /* use stderr */ break; case 'x': agentx_socket = optarg; break; default: fprintf(stderr, "unknown option %c\n", ch); print_usage(); } } if (optind < argc) { int i; /* * There are optional transport addresses on the command line. */ DEBUGMSGTL(("snmpd/main", "optind %d, argc %d\n", optind, argc)); for (i = optind; i < argc; i++) { char* c, *astring; if ((c = netsnmp_ds_get_string(NETSNMP_DS_APPLICATION_ID, NETSNMP_DS_AGENT_PORTS))) { astring = malloc(strlen(c) + 2 + strlen(argv[i])); if (astring == NULL) { fprintf(stderr, "malloc failure processing argv[%d]\n", i); exit(1); } sprintf(astring, "%s,%s", c, argv[i]); netsnmp_ds_set_string(NETSNMP_DS_APPLICATION_ID, NETSNMP_DS_AGENT_PORTS, astring); SNMP_FREE(astring); } else { netsnmp_ds_set_string(NETSNMP_DS_APPLICATION_ID, NETSNMP_DS_AGENT_PORTS, argv[i]); } } DEBUGMSGTL(("snmpd/main", "port spec: %s\n", netsnmp_ds_get_string(NETSNMP_DS_APPLICATION_ID, NETSNMP_DS_AGENT_PORTS))); } /* check directory existence */ DIR* dir = opendir(directory); if (dir) { closedir(dir); } else if (ENOENT == errno) { puts("Directory doesn't not exist."); exit(-1); } else { puts("Cannot open direcory."); exit(-1); } /* we're an agentx subagent? */ if (agentx_subagent) { /* make us a agentx client. */ netsnmp_enable_subagent(); if (NULL != agentx_socket) { netsnmp_ds_set_string(NETSNMP_DS_APPLICATION_ID, NETSNMP_DS_AGENT_X_SOCKET, agentx_socket); } } snmp_disable_log(); if (use_syslog) snmp_enable_calllog(); else snmp_enable_stderrlog(); /* daemonize */ if (!dont_fork) { int rc = netsnmp_daemonize(1, !use_syslog); if (rc) exit(-1); } /* initialize tcp/ip if necessary */ SOCK_STARTUP; /* initialize the agent library */ init_agent(app_name); /* init mib code */ global_settings_t* settings = get_mutable_global_settings(); strcpy(settings->path, directory); void* data[4] = {NULL, NULL, NULL, NULL}; init_Directory(data + 0); init_DirectoryStateNotification(data + 1); init_NumDirectories(data + 2); init_NumFiles(data + 3); init_DirectoryContentTable(); /* read app_name.conf files. */ init_snmp(app_name); /* If we're going to be a snmp master agent, initial the ports */ if (!agentx_subagent) init_master_agent(); /* open the port to listen on (defaults to udp:161) */ /* In case we recevie a request to stop (kill -TERM or kill -INT) */ g_keep_running = 1; signal(SIGTERM, stop_server); signal(SIGINT, stop_server); /* you're main loop here... */ while (g_keep_running) { agent_check_and_process(1); /* 0 == don't block */ } shutdown_DirectoryContentTable(); shutdown_NumFiles(data[3]); shutdown_NumDirectories(data[2]); shutdown_DirectoryStateNotification(data[1]); shutdown_Directory(data[0]); /* at shutdown time */ snmp_shutdown(app_name); SOCK_CLEANUP; exit(0); }
int main (int argc, char *argv[]) { char buffer[10000]; token tokens[500]; int t; //total number of tokens int i; //general use counter int res; FILE *fp; if (argc < 2) { //check if the number of arguments is correct printf("Usage: %s <input_file> -o <output_file>\n", argv[0]); return 1; } else if ((fp = fopen(argv[1], "r")) == NULL) { //try to open the input file printf("%s\n", strerror(errno)); return 2; } else { /*Phase 0: Put instuctions from the input file to the buffer serialized*/ res = serialize_input(buffer, fp); /*If is on DEBUG_MODE print debuggin info*/ if (DEBUG_MODE) { printf("Phase 0: Put instuctions from the input file to the buffer serialized:\n%s\n\n", buffer); } close(fp); if (res == 0) { puts("Empty input file."); return 3; } } /*Phase 1: Parse the code*/ validate_tokens(buffer); /*If is on DEBUG_MODE print debuggin info*/ if (DEBUG_MODE) { printf("Phase 1: Remove lines with invalid tokens:\n%s\n\n", buffer); } /*Phase 2: Extract the tokens*/ t = extract_tokens(buffer, tokens); /*If is on DEBUG_MODE print debuggin info*/ if (DEBUG_MODE) { puts("Phase 2: Extract the tokens:"); print_tokens(tokens, t); putchar('\n'); } /*Phase 3: Do syntax analysis on the tokens*/ t = analize_tokens(tokens, t); /*If is on DEBUG_MODE print debuggin info*/ if (DEBUG_MODE) { puts("Phase 3: Do syntax analysis on the tokens:"); print_tokens(tokens, t); putchar('\n'); } /*Phase 4: Do optimization on the tokens*/ t = optimize_tokens(tokens, t); /*If is on DEBUG_MODE print debuggin info*/ if (DEBUG_MODE) { puts("Phase 4: Do optimization on the tokens:"); print_tokens(tokens, t); putchar('\n'); } /*Phase 5: Do code generation based on the tokens*/ generate_code(buffer, tokens, t); /*If is on DEBUG_MODE print debuggin info*/ if (DEBUG_MODE) { printf("Phase 5: Do code generation based on the tokens:\n%s\n\n", buffer); } /*Print errors buffer*/ if (error_cnt != 0) { puts(error_buffer); } else { puts("No Errors"); } /*Final Phase: Save the code in a file*/ if (argc == 4 && strcmp(argv[2], "-o") == 0) { //check if user provided output file name save_code(buffer, argv[3]); } else { save_code(buffer, "out.c"); } return 0; }
int main(int argc, char **argv) { int ch; int i; #ifdef HAVE_CAP_ENTER int retval; pid_t childpid, pid; #endif FILE *fp; while ((ch = getopt(argc, argv, "d:lnprsx")) != -1) { switch(ch) { case 'd': del = optarg; break; case 'l': oneline = 1; break; case 'n': oflags |= AU_OFLAG_NORESOLVE; break; case 'p': partial = 1; break; case 'r': if (oflags & AU_OFLAG_SHORT) usage(); /* Exclusive from shortfrm. */ oflags |= AU_OFLAG_RAW; break; case 's': if (oflags & AU_OFLAG_RAW) usage(); /* Exclusive from raw. */ oflags |= AU_OFLAG_SHORT; break; case 'x': oflags |= AU_OFLAG_XML; break; case '?': default: usage(); } } #ifdef HAVE_CAP_ENTER /* * Prime group, password, and audit-event files to be opened before we * enter capability mode. */ (void)getgrgid(0); (void)setgroupent(1); (void)getpwuid(0); (void)setpassent(1); (void)getauevent(); #endif if (oflags & AU_OFLAG_XML) au_print_xml_header(stdout); /* For each of the files passed as arguments dump the contents. */ if (optind == argc) { #ifdef HAVE_CAP_ENTER retval = cap_enter(); if (retval != 0 && errno != ENOSYS) err(EXIT_FAILURE, "cap_enter"); #endif print_tokens(stdin); return (1); } for (i = optind; i < argc; i++) { fp = fopen(argv[i], "r"); if (fp == NULL) { perror(argv[i]); continue; } /* * If operating with sandboxing, create a sandbox process for * each trail file we operate on. This avoids the need to do * fancy things with file descriptors, etc, when iterating on * a list of arguments. */ #ifdef HAVE_CAP_ENTER childpid = fork(); if (childpid == 0) { /* Child. */ retval = cap_enter(); if (retval != 0 && errno != ENOSYS) err(EXIT_FAILURE, "cap_enter"); if (print_tokens(fp) == -1) perror(argv[i]); exit(0); } /* Parent. Await child termination. */ while ((pid = waitpid(childpid, NULL, 0)) != childpid); #else if (print_tokens(fp) == -1) perror(argv[i]); #endif fclose(fp); } if (oflags & AU_OFLAG_XML) au_print_xml_footer(stdout); return (0); }
void main(){ void print_tokens(char *line); char str[]="adc,de,fgh"; print_tokens(str); }
/** * @brief Test execution thread function. * * @param[in] stream pointer to a @p BaseSequentialStream object for test * output * @return A failure boolean value casted to @p msg_t. * @retval FALSE if no errors occurred. * @retval TRUE if one or more tests failed. * * @api */ msg_t test_execute(BaseSequentialStream *stream) { int i, j; test_chp = stream; test_println(""); #if defined(TEST_SUITE_NAME) test_println("*** " TEST_SUITE_NAME); #else test_println("*** ChibiOS test suite"); #endif test_println("***"); test_print("*** Compiled: "); test_println(__DATE__ " - " __TIME__); #ifdef PLATFORM_NAME test_print("*** Platform: "); test_println(PLATFORM_NAME); #endif #ifdef BOARD_NAME test_print("*** Test Board: "); test_println(BOARD_NAME); #endif test_println(""); test_global_fail = FALSE; i = 0; while (test_suite[i]) { j = 0; while (test_suite[i][j]) { print_line(); test_print("--- Test Case "); test_printn(i + 1); test_print("."); test_printn(j + 1); test_print(" ("); test_print(test_suite[i][j]->name); test_println(")"); #if TEST_DELAY_BETWEEN_TESTS > 0 osalThreadSleepMilliseconds(TEST_DELAY_BETWEEN_TESTS); #endif execute_test(test_suite[i][j]); if (test_local_fail) { test_print("--- Result: FAILURE (#"); test_printn(test_step); test_print(" ["); print_tokens(); test_print("] \""); test_print(test_failure_message); test_println("\")"); } else test_println("--- Result: SUCCESS"); j++; } i++; } print_line(); test_println(""); test_print("Final result: "); if (test_global_fail) test_println("FAILURE"); else test_println("SUCCESS"); return (msg_t)test_global_fail; }
/** * @brief Test execution thread function. * * @param[in] stream pointer to a @p BaseSequentialStream object for test * output * @param[in] tsp test suite to execute * @return A failure boolean value casted to @p msg_t. * @retval false if no errors occurred. * @retval true if one or more tests failed. * * @api */ msg_t test_execute(BaseSequentialStream *stream, const testsuite_t *tsp) { int tseq, tcase; test_chp = stream; test_println(""); if (tsp->name != NULL) { test_print("*** "); test_println(tsp->name); } else { test_println("*** Test Suite"); } test_println("***"); test_print("*** Compiled: "); test_println(__DATE__ " - " __TIME__); #if defined(PLATFORM_NAME) test_print("*** Platform: "); test_println(PLATFORM_NAME); #endif #if defined(BOARD_NAME) test_print("*** Test Board: "); test_println(BOARD_NAME); #endif #if defined(TEST_SIZE_REPORT) { extern uint8_t __text_base, __text_end, _data_start, _data_end, _bss_start, _bss_end; test_println("***"); test_print("*** Text size: "); test_printn((uint32_t)(&__text_end - &__text_base)); test_println(" bytes"); test_print("*** Data size: "); test_printn((uint32_t)(&_data_end - &_data_start)); test_println(" bytes"); test_print("*** BSS size: "); test_printn((uint32_t)(&_bss_end - &_bss_start)); test_println(" bytes"); } #endif #if defined(TEST_REPORT_HOOK_HEADER) TEST_REPORT_HOOK_HEADER #endif test_println(""); test_global_fail = false; tseq = 0; while (tsp->sequences[tseq] != NULL) { #if TEST_SHOW_SEQUENCES == TRUE print_fat_line(); test_print("=== Test Sequence "); test_printn(tseq + 1); test_print(" ("); test_print(tsp->sequences[tseq]->name); test_println(")"); #endif tcase = 0; while (tsp->sequences[tseq]->cases[tcase] != NULL) { print_line(); test_print("--- Test Case "); test_printn(tseq + 1); test_print("."); test_printn(tcase + 1); test_print(" ("); test_print(tsp->sequences[tseq]->cases[tcase]->name); test_println(")"); #if TEST_DELAY_BETWEEN_TESTS > 0 osalThreadSleepMilliseconds(TEST_DELAY_BETWEEN_TESTS); #endif execute_test(tsp->sequences[tseq]->cases[tcase]); if (test_local_fail) { test_print("--- Result: FAILURE (#"); test_printn(test_step); test_print(" ["); print_tokens(); test_print("] \""); test_print(test_failure_message); test_println("\")"); } else { test_println("--- Result: SUCCESS"); } tcase++; } tseq++; } print_line(); test_println(""); test_print("Final result: "); if (test_global_fail) test_println("FAILURE"); else test_println("SUCCESS"); #if defined(TEST_REPORT_HOOK_END) TEST_REPORT_HOOK_END #endif return (msg_t)test_global_fail; }