int Parser::parse() try { push__(0); // initial state clearin(); // clear the tokens. while (true) { try { if (s_state[d_state__]->d_type & REQ_TOKEN) nextToken(); // obtain next token int action = lookup(false); // lookup d_token__ in d_state__ if (action > 0) // SHIFT: push a new state { push__(action); popToken__(); // token processed } else if (action < 0) // REDUCE: execute and pop. { executeAction(-action); // next token is the rule's LHS reduce__(s_productionInfo[-action]); } else ACCEPT(); } catch (ErrorRecovery__) { errorRecovery(); } } } catch (Return__ retValue) { return retValue; }
int write_image (struct lev_storage_file *E, char *data, crc32_logpos_t *P) { clearin (); assert (E->type == LEV_STORAGE_FILE || (E->type == LEV_STORAGE_HIDE_FILE && !E->size)); const unsigned zero = 0; int l = (E->size + 3) & -4; int padded_zero_bytes = l - E->size; assert (padded_zero_bytes >= 0 && padded_zero_bytes < 4); if (padded_zero_bytes) { assert (!memcmp (data + E->size, &zero, padded_zero_bytes)); } P->crc32_complement = crc32_partial (E, sizeof (*E), P->crc32_complement); P->log_pos += sizeof (*E); P->crc32_complement = crc32_partial (data, l, P->crc32_complement); P->log_pos += l; struct lev_crc32 C; C.type = LEV_CRC32; C.pos = P->log_pos; C.crc32 = ~P->crc32_complement; C.timestamp = last_mtime; P->crc32_complement = crc32_partial (&C, sizeof (C), P->crc32_complement); P->log_pos += sizeof (C); if (writeout (E, sizeof (*E)) < 0 || writeout (data, l) < 0 || writeout (&C, sizeof (C)) < 0 || flushout () < 0) { return -1; } if (test_mode) { char base64url_secret[12]; int r = base64url_encode ((unsigned char *) &E->secret, 8, base64url_secret, 12); assert (!r); printf ("wget -O %d.jpg http://127.0.0.1:%d/v%lld/%x/%s.jpg\n", E->local_id, http_port, volume_id, E->local_id, base64url_secret); } return 0; }
void storage_binlog_append (char *input_filename, char *output_filename) { vkprintf (3, "storage_binlog_append (%s, %s)\n", input_filename, output_filename); file_t A, B; file_open (&A, input_filename, O_RDONLY, 1); file_open (&B, output_filename, O_RDWR, 0); if (B.fd < 0) { unsigned char a[STORAGE_LEV_START_SIZE]; if (A.size < STORAGE_LEV_START_SIZE) { kprintf ("%s too short (couldn't contain LEV_START logevent).\n", A.filename); exit (1); } if (vk_pread (&A, a, STORAGE_LEV_START_SIZE, 0) < 0) { exit (1); } int fd = open (output_filename, O_WRONLY | O_CREAT, 0660); if (fd < 0) { kprintf ("Couldn't create %s\n", output_filename); } if (lock_whole_file (fd, F_WRLCK) <= 0) { kprintf ("lock_whole_file (%s, F_WRLCK) fail.\n", output_filename); exit (1); } if (STORAGE_LEV_START_SIZE != write (fd, a, STORAGE_LEV_START_SIZE)) { kprintf ("writing LEV_START to %s fail. %m\n", output_filename); exit (1); } close (fd); file_open (&B, output_filename, O_RDWR, 1); } if (lock_whole_file (B.fd, F_WRLCK) <= 0) { kprintf ("lock_whole_file (%s, F_WRLCK) fail.\n", B.filename); close (A.fd); close (B.fd); exit (1); } if (prefix_check (&A, &B) < 0) { close (A.fd); close (B.fd); exit (1); } long long cur_log_pos = B.size; int max_file_buf_size = -1; unsigned char *a = NULL; wfd = B.fd; if (B.size != lseek (B.fd, B.size, SEEK_SET)) { kprintf ("[%s:%lld] lseek failed. %m\n", B.filename, B.size); exit (1); } recover_stat_t recover_stat; memset (&recover_stat, 0, sizeof (recover_stat_t)); int records = 0; while (cur_log_pos < A.size) { recover_stat_t w; memcpy (&w, &recover_stat, sizeof (recover_stat_t)); const long long off = cur_log_pos; const unsigned old_crc32_complement = crc32_complement; struct lev_storage_file E; int sz = sizeof (E); if (vk_pread (&A, &E, sz, off) < 0) { break; } if (E.type != LEV_STORAGE_FILE && E.type != LEV_STORAGE_HIDE_FILE) { kprintf ("[%s:%lld] Expected LEV_STORAGE_FILE|LEV_STORAGE_HIDE_FILE, but %x found.\n", A.filename, off, E.type); break; } if (E.size > MAX_FILESIZE) { kprintf ("[%s:%lld] E.size = %u.\n", A.filename, off, E.size); break; } const int l = (E.size + 3) & -4, L = l + sizeof (struct lev_crc32); if (cur_log_pos + sz + L > A.size) { kprintf ("[%s:%lld] Illegal E.size = %u, input binlog too small.\n", A.filename, off, E.size);; break; } if (L > max_file_buf_size) { if (a != NULL) { free (a); } a = malloc (L); if (a == NULL) { kprintf ("Not enough memory for allocate file body, malloc (%d) failed.\n", L); break; } max_file_buf_size = L; } crc32_complement = crc32_partial (&E, sz, crc32_complement); if (vk_pread (&A, a, L, off + sz) < 0) { break; } //const unsigned computed_crc32 = crc32 (a, E.size); int cur_file_body_corrupted = 0; unsigned old_E_crc32 = E.crc32; int r = crc32_check_and_repair (a, E.size, &E.crc32, 0); if (r == 1) { if (md5_check (&E, a)) { w.files++; } else { vkprintf (3, "[%s:%lld] crc32_check_and_repair returns %d.\n", A.filename, off, r); cur_file_body_corrupted = 1; } } else if (r == 2 || r < 0) { if (md5_check (&E, a)) { w.file_crc32++; } else { vkprintf (3, "[%s:%lld] crc32_check_and_repair returns %d.\n", A.filename, off, r); cur_file_body_corrupted = 1; } } if (cur_file_body_corrupted) { kprintf ("[%s:%lld] E.crc32 = %x, but computed crc32 = %x\n", A.filename, off, old_E_crc32, E.crc32); if (exit_on_file_body_error) { break; } else { cur_file_body_corrupted = 1; w.bad_file_bodies++; } } /* if (computed_crc32 != E.crc32) { int r = recover_file (&E, a, computed_crc32); if (r < 0) { vkprintf (3, "[%s:%lld] recover_file returns error code %d.\n", A.filename, off, r); if (md5_check (&E, a)) { E.crc32 = computed_crc32; w.file_crc32++; } else { kprintf ("[%s:%lld] E.crc32 = %x, but computed crc32 = %x\n", A.filename, off, E.crc32, computed_crc32); if (exit_on_file_body_error) { break; } else { cur_file_body_corrupted = 1; w.bad_file_bodies++; } } } else { w.files++; } } */ crc32_complement = ~compute_crc32_combine (~crc32_complement, E.crc32, E.size); const unsigned zero = 0; int padded_zero_bytes = 0; if (l != E.size) { padded_zero_bytes = l - E.size; crc32_complement = crc32_partial (a + E.size, padded_zero_bytes, crc32_complement); } cur_log_pos += sz + l; struct lev_crc32 *C = (struct lev_crc32 *) (a + l); sz = sizeof (struct lev_crc32); if (C->type != LEV_CRC32) { kprintf ("[%s:%lld] Expected LEV_CRC32, but %x found.\n", A.filename, cur_log_pos, C->type); break; } if (C->pos != cur_log_pos) { kprintf ("[%s:%lld] C->pos (%lld) != cur_log_pos (%lld).\n", A.filename, cur_log_pos, C->pos, cur_log_pos); break; } int header_fix_attempts = 0, ct; unsigned char m[16]; while (C->crc32 != ~crc32_complement && header_fix_attempts < 4) { int fix = 0; switch (header_fix_attempts) { case 0: if (padded_zero_bytes && memcmp (a + E.size, &zero, padded_zero_bytes)) { memcpy (a + E.size, &zero, padded_zero_bytes); fix = 1; w.padded_zero++; } break; case 1: if (local_id >= 0 && E.local_id != local_id + 1) { E.local_id = local_id + 1; fix = 1; w.file_local_id++; } break; case 2: if (!cur_file_body_corrupted) { ct = detect_content_type (a, l); if (E.content_type != ct) { E.content_type = ct; fix = 1; w.content_type++; } } break; case 3: if (!cur_file_body_corrupted) { md5 (a, E.size, m); if (memcmp (m, E.md5, 16)) { vkprintf (1, "Try fix file header md5, offset: %lld\n", off); memcpy (E.md5, m, 16); fix = 1; w.file_md5++; } } break; } if (fix) { crc32_complement = crc32_partial (&E, sizeof (struct lev_storage_file), old_crc32_complement); crc32_complement = ~compute_crc32_combine (~crc32_complement, E.crc32, E.size); crc32_complement = crc32_partial (a + E.size, padded_zero_bytes, crc32_complement); } header_fix_attempts++; } if (C->crc32 != ~crc32_complement) { kprintf ("[%s:%lld] C.crc32 (%x) != ~crc32_complement (%x), E.type = %x\n", A.filename, cur_log_pos, C->crc32, ~crc32_complement, E.type); break; } /* if (E.content_type < 0 || E.content_type >= ct_last) { kprintf ("[%s:%lld] E.content_type (%d)\n", A.filename, off, E.content_type); break; } */ crc32_complement = crc32_partial (C, sz, crc32_complement); clearin (); if (writeout (&E, sizeof (struct lev_storage_file)) < 0 || writeout (a, L) < 0 || flushout () < 0) { ftruncate (B.fd, off); break; } local_id = E.local_id; memcpy (&recover_stat, &w, sizeof (recover_stat_t)); cur_log_pos += sizeof (struct lev_crc32); records++; } if (a != NULL) { free (a); } close (A.fd); assert (!fsync (B.fd)); close (B.fd); vkprintf (1, "%d records were added.\n", records); vkprintf (1, "%d file(s) was succesfully recovered using crc32.\n", recover_stat.files); vkprintf (1, "%d crc32 file header field(s) was succesfully recovered using md5.\n", recover_stat.file_crc32); vkprintf (1, "%d md5 file header field(s) was succesfully recovered using LEV_CRC32.\n", recover_stat.file_md5); vkprintf (1, "%d local_id file header field(s) was succesfully recovered using LEV_CRC32.\n", recover_stat.file_local_id); vkprintf (1, "%d file's body zero padding was succesfully recovered using LEV_CRC32.\n", recover_stat.padded_zero); vkprintf (1, "%d content_type header field(s) was succesfully recovered using LEV_CRC32.\n", recover_stat.content_type); if (recover_stat.bad_file_bodies) { kprintf ("Copy %d files with wrong body.\n", recover_stat.bad_file_bodies); } vkprintf (2, "Max appended file size = %d.\n", max_file_buf_size); if (cur_log_pos != A.size) { kprintf ("Original file size = %lld, recovered file size = %lld.\n", A.size, cur_log_pos); exit (1); } }