ya_result zdb_icmtl_replay(zdb_zone *zone) { ya_result return_value; u32 serial; zdb_zone_double_lock(zone, ZDB_ZONE_MUTEX_SIMPLEREADER, ZDB_ZONE_MUTEX_LOAD); return_value = zdb_zone_getserial(zone, &serial); // zone is locked if(FAIL(return_value)) { zdb_zone_double_unlock(zone, ZDB_ZONE_MUTEX_SIMPLEREADER, ZDB_ZONE_MUTEX_LOAD); log_err("journal: %{dnsname}: error reading serial for zone: %r", zone->origin, return_value); return return_value; } input_stream is; #if ICMTL_DUMP_JOURNAL_RECORDS log_debug("journal: zdb_icmtl_replay(%{dnsname})", zone->origin); logger_flush(); #endif u32 first_serial; u32 last_serial; if(FAIL(return_value = zdb_zone_journal_get_serial_range(zone, &first_serial, &last_serial))) { zdb_zone_double_unlock(zone, ZDB_ZONE_MUTEX_SIMPLEREADER, ZDB_ZONE_MUTEX_LOAD); if(return_value == ZDB_ERROR_ICMTL_NOTFOUND) { return_value = SUCCESS; } else { log_err("journal: %{dnsname}: error opening journal for zone: %r", zone->origin, return_value); } return return_value; } log_debug("journal: %{dnsname}: zone serial is %i, journal covers serials from %i to %i", zone->origin, serial, first_serial, last_serial); if(last_serial == serial) { zdb_zone_double_unlock(zone, ZDB_ZONE_MUTEX_SIMPLEREADER, ZDB_ZONE_MUTEX_LOAD); log_debug("journal: %{dnsname}: nothing to read from the journal", zone->origin); return 0; } if(serial_lt(serial, first_serial)) { zdb_zone_double_unlock(zone, ZDB_ZONE_MUTEX_SIMPLEREADER, ZDB_ZONE_MUTEX_LOAD); log_warn("journal: %{dnsname}: first serial from the journal is after the zone", zone->origin); // should invalidate the journal zdb_zone_journal_delete(zone); return 0; } if(serial_gt(serial, last_serial)) { zdb_zone_double_unlock(zone, ZDB_ZONE_MUTEX_SIMPLEREADER, ZDB_ZONE_MUTEX_LOAD); log_warn("journal: %{dnsname}: last serial from the journal is before the zone", zone->origin); // should invalidate the journal zdb_zone_journal_delete(zone); return 0; } if(FAIL(return_value = zdb_zone_journal_get_ixfr_stream_at_serial(zone, serial, &is, NULL))) { zdb_zone_double_unlock(zone, ZDB_ZONE_MUTEX_SIMPLEREADER, ZDB_ZONE_MUTEX_LOAD); log_err("journal: %{dnsname}: error reading journal from serial %d: %r",zone->origin, serial, return_value); return return_value; } log_info("journal: %{dnsname}: replaying from serial %u",zone->origin, serial); buffer_input_stream_init(&is, &is, ZDB_ICMTL_REPLAY_BUFFER_SIZE); u16 shutdown_test_countdown = ZDB_ICMTL_REPLAY_SHUTDOWN_POLL_PERIOD; u32 current_serial = serial; /* * Read all records from [ SOA ... SOA ... [ SOA in memory */ output_stream baos; input_stream bais; dns_resource_record rr; int baos_rr_count = 0; int baos_soa_count = 0; bool was_nsec3 = zdb_zone_is_nsec3(zone); bytearray_output_stream_init_ex(&baos, NULL, ZDB_ICMTL_REPLAY_BUFFER_SIZE, BYTEARRAY_DYNAMIC); dns_resource_record_init(&rr); // 0: gather, 1: commit, 2: commit & stop for(int replay_state = ZDB_ICMTL_REPLAY_GATHER; replay_state != ZDB_ICMTL_REPLAY_COMMIT_AND_STOP;) { // ensure it's not supposed to shutdown (every few iterations) if(--shutdown_test_countdown <= 0) { if(dnscore_shuttingdown()) { return_value = STOPPED_BY_APPLICATION_SHUTDOWN; break; } shutdown_test_countdown = ZDB_ICMTL_REPLAY_SHUTDOWN_POLL_PERIOD; } // read the next record if((return_value = dns_resource_record_read(&rr, &is)) <= 0) { if(ISOK(return_value)) { log_info("journal: %{dnsname}: reached the end of the journal file", zone->origin); replay_state = ZDB_ICMTL_REPLAY_COMMIT_AND_STOP; } else { log_err("journal: broken journal: %r", return_value); logger_flush(); // broken journal (flush is slow, but this is bad, so : keep it) replay_state = ZDB_ICMTL_REPLAY_STOP; } } else // first record must be an SOA (or it's wrong) if(baos_rr_count == 0) // first record ? { if(rr.tctr.qtype != TYPE_SOA) // must be SOA { // expected an SOA return_value = ERROR; break; } ++baos_soa_count; // 0 -> 1 // this is not mandatory but clearer to read } else // the page ends with an SOA or end of stream if(rr.tctr.qtype == TYPE_SOA) { if(baos_soa_count == 2) { // this record is the start of the next stream, keep it for the next iteration replay_state = ZDB_ICMTL_REPLAY_COMMIT; } ++baos_soa_count; } ++baos_rr_count; if((replay_state & ZDB_ICMTL_REPLAY_COMMIT) != 0) { log_info("journal: %{dnsname}: committing changes", zone->origin); u64 ts_start = timeus(); zdb_zone_exchange_locks(zone, ZDB_ZONE_MUTEX_SIMPLEREADER, ZDB_ZONE_MUTEX_LOAD); bytearray_input_stream_init_const(&bais, bytearray_output_stream_buffer(&baos), bytearray_output_stream_size(&baos)); return_value = zdb_icmtl_replay_commit(zone, &bais, ¤t_serial); zdb_zone_exchange_locks(zone, ZDB_ZONE_MUTEX_LOAD, ZDB_ZONE_MUTEX_SIMPLEREADER); input_stream_close(&bais); u64 ts_stop = timeus(); if(ts_stop < ts_start) // time change { ts_stop = ts_start; } u64 ts_delta = ts_stop - ts_start; if(ISOK(return_value)) { if(ts_delta < 1000) { log_info("journal: %{dnsname}: committed changes (%lluus)", zone->origin, ts_delta); } else if(ts_delta < 1000000) { double ts_delta_s = ts_delta; ts_delta_s /= 1000.0; log_info("journal: %{dnsname}: committed changes (%5.2fms)", zone->origin, ts_delta_s); } else { double ts_delta_s = ts_delta; ts_delta_s /= 1000000.0; log_info("journal: %{dnsname}: committed changes (%5.2fs)", zone->origin, ts_delta_s); } } else { log_err("journal: %{dnsname}: failed to committed changes", zone->origin); break; } // the current page has been processed if(replay_state == ZDB_ICMTL_REPLAY_COMMIT_AND_STOP) { // no more page to read break; } // reset the state for the next page // note: the next written record will be the last read SOA baos_rr_count = 1; baos_soa_count = 1; replay_state = ZDB_ICMTL_REPLAY_GATHER; bytearray_output_stream_reset(&baos); } // end if replay_state is ZDB_ICMTL_REPLAY_COMMIT (mask) dns_resource_record_write(&rr, &baos); } input_stream_close(&is); output_stream_close(&baos); dns_resource_record_clear(&rr); // cleanup destroyed nsec3 chains if(ISOK(return_value)) { bool is_nsec3 = zdb_zone_is_nsec3(zone); if(is_nsec3 && !was_nsec3) { // the chain has just been created, but is probably missing internal links log_debug("journal: %{dnsname}: zone switched to NSEC3 by reading the journal: updating links", zone->origin); zdb_zone_exchange_locks(zone, ZDB_ZONE_MUTEX_SIMPLEREADER, ZDB_ZONE_MUTEX_LOAD); nsec3_zone_update_chain0_links(zone); zdb_zone_exchange_locks(zone, ZDB_ZONE_MUTEX_LOAD, ZDB_ZONE_MUTEX_SIMPLEREADER); log_debug("journal: %{dnsname}: zone switched to NSEC3 by reading the journal: links updated", zone->origin); } if(FAIL(return_value = zdb_zone_getserial(zone, &serial))) // zone is locked { zdb_zone_double_unlock(zone, ZDB_ZONE_MUTEX_SIMPLEREADER, ZDB_ZONE_MUTEX_LOAD); log_err("journal: %{dnsname}: error reading confirmation serial for zone: %r",zone->origin, return_value); return return_value; } if(serial != last_serial) { log_warn("journal: %{dnsname}: expected serial to be %i but it is %i instead",zone->origin, last_serial, serial); } #if 0 // ICMTL_DUMP_JOURNAL_RECORDS if(is_nsec) { nsec_logdump_tree(zone); logger_flush(); } #endif } zdb_zone_double_unlock(zone, ZDB_ZONE_MUTEX_SIMPLEREADER, ZDB_ZONE_MUTEX_LOAD); log_info("journal: %{dnsname}: done", zone->origin); return return_value; }
static void* scheduler_queue_zone_write_thread(void* data_) { char fullname_tmp[MAX_PATH]; zone_write_param *zwp = (zone_write_param*)data_; zdb_zone *zone = zwp->zone; u32 serial; log_info("zone write text: writing %{dnsname} zone file", zone->origin); if(ZDB_ZONE_INVALID(zone)) { log_err("zone write text: zone %{dnsname} marked as invalid", zone->origin); scheduler_schedule_task(scheduler_queue_zone_write_callback, zwp); return NULL; } if(FAIL(zdb_zone_getserial(zone, &serial))) { log_err("zone write text: no SOA in %{dnsname}", zone->origin); scheduler_schedule_task(scheduler_queue_zone_write_callback, zwp); return NULL; } if(FAIL(snformat(fullname_tmp, sizeof (fullname_tmp), "%s.%d.tmp", zwp->file_path, serial))) { log_err("zone write text: path '%s.%d.tmp' is too big", zwp->file_path, serial); scheduler_schedule_task(scheduler_queue_zone_write_callback, zwp); /* WARNING: From this point forward, 'zone' cannot be used anymore */ return NULL; } /** * @todo check there is not already a zone file writer working here ... * */ /* * delete the temp file if it exists already */ if(unlink(fullname_tmp) < 0) { int err = errno; if(err != ENOENT) { log_err("zone write text: cannot cleanup '%s': %r", MAKE_ERRNO_ERROR(err)); scheduler_schedule_task(scheduler_queue_zone_write_callback, zwp); /* WARNING: From this point forward, 'zone' cannot be used anymore */ return NULL; } } log_info("zone write text: writing '%s'", fullname_tmp); zdb_zone_lock(zone, ZDB_ZONE_MUTEX_SIMPLEREADER); zdb_zone_write_text_file(zone, fullname_tmp, FALSE); zdb_zone_unlock(zone, ZDB_ZONE_MUTEX_SIMPLEREADER); log_info("zone write text: renaming '%s' to '%s'", fullname_tmp, zwp->file_path); if(rename(fullname_tmp, zwp->file_path) < 0) { log_err("zone write text: unable to rename tmp zone file into '%s': %r", zwp->file_path, ERRNO_ERROR); scheduler_schedule_task(scheduler_queue_zone_write_callback, zwp); /** @note Calling this so the scheduler gets a SCHEDULER_TASK_FINISHED is mandatory. */ return NULL; } log_info("zone write text: %{dnsname} zone file written", zone->origin); scheduler_schedule_task(scheduler_queue_zone_write_callback, zwp); /* WARNING: From this point forward, 'zwp' cannot be used anymore */ return NULL; }
ya_result zdb_zone_update_ixfr(zdb* db, input_stream* is) { u8 rname[MAX_DOMAIN_LENGTH]; u16 rtype; u16 rclass; u32 rttl; u16 rdata_size; u8* rdata; zdb_packed_ttlrdata* soa_ttlrdata; zdb_packed_ttlrdata* tmp_ttlrdata; zdb_packed_ttlrdata* ttlrdata; dnsname_vector name; dnsname_vector entry_name; ya_result err; /* Get the first SOA */ if(FAIL(err = input_stream_read_rr_header(is, rname, sizeof (rname), &rtype, &rclass, &rttl, &rdata_size))) { return err; } if(rtype != TYPE_SOA) { return ZDB_ERROR_GENERAL; } DEBUG_RESET_dnsname(name); dnsname_to_dnsname_vector(rname, &name); ZDB_RECORD_ZALLOC_EMPTY(soa_ttlrdata, rttl, rdata_size); if(FAIL(err = input_stream_read_fully(is, ZDB_PACKEDRECORD_PTR_RDATAPTR(soa_ttlrdata), rdata_size))) { return err; } #ifndef NDEBUG #if DUMP_ICMTL_UPDATE != 0 if(err != 0) { format("zdb_zone_update_ixfr H: %{dnsname} %d %{dnsclass} %{dnstype} ", rname, rttl, &rclass, &rtype); print_rdata(rtype, ZDB_PACKEDRECORD_PTR_RDATAPTR(soa_ttlrdata), rdata_size); println(""); } #endif #endif u32 serial_first; if(FAIL(err = rr_soa_get_serial(soa_ttlrdata->rdata_start, soa_ttlrdata->rdata_size, &serial_first))) { return err; } zdb_zone_label* zone_label = zdb_zone_label_find(db, &name, rclass); if((zone_label == NULL) || (zone_label->zone == NULL)) { /* Not loaded */ return ZDB_ERROR_GENERAL; } zdb_zone* zone = zone_label->zone; u32 serial_current; if(FAIL(err = zdb_zone_getserial(zone, &serial_current))) { return err; } #if ZDB_NSEC3_SUPPORT != 0 nsec3_load_context nsec3_context; nsec3_load_init(&nsec3_context, zone); #endif MALLOC_OR_DIE(zdb_packed_ttlrdata*, tmp_ttlrdata, sizeof (zdb_ttlrdata) + RDATA_MAX_LENGTH, ZDB_RDATABUF_TAG); /* We do not need tmp_ttlrdata and rdata at the same time, let's spare memory */ rdata = ZDB_PACKEDRECORD_PTR_RDATAPTR(tmp_ttlrdata); /* Read the next SOA (sub or end) */ if(FAIL(err = input_stream_read_rr_header(is, rname, sizeof (rname), &rtype, &rclass, &rttl, &rdata_size))) { return err; } if(rtype != TYPE_SOA) { return ZDB_ERROR_GENERAL; } for(;;) { if(FAIL(err = input_stream_read_fully(is, rdata, rdata_size))) { break; } #ifndef NDEBUG #if DUMP_ICMTL_UPDATE != 0 if(err != 0) { format("zdb_zone_update_ixfr F: %{dnsname} %d %{dnsclass} %{dnstype} ", rname, rttl, &rclass, &rtype); print_rdata(rtype, rdata, rdata_size); println(""); } #endif #endif /* The SOA serial is supposed to match our current one or the first one * * If it's the first one, then the task is done * * If it's the current one, we are moving forward * * If it's something else, there is an error * */ u32 serial_from; if(FAIL(err = rr_soa_get_serial(rdata, rdata_size, &serial_from))) { break; } /* Check the serial */ if(serial_from != serial_current) { if(serial_from == serial_first) { /* IXFR done */ err = SUCCESS; break; } /* Serial sequence is not right */ err = ZDB_ERROR_GENERAL; break; } tmp_ttlrdata->ttl = rttl; tmp_ttlrdata->rdata_size = rdata_size; zdb_zone_record_delete(zone, NULL, -1, TYPE_SOA, tmp_ttlrdata); for(;;) { /* Load the next record without the data (sub) */ if(FAIL(err = input_stream_read_rr_header(is, rname, sizeof (rname), &rtype, &rclass, &rttl, &rdata_size))) { break; } /* If we got an SOA, it's the one that starts the "add" sequence */ if(rtype == TYPE_SOA) { break; } /* Load the data */ if(FAIL(err = input_stream_read_fully(is, rdata, rdata_size))) { break; } #ifndef NDEBUG #if DUMP_ICMTL_UPDATE != 0 if(err != 0) { format("zdb_zone_update_ixfr R: %{dnsname} %d %{dnsclass} %{dnstype} ", rname, rttl, &rclass, &rtype); print_rdata(rtype, rdata, rdata_size); println(""); } #endif #endif tmp_ttlrdata->ttl = rttl; tmp_ttlrdata->rdata_size = rdata_size; /* The vector is not always required, but it's so much easier to * read putting it here */ DEBUG_RESET_dnsname(entry_name); dnsname_to_dnsname_vector(rname, &entry_name); #if ZDB_NSEC3_SUPPORT != 0 if(rtype == TYPE_NSEC3PARAM) { /* Remove it from the zone */ zdb_zone_record_delete(zone, name.labels, (entry_name.size - name.size) - 1, rtype, tmp_ttlrdata); /* Destroy the whole NSEC3 collection associated to the NSEC3PARAM */ nsec3_remove_nsec3param_by_record(zone, tmp_ttlrdata); continue; } if(rtype == TYPE_NSEC3) { /* Remove the NSEC3 label (and its signature) * * The previous record will have its signature changed, no doubt. * But I cannot edit the previous one about this. * Since we are in an IXFR, the previous NSEC3 is supposed to be * removed too, until one of the previous is also added in the * next section (soa add) * * zdb_zone_record_delete is not the right call, it's * * nsec3_... * */ nsec3_remove_nsec3(zone, tmp_ttlrdata); continue; } if(rtype == TYPE_RRSIG) { if((GET_U16_AT(*rdata)) == TYPE_NSEC3) /** @note : NATIVETYPE */ { /* Remove the RRSIG from the NSEC3 label * * zdb_zone_record_delete is not the right call, it's * * nsec3_... * */ nsec3_remove_rrsig(zone, tmp_ttlrdata); continue; } } #endif /* Remove from the zone */ zdb_zone_record_delete(zone, name.labels, (entry_name.size - name.size) - 1, rtype, tmp_ttlrdata); } /* Remove records */ /* The current header is the "ADD" SOA */ if(FAIL(err = input_stream_read_fully(is, rdata, rdata_size))) { break; } #ifndef NDEBUG #if DUMP_ICMTL_UPDATE != 0 if(err != 0) { format("zdb_zone_update_ixfr T: %{dnsname} %d %{dnsclass} %{dnstype} ", rname, rttl, &rclass, &rtype); print_rdata(rtype, rdata, rdata_size); println(""); } #endif #endif /* The SOA serial is supposed to be bigger than the previous one */ u32 serial_to; if(FAIL(err = rr_soa_get_serial(rdata, rdata_size, &serial_to))) { break; } /* * After the "add" sequence is done, serial_current will be serial_to */ do { /* Load the record without the data (add) */ if(FAIL(err = input_stream_read_rr_header(is, rname, sizeof (rname), &rtype, &rclass, &rttl, &rdata_size))) { break; } #if ZDB_NSEC3_SUPPORT != 0 if(rtype == TYPE_NSEC3PARAM) { /* Load the data */ if(FAIL(err = input_stream_read_fully(is, rdata, rdata_size))) { break; } #ifndef NDEBUG #if DUMP_ICMTL_UPDATE != 0 if(err != 0) { format("zdb_zone_update_ixfr A: %{dnsname} %d %{dnsclass} %{dnstype} ", rname, rttl, &rclass, &rtype); print_rdata(rtype, rdata, rdata_size); println(""); } #endif #endif /* Add it to the nsec3 context */ nsec3_load_add_nsec3param(&nsec3_context, rdata, rdata_size); /* Add it to the zone */ DEBUG_RESET_dnsname(entry_name); dnsname_to_dnsname_vector(rname, &entry_name); ZDB_RECORD_ZALLOC(ttlrdata, rttl, rdata_size, rdata); zdb_zone_record_add(zone, entry_name.labels, (entry_name.size - name.size) - 1, rtype, ttlrdata); } else if(rtype == TYPE_NSEC3) { /* Load the data */ if(FAIL(err = input_stream_read_fully(is, rdata, rdata_size))) { break; } #ifndef NDEBUG #if DUMP_ICMTL_UPDATE != 0 if(err != 0) { format("zdb_zone_update_ixfr A: %{dnsname} %d %{dnsclass} %{dnstype} ", rname, rttl, &rclass, &rtype); print_rdata(rtype, rdata, rdata_size); println(""); } #endif #endif /* Add it to the nsec3 context */ if(FAIL(err = nsec3_load_add_nsec3(&nsec3_context, rname, rttl, rdata, rdata_size))) { break; } } else if(rtype == TYPE_RRSIG) { /* Load the data */ if(FAIL(err = input_stream_read_fully(is, rdata, rdata_size))) { break; } #if DUMP_ICMTL_UPDATE != 0 if(err != 0) { format("zdb_zone_update_ixfr A: %{dnsname} %d %{dnsclass} %{dnstype} ", rname, rttl, &rclass, &rtype); print_rdata(rtype, rdata, rdata_size); println(""); } #endif if((GET_U16_AT(*rdata)) == TYPE_NSEC3) /** @note : NATIVETYPE */ { /* Add it to the nsec3 context */ if(FAIL(err = nsec3_load_add_rrsig(&nsec3_context, rname, rttl, rdata, rdata_size))) { break; } } else { /* Add it to the zone */ DEBUG_RESET_dnsname(entry_name); dnsname_to_dnsname_vector(rname, &entry_name); ZDB_RECORD_ZALLOC(ttlrdata, rttl, rdata_size, rdata); zdb_zone_record_add(zone, entry_name.labels, (entry_name.size - name.size) - 1, rtype, ttlrdata); } } else { #endif /* Add it to the zone */ ZDB_RECORD_ZALLOC_EMPTY(ttlrdata, rttl, rdata_size); if(FAIL(err = input_stream_read_fully(is, ZDB_PACKEDRECORD_PTR_RDATAPTR(ttlrdata), rdata_size))) { break; } #ifndef NDEBUG #if DUMP_ICMTL_UPDATE != 0 if(err != 0) { format("zdb_zone_update_ixfr A: %{dnsname} %d %{dnsclass} %{dnstype} ", rname, rttl, &rclass, &rtype); print_rdata(rtype, ttlrdata->rdata_start, rdata_size); println(termout, ""); } #endif #endif DEBUG_RESET_dnsname(entry_name); dnsname_to_dnsname_vector(rname, &entry_name); zdb_zone_record_add(zone, entry_name.labels, (entry_name.size - name.size) - 1, rtype, ttlrdata); /* class is implicit */ #if ZDB_NSEC3_SUPPORT != 0 } #endif } while(rtype != TYPE_SOA); /* * The record is either the first of another SOA pair (sub, add) * Either the final one. */ /* Update the current serial */ serial_current = serial_to; break; } free(tmp_ttlrdata); if(ISOK(err)) { /* * soa_ttlrdata is the new SOA */ zdb_zone_record_add(zone, NULL, -1, TYPE_SOA, soa_ttlrdata); #if ZDB_NSEC3_SUPPORT != 0 /** * Check if there is both NSEC & NSEC3. Reject if yes. * compile NSEC if any * compile NSEC3 if any * * @todo: I'm only doing NSEC3 here. Do NSEC as well */ err = nsec3_load_compile(&nsec3_context); if((nsec3_context.nsec3_rejected > 0)||(nsec3_context.nsec3_discarded > 0)) { err = DNSSEC_ERROR_NSEC3_INVALIDZONESTATE; } if(ISOK(err)) { nsec3_load_destroy(&nsec3_context); #endif zone_label->zone = zone; return err; #if ZDB_NSEC3_SUPPORT != 0 } #endif } #if ZDB_NSEC3_SUPPORT != 0 nsec3_load_destroy(&nsec3_context); #endif /** * @note : do NOT use these to unload a zone. * zdb_zone_label_delete(db, &name, zone->zclass); * zdb_zone_destroy(zone); */ zdb_zone_unload(db, &name, zdb_zone_getclass(zone)); return err; }