/* Make a copy of the given old packet, but in the new copy reserve the * given number of bytes of headroom at the start of the packet->buffer. * This empty headroom can later be filled with outer packet headers. * A slow but simple model. */ static struct packet *packet_copy_with_headroom(struct packet *old_packet, int bytes_headroom) { /* Allocate a new packet and copy link layer header and IP datagram. */ const int bytes_used = packet_end(old_packet) - old_packet->buffer; assert(bytes_used >= 0); assert(bytes_used <= 128*1024); struct packet *packet = packet_new(bytes_headroom + bytes_used); u8 *old_base = old_packet->buffer; u8 *new_base = packet->buffer + bytes_headroom; memcpy(new_base, old_base, bytes_used); packet->ip_bytes = old_packet->ip_bytes; packet->direction = old_packet->direction; packet->time_usecs = old_packet->time_usecs; packet->flags = old_packet->flags; packet->ecn = old_packet->ecn; packet->socket_script_fd = old_packet->socket_script_fd; packet_copy_headers(packet, old_packet, bytes_headroom); /* Set up layer 3 header pointer. */ packet->ipv4 = offset_ptr(old_base, new_base, old_packet->ipv4); packet->ipv6 = offset_ptr(old_base, new_base, old_packet->ipv6); packet->tcp = offset_ptr(old_base, new_base, old_packet->tcp); packet->udp = offset_ptr(old_base, new_base, old_packet->udp); packet->icmpv4 = offset_ptr(old_base, new_base, old_packet->icmpv4); packet->icmpv6 = offset_ptr(old_base, new_base, old_packet->icmpv6); packet->tcp_ts_val = offset_ptr(old_base, new_base, old_packet->tcp_ts_val); packet->tcp_ts_ecr = offset_ptr(old_base, new_base, old_packet->tcp_ts_ecr); return packet; }
/* * Copy a term that is guaranteed to be contained in a single * heap block. The heap block is copied word by word, and any * pointers are offsetted to point correctly in the new location. * * Typically used to copy a term from an ets table. * * NOTE: Assumes that term is a tuple (ptr is an untagged tuple ptr). */ Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) { Eterm* tp = ptr; Eterm* hp = *hpp; Sint offs = hp - tp; while (sz--) { Eterm val = *tp++; switch (primary_tag(val)) { case TAG_PRIMARY_IMMED1: *hp++ = val; break; case TAG_PRIMARY_LIST: case TAG_PRIMARY_BOXED: *hp++ = offset_ptr(val, offs); break; case TAG_PRIMARY_HEADER: *hp++ = val; switch (val & _HEADER_SUBTAG_MASK) { case ARITYVAL_SUBTAG: break; case REFC_BINARY_SUBTAG: { ProcBin* pb = (ProcBin *) (hp-1); int tari = thing_arityval(val); sz -= tari; while (tari--) { *hp++ = *tp++; } erts_refc_inc(&pb->val->refc, 2); pb->next = off_heap->mso; off_heap->mso = pb; off_heap->overhead += pb->size / sizeof(Eterm); } break; case FUN_SUBTAG: { #ifndef HYBRID /* FIND ME! */ ErlFunThing* funp = (ErlFunThing *) (hp-1); #endif int tari = thing_arityval(val); sz -= tari; while (tari--) { *hp++ = *tp++; } #ifndef HYBRID /* FIND ME! */ funp->next = off_heap->funs; off_heap->funs = funp; erts_refc_inc(&funp->fe->refc, 2); #endif } break; case EXTERNAL_PID_SUBTAG: case EXTERNAL_PORT_SUBTAG: case EXTERNAL_REF_SUBTAG: { ExternalThing* etp = (ExternalThing *) (hp-1); int tari = thing_arityval(val); sz -= tari; while (tari--) { *hp++ = *tp++; } etp->next = off_heap->externals; off_heap->externals = etp; erts_refc_inc(&etp->node->refc, 2); } break; default: { int tari = header_arity(val); sz -= tari; while (tari--) { *hp++ = *tp++; } } break; } break; } } *hpp = hp; return make_tuple(ptr + offs); }
/* * Moves content of message buffer attached to a message into a heap. * The message buffer is deallocated. */ void erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) { struct erl_off_heap_header* oh; Eterm term, token, *fhp, *hp; Sint offs; Uint sz; ErlHeapFragment *bp; #ifdef USE_VM_PROBES Eterm utag; #endif #ifdef HARD_DEBUG struct erl_off_heap_header* dbg_oh_start = off_heap->first; Eterm dbg_term, dbg_token; ErlHeapFragment *dbg_bp; Uint *dbg_hp, *dbg_thp_start; Uint dbg_term_sz, dbg_token_sz; #ifdef USE_VM_PROBES Eterm dbg_utag; Uint dbg_utag_sz; #endif #endif bp = msg->data.heap_frag; term = ERL_MESSAGE_TERM(msg); token = ERL_MESSAGE_TOKEN(msg); #ifdef USE_VM_PROBES utag = ERL_MESSAGE_DT_UTAG(msg); #endif if (!bp) { #ifdef USE_VM_PROBES ASSERT(is_immed(term) && is_immed(token) && is_immed(utag)); #else ASSERT(is_immed(term) && is_immed(token)); #endif return; } #ifdef HARD_DEBUG dbg_term_sz = size_object(term); dbg_token_sz = size_object(token); dbg_bp = new_message_buffer(dbg_term_sz + dbg_token_sz); #ifdef USE_VM_PROBES dbg_utag_sz = size_object(utag); dbg_bp = new_message_buffer(dbg_term_sz + dbg_token_sz + dbg_utag_sz ); #endif /*ASSERT(dbg_term_sz + dbg_token_sz == erts_msg_used_frag_sz(msg)); Copied size may be smaller due to removed SubBins's or garbage. Copied size may be larger due to duplicated shared terms. */ dbg_hp = dbg_bp->mem; dbg_term = copy_struct(term, dbg_term_sz, &dbg_hp, &dbg_bp->off_heap); dbg_token = copy_struct(token, dbg_token_sz, &dbg_hp, &dbg_bp->off_heap); #ifdef USE_VM_PROBES dbg_utag = copy_struct(utag, dbg_utag_sz, &dbg_hp, &dbg_bp->off_heap); #endif dbg_thp_start = *hpp; #endif if (bp->next != NULL) { move_multi_frags(hpp, off_heap, bp, msg->m, #ifdef USE_VM_PROBES 3 #else 2 #endif ); goto copy_done; } OH_OVERHEAD(off_heap, bp->off_heap.overhead); sz = bp->used_size; ASSERT(is_immed(term) || in_heapfrag(ptr_val(term),bp)); ASSERT(is_immed(token) || in_heapfrag(ptr_val(token),bp)); fhp = bp->mem; hp = *hpp; offs = hp - fhp; oh = NULL; while (sz--) { Uint cpy_sz; Eterm val = *fhp++; switch (primary_tag(val)) { case TAG_PRIMARY_IMMED1: *hp++ = val; break; case TAG_PRIMARY_LIST: case TAG_PRIMARY_BOXED: ASSERT(in_heapfrag(ptr_val(val), bp)); *hp++ = offset_ptr(val, offs); break; case TAG_PRIMARY_HEADER: *hp++ = val; switch (val & _HEADER_SUBTAG_MASK) { case ARITYVAL_SUBTAG: break; case REFC_BINARY_SUBTAG: case FUN_SUBTAG: case EXTERNAL_PID_SUBTAG: case EXTERNAL_PORT_SUBTAG: case EXTERNAL_REF_SUBTAG: oh = (struct erl_off_heap_header*) (hp-1); cpy_sz = thing_arityval(val); goto cpy_words; default: cpy_sz = header_arity(val); cpy_words: ASSERT(sz >= cpy_sz); sz -= cpy_sz; while (cpy_sz >= 8) { cpy_sz -= 8; *hp++ = *fhp++; *hp++ = *fhp++; *hp++ = *fhp++; *hp++ = *fhp++; *hp++ = *fhp++; *hp++ = *fhp++; *hp++ = *fhp++; *hp++ = *fhp++; } switch (cpy_sz) { case 7: *hp++ = *fhp++; case 6: *hp++ = *fhp++; case 5: *hp++ = *fhp++; case 4: *hp++ = *fhp++; case 3: *hp++ = *fhp++; case 2: *hp++ = *fhp++; case 1: *hp++ = *fhp++; default: break; } if (oh) { /* Add to offheap list */ oh->next = off_heap->first; off_heap->first = oh; ASSERT(*hpp <= (Eterm*)oh); ASSERT(hp > (Eterm*)oh); oh = NULL; } break; } break; } } ASSERT(bp->used_size == hp - *hpp); *hpp = hp; if (is_not_immed(token)) { ASSERT(in_heapfrag(ptr_val(token), bp)); ERL_MESSAGE_TOKEN(msg) = offset_ptr(token, offs); #ifdef HARD_DEBUG ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_TOKEN(msg))); ASSERT(hp > ptr_val(ERL_MESSAGE_TOKEN(msg))); #endif } if (is_not_immed(term)) { ASSERT(in_heapfrag(ptr_val(term),bp)); ERL_MESSAGE_TERM(msg) = offset_ptr(term, offs); #ifdef HARD_DEBUG ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_TERM(msg))); ASSERT(hp > ptr_val(ERL_MESSAGE_TERM(msg))); #endif } #ifdef USE_VM_PROBES if (is_not_immed(utag)) { ASSERT(in_heapfrag(ptr_val(utag), bp)); ERL_MESSAGE_DT_UTAG(msg) = offset_ptr(utag, offs); #ifdef HARD_DEBUG ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_DT_UTAG(msg))); ASSERT(hp > ptr_val(ERL_MESSAGE_DT_UTAG(msg))); #endif } #endif copy_done: #ifdef HARD_DEBUG { int i, j; ErlHeapFragment* frag; { struct erl_off_heap_header* dbg_oh = off_heap->first; i = j = 0; while (dbg_oh != dbg_oh_start) { dbg_oh = dbg_oh->next; i++; } for (frag=bp; frag; frag=frag->next) { dbg_oh = frag->off_heap.first; while (dbg_oh) { dbg_oh = dbg_oh->next; j++; } } ASSERT(i == j); } } #endif bp->off_heap.first = NULL; free_message_buffer(bp); msg->data.heap_frag = NULL; #ifdef HARD_DEBUG ASSERT(eq(ERL_MESSAGE_TERM(msg), dbg_term)); ASSERT(eq(ERL_MESSAGE_TOKEN(msg), dbg_token)); #ifdef USE_VM_PROBES ASSERT(eq(ERL_MESSAGE_DT_UTAG(msg), dbg_utag)); #endif free_message_buffer(dbg_bp); #endif }
/* * Moves content of message buffer attached to a message into a heap. * The message buffer is deallocated. */ void erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) { /* Unions for typecasts avoids warnings about type-punned pointers and aliasing */ union { Uint** upp; ProcBin **pbpp; ErlFunThing **efpp; ExternalThing **etpp; } oh_list_pp, oh_el_next_pp; union { Uint *up; ProcBin *pbp; ErlFunThing *efp; ExternalThing *etp; } oh_el_p; Eterm term, token, *fhp, *hp; Sint offs; Uint sz; ErlHeapFragment *bp; #ifdef HARD_DEBUG ProcBin *dbg_mso_start = off_heap->mso; ErlFunThing *dbg_fun_start = off_heap->funs; ExternalThing *dbg_external_start = off_heap->externals; Eterm dbg_term, dbg_token; ErlHeapFragment *dbg_bp; Uint *dbg_hp, *dbg_thp_start; Uint dbg_term_sz, dbg_token_sz; #endif bp = msg->data.heap_frag; term = ERL_MESSAGE_TERM(msg); token = ERL_MESSAGE_TOKEN(msg); if (!bp) { ASSERT(is_immed(term) && is_immed(token)); return; } #ifdef HARD_DEBUG dbg_term_sz = size_object(term); dbg_token_sz = size_object(token); ASSERT(bp->size == dbg_term_sz + dbg_token_sz); dbg_bp = new_message_buffer(bp->size); dbg_hp = dbg_bp->mem; dbg_term = copy_struct(term, dbg_term_sz, &dbg_hp, &dbg_bp->off_heap); dbg_token = copy_struct(token, dbg_token_sz, &dbg_hp, &dbg_bp->off_heap); dbg_thp_start = *hpp; #endif ASSERT(bp); msg->data.attached = NULL; off_heap->overhead += bp->off_heap.overhead; sz = bp->size; #ifdef DEBUG if (is_not_immed(term)) { ASSERT(bp->mem <= ptr_val(term)); ASSERT(bp->mem + bp->size > ptr_val(term)); } if (is_not_immed(token)) { ASSERT(bp->mem <= ptr_val(token)); ASSERT(bp->mem + bp->size > ptr_val(token)); } #endif fhp = bp->mem; hp = *hpp; offs = hp - fhp; oh_list_pp.upp = NULL; oh_el_next_pp.upp = NULL; /* Shut up compiler warning */ oh_el_p.up = NULL; /* Shut up compiler warning */ while (sz--) { Uint cpy_sz; Eterm val = *fhp++; switch (primary_tag(val)) { case TAG_PRIMARY_IMMED1: *hp++ = val; break; case TAG_PRIMARY_LIST: case TAG_PRIMARY_BOXED: ASSERT(bp->mem <= ptr_val(val)); ASSERT(bp->mem + bp->size > ptr_val(val)); *hp++ = offset_ptr(val, offs); break; case TAG_PRIMARY_HEADER: *hp++ = val; switch (val & _HEADER_SUBTAG_MASK) { case ARITYVAL_SUBTAG: break; case REFC_BINARY_SUBTAG: oh_list_pp.pbpp = &off_heap->mso; oh_el_p.up = (hp-1); oh_el_next_pp.pbpp = &(oh_el_p.pbp)->next; cpy_sz = thing_arityval(val); goto cpy_words; case FUN_SUBTAG: #ifndef HYBRID oh_list_pp.efpp = &off_heap->funs; oh_el_p.up = (hp-1); oh_el_next_pp.efpp = &(oh_el_p.efp)->next; #endif cpy_sz = thing_arityval(val); goto cpy_words; case EXTERNAL_PID_SUBTAG: case EXTERNAL_PORT_SUBTAG: case EXTERNAL_REF_SUBTAG: oh_list_pp.etpp = &off_heap->externals; oh_el_p.up = (hp-1); oh_el_next_pp.etpp = &(oh_el_p.etp)->next; cpy_sz = thing_arityval(val); goto cpy_words; default: cpy_sz = header_arity(val); cpy_words: sz -= cpy_sz; while (cpy_sz >= 8) { cpy_sz -= 8; *hp++ = *fhp++; *hp++ = *fhp++; *hp++ = *fhp++; *hp++ = *fhp++; *hp++ = *fhp++; *hp++ = *fhp++; *hp++ = *fhp++; *hp++ = *fhp++; } switch (cpy_sz) { case 7: *hp++ = *fhp++; case 6: *hp++ = *fhp++; case 5: *hp++ = *fhp++; case 4: *hp++ = *fhp++; case 3: *hp++ = *fhp++; case 2: *hp++ = *fhp++; case 1: *hp++ = *fhp++; default: break; } if (oh_list_pp.upp) { #ifdef HARD_DEBUG Uint *dbg_old_oh_list_p = *oh_list_pp.upp; #endif /* Add to offheap list */ *oh_el_next_pp.upp = *oh_list_pp.upp; *oh_list_pp.upp = oh_el_p.up; ASSERT(*hpp <= oh_el_p.up); ASSERT(hp > oh_el_p.up); #ifdef HARD_DEBUG switch (val & _HEADER_SUBTAG_MASK) { case REFC_BINARY_SUBTAG: ASSERT(off_heap->mso == *oh_list_pp.pbpp); ASSERT(off_heap->mso->next == (ProcBin *) dbg_old_oh_list_p); break; #ifndef HYBRID case FUN_SUBTAG: ASSERT(off_heap->funs == *oh_list_pp.efpp); ASSERT(off_heap->funs->next == (ErlFunThing *) dbg_old_oh_list_p); break; #endif case EXTERNAL_PID_SUBTAG: case EXTERNAL_PORT_SUBTAG: case EXTERNAL_REF_SUBTAG: ASSERT(off_heap->externals == *oh_list_pp.etpp); ASSERT(off_heap->externals->next == (ExternalThing *) dbg_old_oh_list_p); break; default: ASSERT(0); } #endif oh_list_pp.upp = NULL; } break; } break; } } ASSERT(bp->size == hp - *hpp); *hpp = hp; if (is_not_immed(token)) { ASSERT(bp->mem <= ptr_val(token)); ASSERT(bp->mem + bp->size > ptr_val(token)); ERL_MESSAGE_TOKEN(msg) = offset_ptr(token, offs); #ifdef HARD_DEBUG ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_TOKEN(msg))); ASSERT(hp > ptr_val(ERL_MESSAGE_TOKEN(msg))); #endif } if (is_not_immed(term)) { ASSERT(bp->mem <= ptr_val(term)); ASSERT(bp->mem + bp->size > ptr_val(term)); ERL_MESSAGE_TERM(msg) = offset_ptr(term, offs); #ifdef HARD_DEBUG ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_TERM(msg))); ASSERT(hp > ptr_val(ERL_MESSAGE_TERM(msg))); #endif } #ifdef HARD_DEBUG { int i, j; { ProcBin *mso = off_heap->mso; i = j = 0; while (mso != dbg_mso_start) { mso = mso->next; i++; } mso = bp->off_heap.mso; while (mso) { mso = mso->next; j++; } ASSERT(i == j); } { ErlFunThing *fun = off_heap->funs; i = j = 0; while (fun != dbg_fun_start) { fun = fun->next; i++; } fun = bp->off_heap.funs; while (fun) { fun = fun->next; j++; } ASSERT(i == j); } { ExternalThing *external = off_heap->externals; i = j = 0; while (external != dbg_external_start) { external = external->next; i++; } external = bp->off_heap.externals; while (external) { external = external->next; j++; } ASSERT(i == j); } } #endif bp->off_heap.mso = NULL; #ifndef HYBRID bp->off_heap.funs = NULL; #endif bp->off_heap.externals = NULL; free_message_buffer(bp); #ifdef HARD_DEBUG ASSERT(eq(ERL_MESSAGE_TERM(msg), dbg_term)); ASSERT(eq(ERL_MESSAGE_TOKEN(msg), dbg_token)); free_message_buffer(dbg_bp); #endif }