Example #1
0
int judyCreateBranchU(Pjp_t Pjp, void *Pjpm)
{
	jp_t JPNull;
	Pjbu_t PjbuRaw;
	Pjbu_t Pjbu;
	Pjbb_t PjbbRaw;
	Pjbb_t Pjbb;
	Word_t ii, jj;
	BITMAPB_t BitMap;
	Pjp_t PDstJP;

	PjbuRaw = judyLAllocJBU(Pjpm);
	if (PjbuRaw == NULL)
		return -1;
	Pjbu = P_JBU(PjbuRaw);
	JL_JPSETADT(&JPNull, 0, 0, JL_JPTYPE(Pjp) - cJL_JPBRANCH_B2 + cJL_JPNULL1);

	PjbbRaw = (Pjbb_t) (Pjp->jp_Addr);
	Pjbb = P_JBB(PjbbRaw);

	PDstJP = Pjbu->jbu_jp;
	for (ii = 0; ii < cJL_NUMSUBEXPB; ii++) {
		Pjp_t PjpA;
		Pjp_t PjpB;

		PjpB = PjpA = P_JP(JL_JBB_PJP(Pjbb, ii));

		BitMap = JL_JBB_BITMAP(Pjbb, ii);

		if (BitMap == 0) {
			for (jj = 0; jj < cJL_BITSPERSUBEXPB; jj++)
				PDstJP[jj] = JPNull;
			PDstJP += cJL_BITSPERSUBEXPB;
			continue;
		}
		if (BitMap == cJL_FULLBITMAPB) {
			JL_COPYMEM(PDstJP, PjpA, cJL_BITSPERSUBEXPB);
			PDstJP += cJL_BITSPERSUBEXPB;
			jj = cJL_BITSPERSUBEXPB;
		} else {
			for (jj = 0; jj < cJL_BITSPERSUBEXPB; jj++) {
				if (BitMap & 1)
					*PDstJP = *PjpA++;
				else
					*PDstJP = JPNull;

				PDstJP++;
				BitMap >>= 1;
			}
			jj = PjpA - PjpB;
		}
		judyLFreeJBBJP(JL_JBB_PJP(Pjbb, ii), jj, Pjpm);
	}

	judyLFreeJBB(PjbbRaw, Pjpm);
	Pjp->jp_Addr = (Word_t) PjbuRaw;
	Pjp->jp_Type += cJL_JPBRANCH_U - cJL_JPBRANCH_B;

	return 1;
}
Example #2
0
/**
 * Build a BranchB from an array of JPs and associated 1 byte digits
 * (expanses).  Return with Pjp pointing to the BranchB.  Caller must
 * deallocate passed arrays, if necessary.
 * We have no idea what kind of BranchB it is, so caller must set the jp_Type.
 * Return -1 if error (details in Pjpm), otherwise return 1.
 *
 * @Pjp		Build JPs from this place
 * @PJPs	Array of JPs to put into Bitmap branch
 * @Exp		Array of expanses to put into bitmap
 * @ExpCnt	Number of above JPs and Expanses
 */
int judyCreateBranchB(Pjp_t Pjp, Pjp_t PJPs, uint8_t Exp[], Word_t ExpCnt, void *Pjpm)
{
	Pjbb_t PjbbRaw, Pjbb;
	Word_t ii, jj;
	uint8_t CurrSubExp;

	assert(ExpCnt <= cJL_BRANCHBMAXJPS);

	PjbbRaw = judyLAllocJBB(Pjpm);
	if (PjbbRaw == NULL)
		return -1;
	Pjbb = P_JBB(PjbbRaw);

	CurrSubExp = Exp[0] / cJL_BITSPERSUBEXPB;

	for (jj = ii = 0; ii <= ExpCnt; ii++) {
		Word_t SubExp;

		if (ii == ExpCnt) {
			SubExp = cJL_ALLONES;
		} else {
			SubExp = Exp[ii] / cJL_BITSPERSUBEXPB;
			JL_JBB_BITMAP(Pjbb, SubExp) |= JL_BITPOSMASKB(Exp[ii]);
		}
		if (SubExp != CurrSubExp) {
			Word_t NumJP = ii - jj;
			Pjp_t PjpRaw, Pjp;

			PjpRaw = judyLAllocJBBJP(NumJP, Pjpm);
			Pjp = P_JP(PjpRaw);

			if (PjpRaw == NULL) {
				while (CurrSubExp--) {
					NumJP = judyCountBits(JL_JBB_BITMAP(Pjbb, CurrSubExp));
					if (NumJP == 0)
						continue;
					judyLFreeJBBJP(JL_JBB_PJP(Pjbb, CurrSubExp), NumJP, Pjpm);
				}
				judyLFreeJBB(PjbbRaw, Pjpm);
				return -1;
			}
			JL_JBB_PJP(Pjbb, CurrSubExp) = PjpRaw;
			JL_COPYMEM(Pjp, PJPs + jj, NumJP);
			jj = ii;
			CurrSubExp = SubExp;
		}
	}

	Pjp->jp_Addr = (Word_t) PjbbRaw;
	return 1;
}
FUNCTION Pjbb_t j__udyAllocJBB(Pjpm_t Pjpm)
{
        Word_t Words   = sizeof(jbb_t) / cJU_BYTESPERWORD;
        Pjbb_t PjbbRaw = (Pjbb_t) MALLOC(JudyMallocVirtual,
                                         Pjpm->jpm_TotalMemWords, Words);

        assert((Words * cJU_BYTESPERWORD) == sizeof(jbb_t));

        if ((Word_t) PjbbRaw > sizeof(Word_t))
        {
            ZEROWORDS(P_JBB(PjbbRaw), Words);
            Pjpm->jpm_TotalMemWords += Words;
        }
        else { J__UDYSETALLOCERROR(PjbbRaw); }

        TRACE_ALLOC5("0x%x %8lu = j__udyAllocJBB(), Words = %lu\n", PjbbRaw,
                     j__udyMemSequence++, Words, (Pjpm->jpm_Pop0) + 2);
        MALLOCBITS_SET(Pjbb_t, PjbbRaw);
        return(PjbbRaw);

} // j__udyAllocJBB()
Example #4
0
FUNCTION static Word_t j__udy1LCountSM(
const	Pjp_t	Pjp,		// top of Judy (sub)SM.
const	Word_t	Index,		// count at or above this Index.
const	Pjpm_t	Pjpm)		// for returning error info.
{
	Pjbl_t	Pjbl;		// Pjp->jp_Addr masked and cast to types:
	Pjbb_t	Pjbb;
	Pjbu_t	Pjbu;
	Pjll_t	Pjll;		// a Judy lower-level linear leaf.

	Word_t	digit;		// next digit to decode from Index.
	long	jpnum;		// JP number in a branch (base 0).
	int	offset;		// index ordinal within a leaf, base 0.
	Word_t	pop1;		// total population of an expanse.
	Word_t	pop1above;	// to return.

// Common code to check Decode bits in a JP against the equivalent portion of
// Index; XOR together, then mask bits of interest; must be all 0:
//
// Note:  Why does this code only assert() compliance rather than actively
// checking for outliers?  Its because Index is supposed to be valid, hence
// always match any Dcd bits traversed.
//
// Note:  This assertion turns out to be always true for cState = 3 on 32-bit
// and 7 on 64-bit, but its harmless, probably removed by the compiler.

#define	CHECKDCD(Pjp,cState) \
	assert(! JU_DCDNOTMATCHINDEX(Index, Pjp, cState))

// Common code to prepare to handle a root-level or lower-level branch:
// Extract a state-dependent digit from Index in a "constant" way, obtain the
// total population for the branch in a state-dependent way, and then branch to
// common code for multiple cases:
//
// For root-level branches, the state is always cJU_ROOTSTATE, and the
// population is received in Pjpm->jpm_Pop0.
//
// Note:  The total population is only needed in cases where the common code
// "counts up" instead of down to minimize cache line fills.  However, its
// available cheaply, and its better to do it with a constant shift (constant
// state value) instead of a variable shift later "when needed".

#define	PREPB_ROOT(Pjp,Next)				\
	digit = JU_DIGITATSTATE(Index, cJU_ROOTSTATE);	\
	pop1  = (Pjpm->jpm_Pop0) + 1;			\
	goto Next

#define	PREPB(Pjp,cState,Next)				\
	digit = JU_DIGITATSTATE(Index, cState);		\
	pop1  = JU_JPBRANCH_POP0(Pjp, (cState)) + 1;    \
	goto Next


// SWITCH ON JP TYPE:
//
// WARNING:  For run-time efficiency the following cases replicate code with
// varying constants, rather than using common code with variable values!

	switch (JU_JPTYPE(Pjp))
	{


// ----------------------------------------------------------------------------
// ROOT-STATE LEAF that starts with a Pop0 word; just count within the leaf:

	case cJU_LEAFW:
	{
	    Pjlw_t Pjlw = P_JLW(Pjp->jp_Addr);		// first word of leaf.

	    assert((Pjpm->jpm_Pop0) + 1 == Pjlw[0] + 1);  // sent correctly.
	    offset = j__udySearchLeafW(Pjlw + 1, Pjpm->jpm_Pop0 + 1, Index);
	    assert(offset >= 0);			// Index must exist.
	    assert(offset < (Pjpm->jpm_Pop0) + 1);	// Index be in range.
	    return((Pjpm->jpm_Pop0) + 1 - offset);	// INCLUSIVE of Index.
	}

// ----------------------------------------------------------------------------
// LINEAR BRANCH; count populations in JPs in the JBL ABOVE the next digit in
// Index, and recurse for the next digit in Index:
//
// Note:  There are no null JPs in a JBL; watch out for pop1 == 0.
//
// Note:  A JBL should always fit in one cache line => no need to count up
// versus down to save cache line fills.  (PREPB() sets pop1 for no reason.)

	case cJU_JPBRANCH_L2:  CHECKDCD(Pjp, 2); PREPB(Pjp, 2, BranchL);
	case cJU_JPBRANCH_L3:  CHECKDCD(Pjp, 3); PREPB(Pjp, 3, BranchL);

#ifdef JU_64BIT
	case cJU_JPBRANCH_L4:  CHECKDCD(Pjp, 4); PREPB(Pjp, 4, BranchL);
	case cJU_JPBRANCH_L5:  CHECKDCD(Pjp, 5); PREPB(Pjp, 5, BranchL);
	case cJU_JPBRANCH_L6:  CHECKDCD(Pjp, 6); PREPB(Pjp, 6, BranchL);
	case cJU_JPBRANCH_L7:  CHECKDCD(Pjp, 7); PREPB(Pjp, 7, BranchL);
#endif
	case cJU_JPBRANCH_L:   PREPB_ROOT(Pjp, BranchL);

// Common code (state-independent) for all cases of linear branches:

BranchL:

	Pjbl      = P_JBL(Pjp->jp_Addr);
	jpnum     = Pjbl->jbl_NumJPs;			// above last JP.
	pop1above = 0;

	while (digit < (Pjbl->jbl_Expanse[--jpnum]))	 // still ABOVE digit.
	{
	    if ((pop1 = j__udyJPPop1((Pjbl->jbl_jp) + jpnum)) == cJU_ALLONES)
	    {
		JU_SET_ERRNO_NONNULL(Pjpm, JU_ERRNO_CORRUPT);
		return(C_JERR);
	    }

	    pop1above += pop1;
	    assert(jpnum > 0);				// should find digit.
	}

	assert(digit == (Pjbl->jbl_Expanse[jpnum]));	// should find digit.

	pop1 = j__udy1LCountSM((Pjbl->jbl_jp) + jpnum, Index, Pjpm);
	if (pop1 == C_JERR) return(C_JERR);		// pass error up.

	assert(pop1above + pop1);
	return(pop1above + pop1);


// ----------------------------------------------------------------------------
// BITMAP BRANCH; count populations in JPs in the JBB ABOVE the next digit in
// Index, and recurse for the next digit in Index:
//
// Note:  There are no null JPs in a JBB; watch out for pop1 == 0.

	case cJU_JPBRANCH_B2:  CHECKDCD(Pjp, 2); PREPB(Pjp, 2, BranchB);
	case cJU_JPBRANCH_B3:  CHECKDCD(Pjp, 3); PREPB(Pjp, 3, BranchB);
#ifdef JU_64BIT
	case cJU_JPBRANCH_B4:  CHECKDCD(Pjp, 4); PREPB(Pjp, 4, BranchB);
	case cJU_JPBRANCH_B5:  CHECKDCD(Pjp, 5); PREPB(Pjp, 5, BranchB);
	case cJU_JPBRANCH_B6:  CHECKDCD(Pjp, 6); PREPB(Pjp, 6, BranchB);
	case cJU_JPBRANCH_B7:  CHECKDCD(Pjp, 7); PREPB(Pjp, 7, BranchB);
#endif
	case cJU_JPBRANCH_B:   PREPB_ROOT(Pjp, BranchB);

// Common code (state-independent) for all cases of bitmap branches:

BranchB:
	{
	    intptr_t subexp;	// for stepping through layer 1 (subexpanses).
	    intptr_t  findsub;	// subexpanse containing   Index (digit).
	    Word_t findbit;	// bit	      representing Index (digit).
	    Word_t lowermask;	// bits for indexes at or below Index.
	    Word_t jpcount;	// JPs in a subexpanse.
	    Word_t clbelow;	// cache lines below digits cache line.
	    Word_t clabove;	// cache lines above digits cache line.

	    Pjbb      = P_JBB(Pjp->jp_Addr);
	    findsub   = digit / cJU_BITSPERSUBEXPB;
	    findbit   = digit % cJU_BITSPERSUBEXPB;
	    lowermask = JU_MASKLOWERINC(JU_BITPOSMASKB(findbit));
	    clbelow   = clabove = 0;	// initial/default => always downward.

	    assert(JU_BITMAPTESTB(Pjbb, digit)); // digit must have a JP.
	    assert(findsub < cJU_NUMSUBEXPB);	 // falls in expected range.

// Shorthand for one subexpanse in a bitmap and for one JP in a bitmap branch:
//
// Note: BMPJP0 exists separately to support assertions.

#define	BMPJP0(Subexp)       (P_JP(JU_JBB_PJP(Pjbb, Subexp)))
#define	BMPJP(Subexp,JPnum)  (BMPJP0(Subexp) + (JPnum))

#ifndef NOSMARTJBB  // enable to turn off smart code for comparison purposes.

// FIGURE OUT WHICH DIRECTION CAUSES FEWER CACHE LINE FILLS; adding the pop1s
// in JPs above Indexs JP, or subtracting the pop1s in JPs below Indexs JP.
//
// This is tricky because, while each set bit in the bitmap represents a JP,
// the JPs are scattered over cJU_NUMSUBEXPB subexpanses, each of which can
// contain JPs packed into multiple cache lines, and this code must visit every
// JP either BELOW or ABOVE the JP for Index.
//
// Number of cache lines required to hold a linear list of the given number of
// JPs, assuming the first JP is at the start of a cache line or the JPs in
// jpcount fit wholly within a single cache line, which is ensured by
// JudyMalloc():

#define	CLPERJPS(jpcount) \
	((((jpcount) * cJU_WORDSPERJP) + cJU_WORDSPERCL - 1) / cJU_WORDSPERCL)

// Count cache lines below/above for each subexpanse:

	    for (subexp = 0; subexp < cJU_NUMSUBEXPB; ++subexp)
	    {
		jpcount = j__udyCountBitsB(JU_JBB_BITMAP(Pjbb, subexp));

// When at the subexpanse containing Index (digit), add cache lines
// below/above appropriately, excluding the cache line containing the JP for
// Index itself:

		if	(subexp <  findsub)  clbelow += CLPERJPS(jpcount);
		else if (subexp >  findsub)  clabove += CLPERJPS(jpcount);
		else // (subexp == findsub)
		{
		    Word_t clfind;	// cache line containing Index (digit).

		    clfind = CLPERJPS(j__udyCountBitsB(
				    JU_JBB_BITMAP(Pjbb, subexp) & lowermask));

		    assert(clfind > 0);	 // digit itself should have 1 CL.
		    clbelow += clfind - 1;
		    clabove += CLPERJPS(jpcount) - clfind;
		}
	    }
#endif // ! NOSMARTJBB

// Note:  Its impossible to get through the following "if" without setting
// jpnum -- see some of the assertions below -- but gcc -Wall doesnt know
// this, so preset jpnum to make it happy:

	    jpnum = 0;


// COUNT POPULATION FOR A BITMAP BRANCH, in whichever direction should result
// in fewer cache line fills:
//
// Note:  If the remainder of Index is zero, pop1above is the pop1 of the
// entire expanse and theres no point in recursing to lower levels; but this
// should be so rare that its not worth checking for;
// Judy1Count()/JudyLCount() never even calls the motor for Index == 0 (all
// bytes).


// COUNT UPWARD, subtracting each "below or at" JPs pop1 from the whole
// expanses pop1:
//
// Note:  If this causes clbelow + 1 cache line fills including JPs cache
// line, thats OK; at worst this is the same as clabove.

	    if (clbelow < clabove)
	    {
#ifdef SMARTMETRICS
		++jbb_upward;
#endif
		pop1above = pop1;		// subtract JPs at/below Index.

// Count JPs for which to accrue pop1s in this subexpanse:
//
// TBD:  If JU_JBB_BITMAP is cJU_FULLBITMAPB, dont bother counting.

		for (subexp = 0; subexp <= findsub; ++subexp)
		{
		    jpcount = j__udyCountBitsB((subexp < findsub) ?
				      JU_JBB_BITMAP(Pjbb, subexp) :
				      JU_JBB_BITMAP(Pjbb, subexp) & lowermask);

		    // should always find findbit:
		    assert((subexp < findsub) || jpcount);

// Subtract pop1s from JPs BELOW OR AT Index (digit):
//
// Note:  The pop1 for Indexs JP itself is partially added back later at a
// lower state.
//
// Note:  An empty subexpanse (jpcount == 0) is handled "for free".
//
// Note:  Must be null JP subexp pointer in empty subexpanse and non-empty in
// non-empty subexpanse:

		    assert(   jpcount  || (BMPJP0(subexp) == (Pjp_t) NULL));
		    assert((! jpcount) || (BMPJP0(subexp) != (Pjp_t) NULL));

		    for (jpnum = 0; jpnum < jpcount; ++jpnum)
		    {
			if ((pop1 = j__udyJPPop1(BMPJP(subexp, jpnum)))
			    == cJU_ALLONES)
			{
			    JU_SET_ERRNO_NONNULL(Pjpm, JU_ERRNO_CORRUPT);
			    return(C_JERR);
			}

			pop1above -= pop1;
		    }

		    jpnum = jpcount - 1;	// make correct for digit.
		}
	    }

// COUNT DOWNWARD, adding each "above" JPs pop1:

	    else
	    {
		long jpcountbf;			// below findbit, inclusive.
#ifdef SMARTMETRICS
		++jbb_downward;
#endif
		pop1above = 0;			// add JPs above Index.
		jpcountbf = 0;			// until subexp == findsub.

// Count JPs for which to accrue pop1s in this subexpanse:
//
// This is more complicated than counting upward because the scan of digits
// subexpanse must count ALL JPs, to know where to START counting down, and
// ALSO note the offset of digits JP to know where to STOP counting down.

		for (subexp = cJU_NUMSUBEXPB - 1; subexp >= findsub; --subexp)
		{
		    jpcount = j__udyCountBitsB(JU_JBB_BITMAP(Pjbb, subexp));

		    // should always find findbit:
		    assert((subexp > findsub) || jpcount);

		    if (! jpcount) continue;	// empty subexpanse, save time.

// Count JPs below digit, inclusive:

		    if (subexp == findsub)
		    {
			jpcountbf = j__udyCountBitsB(JU_JBB_BITMAP(Pjbb, subexp)
						  & lowermask);
		    }

		    // should always find findbit:
		    assert((subexp > findsub) || jpcountbf);
		    assert(jpcount >= jpcountbf);	// proper relationship.

// Add pop1s from JPs ABOVE Index (digit):

		    // no null JP subexp pointers:
		    assert(BMPJP0(subexp) != (Pjp_t) NULL);

		    for (jpnum = jpcount - 1; jpnum >= jpcountbf; --jpnum)
		    {
			if ((pop1 = j__udyJPPop1(BMPJP(subexp, jpnum)))
			    == cJU_ALLONES)
			{
			    JU_SET_ERRNO_NONNULL(Pjpm, JU_ERRNO_CORRUPT);
			    return(C_JERR);
			}

			pop1above += pop1;
		    }
		    // jpnum is now correct for digit.
		}
	    } // else.

// Return the net population ABOVE the digits JP at this state (in this JBB)
// plus the population AT OR ABOVE Index in the SM under the digits JP:

	    pop1 = j__udy1LCountSM(BMPJP(findsub, jpnum), Index, Pjpm);
	    if (pop1 == C_JERR) return(C_JERR);		// pass error up.

	    assert(pop1above + pop1);
	    return(pop1above + pop1);

	} // case.


// ----------------------------------------------------------------------------
// UNCOMPRESSED BRANCH; count populations in JPs in the JBU ABOVE the next
// digit in Index, and recurse for the next digit in Index:
//
// Note:  If the remainder of Index is zero, pop1above is the pop1 of the
// entire expanse and theres no point in recursing to lower levels; but this
// should be so rare that its not worth checking for;
// Judy1Count()/JudyLCount() never even calls the motor for Index == 0 (all
// bytes).

	case cJU_JPBRANCH_U2:  CHECKDCD(Pjp, 2); PREPB(Pjp, 2, BranchU);
	case cJU_JPBRANCH_U3:  CHECKDCD(Pjp, 3); PREPB(Pjp, 3, BranchU);
#ifdef JU_64BIT
	case cJU_JPBRANCH_U4:  CHECKDCD(Pjp, 4); PREPB(Pjp, 4, BranchU);
	case cJU_JPBRANCH_U5:  CHECKDCD(Pjp, 5); PREPB(Pjp, 5, BranchU);
	case cJU_JPBRANCH_U6:  CHECKDCD(Pjp, 6); PREPB(Pjp, 6, BranchU);
	case cJU_JPBRANCH_U7:  CHECKDCD(Pjp, 7); PREPB(Pjp, 7, BranchU);
#endif
	case cJU_JPBRANCH_U:   PREPB_ROOT(Pjp, BranchU);

// Common code (state-independent) for all cases of uncompressed branches:

BranchU:
	    Pjbu = P_JBU(Pjp->jp_Addr);

#ifndef NOSMARTJBU  // enable to turn off smart code for comparison purposes.

// FIGURE OUT WHICH WAY CAUSES FEWER CACHE LINE FILLS; adding the JPs above
// Indexs JP, or subtracting the JPs below Indexs JP.
//
// COUNT UPWARD, subtracting the pop1 of each JP BELOW OR AT Index, from the
// whole expanses pop1:

	    if (digit < (cJU_BRANCHUNUMJPS / 2))
	    {
		pop1above = pop1;		// subtract JPs below Index.
#ifdef SMARTMETRICS
		++jbu_upward;
#endif
		for (jpnum = 0; jpnum <= digit; ++jpnum)
		{
		    if ((Pjbu->jbu_jp[jpnum].jp_Type) <= cJU_JPNULLMAX)
			continue;	// shortcut, save a function call.

		    if ((pop1 = j__udyJPPop1(Pjbu->jbu_jp + jpnum))
		     == cJU_ALLONES)
		    {
			JU_SET_ERRNO_NONNULL(Pjpm, JU_ERRNO_CORRUPT);
			return(C_JERR);
		    }

		    pop1above -= pop1;
		}
	    }

// COUNT DOWNWARD, simply adding the pop1 of each JP ABOVE Index:

	    else
#endif // NOSMARTJBU
	    {
		assert(digit < cJU_BRANCHUNUMJPS);
#ifdef SMARTMETRICS
		++jbu_downward;
#endif
		pop1above = 0;			// add JPs above Index.

		for (jpnum = cJU_BRANCHUNUMJPS - 1; jpnum > digit; --jpnum)
		{
		    if ((Pjbu->jbu_jp[jpnum].jp_Type) <= cJU_JPNULLMAX)
			continue;	// shortcut, save a function call.

		    if ((pop1 = j__udyJPPop1(Pjbu->jbu_jp + jpnum))
		     == cJU_ALLONES)
		    {
			JU_SET_ERRNO_NONNULL(Pjpm, JU_ERRNO_CORRUPT);
			return(C_JERR);
		    }

		    pop1above += pop1;
		}
	    }

	    if ((pop1 = j__udy1LCountSM(Pjbu->jbu_jp + digit, Index, Pjpm))
	     == C_JERR) return(C_JERR);		// pass error up.

	    assert(pop1above + pop1);
	    return(pop1above + pop1);


// ----------------------------------------------------------------------------
// LEAF COUNT MACROS:
//
// LEAF*ABOVE() are common code for different JP types (linear leaves, bitmap
// leaves, and immediates) and different leaf Index Sizes, which result in
// calling different leaf search functions.  Linear leaves get the leaf address
// from jp_Addr and the Population from jp_DcdPopO, while immediates use Pjp
// itself as the leaf address and get Population from jp_Type.

#define	LEAFLABOVE(Func)				\
	Pjll = P_JLL(Pjp->jp_Addr);			\
	pop1 = JU_JPLEAF_POP0(Pjp) + 1;	                \
	LEAFABOVE(Func, Pjll, pop1)

#define	LEAFB1ABOVE(Func) LEAFLABOVE(Func)  // different Func, otherwise same.

#ifdef JUDY1
#define	IMMABOVE(Func,Pop1)	\
	Pjll = (Pjll_t) Pjp;	\
	LEAFABOVE(Func, Pjll, Pop1)
#else
// Note:  For JudyL immediates with >= 2 Indexes, the index bytes are in a
// different place than for Judy1:

#define	IMMABOVE(Func,Pop1) \
	LEAFABOVE(Func, (Pjll_t) (Pjp->jp_LIndex), Pop1)
#endif

// For all leaf types, the population AT OR ABOVE is the total pop1 less the
// offset of Index; and Index should always be found:

#define	LEAFABOVE(Func,Pjll,Pop1)		\
	offset = Func(Pjll, Pop1, Index);	\
	assert(offset >= 0);			\
	assert(offset < (Pop1));		\
	return((Pop1) - offset)

// IMMABOVE_01 handles the special case of an immediate JP with 1 index, which
// the search functions arent used for anyway:
//
// The target Index should be the one in this Immediate, in which case the
// count above (inclusive) is always 1.

#define	IMMABOVE_01						\
	assert((JU_JPDCDPOP0(Pjp)) == JU_TRIMTODCDSIZE(Index));	\
	return(1)


// ----------------------------------------------------------------------------
// LINEAR LEAF; search the leaf for Index; size is computed from jp_Type:

#if (defined(JUDYL) || (! defined(JU_64BIT)))
	case cJU_JPLEAF1:  LEAFLABOVE(j__udySearchLeaf1);
#endif
	case cJU_JPLEAF2:  LEAFLABOVE(j__udySearchLeaf2);
	case cJU_JPLEAF3:  LEAFLABOVE(j__udySearchLeaf3);

#ifdef JU_64BIT
	case cJU_JPLEAF4:  LEAFLABOVE(j__udySearchLeaf4);
	case cJU_JPLEAF5:  LEAFLABOVE(j__udySearchLeaf5);
	case cJU_JPLEAF6:  LEAFLABOVE(j__udySearchLeaf6);
	case cJU_JPLEAF7:  LEAFLABOVE(j__udySearchLeaf7);
#endif


// ----------------------------------------------------------------------------
// BITMAP LEAF; search the leaf for Index:
//
// Since the bitmap describes Indexes digitally rather than linearly, this is
// not really a search, but just a count.

	case cJU_JPLEAF_B1:  LEAFB1ABOVE(j__udyCountLeafB1);


#ifdef JUDY1
// ----------------------------------------------------------------------------
// FULL POPULATION:
//
// Return the count of Indexes AT OR ABOVE Index, which is the total population
// of the expanse (a constant) less the value of the undecoded digit remaining
// in Index (its base-0 offset in the expanse), which yields an inclusive count
// above.
//
// TBD:  This only supports a 1-byte full expanse.  Should this extract a
// stored value for pop0 and possibly more LSBs of Index, to handle larger full
// expanses?

	case cJ1_JPFULLPOPU1:
	    return(cJU_JPFULLPOPU1_POP0 + 1 - JU_DIGITATSTATE(Index, 1));
#endif


// ----------------------------------------------------------------------------
// IMMEDIATE:

	case cJU_JPIMMED_1_01:  IMMABOVE_01;
	case cJU_JPIMMED_2_01:  IMMABOVE_01;
	case cJU_JPIMMED_3_01:  IMMABOVE_01;
#ifdef JU_64BIT
	case cJU_JPIMMED_4_01:  IMMABOVE_01;
	case cJU_JPIMMED_5_01:  IMMABOVE_01;
	case cJU_JPIMMED_6_01:  IMMABOVE_01;
	case cJU_JPIMMED_7_01:  IMMABOVE_01;
#endif

	case cJU_JPIMMED_1_02:  IMMABOVE(j__udySearchLeaf1,  2);
	case cJU_JPIMMED_1_03:  IMMABOVE(j__udySearchLeaf1,  3);
#if (defined(JUDY1) || defined(JU_64BIT))
	case cJU_JPIMMED_1_04:  IMMABOVE(j__udySearchLeaf1,  4);
	case cJU_JPIMMED_1_05:  IMMABOVE(j__udySearchLeaf1,  5);
	case cJU_JPIMMED_1_06:  IMMABOVE(j__udySearchLeaf1,  6);
	case cJU_JPIMMED_1_07:  IMMABOVE(j__udySearchLeaf1,  7);
#endif
#if (defined(JUDY1) && defined(JU_64BIT))
	case cJ1_JPIMMED_1_08:  IMMABOVE(j__udySearchLeaf1,  8);
	case cJ1_JPIMMED_1_09:  IMMABOVE(j__udySearchLeaf1,  9);
	case cJ1_JPIMMED_1_10:  IMMABOVE(j__udySearchLeaf1, 10);
	case cJ1_JPIMMED_1_11:  IMMABOVE(j__udySearchLeaf1, 11);
	case cJ1_JPIMMED_1_12:  IMMABOVE(j__udySearchLeaf1, 12);
	case cJ1_JPIMMED_1_13:  IMMABOVE(j__udySearchLeaf1, 13);
	case cJ1_JPIMMED_1_14:  IMMABOVE(j__udySearchLeaf1, 14);
	case cJ1_JPIMMED_1_15:  IMMABOVE(j__udySearchLeaf1, 15);
#endif

#if (defined(JUDY1) || defined(JU_64BIT))
	case cJU_JPIMMED_2_02:  IMMABOVE(j__udySearchLeaf2,  2);
	case cJU_JPIMMED_2_03:  IMMABOVE(j__udySearchLeaf2,  3);
#endif
#if (defined(JUDY1) && defined(JU_64BIT))
	case cJ1_JPIMMED_2_04:  IMMABOVE(j__udySearchLeaf2,  4);
	case cJ1_JPIMMED_2_05:  IMMABOVE(j__udySearchLeaf2,  5);
	case cJ1_JPIMMED_2_06:  IMMABOVE(j__udySearchLeaf2,  6);
	case cJ1_JPIMMED_2_07:  IMMABOVE(j__udySearchLeaf2,  7);
#endif

#if (defined(JUDY1) || defined(JU_64BIT))
	case cJU_JPIMMED_3_02:  IMMABOVE(j__udySearchLeaf3,  2);
#endif
#if (defined(JUDY1) && defined(JU_64BIT))
	case cJ1_JPIMMED_3_03:  IMMABOVE(j__udySearchLeaf3,  3);
	case cJ1_JPIMMED_3_04:  IMMABOVE(j__udySearchLeaf3,  4);
	case cJ1_JPIMMED_3_05:  IMMABOVE(j__udySearchLeaf3,  5);

	case cJ1_JPIMMED_4_02:  IMMABOVE(j__udySearchLeaf4,  2);
	case cJ1_JPIMMED_4_03:  IMMABOVE(j__udySearchLeaf4,  3);

	case cJ1_JPIMMED_5_02:  IMMABOVE(j__udySearchLeaf5,  2);
	case cJ1_JPIMMED_5_03:  IMMABOVE(j__udySearchLeaf5,  3);

	case cJ1_JPIMMED_6_02:  IMMABOVE(j__udySearchLeaf6,  2);

	case cJ1_JPIMMED_7_02:  IMMABOVE(j__udySearchLeaf7,  2);
#endif


// ----------------------------------------------------------------------------
// OTHER CASES:

	default: JU_SET_ERRNO_NONNULL(Pjpm, JU_ERRNO_CORRUPT); return(C_JERR);

	} // switch on JP type

	/*NOTREACHED*/

} // j__udy1LCountSM()
Example #5
0
FUNCTION static Word_t j__udyGetMemActive(
	Pjp_t  Pjp)		// top of subtree.
{
	Word_t offset;		// in a branch.
	Word_t Bytes = 0;	// actual bytes used at this level.
	Word_t IdxSz;		// bytes per index in leaves

	switch (JU_JPTYPE(Pjp))
	{

	case cJU_JPBRANCH_L2:
	case cJU_JPBRANCH_L3:
#ifdef JU_64BIT
	case cJU_JPBRANCH_L4:
	case cJU_JPBRANCH_L5:
	case cJU_JPBRANCH_L6:
	case cJU_JPBRANCH_L7:
#endif
	case cJU_JPBRANCH_L:
	{
	    Pjbl_t Pjbl = P_JBL(Pjp->jp_Addr);

	    for (offset = 0; offset < (Pjbl->jbl_NumJPs); ++offset)
	        Bytes += j__udyGetMemActive((Pjbl->jbl_jp) + offset);

	    return(Bytes + sizeof(jbl_t));
	}

	case cJU_JPBRANCH_B2:
	case cJU_JPBRANCH_B3:
#ifdef JU_64BIT
	case cJU_JPBRANCH_B4:
	case cJU_JPBRANCH_B5:
	case cJU_JPBRANCH_B6:
	case cJU_JPBRANCH_B7:
#endif
	case cJU_JPBRANCH_B:
	{
	    Word_t subexp;
	    Word_t jpcount;
	    Pjbb_t Pjbb = P_JBB(Pjp->jp_Addr);

	    for (subexp = 0; subexp < cJU_NUMSUBEXPB; ++subexp)
	    {
	        jpcount = j__udyCountBitsB(JU_JBB_BITMAP(Pjbb, subexp));
                Bytes  += jpcount * sizeof(jp_t);

		for (offset = 0; offset < jpcount; ++offset)
		{
		    Bytes += j__udyGetMemActive(P_JP(JU_JBB_PJP(Pjbb, subexp))
			   + offset);
		}
	    }

	    return(Bytes + sizeof(jbb_t));
	}

	case cJU_JPBRANCH_U2:
	case cJU_JPBRANCH_U3:
#ifdef JU_64BIT
	case cJU_JPBRANCH_U4:
	case cJU_JPBRANCH_U5:
	case cJU_JPBRANCH_U6:
	case cJU_JPBRANCH_U7:
#endif
	case cJU_JPBRANCH_U:
        {
	    Pjbu_t Pjbu = P_JBU(Pjp->jp_Addr);

            for (offset = 0; offset < cJU_BRANCHUNUMJPS; ++offset)
	    {
		if (((Pjbu->jbu_jp[offset].jp_Type) >= cJU_JPNULL1)
		 && ((Pjbu->jbu_jp[offset].jp_Type) <= cJU_JPNULLMAX))
		{
		    continue;		// skip null JP to save time.
		}

	        Bytes += j__udyGetMemActive(Pjbu->jbu_jp + offset);
	    }

	    return(Bytes + sizeof(jbu_t));
        }


// -- Cases below here terminate and do not recurse. --

#if (defined(JUDYL) || (! defined(JU_64BIT)))
        case cJU_JPLEAF1: IdxSz = 1; goto LeafWords;
#endif
	case cJU_JPLEAF2: IdxSz = 2; goto LeafWords;
	case cJU_JPLEAF3: IdxSz = 3; goto LeafWords;
#ifdef JU_64BIT
	case cJU_JPLEAF4: IdxSz = 4; goto LeafWords;
	case cJU_JPLEAF5: IdxSz = 5; goto LeafWords;
	case cJU_JPLEAF6: IdxSz = 6; goto LeafWords;
	case cJU_JPLEAF7: IdxSz = 7; goto LeafWords;
#endif
LeafWords:

#ifdef JUDY1
            return(IdxSz * (JU_JPLEAF_POP0(Pjp) + 1));
#else
            return((IdxSz + sizeof(Word_t))
		 * (JU_JPLEAF_POP0(Pjp) + 1));
#endif
	case cJU_JPLEAF_B1:
	{
#ifdef JUDY1
            return(sizeof(jlb_t));
#else
            Bytes = (JU_JPLEAF_POP0(Pjp) + 1) * sizeof(Word_t);

	    return(Bytes + sizeof(jlb_t));
#endif
	}

	JUDY1CODE(case cJ1_JPFULLPOPU1: return(0);)

#ifdef JUDY1
#define J__Mpy 0
#else
#define J__Mpy sizeof(Word_t)
#endif

	case cJU_JPIMMED_1_01:	return(0);
	case cJU_JPIMMED_2_01:	return(0);
	case cJU_JPIMMED_3_01:	return(0);
#ifdef JU_64BIT
	case cJU_JPIMMED_4_01:	return(0);
	case cJU_JPIMMED_5_01:	return(0);
	case cJU_JPIMMED_6_01:	return(0);
	case cJU_JPIMMED_7_01:	return(0);
#endif

	case cJU_JPIMMED_1_02:	return(J__Mpy * 2);
	case cJU_JPIMMED_1_03:	return(J__Mpy * 3);
#if (defined(JUDY1) || defined(JU_64BIT))
	case cJU_JPIMMED_1_04:	return(J__Mpy * 4);
	case cJU_JPIMMED_1_05:	return(J__Mpy * 5);
	case cJU_JPIMMED_1_06:	return(J__Mpy * 6);
	case cJU_JPIMMED_1_07:	return(J__Mpy * 7);
#endif
#if (defined(JUDY1) && defined(JU_64BIT))
	case cJ1_JPIMMED_1_08:	return(0);
	case cJ1_JPIMMED_1_09:	return(0);
	case cJ1_JPIMMED_1_10:	return(0);
	case cJ1_JPIMMED_1_11:	return(0);
	case cJ1_JPIMMED_1_12:	return(0);
	case cJ1_JPIMMED_1_13:	return(0);
	case cJ1_JPIMMED_1_14:	return(0);
	case cJ1_JPIMMED_1_15:	return(0);
#endif

#if (defined(JUDY1) || defined(JU_64BIT))
	case cJU_JPIMMED_2_02:	return(J__Mpy * 2);
	case cJU_JPIMMED_2_03:	return(J__Mpy * 3);
#endif
#if (defined(JUDY1) && defined(JU_64BIT))
	case cJ1_JPIMMED_2_04:	return(0);
	case cJ1_JPIMMED_2_05:	return(0);
	case cJ1_JPIMMED_2_06:	return(0);
	case cJ1_JPIMMED_2_07:	return(0);
#endif

#if (defined(JUDY1) || defined(JU_64BIT))
	case cJU_JPIMMED_3_02:	return(J__Mpy * 2);
#endif
#if (defined(JUDY1) && defined(JU_64BIT))
	case cJ1_JPIMMED_3_03:	return(0);
	case cJ1_JPIMMED_3_04:	return(0);
	case cJ1_JPIMMED_3_05:	return(0);

	case cJ1_JPIMMED_4_02:	return(0);
	case cJ1_JPIMMED_4_03:	return(0);
	case cJ1_JPIMMED_5_02:	return(0);
	case cJ1_JPIMMED_5_03:	return(0);
	case cJ1_JPIMMED_6_02:	return(0);
	case cJ1_JPIMMED_7_02:	return(0);
#endif

	} // switch (JU_JPTYPE(Pjp))
Example #6
0
int JudyLDel(void **PPArray, uint32_t Index, void **PPvalue)
{
	Word_t pop1;
	int offset;
	void **PPret;

	if (PPArray == NULL) {
		JL_SET_ERRNO(JLE_NULLPPARRAY);
		return JERR;
	}

	if ((PPret = JudyLGet(*PPArray, Index)) == PPJERR)
		return JERR;

	if (PPret == NULL)
		return 0;

	if (PPvalue)
		*PPvalue = *PPret;

	if (JL_LEAFW_POP0(*PPArray) < cJL_LEAFW_MAXPOP1) {
		Pjv_t Pjv;
		Pjv_t Pjvnew;
		Pjlw_t Pjlw = P_JLW(*PPArray);
		Pjlw_t Pjlwnew;
		pop1 = Pjlw[0] + 1;

		if (pop1 == 1) {
			judyLFreeJLW(Pjlw, /* pop1 = */ 1, NULL);
			*PPArray = NULL;
			return 1;
		}

		offset = judySearchLeafW(Pjlw + 1, pop1, Index);
		assert(offset >= 0);

		Pjv = JL_LEAFWVALUEAREA(Pjlw, pop1);

		if (JL_LEAFWGROWINPLACE(pop1 - 1)) {
			JL_DELETEINPLACE(Pjlw + 1, pop1, offset, ignore);
			JL_DELETEINPLACE(Pjv, pop1, offset, ignore);
			--(Pjlw[0]);
			return 1;
		}

		Pjlwnew = judyLAllocJLW(pop1 - 1);
		JL_CHECKALLOC(Pjlw_t, Pjlwnew, JERR);

		Pjlwnew[0] = (pop1 - 1) - 1;
		JL_DELETECOPY(Pjlwnew + 1, Pjlw + 1, pop1, offset, ignore);

		Pjvnew = JL_LEAFWVALUEAREA(Pjlwnew, pop1 - 1);
		JL_DELETECOPY(Pjvnew, Pjv, pop1, offset, ignore);

		judyLFreeJLW(Pjlw, pop1, NULL);

		*PPArray = (void *) Pjlwnew;
		return 1;
	} else {
		Word_t digit;
		Pjv_t Pjv;
		Pjlw_t Pjlwnew;
		Pjpm_t Pjpm = P_JPM(*PPArray);
		Pjp_t Pjp = &(Pjpm->jpm_JP);

		assert(((Pjpm->jpm_JP.jp_Type) == cJL_JPBRANCH_L)
		       || ((Pjpm->jpm_JP.jp_Type) == cJL_JPBRANCH_B)
		       || ((Pjpm->jpm_JP.jp_Type) == cJL_JPBRANCH_U));

		if (judyDelWalk(Pjp, Index, cJL_ROOTSTATE, Pjpm) == -1)
			return JERR;

		--(Pjpm->jpm_Pop0);

		if ((Pjpm->jpm_Pop0 + 1) != cJL_LEAFW_MAXPOP1)
			return 1;

		Pjlwnew = judyLAllocJLW(cJL_LEAFW_MAXPOP1);
		JL_CHECKALLOC(Pjlw_t, Pjlwnew, JERR);

		*PPArray = (void *) Pjlwnew;
		Pjv = JL_LEAFWVALUEAREA(Pjlwnew, cJL_LEAFW_MAXPOP1);
		*Pjlwnew++ = cJL_LEAFW_MAXPOP1 - 1;

		switch (JL_JPTYPE(Pjp)) {
		case cJL_JPBRANCH_L: {
			Pjbl_t PjblRaw = (Pjbl_t) (Pjp->jp_Addr);
			Pjbl_t Pjbl = P_JBL(PjblRaw);

			for (offset = 0; offset < Pjbl->jbl_NumJPs; ++offset) {
				pop1 = judyLeafM1ToLeafW(Pjlwnew, Pjv, (Pjbl->jbl_jp) + offset,
					JL_DIGITTOSTATE(Pjbl->jbl_Expanse [offset],
					cJL_BYTESPERWORD), (void *) Pjpm);
				Pjlwnew += pop1;
				Pjv += pop1;
			}
			judyLFreeJBL(PjblRaw, Pjpm);

			break;
		}
		case cJL_JPBRANCH_B: {
			Pjbb_t PjbbRaw = (Pjbb_t) (Pjp->jp_Addr);
			Pjbb_t Pjbb = P_JBB(PjbbRaw);
			Word_t subexp;
			BITMAPB_t bitmap;
			Pjp_t Pjp2Raw, Pjp2;

			for (subexp = 0; subexp < cJL_NUMSUBEXPB; ++subexp) {
				if ((bitmap = JL_JBB_BITMAP(Pjbb, subexp)) == 0)
					continue;

				digit = subexp * cJL_BITSPERSUBEXPB;
				Pjp2Raw = JL_JBB_PJP(Pjbb, subexp);
				Pjp2 = P_JP(Pjp2Raw);
				assert(Pjp2 != NULL);

				for (offset = 0; bitmap != 0;
				     bitmap >>= 1, ++digit) {
					if (!(bitmap & 1))
						continue;
					pop1 = judyLeafM1ToLeafW(Pjlwnew, Pjv, Pjp2 + offset,
						JL_DIGITTOSTATE(digit, cJL_BYTESPERWORD), (void *) Pjpm);
					Pjlwnew += pop1;
					Pjv += pop1;
					++offset;
				}
				judyLFreeJBBJP(Pjp2Raw, /* pop1 = */ offset, Pjpm);
			}
			judyLFreeJBB(PjbbRaw, Pjpm);

			break;
		}
		case cJL_JPBRANCH_U: {
			Pjbu_t PjbuRaw = (Pjbu_t) (Pjp->jp_Addr);
			Pjbu_t Pjbu = P_JBU(PjbuRaw);
			Word_t ldigit;

			for (Pjp = Pjbu->jbu_jp, ldigit = 0; ldigit < cJL_BRANCHUNUMJPS; ++Pjp, ++ldigit) {
				if ((JL_JPTYPE(Pjp)) == cJL_JPNULLMAX)
					continue;
				if ((JL_JPTYPE(Pjp)) == cJL_JPIMMED_3_01) {
					*Pjlwnew++ = JL_DIGITTOSTATE(ldigit, cJL_BYTESPERWORD)
							| JL_JPDCDPOP0(Pjp);
					*Pjv++ = Pjp->jp_Addr;
					continue;
				}

				pop1 = judyLeafM1ToLeafW(Pjlwnew, Pjv, Pjp,
					JL_DIGITTOSTATE(ldigit, cJL_BYTESPERWORD), (void *) Pjpm);
				Pjlwnew += pop1;
				Pjv += pop1;
			}
			judyLFreeJBU(PjbuRaw, Pjpm);
			break;
		}
		default:
			JL_SET_ERRNO(JLE_CORRUPT);
			return JERR;
		}
		judyLFreeJPM(Pjpm, NULL);
		return 1;
	}
}
Example #7
0
/* Given a pointer to a JP, an Index known to be valid, the number of bytes
 * left to decode (== level in the tree), and a pointer to a global JPM, walk a
 * Judy (sub)tree to do an unset/delete of that index, and possibly modify the
 * JPM.  This function is only called internally, and recursively.  Unlike
 * Judy1Test() and JudyLGet(), the extra time required for recursion should be
 * negligible compared with the total.
 * Return values:
 * -1 error; details in JPM
 *  0 Index already deleted (should never happen, Index is known to be valid)
 *  1 previously valid Index deleted
 *  2 same as 1, but in addition the JP now points to a BranchL containing a
 *    single JP, which should be compressed into the parent branch (if there
 *    is one, which is not the case for a top-level branch under a JPM) */
static int judyDelWalk(Pjp_t Pjp, Word_t Index,	Word_t ParentLevel, Pjpm_t Pjpm)
{
	Word_t pop1, level;
	Pjll_t PjllnewRaw, Pjllnew;
	Pjv_t PjvRaw, Pjv;
	uint8_t digit;
	int offset, retcode;

      ContinueDelWalk:
	switch (JL_JPTYPE(Pjp)) {
#define JL_BRANCH_KEEP(cLevel,MaxPop1,Next)			\
        if (pop1 > (MaxPop1)) {  /* hysteresis = 1 */		\
            assert((cLevel) >= 2);				\
            level = (cLevel);					\
            digit = JL_DIGITATSTATE(Index, cLevel);		\
            goto Next;						\
        }

#define JL_BRANCH_COPY_IMMED_EVEN(cLevel,Pjp,ignore)            \
        if (JL_JPTYPE(Pjp) == cJL_JPIMMED_1_01 + (cLevel) - 2) {\
            *Pleaf++ = JL_JPDCDPOP0(Pjp);                       \
	    *Pjv++   = (Pjp)->jp_Addr;                          \
            continue;   /* for-loop */                          \
        }

#define JL_BRANCH_COPY_IMMED_ODD(cLevel,Pjp,CopyIndex)          \
        if (JL_JPTYPE(Pjp) == cJL_JPIMMED_1_01 + (cLevel) - 2) {\
            CopyIndex(Pleaf, (Word_t) (JL_JPDCDPOP0(Pjp)));     \
            Pleaf += (cLevel);  /* index size = level */        \
	    *Pjv++ = (Pjp)->jp_Addr;                            \
            continue;   /* for-loop */                          \
        }

#define JL_BRANCHL_COMPRESS(cLevel,LeafType,MaxPop1,NewJPType,  \
                            LeafToLeaf,Alloc,ValueArea,         \
                            CopyImmed,CopyIndex)                \
{                                                               \
    LeafType Pleaf;                                             \
    Pjbl_t   PjblRaw;                                           \
    Pjbl_t   Pjbl;                                              \
    Word_t numJPs;						\
								\
    if ((PjllnewRaw = Alloc(MaxPop1, Pjpm)) == 0) return -1;	\
    Pjllnew = P_JLL(PjllnewRaw);                                \
    Pleaf   = (LeafType) Pjllnew;                               \
    Pjv     = ValueArea(Pleaf, MaxPop1);                        \
								\
    PjblRaw = (Pjbl_t) (Pjp->jp_Addr);                          \
    Pjbl    = P_JBL(PjblRaw);                                   \
    numJPs  = Pjbl->jbl_NumJPs;                                 \
								\
    for (offset = 0; offset < numJPs; ++offset) {               \
	CopyImmed(cLevel, (Pjbl->jbl_jp) + offset, CopyIndex);  \
	pop1 = LeafToLeaf(Pleaf, Pjv, (Pjbl->jbl_jp) + offset,  \
		  JL_DIGITTOSTATE(Pjbl->jbl_Expanse[offset],    \
		  cLevel), (void *) Pjpm);			\
	Pleaf = (LeafType) (((Word_t) Pleaf) + ((cLevel) * pop1));\
	Pjv  += pop1;						\
    }                                                           \
    assert(((((Word_t) Pleaf) - ((Word_t) Pjllnew)) / (cLevel)) == (MaxPop1)); \
    assert((Pjv - ValueArea(Pjllnew, MaxPop1)) == (MaxPop1));	\
    judyLFreeJBL(PjblRaw, Pjpm);				\
    Pjp->jp_Type = (NewJPType);                                 \
    Pjp->jp_Addr = (Word_t) PjllnewRaw;                         \
    goto ContinueDelWalk;       /* delete from new leaf */      \
}

#define JL_BRANCHL(cLevel,MaxPop1,LeafType,NewJPType,                   \
                   LeafToLeaf,Alloc,ValueArea,CopyImmed,CopyIndex)      \
        assert(! JL_DCDNOTMATCHINDEX(Index, Pjp, cLevel));              \
        assert(ParentLevel > (cLevel));                                 \
        pop1 = JL_JPBRANCH_POP0(Pjp, cLevel) + 1;                       \
        JL_BRANCH_KEEP(cLevel, MaxPop1, BranchLKeep);                   \
        assert(pop1 == (MaxPop1));                                      \
        JL_BRANCHL_COMPRESS(cLevel, LeafType, MaxPop1, NewJPType,       \
                            LeafToLeaf, Alloc, ValueArea, CopyImmed, CopyIndex)

	case cJL_JPBRANCH_L2:
		JL_BRANCHL(2, cJL_LEAF2_MAXPOP1, uint16_t *, cJL_JPLEAF2,
			   judyLeaf1ToLeaf2, judyLAllocJLL2, JL_LEAF2VALUEAREA,
			   JL_BRANCH_COPY_IMMED_EVEN, ignore);
	case cJL_JPBRANCH_L3:
		JL_BRANCHL(3, cJL_LEAF3_MAXPOP1, uint8_t *, cJL_JPLEAF3,
			   judyLeaf2ToLeaf3, judyLAllocJLL3, JL_LEAF3VALUEAREA,
			   JL_BRANCH_COPY_IMMED_ODD, JL_COPY3_LONG_TO_PINDEX);
	case cJL_JPBRANCH_L: {
		Pjbl_t Pjbl;
		Word_t numJPs;
		level = cJL_ROOTSTATE;
		digit = JL_DIGITATSTATE(Index, cJL_ROOTSTATE);
	      BranchLKeep:
		Pjbl = P_JBL(Pjp->jp_Addr);
		numJPs = Pjbl->jbl_NumJPs;
		assert(numJPs > 0);
		for (offset = 0; (Pjbl->jbl_Expanse[offset]) != digit; ++offset)
			assert(offset < numJPs - 1);
		Pjp = (Pjbl->jbl_jp) + offset;
		assert(level >= 2);
		if ((JL_JPTYPE(Pjp)) != cJL_JPIMMED_1_01 + level - 2)
			break;
		assert(JL_JPDCDPOP0(Pjp) == JL_TRIMTODCDSIZE(Index));
		JL_DELETEINPLACE(Pjbl->jbl_Expanse, numJPs, offset, ignore);
		JL_DELETEINPLACE(Pjbl->jbl_jp, numJPs, offset, ignore);
		return ((--(Pjbl->jbl_NumJPs) <= 1) ? 2 : 1);
	}
#define JL_BRANCHB_COMPRESS(cLevel,LeafType,MaxPop1,NewJPType,          \
                            LeafToLeaf,Alloc,ValueArea,                 \
                            CopyImmed,CopyIndex)                        \
{									\
    LeafType  Pleaf;							\
    Pjbb_t    PjbbRaw;  /* BranchB to compress */			\
    Pjbb_t    Pjbb;							\
    Word_t subexp;   /* current subexpanse number    */			\
    BITMAPB_t bitmap;   /* portion for this subexpanse  */		\
    Pjp_t     Pjp2Raw;  /* one subexpanses subarray     */		\
    Pjp_t     Pjp2;							\
									\
    if ((PjllnewRaw = Alloc(MaxPop1, Pjpm)) == 0) return -1;		\
    Pjllnew = P_JLL(PjllnewRaw);					\
    Pleaf   = (LeafType) Pjllnew;					\
    Pjv     = ValueArea(Pleaf, MaxPop1);				\
    PjbbRaw = (Pjbb_t) (Pjp->jp_Addr);					\
    Pjbb    = P_JBB(PjbbRaw);						\
									\
    for (subexp = 0; subexp < cJL_NUMSUBEXPB; ++subexp) {		\
	if ((bitmap = JL_JBB_BITMAP(Pjbb, subexp)) == 0)		\
	    continue;           /* empty subexpanse */			\
									\
	digit   = subexp * cJL_BITSPERSUBEXPB;				\
	Pjp2Raw = JL_JBB_PJP(Pjbb, subexp);				\
	Pjp2    = P_JP(Pjp2Raw);					\
	assert(Pjp2 != NULL);						\
									\
	for (offset = 0; bitmap != 0; bitmap >>= 1, ++digit) {		\
	    if (! (bitmap & 1))						\
		continue;       /* empty sub-subexpanse */		\
	    ++offset;           /* before any continue */		\
	    CopyImmed(cLevel, Pjp2 + offset - 1, CopyIndex);		\
	    pop1 = LeafToLeaf(Pleaf, Pjv, Pjp2 + offset - 1,		\
			      JL_DIGITTOSTATE(digit, cLevel),		\
			      (void *) Pjpm);				\
	    Pleaf = (LeafType) (((Word_t) Pleaf) + ((cLevel) * pop1));	\
	    Pjv  += pop1;						\
	}								\
	judyLFreeJBBJP(Pjp2Raw, /* pop1 = */ offset, Pjpm);		\
    }									\
    assert(((((Word_t) Pleaf) - ((Word_t) Pjllnew)) / (cLevel)) == (MaxPop1)); \
    assert((Pjv - ValueArea(Pjllnew, MaxPop1)) == (MaxPop1));		\
    judyLFreeJBB(PjbbRaw, Pjpm);					\
    Pjp->jp_Type = (NewJPType);						\
    Pjp->jp_Addr = (Word_t) PjllnewRaw;					\
    goto ContinueDelWalk;       /* delete from new leaf */		\
}

#define JL_BRANCHB(cLevel,MaxPop1,LeafType,NewJPType,                   \
                   LeafToLeaf,Alloc,ValueArea,CopyImmed,CopyIndex)      \
        assert(! JL_DCDNOTMATCHINDEX(Index, Pjp, cLevel));              \
        assert(ParentLevel > (cLevel));                                 \
        pop1 = JL_JPBRANCH_POP0(Pjp, cLevel) + 1;                       \
        JL_BRANCH_KEEP(cLevel, MaxPop1, BranchBKeep);                   \
        assert(pop1 == (MaxPop1));                                      \
        JL_BRANCHB_COMPRESS(cLevel, LeafType, MaxPop1, NewJPType,       \
                            LeafToLeaf, Alloc, ValueArea, CopyImmed, CopyIndex)

	case cJL_JPBRANCH_B2:
		JL_BRANCHB(2, cJL_LEAF2_MAXPOP1, uint16_t *, cJL_JPLEAF2,
			   judyLeaf1ToLeaf2, judyLAllocJLL2, JL_LEAF2VALUEAREA,
			   JL_BRANCH_COPY_IMMED_EVEN, ignore);
	case cJL_JPBRANCH_B3:
		JL_BRANCHB(3, cJL_LEAF3_MAXPOP1, uint8_t *, cJL_JPLEAF3,
			   judyLeaf2ToLeaf3, judyLAllocJLL3, JL_LEAF3VALUEAREA,
			   JL_BRANCH_COPY_IMMED_ODD, JL_COPY3_LONG_TO_PINDEX);
	case cJL_JPBRANCH_B: {
		Pjbb_t Pjbb;
		Word_t subexp, subexp2;
		BITMAPB_t bitmap, bitmask;
		Pjp_t Pjp2Raw, Pjp2;
		Word_t numJPs;

		level = cJL_ROOTSTATE;
		digit = JL_DIGITATSTATE(Index, cJL_ROOTSTATE);

	      BranchBKeep:
		Pjbb = P_JBB(Pjp->jp_Addr);
		subexp = digit / cJL_BITSPERSUBEXPB;
		bitmap = JL_JBB_BITMAP(Pjbb, subexp);
		bitmask = JL_BITPOSMASKB(digit);
		assert(bitmap & bitmask);

		offset = ((bitmap == (cJL_FULLBITMAPB)) ?
			  digit % cJL_BITSPERSUBEXPB :
			  judyCountBits(bitmap & JL_MASKLOWEREXC(bitmask)));
		Pjp2Raw = JL_JBB_PJP(Pjbb, subexp);
		Pjp2 = P_JP(Pjp2Raw);
		assert(Pjp2 != NULL);

		if (JL_JPTYPE(Pjp2 + offset) != cJL_JPIMMED_1_01 + level - 2) {
			Pjp = Pjp2 + offset;
			break;
		}

		assert(JL_JPDCDPOP0(Pjp2 + offset) == JL_TRIMTODCDSIZE(Index));
		if ((numJPs = judyCountBits(bitmap)) == 1) {
			judyLFreeJBBJP(Pjp2Raw, /* pop1 = */ 1, Pjpm);
			JL_JBB_PJP(Pjbb, subexp) = NULL;
		} else if (JL_BRANCHBJPGROWINPLACE(numJPs - 1)) {
			assert(numJPs > 0);
			JL_DELETEINPLACE(Pjp2, numJPs, offset, ignore);
		} else {
			Pjp_t PjpnewRaw, Pjpnew;

			if ((PjpnewRaw = judyLAllocJBBJP(numJPs - 1, Pjpm)) == NULL)
				return -1;
			Pjpnew = P_JP(PjpnewRaw);

			JL_DELETECOPY(Pjpnew, Pjp2, numJPs, offset, ignore);
			judyLFreeJBBJP(Pjp2Raw, numJPs, Pjpm);

			JL_JBB_PJP(Pjbb, subexp) = PjpnewRaw;
		}
		JL_JBB_BITMAP(Pjbb, subexp) ^= bitmask;
		if (numJPs > cJL_BRANCHLMAXJPS)
			return 1;
		for (subexp2 = 0; subexp2 < cJL_NUMSUBEXPB; ++subexp2) {
			if (subexp2 == subexp)
				continue;

			if ((numJPs == cJL_BRANCHLMAXJPS) ? JL_JBB_BITMAP(Pjbb, subexp2) :
			    ((numJPs += judyCountBits(JL_JBB_BITMAP(Pjbb, subexp2))) > cJL_BRANCHLMAXJPS)) {
				return 1;
			}
		}

		(void)judyBranchBToBranchL(Pjp, Pjpm);
		return 1;
	}

#define JL_BRANCHU_COMPRESS(cLevel,LeafType,MaxPop1,NullJPType,NewJPType,   \
                            LeafToLeaf,Alloc,ValueArea,CopyImmed,CopyIndex) \
        {                                                               \
            LeafType Pleaf;                                             \
            Pjbu_t PjbuRaw = (Pjbu_t) (Pjp->jp_Addr);                   \
            Pjp_t  Pjp2    = JL_JBU_PJP0(Pjp);                          \
            Word_t ldigit;      /* larger than uint8_t */               \
                                                                        \
            if ((PjllnewRaw = Alloc(MaxPop1, Pjpm)) == 0) return -1;	\
            Pjllnew = P_JLL(PjllnewRaw);                                \
            Pleaf   = (LeafType) Pjllnew;                               \
      	    Pjv     = ValueArea(Pleaf, MaxPop1);			\
            for (ldigit = 0; ldigit < cJL_BRANCHUNUMJPS; ++ldigit, ++Pjp2) { \
                /* fast-process common types: */                        \
                if (JL_JPTYPE(Pjp2) == (NullJPType)) continue;          \
                CopyImmed(cLevel, Pjp2, CopyIndex);                     \
                pop1 = LeafToLeaf(Pleaf, Pjv, Pjp2,			\
			JL_DIGITTOSTATE(ldigit, cLevel), (void *)Pjpm); \
                Pleaf = (LeafType) (((Word_t) Pleaf) + ((cLevel) * pop1)); \
                Pjv  += pop1;						\
            }                                                           \
            assert(((((Word_t) Pleaf) - ((Word_t) Pjllnew)) / (cLevel)) == (MaxPop1)); \
	    assert((Pjv - ValueArea(Pjllnew, MaxPop1)) == (MaxPop1));	\
            judyLFreeJBU(PjbuRaw, Pjpm);				\
            Pjp->jp_Type = (NewJPType);                                 \
            Pjp->jp_Addr = (Word_t) PjllnewRaw;                         \
            goto ContinueDelWalk;       /* delete from new leaf */      \
        }

#define JL_BRANCHU(cLevel,MaxPop1,LeafType,NullJPType,NewJPType,        \
                   LeafToLeaf,Alloc,ValueArea,CopyImmed,CopyIndex)      \
        assert(! JL_DCDNOTMATCHINDEX(Index, Pjp, cLevel));              \
        assert(ParentLevel > (cLevel));                                 \
        pop1 = JL_JPBRANCH_POP0(Pjp, cLevel) + 1;                       \
        if (pop1 > (MaxPop1)) {  /* hysteresis = 1 */                   \
            level = (cLevel);                                           \
            Pjp   = P_JP(Pjp->jp_Addr) + JL_DIGITATSTATE(Index, cLevel);\
            break;              /* descend to next level */             \
        }                                                               \
        assert(pop1 == (MaxPop1));                                      \
        JL_BRANCHU_COMPRESS(cLevel, LeafType, MaxPop1, NullJPType, NewJPType, \
                            LeafToLeaf, Alloc, ValueArea, CopyImmed, CopyIndex)

	case cJL_JPBRANCH_U2:
		JL_BRANCHU(2, cJL_LEAF2_MAXPOP1, uint16_t *,
			   cJL_JPNULL1, cJL_JPLEAF2,
			   judyLeaf1ToLeaf2, judyLAllocJLL2, JL_LEAF2VALUEAREA,
			   JL_BRANCH_COPY_IMMED_EVEN, ignore);
	case cJL_JPBRANCH_U3:
		JL_BRANCHU(3, cJL_LEAF3_MAXPOP1, uint8_t *,
			   cJL_JPNULL2, cJL_JPLEAF3,
			   judyLeaf2ToLeaf3, judyLAllocJLL3, JL_LEAF3VALUEAREA,
			   JL_BRANCH_COPY_IMMED_ODD, JL_COPY3_LONG_TO_PINDEX);
	case cJL_JPBRANCH_U:
		level = cJL_ROOTSTATE;
		Pjp = P_JP(Pjp->jp_Addr) + JL_DIGITATSTATE(Index, cJL_ROOTSTATE);
		break;

#define JL_LEAF_UPLEVEL(cIS,LeafType,MaxPop1,NewJPType,LeafToLeaf,      \
                        Alloc,ValueArea)                                \
        assert(((ParentLevel - 1) == (cIS)) || (pop1 >= (MaxPop1)));    \
        if (((ParentLevel - 1) > (cIS))  /* under narrow pointer */     \
         && (pop1 == (MaxPop1))) {         /* hysteresis = 1       */   \
            Word_t D_cdP0;                                              \
            if ((PjllnewRaw = Alloc(MaxPop1, Pjpm)) == 0) return -1;	\
            Pjllnew = P_JLL(PjllnewRaw);                                \
	    Pjv     = ValueArea((LeafType) Pjllnew, MaxPop1);		\
            (void) LeafToLeaf((LeafType) Pjllnew, Pjv, Pjp,		\
                              Index & cJL_DCDMASK(cIS),	(void *) Pjpm);	\
            D_cdP0 = (~cJL_MASKATSTATE((cIS) + 1)) & JL_JPDCDPOP0(Pjp); \
            JL_JPSETADT(Pjp, (Word_t)PjllnewRaw, D_cdP0, NewJPType);    \
            goto ContinueDelWalk;       /* delete from new leaf */      \
        }

#define JL_LEAF_UPLEVEL64(cIS,LeafType,MaxPop1,NewJPType,LeafToLeaf,    \
                          Alloc,ValueArea)	/* null. */
#define JL_TOIMMED_01_EVEN(cIS,ignore1,ignore2)                         \
{                                                                       \
        Word_t D_cdP0;							\
        Word_t A_ddr = 0;						\
        uint8_t T_ype = JL_JPTYPE(Pjp);                                 \
        offset = (Pleaf[0] == JL_LEASTBYTES(Index, cIS)); /* undeleted Ind */ \
        assert(Pleaf[offset ? 0 : 1] == JL_LEASTBYTES(Index, cIS));     \
        D_cdP0 = (Index & cJL_DCDMASK(cIS)) | Pleaf[offset];            \
	A_ddr = Pjv[offset];						\
        JL_JPSETADT(Pjp, A_ddr, D_cdP0, T_ype);                         \
}

#define JL_TOIMMED_01_ODD(cIS,SearchLeaf,CopyPIndex)                    \
        {                                                               \
            Word_t D_cdP0;						\
            Word_t A_ddr = 0;						\
            uint8_t T_ype = JL_JPTYPE(Pjp);                             \
            offset = SearchLeaf(Pleaf, 2, Index);                       \
            assert(offset >= 0);        /* Index must be valid */       \
            CopyPIndex(D_cdP0, & (Pleaf[offset ? 0 : cIS]));            \
            D_cdP0 |= Index & cJL_DCDMASK(cIS);                         \
	    A_ddr = Pjv[offset ? 0 : 1];				\
            JL_JPSETADT(Pjp, A_ddr, D_cdP0, T_ype);                     \
        }

#define JL_LEAF_TOIMMED(cIS,LeafType,MaxPop1,BaseJPType,ignore1,	\
                        ignore2,ignore3,ignore4,			\
                        DeleteCopy,FreeLeaf)				\
        assert(pop1 > (MaxPop1));					\
        if ((pop1 - 1) == (MaxPop1)) {    /* hysteresis = 0 */		\
            Pjll_t PjllRaw = (Pjll_t) (Pjp->jp_Addr);			\
            Pjv_t  PjvnewRaw;						\
            Pjv_t  Pjvnew;						\
									\
            if ((PjvnewRaw = judyLAllocJV(pop1 - 1, Pjpm)) == NULL)	\
		    return -1;						\
	    Pjvnew = P_JV(PjvnewRaw);					\
									\
            DeleteCopy((LeafType) (Pjp->jp_LIndex), Pleaf, pop1, offset, cIS); \
            JL_DELETECOPY(Pjvnew, Pjv, pop1, offset, cIS);		\
            FreeLeaf(PjllRaw, pop1, Pjpm);				\
            Pjp->jp_Addr = (Word_t) PjvnewRaw;				\
            Pjp->jp_Type = (BaseJPType) - 2 + (MaxPop1);		\
            return 1;							\
        }
#define JL_LEAF_TOIMMED_01(cIS,LeafType,MaxPop1,ignore,Immed01JPType,   \
                           ToImmed,SearchLeaf,CopyPIndex,               \
                           DeleteCopy,FreeLeaf)                         \
        assert(pop1 > (MaxPop1));                                       \
        if ((pop1 - 1) == (MaxPop1)) {    /* hysteresis = 0 */          \
            Pjll_t PjllRaw = (Pjll_t) (Pjp->jp_Addr);                   \
            ToImmed(cIS, SearchLeaf, CopyPIndex);                       \
            FreeLeaf(PjllRaw, pop1, Pjpm);                              \
            Pjp->jp_Type = (Immed01JPType);                             \
            return 1;							\
        }
#define JL_LEAF_TOIMMED_23(cIS,LeafType,MaxPop1,BaseJPType,Immed01JPType, \
                           ToImmed,SearchLeaf,CopyPIndex,               \
                           DeleteCopy,FreeLeaf)                         \
        JL_LEAF_TOIMMED_01(cIS,LeafType,MaxPop1,ignore,Immed01JPType,   \
                           ToImmed,SearchLeaf,CopyPIndex,               \
                           DeleteCopy,FreeLeaf)
#define JL_LEAF_INPLACE(cIS,GrowInPlace,DeleteInPlace)			\
        if (GrowInPlace(pop1 - 1)) {      /* hysteresis = 0 */		\
            DeleteInPlace(Pleaf, pop1, offset, cIS);			\
	    JL_DELETEINPLACE(Pjv, pop1, offset, ignore);		\
            return 1;							\
        }

#define JL_LEAF_SHRINK(cIS,LeafType,DeleteCopy,Alloc,FreeLeaf,ValueArea) \
        {								 \
	    Pjv_t Pjvnew;                                                \
            if ((PjllnewRaw = Alloc(pop1 - 1, Pjpm)) == 0) return -1;    \
            Pjllnew = P_JLL(PjllnewRaw);                                 \
	    Pjvnew  = ValueArea(Pjllnew, pop1 - 1);                      \
            DeleteCopy((LeafType) Pjllnew, Pleaf, pop1, offset, cIS);    \
            JL_DELETECOPY(Pjvnew, Pjv, pop1, offset, cIS);               \
            FreeLeaf(PleafRaw, pop1, Pjpm);                              \
            Pjp->jp_Addr = (Word_t) PjllnewRaw;                          \
            return 1;                                                    \
        }

#define JL_LEAF(cIS, UpLevel, LeafTypeUp,MaxPop1Up,LeafJPTypeUp,LeafToLeaf, \
                AllocUp,ValueAreaUp,LeafToImmed,ToImmed,CopyPIndex,     \
                LeafType,ImmedMaxPop1,ImmedBaseJPType,Immed01JPType,	\
                SearchLeaf,GrowInPlace,DeleteInPlace,DeleteCopy,Alloc,	\
		FreeLeaf,ValueArea)					\
        {                                                               \
            Pjll_t   PleafRaw;                                          \
            LeafType Pleaf;                                             \
            assert(! JL_DCDNOTMATCHINDEX(Index, Pjp, cIS));             \
            assert(ParentLevel > (cIS));                                \
            PleafRaw = (Pjll_t) (Pjp->jp_Addr);                         \
            Pleaf    = (LeafType) P_JLL(PleafRaw);                      \
            pop1     = JL_JPLEAF_POP0(Pjp) + 1;                         \
                                                                        \
            UpLevel(cIS, LeafTypeUp, MaxPop1Up, LeafJPTypeUp,           \
                    LeafToLeaf, AllocUp, ValueAreaUp);                  \
            offset = SearchLeaf(Pleaf, pop1, Index);                    \
            assert(offset >= 0);        /* Index must be valid */       \
	    Pjv = ValueArea(Pleaf, pop1);				\
            LeafToImmed(cIS, LeafType, ImmedMaxPop1,                    \
                        ImmedBaseJPType, Immed01JPType,                 \
                        ToImmed, SearchLeaf, CopyPIndex,                \
                        DeleteCopy, FreeLeaf);                          \
            JL_LEAF_INPLACE(cIS, GrowInPlace, DeleteInPlace);           \
            JL_LEAF_SHRINK(cIS, LeafType, DeleteCopy, Alloc, FreeLeaf,  \
                           ValueArea);                                  \
        }

	case cJL_JPLEAF1:
		JL_LEAF(1, JL_LEAF_UPLEVEL, uint16_t *, cJL_LEAF2_MAXPOP1, cJL_JPLEAF2,
			judyLeaf1ToLeaf2, judyLAllocJLL2, JL_LEAF2VALUEAREA,
			JL_LEAF_TOIMMED, ignore, ignore,
			uint8_t *, cJL_IMMED1_MAXPOP1,
			cJL_JPIMMED_1_02, cJL_JPIMMED_1_01, judySearchLeaf1,
			JL_LEAF1GROWINPLACE, JL_DELETEINPLACE, JL_DELETECOPY,
			judyLAllocJLL1, judyLFreeJLL1, JL_LEAF1VALUEAREA);
	case cJL_JPLEAF2:
		JL_LEAF(2, JL_LEAF_UPLEVEL, uint8_t *, cJL_LEAF3_MAXPOP1, cJL_JPLEAF3,
			judyLeaf2ToLeaf3, judyLAllocJLL3, JL_LEAF3VALUEAREA,
			JL_LEAF_TOIMMED_23, JL_TOIMMED_01_EVEN, ignore,
			uint16_t *, cJL_IMMED2_MAXPOP1,
			cJL_JPIMMED_2_02, cJL_JPIMMED_2_01, judySearchLeaf2,
			JL_LEAF2GROWINPLACE, JL_DELETEINPLACE, JL_DELETECOPY,
			judyLAllocJLL2, judyLFreeJLL2, JL_LEAF2VALUEAREA);
	case cJL_JPLEAF3:
		JL_LEAF(3, JL_LEAF_UPLEVEL64, uint32_t *, cJL_LEAF4_MAXPOP1,
			cJL_JPLEAF4, judyLLeaf3ToLeaf4, judyLAllocJLL4, JL_LEAF4VALUEAREA,
			JL_LEAF_TOIMMED_23, JL_TOIMMED_01_ODD, JL_COPY3_PINDEX_TO_LONG,
			uint8_t *, cJL_IMMED3_MAXPOP1, cJL_JPIMMED_3_02, cJL_JPIMMED_3_01,
			judySearchLeaf3, JL_LEAF3GROWINPLACE, JL_DELETEINPLACE_ODD,
			JL_DELETECOPY_ODD, judyLAllocJLL3, judyLFreeJLL3, JL_LEAF3VALUEAREA);
	case cJL_JPLEAF_B1: {
		Pjv_t PjvnewRaw, Pjvnew;
		Word_t subexp;
		Pjlb_t Pjlb;
		BITMAPL_t bitmap, bitmask;

		assert(!JL_DCDNOTMATCHINDEX(Index, Pjp, 1));
		assert(ParentLevel > 1);
		assert(JL_BITMAPTESTL(P_JLB(Pjp->jp_Addr), Index));
		pop1 = JL_JPLEAF_POP0(Pjp) + 1;

		JL_LEAF_UPLEVEL(1, uint16_t *, cJL_LEAF2_MAXPOP1, cJL_JPLEAF2,
				judyLeaf1ToLeaf2, judyLAllocJLL2, JL_LEAF2VALUEAREA);
		if (pop1 == cJL_LEAF1_MAXPOP1) {
			if (judyLeafB1ToLeaf1(Pjp, Pjpm) == -1)
				return -1;
			goto ContinueDelWalk;
		}

		digit = JL_DIGITATSTATE(Index, 1);
		Pjlb = P_JLB(Pjp->jp_Addr);

		subexp = digit / cJL_BITSPERSUBEXPL;
		bitmap = JL_JLB_BITMAP(Pjlb, subexp);
		PjvRaw = JL_JLB_PVALUE(Pjlb, subexp);
		Pjv = P_JV(PjvRaw);
		bitmask = JL_BITPOSMASKL(digit);

		assert(bitmap & bitmask);
		if (bitmap == cJL_FULLBITMAPL) {
			pop1 = cJL_BITSPERSUBEXPL;
			offset = digit % cJL_BITSPERSUBEXPL;
		} else {
			pop1 = judyCountBits(bitmap);
			offset = judyCountBits(bitmap & (bitmask - 1));
		}

		if (pop1 == 1) {
			judyLFreeJV(PjvRaw, 1, Pjpm);
			JL_JLB_PVALUE(Pjlb, subexp) = NULL;
			JL_JLB_BITMAP(Pjlb, subexp) = 0;
			return 1;
		}

		if (JL_LEAFVGROWINPLACE(pop1 - 1)) {
			JL_DELETEINPLACE(Pjv, pop1, offset, ignore);
		} else {
			if ((PjvnewRaw = judyLAllocJV(pop1 - 1, Pjpm)) == NULL)
				return -1;
			Pjvnew = P_JV(PjvnewRaw);

			JL_DELETECOPY(Pjvnew, Pjv, pop1, offset, ignore);
			judyLFreeJV(PjvRaw, pop1, Pjpm);
			JL_JLB_PVALUE(Pjlb, subexp) = (Pjv_t) PjvnewRaw;
		}

		JL_JLB_BITMAP(Pjlb, subexp) ^= bitmask;
		return 1;
	}

#define JL_IMMED_01(NewJPType,ParentJPType)                             \
            assert(JL_JPDCDPOP0(Pjp) == JL_TRIMTODCDSIZE(Index));       \
            JL_JPSETADT(Pjp, 0, 0, NewJPType);                          \
            return 1

#define JL_IMMED_02(cIS,LeafType,NewJPType) do {			\
            LeafType Pleaf;						\
            assert((ParentLevel - 1) == (cIS));				\
	    Pleaf  = (LeafType) (Pjp->jp_LIndex);			\
	    PjvRaw = (Pjv_t) (Pjp->jp_Addr);				\
	    Pjv    = P_JV(PjvRaw);					\
            JL_TOIMMED_01_EVEN(cIS, ignore, ignore);			\
	    judyLFreeJV(PjvRaw, 2, Pjpm);				\
            Pjp->jp_Type = (NewJPType);					\
            return 1;							\
} while (0)

#define JL_IMMED_DEL(cIS,DeleteInPlace)					\
        if (JL_LEAFVGROWINPLACE(pop1 - 1)) { /* hysteresis = 0 */	\
            DeleteInPlace(   Pleaf,  pop1, offset, cIS);		\
            JL_DELETEINPLACE(Pjv, pop1, offset, ignore);		\
	} else {							\
            Pjv_t PjvnewRaw;						\
            Pjv_t Pjvnew;						\
									\
            if ((PjvnewRaw = judyLAllocJV(pop1 - 1, Pjpm))		\
                == NULL) return -1;					\
            Pjvnew = P_JV(PjvnewRaw);					\
            DeleteInPlace(Pleaf, pop1, offset, cIS);			\
            JL_DELETECOPY(Pjvnew, Pjv, pop1, offset, ignore);		\
            judyLFreeJV(PjvRaw, pop1, Pjpm);				\
            (Pjp->jp_Addr) = (Word_t) PjvnewRaw;			\
        }

#define JL_IMMED(cIS,LeafType,BaseJPType,SearchLeaf,DeleteInPlace) do {	\
            LeafType Pleaf;                                             \
            assert((ParentLevel - 1) == (cIS));                         \
	    Pleaf  = (LeafType) (Pjp->jp_LIndex);			\
	    PjvRaw = (Pjv_t) (Pjp->jp_Addr);				\
	    Pjv    = P_JV(PjvRaw);					\
            pop1   = (JL_JPTYPE(Pjp)) - (BaseJPType) + 2;               \
            offset = SearchLeaf(Pleaf, pop1, Index);                    \
            assert(offset >= 0);        /* Index must be valid */       \
            JL_IMMED_DEL(cIS, DeleteInPlace);                           \
            --(Pjp->jp_Type);                                           \
            return 1;							\
} while (0)

	case cJL_JPIMMED_1_01: JL_IMMED_01(cJL_JPNULL1, cJL_JPBRANCH_U2);
	case cJL_JPIMMED_2_01: JL_IMMED_01(cJL_JPNULL2, cJL_JPBRANCH_U3);
	case cJL_JPIMMED_3_01: JL_IMMED_01(cJL_JPNULL3, cJL_JPBRANCH_U);
	case cJL_JPIMMED_1_02: JL_IMMED_02(1, uint8_t *, cJL_JPIMMED_1_01);
	case cJL_JPIMMED_1_03:
		JL_IMMED(1, uint8_t *, cJL_JPIMMED_1_02, judySearchLeaf1, JL_DELETEINPLACE);
	default:
		JL_SET_ERRNO(JLE_CORRUPT);
		return -1;
	}

	assert(level);
	retcode = judyDelWalk(Pjp, Index, level, Pjpm);
	assert(retcode != 0);
	if ((JL_JPTYPE(Pjp)) < cJL_JPIMMED_1_01) {
		switch (retcode) {
		case 1: {
			jp_t JP = *Pjp;
			Word_t DcdP0;

			DcdP0 = JL_JPDCDPOP0(Pjp) - 1;
			JL_JPSETADT(Pjp, JP.jp_Addr, DcdP0, JL_JPTYPE(&JP));
			break;
		}
		case 2:	{
			Pjbl_t PjblRaw = (Pjbl_t) (Pjp->jp_Addr);
			Pjbl_t Pjbl = P_JBL(PjblRaw);

			*Pjp = Pjbl->jbl_jp[0];
			judyLFreeJBL(PjblRaw, Pjpm);
			retcode = 1;
		}
		}
	}

	return retcode;
}
Example #8
0
FUNCTION int Judy1PrevEmpty
#else
FUNCTION int Judy1NextEmpty
#endif
#else
#ifdef JUDYPREV
FUNCTION int JudyLPrevEmpty
#else
FUNCTION int JudyLNextEmpty
#endif
#endif
(
    Pcvoid_t  PArray,	// Judy array to search.
    Word_t *  PIndex,	// starting point and result.
    PJError_t PJError	// optional, for returning error info.
)
{
    Word_t	  Index;	// fast copy, in a register.
    Pjp_t	  Pjp;		// current JP.
    Pjbl_t	  Pjbl;		// Pjp->jp_Addr masked and cast to types:
    Pjbb_t	  Pjbb;
    Pjbu_t	  Pjbu;
    Pjlb_t	  Pjlb;
    PWord_t	  Pword;	// alternate name for use by GET* macros.

    Word_t	  digit;	// next digit to decode from Index.
    Word_t	  digits;	// current state in SM = digits left to decode.
    Word_t	  pop0;		// in a leaf.
    Word_t	  pop0mask;	// precalculated to avoid variable shifts.
    long	  offset;	// within a branch or leaf (can be large).
    int	  subexp;	// subexpanse in a bitmap branch.
    BITMAPB_t bitposmaskB;	// bit in bitmap for bitmap branch.
    BITMAPL_t bitposmaskL;	// bit in bitmap for bitmap leaf.
    Word_t	  possfullJP1;	// JP types for possibly full subexpanses:
    Word_t	  possfullJP2;
    Word_t	  possfullJP3;


// ----------------------------------------------------------------------------
// M A C R O S
//
// These are intended to make the code a bit more readable and less redundant.


// CHECK FOR NULL JP:
//
// TBD:  In principle this can be reduced (here and in other *.c files) to just
// the latter clause since no Type should ever be below cJU_JPNULL1, but in
// fact some root pointer types can be lower, so for safety do both checks.

#define	JPNULL(Type)  (((Type) >= cJU_JPNULL1) && ((Type) <= cJU_JPNULLMAX))


// CHECK FOR A FULL JP:
//
// Given a JP, indicate if it is fully populated.  Use digits, pop0mask, and
// possfullJP1..3 in the context.
//
// This is a difficult problem because it requires checking the Pop0 bits for
// all-ones, but the number of bytes depends on the JP type, which is not
// directly related to the parent branchs type or level -- the JPs child
// could be under a narrow pointer (hence not full).  The simple answer
// requires switching on or otherwise calculating the JP type, which could be
// slow.  Instead, in SMPREPB* precalculate pop0mask and also record in
// possfullJP1..3 the child JP (branch) types that could possibly be full (one
// level down), and use them here.  For level-2 branches (with digits == 2),
// the test for a full child depends on Judy1/JudyL.
//
// Note:  This cannot be applied to the JP in a JPM because it doesnt have
// enough pop0 digits.
//
// TBD:  JPFULL_BRANCH diligently checks for BranchL or BranchB, where neither
// of those can ever be full as it turns out.  Could just check for a BranchU
// at the right level.  Also, pop0mask might be overkill, its not used much,
// so perhaps just call cJU_POP0MASK(digits - 1) here?
//
// First, JPFULL_BRANCH checks for a full expanse for a JP whose child can be a
// branch, that is, a JP in a branch at level 3 or higher:

#define	JPFULL_BRANCH(Pjp)						\
	  ((((JU_JPDCDPOP0(Pjp) ^ cJU_ALLONES) & pop0mask) == 0)	\
	&& ((JU_JPTYPE(Pjp) == possfullJP1)				\
	 || (JU_JPTYPE(Pjp) == possfullJP2)				\
	 || (JU_JPTYPE(Pjp) == possfullJP3)))

#ifdef JUDY1
#define	JPFULL(Pjp)							\
	((digits == 2) ?						\
	 (JU_JPTYPE(Pjp) == cJ1_JPFULLPOPU1) : JPFULL_BRANCH(Pjp))
#else
#define	JPFULL(Pjp)							\
	((digits == 2) ?						\
	   (JU_JPTYPE(Pjp) == cJU_JPLEAF_B1)				\
	 && (((JU_JPDCDPOP0(Pjp) & cJU_POP0MASK(1)) == cJU_POP0MASK(1))) : \
	 JPFULL_BRANCH(Pjp))
#endif


// RETURN SUCCESS:
//
// This hides the need to set *PIndex back to the local value of Index -- use a
// local value for faster operation.  Note that the callers *PIndex is ALWAYS
// modified upon success, at least decremented/incremented.

#define	RET_SUCCESS { *PIndex = Index; return(1); }


// RETURN A CORRUPTION:

#define	RET_CORRUPT { JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); return(JERRI); }


// SEARCH A BITMAP BRANCH:
//
// This is a weak analog of j__udySearchLeaf*() for bitmap branches.  Return
// the actual or next-left position, base 0, of Digit in a BITMAPB_t bitmap
// (subexpanse of a full bitmap), also given a Bitposmask for Digit.  The
// position is the offset within the set bits.
//
// Unlike j__udySearchLeaf*(), the offset is not returned bit-complemented if
// Digits bit is unset, because the caller can check the bitmap themselves to
// determine that.  Also, if Digits bit is unset, the returned offset is to
// the next-left JP or index (including -1), not to the "ideal" position for
// the index = next-right JP or index.
//
// Shortcut and skip calling j__udyCountBitsB() if the bitmap is full, in which
// case (Digit % cJU_BITSPERSUBEXPB) itself is the base-0 offset.

#define	SEARCHBITMAPB(Bitmap,Digit,Bitposmask)				\
	(((Bitmap) == cJU_FULLBITMAPB) ? (Digit % cJU_BITSPERSUBEXPB) :	\
	 j__udyCountBitsB((Bitmap) & JU_MASKLOWERINC(Bitposmask)) - 1)

#ifdef JUDYPREV
// Equivalent to search for the highest offset in Bitmap, that is, one less
// than the number of bits set:

#define	SEARCHBITMAPMAXB(Bitmap)					\
	(((Bitmap) == cJU_FULLBITMAPB) ? cJU_BITSPERSUBEXPB - 1 :	\
	 j__udyCountBitsB(Bitmap) - 1)
#endif


// CHECK DECODE BYTES:
//
// Check Decode bytes in a JP against the equivalent portion of Index.  If they
// dont match, Index is outside the subexpanse of a narrow pointer, hence is
// empty.

#define	CHECKDCD(cDigits) \
	if (JU_DCDNOTMATCHINDEX(Index, Pjp, cDigits)) RET_SUCCESS


// REVISE REMAINDER OF INDEX:
//
// Put one digit in place in Index and clear/set the lower digits, if any, so
// the resulting Index is at the start/end of an expanse, or just clear/set the
// least digits.
//
// Actually, to make simple use of JU_LEASTBYTESMASK, first clear/set all least
// digits of Index including the digit to be overridden, then set the value of
// that one digit.  If Digits == 1 the first operation is redundant, but either
// very fast or even removed by the optimizer.

#define	CLEARLEASTDIGITS(Digits) Index &= ~JU_LEASTBYTESMASK(Digits)
#define	SETLEASTDIGITS(  Digits) Index |=  JU_LEASTBYTESMASK(Digits)

#define	CLEARLEASTDIGITS_D(Digit,Digits)	\
	{					\
	    CLEARLEASTDIGITS(Digits);		\
	    JU_SETDIGIT(Index, Digit, Digits);	\
	}

#define	SETLEASTDIGITS_D(Digit,Digits)		\
	{					\
	    SETLEASTDIGITS(Digits);		\
	    JU_SETDIGIT(Index, Digit, Digits);	\
	}


// SET REMAINDER OF INDEX AND THEN RETURN OR CONTINUE:

#define	SET_AND_RETURN(OpLeastDigits,Digit,Digits)	\
	{						\
	    OpLeastDigits(Digit, Digits);		\
	    RET_SUCCESS;				\
	}

#define	SET_AND_CONTINUE(OpLeastDigits,Digit,Digits)	\
	{						\
	    OpLeastDigits(Digit, Digits);		\
	    goto SMGetContinue;				\
	}


// PREPARE TO HANDLE A LEAFW OR JP BRANCH IN THE STATE MACHINE:
//
// Extract a state-dependent digit from Index in a "constant" way, then jump to
// common code for multiple cases.
//
// TBD:  Should this macro do more, such as preparing variable-shift masks for
// use in CLEARLEASTDIGITS and SETLEASTDIGITS?

#define	SMPREPB(cDigits,Next,PossFullJP1,PossFullJP2,PossFullJP3)	\
	digits	 = (cDigits);						\
	digit	 = JU_DIGITATSTATE(Index, cDigits);			\
	pop0mask = cJU_POP0MASK((cDigits) - 1);	 /* for branchs JPs */	\
	possfullJP1 = (PossFullJP1);					\
	possfullJP2 = (PossFullJP2);					\
	possfullJP3 = (PossFullJP3);					\
	goto Next

// Variations for specific-level branches and for shorthands:
//
// Note:  SMPREPB2 need not initialize possfullJP* because JPFULL does not use
// them for digits == 2, but gcc -Wall isnt quite smart enough to see this, so
// waste a bit of time and space to get rid of the warning:

#define	SMPREPB2(Next)				\
	digits	 = 2;				\
	digit	 = JU_DIGITATSTATE(Index, 2);	\
	pop0mask = cJU_POP0MASK(1);  /* for branchs JPs */ \
	possfullJP1 = possfullJP2 = possfullJP3 = 0;	    \
	goto Next

#define	SMPREPB3(Next) SMPREPB(3,	      Next, cJU_JPBRANCH_L2, \
						    cJU_JPBRANCH_B2, \
						    cJU_JPBRANCH_U2)
#ifndef JU_64BIT
#define	SMPREPBL(Next) SMPREPB(cJU_ROOTSTATE, Next, cJU_JPBRANCH_L3, \
						    cJU_JPBRANCH_B3, \
						    cJU_JPBRANCH_U3)
#else
#define	SMPREPB4(Next) SMPREPB(4,	      Next, cJU_JPBRANCH_L3, \
						    cJU_JPBRANCH_B3, \
						    cJU_JPBRANCH_U3)
#define	SMPREPB5(Next) SMPREPB(5,	      Next, cJU_JPBRANCH_L4, \
						    cJU_JPBRANCH_B4, \
						    cJU_JPBRANCH_U4)
#define	SMPREPB6(Next) SMPREPB(6,	      Next, cJU_JPBRANCH_L5, \
						    cJU_JPBRANCH_B5, \
						    cJU_JPBRANCH_U5)
#define	SMPREPB7(Next) SMPREPB(7,	      Next, cJU_JPBRANCH_L6, \
						    cJU_JPBRANCH_B6, \
						    cJU_JPBRANCH_U6)
#define	SMPREPBL(Next) SMPREPB(cJU_ROOTSTATE, Next, cJU_JPBRANCH_L7, \
						    cJU_JPBRANCH_B7, \
						    cJU_JPBRANCH_U7)
#endif


// RESTART AFTER SECONDARY DEAD END:
//
// Set Index to the first/last index in the branch or leaf subexpanse and start
// over at the top of the tree.

#ifdef JUDYPREV
#define	SMRESTART(Digits) { CLEARLEASTDIGITS(Digits); goto SMGetRestart; }
#else
#define	SMRESTART(Digits) { SETLEASTDIGITS(  Digits); goto SMGetRestart; }
#endif


// CHECK EDGE OF LEAFS EXPANSE:
//
// Given the LSBs of the lowest/highest valid index in a leaf (or equivalently
// in an immediate JP), the level (index size) of the leaf, and the full index
// to return (as Index in the context) already set to the full index matching
// the lowest/highest one, determine if there is an empty index in the leafs
// expanse below/above the lowest/highest index, which is true if the
// lowest/highest index is not at the "edge" of the leafs expanse based on its
// LSBs.  If so, return Index decremented/incremented; otherwise restart at the
// top of the tree.
//
// Note:  In many cases Index is already at the right spot and calling
// SMRESTART instead of just going directly to SMGetRestart is a bit of
// overkill.
//
// Note:  Variable shift occurs if Digits is not a constant.

#ifdef JUDYPREV
#define	LEAF_EDGE(MinIndex,Digits)			\
	{						\
	    if (MinIndex) { --Index; RET_SUCCESS; }	\
	    SMRESTART(Digits);				\
	}
#else
#define	LEAF_EDGE(MaxIndex,Digits)			\
	{						\
	    if ((MaxIndex) != JU_LEASTBYTES(cJU_ALLONES, Digits)) \
	    { ++Index; RET_SUCCESS; }			\
	    SMRESTART(Digits);				\
	}
#endif

// Same as above except Index is not already set to match the lowest/highest
// index, so do that before decrementing/incrementing it:

#ifdef JUDYPREV
#define	LEAF_EDGE_SET(MinIndex,Digits)	\
	{				\
	    if (MinIndex)		\
	    { JU_SETDIGITS(Index, MinIndex, Digits); --Index; RET_SUCCESS; } \
	    SMRESTART(Digits);		\
	}
#else
#define	LEAF_EDGE_SET(MaxIndex,Digits)	\
	{				\
	    if ((MaxIndex) != JU_LEASTBYTES(cJU_ALLONES, Digits))	    \
	    { JU_SETDIGITS(Index, MaxIndex, Digits); ++Index; RET_SUCCESS; } \
	    SMRESTART(Digits);		\
	}
#endif


// FIND A HOLE (EMPTY INDEX) IN AN IMMEDIATE OR LEAF:
//
// Given an index location in a leaf (or equivalently an immediate JP) known to
// contain a usable hole (an empty index less/greater than Index), and the LSBs
// of a minimum/maximum index to locate, find the previous/next empty index and
// return it.
//
// Note:  "Even" index sizes (1,2,4[,8] bytes) have corresponding native C
// types; "odd" index sizes dont, but they are not represented here because
// they are handled completely differently; see elsewhere.

#ifdef JUDYPREV

#define	LEAF_HOLE_EVEN(cDigits,Pjll,IndexLSB)				\
	{								\
	    while (*(Pjll) > (IndexLSB)) --(Pjll); /* too high */	\
	    if (*(Pjll) < (IndexLSB)) RET_SUCCESS  /* Index is empty */	\
	    while (*(--(Pjll)) == --(IndexLSB)) /* null, find a hole */;\
	    JU_SETDIGITS(Index, IndexLSB, cDigits);			\
	    RET_SUCCESS;						\
	}
#else
#define	LEAF_HOLE_EVEN(cDigits,Pjll,IndexLSB)				\
	{								\
	    while (*(Pjll) < (IndexLSB)) ++(Pjll); /* too low */	\
	    if (*(Pjll) > (IndexLSB)) RET_SUCCESS  /* Index is empty */	\
	    while (*(++(Pjll)) == ++(IndexLSB)) /* null, find a hole */;\
	    JU_SETDIGITS(Index, IndexLSB, cDigits);			\
	    RET_SUCCESS;						\
	}
#endif


// SEARCH FOR AN EMPTY INDEX IN AN IMMEDIATE OR LEAF:
//
// Given a pointer to the first index in a leaf (or equivalently an immediate
// JP), the population of the leaf, and a first empty Index to find (inclusive,
// as Index in the context), where Index is known to fall within the expanse of
// the leaf to search, efficiently find the previous/next empty index in the
// leaf, if any.  For simplicity the following overview is stated in terms of
// Judy*NextEmpty() only, but the same concepts apply symmetrically for
// Judy*PrevEmpty().  Also, in each case the comparisons are for the LSBs of
// Index and leaf indexes, according to the leafs level.
//
// 1.  If Index is GREATER than the last (highest) index in the leaf
//     (maxindex), return success, Index is empty.  (Remember, Index is known
//     to be in the leafs expanse.)
//
// 2.  If Index is EQUAL to maxindex:  If maxindex is not at the edge of the
//     leafs expanse, increment Index and return success, there is an empty
//     Index one higher than any in the leaf; otherwise restart with Index
//     reset to the upper edge of the leafs expanse.  Note:  This might cause
//     an extra cache line fill, but this is OK for repeatedly-called search
//     code, and it saves CPU time.
//
// 3.  If Index is LESS than maxindex, check for "dense to end of leaf":
//     Subtract Index from maxindex, and back up that many slots in the leaf.
//     If the resulting offset is not before the start of the leaf then compare
//     the index at this offset (baseindex) with Index:
//
// 3a.  If GREATER, the leaf must be corrupt, since indexes are sorted and
//      there are no duplicates.
//
// 3b.  If EQUAL, the leaf is "dense" from Index to maxindex, meaning there is
//      no reason to search it.  "Slide right" to the high end of the leaf
//      (modify Index to maxindex) and continue with step 2 above.
//
// 3c.  If LESS, continue with step 4.
//
// 4.  If the offset based on maxindex minus Index falls BEFORE the start of
//     the leaf, or if, per 3c above, baseindex is LESS than Index, the leaf is
//     guaranteed "not dense to the end" and a usable empty Index must exist.
//     This supports a more efficient search loop.  Start at the FIRST index in
//     the leaf, or one BEYOND baseindex, respectively, and search the leaf as
//     follows, comparing each current index (currindex) with Index:
//
// 4a.  If LESS, keep going to next index.  Note:  This is certain to terminate
//      because maxindex is known to be greater than Index, hence the loop can
//      be small and fast.
//
// 4b.  If EQUAL, loop and increment Index until finding currindex greater than
//      Index, and return success with the modified Index.
//
// 4c.  If GREATER, return success, Index (unmodified) is empty.
//
// Note:  These are macros rather than functions for speed.

#ifdef JUDYPREV

#define	JSLE_EVEN(Addr,Pop0,cDigits,LeafType)				\
	{								\
	    LeafType * PjllLSB  = (LeafType *) (Addr);			\
	    LeafType   IndexLSB = Index;	/* auto-masking */	\
									\
	/* Index before or at start of leaf: */				\
									\
	    if (*PjllLSB >= IndexLSB)		/* no need to search */	\
	    {								\
		if (*PjllLSB > IndexLSB) RET_SUCCESS; /* Index empty */	\
		LEAF_EDGE(*PjllLSB, cDigits);				\
	    }								\
									\
	/* Index in or after leaf: */					\
									\
	    offset = IndexLSB - *PjllLSB;	/* tentative offset  */	\
	    if (offset <= (Pop0))		/* can check density */	\
	    {								\
		PjllLSB += offset;		/* move to slot */	\
									\
		if (*PjllLSB <= IndexLSB)	/* dense or corrupt */	\
		{							\
		    if (*PjllLSB == IndexLSB)	/* dense, check edge */	\
			LEAF_EDGE_SET(PjllLSB[-offset], cDigits);	\
		    RET_CORRUPT;					\
		}							\
		--PjllLSB;	/* not dense, start at previous */	\
	    }								\
	    else PjllLSB = ((LeafType *) (Addr)) + (Pop0); /* start at max */ \
									\
	    LEAF_HOLE_EVEN(cDigits, PjllLSB, IndexLSB);			\
	}

// JSLE_ODD is completely different from JSLE_EVEN because its important to
// minimize copying odd indexes to compare them (see 4.14).  Furthermore, a
// very complex version (4.17, but abandoned before fully debugged) that
// avoided calling j__udySearchLeaf*() ran twice as fast as 4.14, but still
// half as fast as SearchValid.  Doug suggested that to minimize complexity and
// share common code we should use j__udySearchLeaf*() for the initial search
// to establish if Index is empty, which should be common.  If Index is valid
// in a leaf or immediate indexes, odds are good that an empty Index is nearby,
// so for simplicity just use a *COPY* function to linearly search the
// remainder.
//
// TBD:  Pathological case?  Average performance should be good, but worst-case
// might suffer.  When Search says the initial Index is valid, so a linear
// copy-and-compare is begun, if the caller builds fairly large leaves with
// dense clusters AND frequently does a SearchEmpty at one end of such a
// cluster, performance wont be very good.  Might a dense-check help?  This
// means checking offset against the index at offset, and then against the
// first/last index in the leaf.  We doubt the pathological case will appear
// much in real applications because they will probably alternate SearchValid
// and SearchEmpty calls.

#define	JSLE_ODD(cDigits,Pjll,Pop0,Search,Copy)				\
	{								\
	    Word_t IndexLSB;		/* least bytes only */		\
	    Word_t IndexFound;		/* in leaf	    */		\
									\
	    if ((offset = Search(Pjll, (Pop0) + 1, Index)) < 0)		\
		RET_SUCCESS;		/* Index is empty */		\
									\
	    IndexLSB = JU_LEASTBYTES(Index, cDigits);			\
	    offset  *= (cDigits);					\
									\
	    while ((offset -= (cDigits)) >= 0)				\
	    {				/* skip until empty or start */	\
		Copy(IndexFound, ((uint8_t *) (Pjll)) + offset);	\
		if (IndexFound != (--IndexLSB))	/* found an empty */	\
		{ JU_SETDIGITS(Index, IndexLSB, cDigits); RET_SUCCESS; }\
	    }								\
	    LEAF_EDGE_SET(IndexLSB, cDigits);				\
	}

#else // JUDYNEXT

#define	JSLE_EVEN(Addr,Pop0,cDigits,LeafType)				\
	{								\
	    LeafType * PjllLSB   = ((LeafType *) (Addr)) + (Pop0);	\
	    LeafType   IndexLSB = Index;	/* auto-masking */	\
									\
	/* Index at or after end of leaf: */				\
									\
	    if (*PjllLSB <= IndexLSB)		/* no need to search */	\
	    {								\
		if (*PjllLSB < IndexLSB) RET_SUCCESS;  /* Index empty */\
		LEAF_EDGE(*PjllLSB, cDigits);				\
	    }								\
									\
	/* Index before or in leaf: */					\
									\
	    offset = *PjllLSB - IndexLSB;	/* tentative offset  */	\
	    if (offset <= (Pop0))		/* can check density */	\
	    {								\
		PjllLSB -= offset;		/* move to slot */	\
									\
		if (*PjllLSB >= IndexLSB)	/* dense or corrupt */	\
		{							\
		    if (*PjllLSB == IndexLSB)	/* dense, check edge */	\
			LEAF_EDGE_SET(PjllLSB[offset], cDigits);	\
		    RET_CORRUPT;					\
		}							\
		++PjllLSB;		/* not dense, start at next */	\
	    }								\
	    else PjllLSB = (LeafType *) (Addr);	/* start at minimum */	\
									\
	    LEAF_HOLE_EVEN(cDigits, PjllLSB, IndexLSB);			\
	}

#define	JSLE_ODD(cDigits,Pjll,Pop0,Search,Copy)				\
	{								\
	    Word_t IndexLSB;		/* least bytes only */		\
	    Word_t IndexFound;		/* in leaf	    */		\
	    int	   offsetmax;		/* in bytes	    */		\
									\
	    if ((offset = Search(Pjll, (Pop0) + 1, Index)) < 0)		\
		RET_SUCCESS;			/* Index is empty */	\
									\
	    IndexLSB  = JU_LEASTBYTES(Index, cDigits);			\
	    offset   *= (cDigits);					\
	    offsetmax = (Pop0) * (cDigits);	/* single multiply */	\
									\
	    while ((offset += (cDigits)) <= offsetmax)			\
	    {				/* skip until empty or end */	\
		Copy(IndexFound, ((uint8_t *) (Pjll)) + offset);	\
		if (IndexFound != (++IndexLSB))	/* found an empty */	\
		{ JU_SETDIGITS(Index, IndexLSB, cDigits); RET_SUCCESS; } \
	    }								\
	    LEAF_EDGE_SET(IndexLSB, cDigits);				\
	}

#endif // JUDYNEXT

// Note:  Immediate indexes never fill a single index group, so for odd index
// sizes, save time by calling JSLE_ODD_IMM instead of JSLE_ODD.

#define	j__udySearchLeafEmpty1(Addr,Pop0) \
	JSLE_EVEN(Addr, Pop0, 1, uint8_t)

#define	j__udySearchLeafEmpty2(Addr,Pop0) \
	JSLE_EVEN(Addr, Pop0, 2, uint16_t)

#define	j__udySearchLeafEmpty3(Addr,Pop0) \
	JSLE_ODD(3, Addr, Pop0, j__udySearchLeaf3, JU_COPY3_PINDEX_TO_LONG)

#ifndef JU_64BIT

#define	j__udySearchLeafEmptyL(Addr,Pop0) \
	JSLE_EVEN(Addr, Pop0, 4, Word_t)

#else

#define	j__udySearchLeafEmpty4(Addr,Pop0) \
	JSLE_EVEN(Addr, Pop0, 4, uint32_t)

#define	j__udySearchLeafEmpty5(Addr,Pop0) \
	JSLE_ODD(5, Addr, Pop0, j__udySearchLeaf5, JU_COPY5_PINDEX_TO_LONG)

#define	j__udySearchLeafEmpty6(Addr,Pop0) \
	JSLE_ODD(6, Addr, Pop0, j__udySearchLeaf6, JU_COPY6_PINDEX_TO_LONG)

#define	j__udySearchLeafEmpty7(Addr,Pop0) \
	JSLE_ODD(7, Addr, Pop0, j__udySearchLeaf7, JU_COPY7_PINDEX_TO_LONG)

#define	j__udySearchLeafEmptyL(Addr,Pop0) \
	JSLE_EVEN(Addr, Pop0, 8, Word_t)

#endif // JU_64BIT


// ----------------------------------------------------------------------------
// START OF CODE:
//
// CHECK FOR SHORTCUTS:
//
// Error out if PIndex is null.

    if (PIndex == (PWord_t) NULL)
    {
        JU_SET_ERRNO(PJError, JU_ERRNO_NULLPINDEX);
        return(JERRI);
    }

    Index = *PIndex;			// fast local copy.

// Set and pre-decrement/increment Index, watching for underflow/overflow:
//
// An out-of-bounds Index means failure:  No previous/next empty index.

SMGetRestart:		// return here with revised Index.

#ifdef JUDYPREV
    if (Index-- == 0) return(0);
#else
    if (++Index == 0) return(0);
#endif

// An empty array with an in-bounds (not underflowed/overflowed) Index means
// success:
//
// Note:  This check is redundant after restarting at SMGetRestart, but should
// take insignificant time.

    if (PArray == (Pvoid_t) NULL) RET_SUCCESS;

// ----------------------------------------------------------------------------
// ROOT-LEVEL LEAF that starts with a Pop0 word; just look within the leaf:
//
// If Index is not in the leaf, return success; otherwise return the first
// empty Index, if any, below/above where it would belong.

    if (JU_LEAFW_POP0(PArray) < cJU_LEAFW_MAXPOP1) // must be a LEAFW
    {
        Pjlw_t Pjlw = P_JLW(PArray);	// first word of leaf.
        pop0 = Pjlw[0];

#ifdef	JUDY1
        if (pop0 == 0)			// special case.
        {
#ifdef JUDYPREV
            if ((Index != Pjlw[1]) || (Index-- != 0)) RET_SUCCESS;
#else
            if ((Index != Pjlw[1]) || (++Index != 0)) RET_SUCCESS;
#endif
            return(0);		// no previous/next empty index.
        }
#endif // JUDY1

        j__udySearchLeafEmptyL(Pjlw + 1, pop0);

//  No return -- thanks ALAN

    }
    else

// ----------------------------------------------------------------------------
// HANDLE JRP Branch:
//
// For JRP branches, traverse the JPM; handle LEAFW
// directly; but look for the most common cases first.

    {
        Pjpm_t Pjpm = P_JPM(PArray);
        Pjp = &(Pjpm->jpm_JP);

//	    goto SMGetContinue;
    }


// ============================================================================
// STATE MACHINE -- GET INDEX:
//
// Search for Index (already decremented/incremented so as to be an inclusive
// search).  If not found (empty index), return success.  Otherwise do a
// previous/next search, and if successful modify Index to the empty index
// found.  See function header comments.
//
// ENTRY:  Pjp points to next JP to interpret, whose Decode bytes have not yet
// been checked.
//
// Note:  Check Decode bytes at the start of each loop, not after looking up a
// new JP, so its easy to do constant shifts/masks.
//
// EXIT:  Return, or branch to SMGetRestart with modified Index, or branch to
// SMGetContinue with a modified Pjp, as described elsewhere.
//
// WARNING:  For run-time efficiency the following cases replicate code with
// varying constants, rather than using common code with variable values!

SMGetContinue:			// return here for next branch/leaf.

#ifdef TRACEJPSE
    JudyPrintJP(Pjp, "sf", __LINE__);
#endif

    switch (JU_JPTYPE(Pjp))
    {


// ----------------------------------------------------------------------------
// LINEAR BRANCH:
//
// Check Decode bytes, if any, in the current JP, then search for a JP for the
// next digit in Index.

    case cJU_JPBRANCH_L2:
        CHECKDCD(2);
        SMPREPB2(SMBranchL);
    case cJU_JPBRANCH_L3:
        CHECKDCD(3);
        SMPREPB3(SMBranchL);
#ifdef JU_64BIT
    case cJU_JPBRANCH_L4:
        CHECKDCD(4);
        SMPREPB4(SMBranchL);
    case cJU_JPBRANCH_L5:
        CHECKDCD(5);
        SMPREPB5(SMBranchL);
    case cJU_JPBRANCH_L6:
        CHECKDCD(6);
        SMPREPB6(SMBranchL);
    case cJU_JPBRANCH_L7:
        CHECKDCD(7);
        SMPREPB7(SMBranchL);
#endif
    case cJU_JPBRANCH_L:
        SMPREPBL(SMBranchL);

// Common code (state-independent) for all cases of linear branches:

SMBranchL:
        Pjbl = P_JBL(Pjp->jp_Addr);

// First, check if Indexs expanse (digit) is below/above the first/last
// populated expanse in the BranchL, in which case Index is empty; otherwise
// find the offset of the lowest/highest populated expanse at or above/below
// digit, if any:
//
// Note:  The for-loop is guaranteed to exit eventually because the first/last
// expanse is known to be a terminator.
//
// Note:  Cannot use j__udySearchLeaf*Empty1() here because it only applies to
// leaves and does not know about partial versus full JPs, unlike the use of
// j__udySearchLeaf1() for BranchLs in SearchValid code.  Also, since linear
// leaf expanse lists are small, dont waste time calling j__udySearchLeaf1(),
// just scan the expanse list.

#ifdef JUDYPREV
        if ((Pjbl->jbl_Expanse[0]) > digit) RET_SUCCESS;

        for (offset = (Pjbl->jbl_NumJPs) - 1; /* null */; --offset)
#else
        if ((Pjbl->jbl_Expanse[(Pjbl->jbl_NumJPs) - 1]) < digit)
            RET_SUCCESS;

        for (offset = 0; /* null */; ++offset)
#endif
        {

// Too low/high, keep going; or too high/low, meaning the loop passed a hole
// and the initial Index is empty:

#ifdef JUDYPREV
            if ((Pjbl->jbl_Expanse[offset]) > digit) continue;
            if ((Pjbl->jbl_Expanse[offset]) < digit) RET_SUCCESS;
#else
            if ((Pjbl->jbl_Expanse[offset]) < digit) continue;
            if ((Pjbl->jbl_Expanse[offset]) > digit) RET_SUCCESS;
#endif

// Found expanse matching digit; if its not full, traverse through it:

            if (! JPFULL((Pjbl->jbl_jp) + offset))
            {
                Pjp = (Pjbl->jbl_jp) + offset;
                goto SMGetContinue;
            }

// Common code:  While searching for a lower/higher hole or a non-full JP, upon
// finding a lower/higher hole, adjust Index using the revised digit and
// return; or upon finding a consecutive lower/higher expanse, if the expanses
// JP is non-full, modify Index and traverse through the JP:

#define	BRANCHL_CHECK(OpIncDec,OpLeastDigits,Digit,Digits)	\
	{							\
	    if ((Pjbl->jbl_Expanse[offset]) != OpIncDec digit)	\
		SET_AND_RETURN(OpLeastDigits, Digit, Digits);	\
								\
	    if (! JPFULL((Pjbl->jbl_jp) + offset))		\
	    {							\
		Pjp = (Pjbl->jbl_jp) + offset;			\
		SET_AND_CONTINUE(OpLeastDigits, Digit, Digits);	\
	    }							\
	}

// BranchL primary dead end:  Expanse matching Index/digit is full (rare except
// for dense/sequential indexes):
//
// Search for a lower/higher hole, a non-full JP, or the end of the expanse
// list, while decrementing/incrementing digit.

#ifdef JUDYPREV
            while (--offset >= 0)
                BRANCHL_CHECK(--, SETLEASTDIGITS_D, digit, digits)
#else
            while (++offset < Pjbl->jbl_NumJPs)
                BRANCHL_CHECK(++, CLEARLEASTDIGITS_D, digit, digits)
#endif

// Passed end of BranchL expanse list after finding a matching but full
// expanse:
//
// Digit now matches the lowest/highest expanse, which is a full expanse; if
// digit is at the end of BranchLs expanse (no hole before/after), break out
// of the loop; otherwise modify Index to the next lower/higher digit and
// return success:

#ifdef JUDYPREV
                if (digit == 0) break;
            --digit;
            SET_AND_RETURN(SETLEASTDIGITS_D, digit, digits);
#else
                if (digit == JU_LEASTBYTES(cJU_ALLONES, 1)) break;
            ++digit;
            SET_AND_RETURN(CLEARLEASTDIGITS_D, digit, digits);
#endif
        } // for-loop

// BranchL secondary dead end, no non-full previous/next JP:

        SMRESTART(digits);


// ----------------------------------------------------------------------------
// BITMAP BRANCH:
//
// Check Decode bytes, if any, in the current JP, then search for a JP for the
// next digit in Index.

    case cJU_JPBRANCH_B2:
        CHECKDCD(2);
        SMPREPB2(SMBranchB);
    case cJU_JPBRANCH_B3:
        CHECKDCD(3);
        SMPREPB3(SMBranchB);
#ifdef JU_64BIT
    case cJU_JPBRANCH_B4:
        CHECKDCD(4);
        SMPREPB4(SMBranchB);
    case cJU_JPBRANCH_B5:
        CHECKDCD(5);
        SMPREPB5(SMBranchB);
    case cJU_JPBRANCH_B6:
        CHECKDCD(6);
        SMPREPB6(SMBranchB);
    case cJU_JPBRANCH_B7:
        CHECKDCD(7);
        SMPREPB7(SMBranchB);
#endif
    case cJU_JPBRANCH_B:
        SMPREPBL(SMBranchB);

// Common code (state-independent) for all cases of bitmap branches:

SMBranchB:
        Pjbb = P_JBB(Pjp->jp_Addr);

// Locate the digits JP in the subexpanse list, if present:

        subexp     = digit / cJU_BITSPERSUBEXPB;
        assert(subexp < cJU_NUMSUBEXPB);	// falls in expected range.
        bitposmaskB = JU_BITPOSMASKB(digit);

// Absent JP = no JP matches current digit in Index:

//	    if (! JU_BITMAPTESTB(Pjbb, digit))			// slower.
        if (! (JU_JBB_BITMAP(Pjbb, subexp) & bitposmaskB))	// faster.
            RET_SUCCESS;

// Non-full JP matches current digit in Index:
//
// Iterate to the subsidiary non-full JP.

        offset = SEARCHBITMAPB(JU_JBB_BITMAP(Pjbb, subexp), digit,
                               bitposmaskB);
        // not negative since at least one bit is set:
        assert(offset >= 0);
        assert(offset < (int) cJU_BITSPERSUBEXPB);

// Watch for null JP subarray pointer with non-null bitmap (a corruption):

        if ((Pjp = P_JP(JU_JBB_PJP(Pjbb, subexp)))
                == (Pjp_t) NULL) RET_CORRUPT;

        Pjp += offset;
        if (! JPFULL(Pjp)) goto SMGetContinue;

// BranchB primary dead end:
//
// Upon hitting a full JP in a BranchB for the next digit in Index, search
// sideways for a previous/next absent JP (unset bit) or non-full JP (set bit
// with non-full JP); first in the current bitmap subexpanse, then in
// lower/higher subexpanses.  Upon entry, Pjp points to a known-unusable JP,
// ready to decrement/increment.
//
// Note:  The preceding code is separate from this loop because Index does not
// need revising (see SET_AND_*()) if the initial index is an empty index.
//
// TBD:  For speed, shift bitposmaskB instead of using JU_BITMAPTESTB or
// JU_BITPOSMASKB, but this shift has knowledge of bit order that really should
// be encapsulated in a header file.

#define	BRANCHB_CHECKBIT(OpLeastDigits)					\
    if (! (JU_JBB_BITMAP(Pjbb, subexp) & bitposmaskB))  /* absent JP */	\
	SET_AND_RETURN(OpLeastDigits, digit, digits)

#define	BRANCHB_CHECKJPFULL(OpLeastDigits)				\
    if (! JPFULL(Pjp))							\
	SET_AND_CONTINUE(OpLeastDigits, digit, digits)

#define	BRANCHB_STARTSUBEXP(OpLeastDigits)				\
    if (! JU_JBB_BITMAP(Pjbb, subexp)) /* empty subexpanse, shortcut */ \
	SET_AND_RETURN(OpLeastDigits, digit, digits)			\
    if ((Pjp = P_JP(JU_JBB_PJP(Pjbb, subexp))) == (Pjp_t) NULL) RET_CORRUPT

#ifdef JUDYPREV

        --digit;				// skip initial digit.
        bitposmaskB >>= 1;			// see TBD above.

BranchBNextSubexp:	// return here to check next bitmap subexpanse.

        while (bitposmaskB)			// more bits to check in subexp.
        {
            BRANCHB_CHECKBIT(SETLEASTDIGITS_D);
            --Pjp;				// previous in subarray.
            BRANCHB_CHECKJPFULL(SETLEASTDIGITS_D);
            assert(digit >= 0);
            --digit;
            bitposmaskB >>= 1;
        }

        if (subexp-- > 0)			// more subexpanses.
        {
            BRANCHB_STARTSUBEXP(SETLEASTDIGITS_D);
            Pjp += SEARCHBITMAPMAXB(JU_JBB_BITMAP(Pjbb, subexp)) + 1;
            bitposmaskB = (1U << (cJU_BITSPERSUBEXPB - 1));
            goto BranchBNextSubexp;
        }

#else // JUDYNEXT

        ++digit;				// skip initial digit.
        bitposmaskB <<= 1;			// note:  BITMAPB_t.

BranchBNextSubexp:	// return here to check next bitmap subexpanse.

        while (bitposmaskB)			// more bits to check in subexp.
        {
            BRANCHB_CHECKBIT(CLEARLEASTDIGITS_D);
            ++Pjp;				// previous in subarray.
            BRANCHB_CHECKJPFULL(CLEARLEASTDIGITS_D);
            assert(digit < cJU_SUBEXPPERSTATE);
            ++digit;
            bitposmaskB <<= 1;		// note:  BITMAPB_t.
        }

        if (++subexp < cJU_NUMSUBEXPB)	// more subexpanses.
        {
            BRANCHB_STARTSUBEXP(CLEARLEASTDIGITS_D);
            --Pjp;				// pre-decrement.
            bitposmaskB = 1;
            goto BranchBNextSubexp;
        }

#endif // JUDYNEXT

// BranchB secondary dead end, no non-full previous/next JP:

        SMRESTART(digits);


// ----------------------------------------------------------------------------
// UNCOMPRESSED BRANCH:
//
// Check Decode bytes, if any, in the current JP, then search for a JP for the
// next digit in Index.

    case cJU_JPBRANCH_U2:
        CHECKDCD(2);
        SMPREPB2(SMBranchU);
    case cJU_JPBRANCH_U3:
        CHECKDCD(3);
        SMPREPB3(SMBranchU);
#ifdef JU_64BIT
    case cJU_JPBRANCH_U4:
        CHECKDCD(4);
        SMPREPB4(SMBranchU);
    case cJU_JPBRANCH_U5:
        CHECKDCD(5);
        SMPREPB5(SMBranchU);
    case cJU_JPBRANCH_U6:
        CHECKDCD(6);
        SMPREPB6(SMBranchU);
    case cJU_JPBRANCH_U7:
        CHECKDCD(7);
        SMPREPB7(SMBranchU);
#endif
    case cJU_JPBRANCH_U:
        SMPREPBL(SMBranchU);

// Common code (state-independent) for all cases of uncompressed branches:

SMBranchU:
        Pjbu = P_JBU(Pjp->jp_Addr);
        Pjp	 = (Pjbu->jbu_jp) + digit;

// Absent JP = null JP for current digit in Index:

        if (JPNULL(JU_JPTYPE(Pjp))) RET_SUCCESS;

// Non-full JP matches current digit in Index:
//
// Iterate to the subsidiary JP.

        if (! JPFULL(Pjp)) goto SMGetContinue;

// BranchU primary dead end:
//
// Upon hitting a full JP in a BranchU for the next digit in Index, search
// sideways for a previous/next null or non-full JP.  BRANCHU_CHECKJP() is
// shorthand for common code.
//
// Note:  The preceding code is separate from this loop because Index does not
// need revising (see SET_AND_*()) if the initial index is an empty index.

#define	BRANCHU_CHECKJP(OpIncDec,OpLeastDigits)			\
	{							\
	    OpIncDec Pjp;					\
								\
	    if (JPNULL(JU_JPTYPE(Pjp)))				\
		SET_AND_RETURN(OpLeastDigits, digit, digits)	\
								\
	    if (! JPFULL(Pjp))					\
		SET_AND_CONTINUE(OpLeastDigits, digit, digits)	\
	}

#ifdef JUDYPREV
        while (digit-- > 0)
            BRANCHU_CHECKJP(--, SETLEASTDIGITS_D);
#else
        while (++digit < cJU_BRANCHUNUMJPS)
            BRANCHU_CHECKJP(++, CLEARLEASTDIGITS_D);
#endif

// BranchU secondary dead end, no non-full previous/next JP:

        SMRESTART(digits);


// ----------------------------------------------------------------------------
// LINEAR LEAF:
//
// Check Decode bytes, if any, in the current JP, then search the leaf for the
// previous/next empty index starting at Index.  Primary leaf dead end is
// hidden within j__udySearchLeaf*Empty*().  In case of secondary leaf dead
// end, restart at the top of the tree.
//
// Note:  Pword is the name known to GET*; think of it as Pjlw.

#define	SMLEAFL(cDigits,Func)                   \
	Pword = (PWord_t) P_JLW(Pjp->jp_Addr);  \
	pop0  = JU_JPLEAF_POP0(Pjp);            \
	Func(Pword, pop0)

#if (defined(JUDYL) || (! defined(JU_64BIT)))
    case cJU_JPLEAF1:
        CHECKDCD(1);
        SMLEAFL(1, j__udySearchLeafEmpty1);
#endif
    case cJU_JPLEAF2:
        CHECKDCD(2);
        SMLEAFL(2, j__udySearchLeafEmpty2);
    case cJU_JPLEAF3:
        CHECKDCD(3);
        SMLEAFL(3, j__udySearchLeafEmpty3);

#ifdef JU_64BIT
    case cJU_JPLEAF4:
        CHECKDCD(4);
        SMLEAFL(4, j__udySearchLeafEmpty4);
    case cJU_JPLEAF5:
        CHECKDCD(5);
        SMLEAFL(5, j__udySearchLeafEmpty5);
    case cJU_JPLEAF6:
        CHECKDCD(6);
        SMLEAFL(6, j__udySearchLeafEmpty6);
    case cJU_JPLEAF7:
        CHECKDCD(7);
        SMLEAFL(7, j__udySearchLeafEmpty7);
#endif


// ----------------------------------------------------------------------------
// BITMAP LEAF:
//
// Check Decode bytes, if any, in the current JP, then search the leaf for the
// previous/next empty index starting at Index.

    case cJU_JPLEAF_B1:

        CHECKDCD(1);

        Pjlb	= P_JLB(Pjp->jp_Addr);
        digit	= JU_DIGITATSTATE(Index, 1);
        subexp	= digit / cJU_BITSPERSUBEXPL;
        bitposmaskL	= JU_BITPOSMASKL(digit);
        assert(subexp < cJU_NUMSUBEXPL);	// falls in expected range.

// Absent index = no index matches current digit in Index:

//	    if (! JU_BITMAPTESTL(Pjlb, digit))			// slower.
        if (! (JU_JLB_BITMAP(Pjlb, subexp) & bitposmaskL))	// faster.
            RET_SUCCESS;

// LeafB1 primary dead end:
//
// Upon hitting a valid (non-empty) index in a LeafB1 for the last digit in
// Index, search sideways for a previous/next absent index, first in the
// current bitmap subexpanse, then in lower/higher subexpanses.
// LEAFB1_CHECKBIT() is shorthand for common code to handle one bit in one
// bitmap subexpanse.
//
// Note:  The preceding code is separate from this loop because Index does not
// need revising (see SET_AND_*()) if the initial index is an empty index.
//
// TBD:  For speed, shift bitposmaskL instead of using JU_BITMAPTESTL or
// JU_BITPOSMASKL, but this shift has knowledge of bit order that really should
// be encapsulated in a header file.

#define	LEAFB1_CHECKBIT(OpLeastDigits)				\
	if (! (JU_JLB_BITMAP(Pjlb, subexp) & bitposmaskL))	\
	    SET_AND_RETURN(OpLeastDigits, digit, 1)

#define	LEAFB1_STARTSUBEXP(OpLeastDigits)			\
	if (! JU_JLB_BITMAP(Pjlb, subexp)) /* empty subexp */	\
	    SET_AND_RETURN(OpLeastDigits, digit, 1)

#ifdef JUDYPREV

        --digit;				// skip initial digit.
        bitposmaskL >>= 1;			// see TBD above.

LeafB1NextSubexp:	// return here to check next bitmap subexpanse.

        while (bitposmaskL)			// more bits to check in subexp.
        {
            LEAFB1_CHECKBIT(SETLEASTDIGITS_D);
            assert(digit >= 0);
            --digit;
            bitposmaskL >>= 1;
        }

        if (subexp-- > 0)		// more subexpanses.
        {
            LEAFB1_STARTSUBEXP(SETLEASTDIGITS_D);
#ifdef JU_64BIT
            bitposmaskL = (1ULL << (cJU_BITSPERSUBEXPL - 1));
#else
            bitposmaskL = (1UL << (cJU_BITSPERSUBEXPL - 1));
#endif
            goto LeafB1NextSubexp;
        }

#else // JUDYNEXT

        ++digit;				// skip initial digit.
        bitposmaskL <<= 1;			// note:  BITMAPL_t.

LeafB1NextSubexp:	// return here to check next bitmap subexpanse.

        while (bitposmaskL)			// more bits to check in subexp.
        {
            LEAFB1_CHECKBIT(CLEARLEASTDIGITS_D);
            assert(digit < cJU_SUBEXPPERSTATE);
            ++digit;
            bitposmaskL <<= 1;		// note:  BITMAPL_t.
        }

        if (++subexp < cJU_NUMSUBEXPL)	// more subexpanses.
        {
            LEAFB1_STARTSUBEXP(CLEARLEASTDIGITS_D);
            bitposmaskL = 1;
            goto LeafB1NextSubexp;
        }

#endif // JUDYNEXT

// LeafB1 secondary dead end, no empty index:

        SMRESTART(1);


#ifdef JUDY1
// ----------------------------------------------------------------------------
// FULL POPULATION:
//
// If the Decode bytes do not match, Index is empty (without modification);
// otherwise restart.

    case cJ1_JPFULLPOPU1:

        CHECKDCD(1);
        SMRESTART(1);
#endif


// ----------------------------------------------------------------------------
// IMMEDIATE:
//
// Pop1 = 1 Immediate JPs:
//
// If Index is not in the immediate JP, return success; otherwise check if
// there is an empty index below/above the immediate JPs index, and if so,
// return success with modified Index, else restart.
//
// Note:  Doug says its fast enough to calculate the index size (digits) in
// the following; no need to set it separately for each case.

    case cJU_JPIMMED_1_01:
    case cJU_JPIMMED_2_01:
    case cJU_JPIMMED_3_01:
#ifdef JU_64BIT
    case cJU_JPIMMED_4_01:
    case cJU_JPIMMED_5_01:
    case cJU_JPIMMED_6_01:
    case cJU_JPIMMED_7_01:
#endif
        if (JU_JPDCDPOP0(Pjp) != JU_TRIMTODCDSIZE(Index)) RET_SUCCESS;
        digits = JU_JPTYPE(Pjp) - cJU_JPIMMED_1_01 + 1;
        LEAF_EDGE(JU_LEASTBYTES(JU_JPDCDPOP0(Pjp), digits), digits);

// Immediate JPs with Pop1 > 1:

#define	IMM_MULTI(Func,BaseJPType)			\
	JUDY1CODE(Pword = (PWord_t) (Pjp->jp_1Index);)	\
	JUDYLCODE(Pword = (PWord_t) (Pjp->jp_LIndex);)	\
	Func(Pword, JU_JPTYPE(Pjp) - (BaseJPType) + 1)

    case cJU_JPIMMED_1_02:
    case cJU_JPIMMED_1_03:
#if (defined(JUDY1) || defined(JU_64BIT))
    case cJU_JPIMMED_1_04:
    case cJU_JPIMMED_1_05:
    case cJU_JPIMMED_1_06:
    case cJU_JPIMMED_1_07:
#endif
#if (defined(JUDY1) && defined(JU_64BIT))
    case cJ1_JPIMMED_1_08:
    case cJ1_JPIMMED_1_09:
    case cJ1_JPIMMED_1_10:
    case cJ1_JPIMMED_1_11:
    case cJ1_JPIMMED_1_12:
    case cJ1_JPIMMED_1_13:
    case cJ1_JPIMMED_1_14:
    case cJ1_JPIMMED_1_15:
#endif
        IMM_MULTI(j__udySearchLeafEmpty1, cJU_JPIMMED_1_02);

#if (defined(JUDY1) || defined(JU_64BIT))
    case cJU_JPIMMED_2_02:
    case cJU_JPIMMED_2_03:
#endif
#if (defined(JUDY1) && defined(JU_64BIT))
    case cJ1_JPIMMED_2_04:
    case cJ1_JPIMMED_2_05:
    case cJ1_JPIMMED_2_06:
    case cJ1_JPIMMED_2_07:
#endif
#if (defined(JUDY1) || defined(JU_64BIT))
        IMM_MULTI(j__udySearchLeafEmpty2, cJU_JPIMMED_2_02);
#endif

#if (defined(JUDY1) || defined(JU_64BIT))
    case cJU_JPIMMED_3_02:
#endif
#if (defined(JUDY1) && defined(JU_64BIT))
    case cJ1_JPIMMED_3_03:
    case cJ1_JPIMMED_3_04:
    case cJ1_JPIMMED_3_05:
#endif
#if (defined(JUDY1) || defined(JU_64BIT))
        IMM_MULTI(j__udySearchLeafEmpty3, cJU_JPIMMED_3_02);
#endif

#if (defined(JUDY1) && defined(JU_64BIT))
    case cJ1_JPIMMED_4_02:
    case cJ1_JPIMMED_4_03:
        IMM_MULTI(j__udySearchLeafEmpty4, cJ1_JPIMMED_4_02);

    case cJ1_JPIMMED_5_02:
    case cJ1_JPIMMED_5_03:
        IMM_MULTI(j__udySearchLeafEmpty5, cJ1_JPIMMED_5_02);

    case cJ1_JPIMMED_6_02:
        IMM_MULTI(j__udySearchLeafEmpty6, cJ1_JPIMMED_6_02);

    case cJ1_JPIMMED_7_02:
        IMM_MULTI(j__udySearchLeafEmpty7, cJ1_JPIMMED_7_02);
#endif


// ----------------------------------------------------------------------------
// INVALID JP TYPE:

    default:
        RET_CORRUPT;

    } // SMGet switch.

} // Judy1PrevEmpty() / Judy1NextEmpty() / JudyLPrevEmpty() / JudyLNextEmpty()
FUNCTION int j__udyCreateBranchB(
    Pjp_t    Pjp,        // Build JPs from this place
    Pjp_t    PJPs,        // Array of JPs to put into Bitmap branch
    uint8_t Exp[],        // Array of expanses to put into bitmap
    Word_t  ExpCnt,        // Number of above JPs and Expanses
    Pvoid_t    Pjpm)
{
    Pjbb_t    PjbbRaw;    // pointer to bitmap branch.
    Pjbb_t    Pjbb;
    Word_t  ii, jj;        // Temps
    uint8_t CurrSubExp;    // Current sub expanse for BM

// This assertion says the number of populated subexpanses is not too large.
// This function is only called when a BranchL overflows to a BranchB or when a
// cascade occurs, meaning a leaf overflows.  Either way ExpCnt cant be very
// large, in fact a lot smaller than cJU_BRANCHBMAXJPS.  (Otherwise a BranchU
// would be used.)  Popping this assertion means something (unspecified) has
// gone very wrong, or else Judys design criteria have changed, although in
// fact there should be no HARM in creating a BranchB with higher actual
// fanout.

    assert(ExpCnt <= cJU_BRANCHBMAXJPS);

//    Get memory for a Bitmap branch
    PjbbRaw    = j__udyAllocJBB(Pjpm);
    if (PjbbRaw == (Pjbb_t) NULL) return(-1);
    Pjbb = P_JBB(PjbbRaw);

//    Get 1st "sub" expanse (0..7) of bitmap branch
    CurrSubExp = Exp[0] / cJU_BITSPERSUBEXPB;

// Index thru all 1 byte sized expanses:

    for (jj = ii = 0; ii <= ExpCnt; ii++)
    {
        Word_t SubExp;    // Cannot be a uint8_t

//        Make sure we cover the last one
        if (ii == ExpCnt)
        {
            SubExp = cJU_ALLONES;    // Force last one
        }
        else
        {
//            Calculate the "sub" expanse of the byte expanse
            SubExp = Exp[ii] / cJU_BITSPERSUBEXPB;  // Bits 5..7.

//            Set the bit that represents the expanse in Exp[]
            JU_JBB_BITMAP(Pjbb, SubExp) |= JU_BITPOSMASKB(Exp[ii]);
        }
//        Check if a new "sub" expanse range needed
        if (SubExp != CurrSubExp)
        {
//            Get number of JPs in this sub expanse
            Word_t NumJP = ii - jj;
            Pjp_t  PjpRaw;
            Pjp_t  Pjp;

            PjpRaw = j__udyAllocJBBJP(NumJP, Pjpm);
                        Pjp    = P_JP(PjpRaw);

            if (PjpRaw == (Pjp_t) NULL)    // out of memory.
            {

// Free any previous allocations:

                while(CurrSubExp--)
                {
                NumJP = j__udyCountBitsB(JU_JBB_BITMAP(Pjbb,
                                  CurrSubExp));
                if (NumJP)
                {
                    j__udyFreeJBBJP(JU_JBB_PJP(Pjbb,
                            CurrSubExp), NumJP, Pjpm);
                }
                }
                j__udyFreeJBB(PjbbRaw, Pjpm);
                return(-1);
            }

// Place the array of JPs in bitmap branch:

            JU_JBB_PJP(Pjbb, CurrSubExp) = PjpRaw;

// Copy the JPs to new leaf:

            JU_COPYMEM(Pjp, PJPs + jj, NumJP);

// On to the next bitmap branch "sub" expanse:

            jj       = ii;
            CurrSubExp = SubExp;
        }
    } // for each 1-byte expanse

// Pass back some of the JP to the new Bitmap branch:

    Pjp->jp_Addr = (Word_t) PjbbRaw;

    return(1);

} // j__udyCreateBranchB()
FUNCTION int j__udyCreateBranchU(
    Pjp_t      Pjp,
    Pvoid_t      Pjpm)
{
    jp_t      JPNull;
        Pjbu_t    PjbuRaw;
        Pjbu_t    Pjbu;
    Pjbb_t      PjbbRaw;
    Pjbb_t      Pjbb;
    Word_t      ii, jj;
    BITMAPB_t BitMap;
    Pjp_t      PDstJP;
#ifdef JU_STAGED_EXP
    jbu_t      BranchU;    // Staged uncompressed branch
#else

// Allocate memory for a BranchU:

    PjbuRaw = j__udyAllocJBU(Pjpm);
    if (PjbuRaw == (Pjbu_t) NULL) return(-1);
        Pjbu = P_JBU(PjbuRaw);
#endif
        JU_JPSETADT(&JPNull, 0, 0, JU_JPTYPE(Pjp) - cJU_JPBRANCH_B2 + cJU_JPNULL1);

// Get the pointer to the BranchB:

    PjbbRaw    = (Pjbb_t) (Pjp->jp_Addr);
    Pjbb    = P_JBB(PjbbRaw);

//    Set the pointer to the Uncompressed branch
#ifdef JU_STAGED_EXP
    PDstJP = BranchU.jbu_jp;
#else
        PDstJP = Pjbu->jbu_jp;
#endif
    for (ii = 0; ii < cJU_NUMSUBEXPB; ii++)
    {
        Pjp_t    PjpA;
        Pjp_t    PjpB;

        PjpB = PjpA = P_JP(JU_JBB_PJP(Pjbb, ii));

//        Get the bitmap for this subexpanse
        BitMap    = JU_JBB_BITMAP(Pjbb, ii);

//        NULL empty subexpanses
        if (BitMap == 0)
        {
//            But, fill with NULLs
            for (jj = 0; jj < cJU_BITSPERSUBEXPB; jj++)
            {
                PDstJP[jj] = JPNull;
            }
            PDstJP += cJU_BITSPERSUBEXPB;
            continue;
        }
//        Check if Uncompressed subexpanse
        if (BitMap == cJU_FULLBITMAPB)
        {
//            Copy subexpanse to the Uncompressed branch intact
            JU_COPYMEM(PDstJP, PjpA, cJU_BITSPERSUBEXPB);

//            Bump to next subexpanse
            PDstJP += cJU_BITSPERSUBEXPB;

//            Set length of subexpanse
            jj = cJU_BITSPERSUBEXPB;
        }
        else
        {
            for (jj = 0; jj < cJU_BITSPERSUBEXPB; jj++)
            {
//                Copy JP or NULLJP depending on bit
                if (BitMap & 1) { *PDstJP = *PjpA++; }
                else        { *PDstJP = JPNull; }

                PDstJP++;    // advance to next JP
                BitMap >>= 1;
            }
            jj = PjpA - PjpB;
        }

// Free the subexpanse:

        j__udyFreeJBBJP(JU_JBB_PJP(Pjbb, ii), jj, Pjpm);

    } // for each JP in BranchU

#ifdef JU_STAGED_EXP

// Allocate memory for a BranchU:

    PjbuRaw = j__udyAllocJBU(Pjpm);
    if (PjbuRaw == (Pjbu_t) NULL) return(-1);
        Pjbu = P_JBU(PjbuRaw);

// Copy staged branch to newly allocated branch:
//
// TBD:  I think this code is broken.

    *Pjbu = BranchU;

#endif // JU_STAGED_EXP

// Finally free the BranchB and put the BranchU in its place:

    j__udyFreeJBB(PjbbRaw, Pjpm);

    Pjp->jp_Addr  = (Word_t) PjbuRaw;
    Pjp->jp_Type += cJU_JPBRANCH_U - cJU_JPBRANCH_B;

    return(1);

} // j__udyCreateBranchU()