U_CDECL_END U_CAPI void U_EXPORT2 ucm_optimizeStates(UCMStates *states, uint16_t **pUnicodeCodeUnits, _MBCSToUFallback *toUFallbacks, int32_t countToUFallbacks, UBool verbose) { UErrorCode errorCode; int32_t state, cell, entry; /* test each state table entry */ for(state=0; state<states->countStates; ++state) { for(cell=0; cell<256; ++cell) { entry=states->stateTable[state][cell]; /* * if the entry is a final one with an MBCS_STATE_VALID_DIRECT_16 action code * and the code point is "unassigned" (0xfffe), then change it to * the "unassigned" action code with bits 26..23 set to zero and U+fffe. */ if(MBCS_ENTRY_SET_STATE(entry, 0)==MBCS_ENTRY_FINAL(0, MBCS_STATE_VALID_DIRECT_16, 0xfffe)) { states->stateTable[state][cell]=MBCS_ENTRY_FINAL_SET_ACTION(entry, MBCS_STATE_UNASSIGNED); } } } /* try to compact the toUnicode tables */ if(states->maxCharLength==2) { compactToUnicode2(states, pUnicodeCodeUnits, toUFallbacks, countToUFallbacks, verbose); } else if(states->maxCharLength>2) { if(verbose) { compactToUnicodeHelper(states, *pUnicodeCodeUnits, toUFallbacks, countToUFallbacks); } } /* sort toUFallbacks */ /* * It should be safe to sort them before compactToUnicode2() is called, * because it should not change the relative order of the offset values * that it adjusts, but they need to be sorted at some point, and * it is safest here. */ if(countToUFallbacks>0) { errorCode=U_ZERO_ERROR; /* nothing bad will happen... */ uprv_sortArray(toUFallbacks, countToUFallbacks, sizeof(_MBCSToUFallback), compareFallbacks, NULL, FALSE, &errorCode); } }
/* * This function tries to compact toUnicode tables for 2-byte codepages * by finding lead bytes with all-unassigned trail bytes and adding another state * for them. */ static void compactToUnicode2(UCMStates *states, uint16_t **pUnicodeCodeUnits, _MBCSToUFallback *toUFallbacks, int32_t countToUFallbacks, UBool verbose) { int32_t (*oldStateTable)[256]; uint16_t count[256]; uint16_t *oldUnicodeCodeUnits; int32_t entry, offset, oldOffset, trailOffset, oldTrailOffset, savings, sum; int32_t i, j, leadState, trailState, newState, fallback; uint16_t unit; /* find the lead state */ if(states->outputType==MBCS_OUTPUT_2_SISO) { /* use the DBCS lead state for SI/SO codepages */ leadState=1; } else { leadState=0; } /* find the main trail state: the most used target state */ uprv_memset(count, 0, sizeof(count)); for(i=0; i<256; ++i) { entry=states->stateTable[leadState][i]; if(MBCS_ENTRY_IS_TRANSITION(entry)) { ++count[MBCS_ENTRY_TRANSITION_STATE(entry)]; } } trailState=0; for(i=1; i<states->countStates; ++i) { if(count[i]>count[trailState]) { trailState=i; } } /* count possible savings from lead bytes with all-unassigned results in all trail bytes */ uprv_memset(count, 0, sizeof(count)); savings=0; /* for each lead byte */ for(i=0; i<256; ++i) { entry=states->stateTable[leadState][i]; if(MBCS_ENTRY_IS_TRANSITION(entry) && (MBCS_ENTRY_TRANSITION_STATE(entry))==trailState) { /* the offset is different for each lead byte */ offset=MBCS_ENTRY_TRANSITION_OFFSET(entry); /* for each trail byte for this lead byte */ for(j=0; j<256; ++j) { entry=states->stateTable[trailState][j]; switch(MBCS_ENTRY_FINAL_ACTION(entry)) { case MBCS_STATE_VALID_16: entry=offset+MBCS_ENTRY_FINAL_VALUE_16(entry); if((*pUnicodeCodeUnits)[entry]==0xfffe && ucm_findFallback(toUFallbacks, countToUFallbacks, entry)<0) { ++count[i]; } else { j=999; /* do not count for this lead byte because there are assignments */ } break; case MBCS_STATE_VALID_16_PAIR: entry=offset+MBCS_ENTRY_FINAL_VALUE_16(entry); if((*pUnicodeCodeUnits)[entry]==0xfffe) { count[i]+=2; } else { j=999; /* do not count for this lead byte because there are assignments */ } break; default: break; } } if(j==256) { /* all trail bytes for this lead byte are unassigned */ savings+=count[i]; } else { count[i]=0; } } } /* subtract from the possible savings the cost of an additional state */ savings=savings*2-1024; /* count bytes, not 16-bit words */ if(savings<=0) { return; } if(verbose) { printf("compacting toUnicode data saves %ld bytes\n", (long)savings); } if(states->countStates>=MBCS_MAX_STATE_COUNT) { fprintf(stderr, "cannot compact toUnicode because the maximum number of states is reached\n"); return; } /* make a copy of the state table */ oldStateTable=(int32_t (*)[256])uprv_malloc(states->countStates*1024); if(oldStateTable==NULL) { fprintf(stderr, "cannot compact toUnicode: out of memory\n"); return; } uprv_memcpy(oldStateTable, states->stateTable, states->countStates*1024); /* add the new state */ /* * this function does not catch the degenerate case where all lead bytes * have all-unassigned trail bytes and the lead state could be removed */ newState=states->countStates++; states->stateFlags[newState]=0; /* copy the old trail state, turning all assigned states into unassigned ones */ for(i=0; i<256; ++i) { entry=states->stateTable[trailState][i]; switch(MBCS_ENTRY_FINAL_ACTION(entry)) { case MBCS_STATE_VALID_16: case MBCS_STATE_VALID_16_PAIR: states->stateTable[newState][i]=MBCS_ENTRY_FINAL_SET_ACTION_VALUE(entry, MBCS_STATE_UNASSIGNED, 0xfffe); break; default: states->stateTable[newState][i]=entry; break; } } /* in the lead state, redirect all lead bytes with all-unassigned trail bytes to the new state */ for(i=0; i<256; ++i) { if(count[i]>0) { states->stateTable[leadState][i]=MBCS_ENTRY_SET_STATE(states->stateTable[leadState][i], newState); } } /* sum up the new state table */ for(i=0; i<states->countStates; ++i) { states->stateFlags[i]&=~MBCS_STATE_FLAG_READY; } sum=sumUpStates(states); /* allocate a new, smaller code units array */ oldUnicodeCodeUnits=*pUnicodeCodeUnits; if(sum==0) { *pUnicodeCodeUnits=NULL; if(oldUnicodeCodeUnits!=NULL) { uprv_free(oldUnicodeCodeUnits); } uprv_free(oldStateTable); return; } *pUnicodeCodeUnits=(uint16_t *)uprv_malloc(sum*sizeof(uint16_t)); if(*pUnicodeCodeUnits==NULL) { fprintf(stderr, "cannot compact toUnicode: out of memory allocating %ld 16-bit code units\n", (long)sum); /* revert to the old state table */ *pUnicodeCodeUnits=oldUnicodeCodeUnits; --states->countStates; uprv_memcpy(states->stateTable, oldStateTable, states->countStates*1024); uprv_free(oldStateTable); return; } for(i=0; i<sum; ++i) { (*pUnicodeCodeUnits)[i]=0xfffe; } /* copy the code units for all assigned characters */ /* * The old state table has the same lead _and_ trail states for assigned characters! * The differences are in the offsets, and in the trail states for some unassigned characters. * For each character with an assigned state in the new table, it was assigned in the old one. * Only still-assigned characters are copied. * Note that fallback mappings need to get their offset values adjusted. */ /* for each initial state */ for(leadState=0; leadState<states->countStates; ++leadState) { if((states->stateFlags[leadState]&0xf)==MBCS_STATE_FLAG_DIRECT) { /* for each lead byte from there */ for(i=0; i<256; ++i) { entry=states->stateTable[leadState][i]; if(MBCS_ENTRY_IS_TRANSITION(entry)) { trailState=(uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry); /* the new state does not have assigned states */ if(trailState!=newState) { trailOffset=MBCS_ENTRY_TRANSITION_OFFSET(entry); oldTrailOffset=MBCS_ENTRY_TRANSITION_OFFSET(oldStateTable[leadState][i]); /* for each trail byte */ for(j=0; j<256; ++j) { entry=states->stateTable[trailState][j]; /* copy assigned-character code units and adjust fallback offsets */ switch(MBCS_ENTRY_FINAL_ACTION(entry)) { case MBCS_STATE_VALID_16: offset=trailOffset+MBCS_ENTRY_FINAL_VALUE_16(entry); /* find the old offset according to the old state table */ oldOffset=oldTrailOffset+MBCS_ENTRY_FINAL_VALUE_16(oldStateTable[trailState][j]); unit=(*pUnicodeCodeUnits)[offset]=oldUnicodeCodeUnits[oldOffset]; if(unit==0xfffe && (fallback=ucm_findFallback(toUFallbacks, countToUFallbacks, oldOffset))>=0) { toUFallbacks[fallback].offset=0x80000000|offset; } break; case MBCS_STATE_VALID_16_PAIR: offset=trailOffset+MBCS_ENTRY_FINAL_VALUE_16(entry); /* find the old offset according to the old state table */ oldOffset=oldTrailOffset+MBCS_ENTRY_FINAL_VALUE_16(oldStateTable[trailState][j]); (*pUnicodeCodeUnits)[offset++]=oldUnicodeCodeUnits[oldOffset++]; (*pUnicodeCodeUnits)[offset]=oldUnicodeCodeUnits[oldOffset]; break; default: break; } } } } } } } /* remove temporary flags from fallback offsets that protected them from being modified twice */ for(i=0; i<countToUFallbacks; ++i) { toUFallbacks[i].offset&=0x7fffffff; } /* free temporary memory */ uprv_free(oldUnicodeCodeUnits); uprv_free(oldStateTable); }
/* * state table row grammar (ebnf-style): * (whitespace is allowed between all tokens) * * row=[[firstentry ','] entry (',' entry)*] * firstentry="initial" | "surrogates" * (initial state (default for state 0), output is all surrogate pairs) * entry=range [':' nextstate] ['.' action] * range=number ['-' number] * nextstate=number * (0..7f) * action='u' | 's' | 'p' | 'i' * (unassigned, state change only, surrogate pair, illegal) * number=(1- or 2-digit hexadecimal number) */ static const char * parseState(const char *s, int32_t state[256], uint32_t *pFlags) { const char *t; uint32_t start, end, i; int32_t entry; /* initialize the state: all illegal with U+ffff */ for(i=0; i<256; ++i) { state[i]=MBCS_ENTRY_FINAL(0, MBCS_STATE_ILLEGAL, 0xffff); } /* skip leading white space */ s=u_skipWhitespace(s); /* is there an "initial" or "surrogates" directive? */ if(uprv_strncmp("initial", s, 7)==0) { *pFlags=MBCS_STATE_FLAG_DIRECT; s=u_skipWhitespace(s+7); if(*s++!=',') { return s-1; } } else if(*pFlags==0 && uprv_strncmp("surrogates", s, 10)==0) { *pFlags=MBCS_STATE_FLAG_SURROGATES; s=u_skipWhitespace(s+10); if(*s++!=',') { return s-1; } } else if(*s==0) { /* empty state row: all-illegal */ return NULL; } for(;;) { /* read an entry, the start of the range first */ s=u_skipWhitespace(s); start=uprv_strtoul(s, (char **)&t, 16); if(s==t || 0xff<start) { return s; } s=u_skipWhitespace(t); /* read the end of the range if there is one */ if(*s=='-') { s=u_skipWhitespace(s+1); end=uprv_strtoul(s, (char **)&t, 16); if(s==t || end<start || 0xff<end) { return s; } s=u_skipWhitespace(t); } else { end=start; } /* determine the state entrys for this range */ if(*s!=':' && *s!='.') { /* the default is: final state with valid entries */ entry=MBCS_ENTRY_FINAL(0, MBCS_STATE_VALID_16, 0); } else { entry=MBCS_ENTRY_TRANSITION(0, 0); if(*s==':') { /* get the next state, default to 0 */ s=u_skipWhitespace(s+1); i=uprv_strtoul(s, (char **)&t, 16); if(s!=t) { if(0x7f<i) { return s; } s=u_skipWhitespace(t); entry=MBCS_ENTRY_SET_STATE(entry, i); } } /* get the state action, default to valid */ if(*s=='.') { /* this is a final state */ entry=MBCS_ENTRY_SET_FINAL(entry); s=u_skipWhitespace(s+1); if(*s=='u') { /* unassigned set U+fffe */ entry=MBCS_ENTRY_FINAL_SET_ACTION_VALUE(entry, MBCS_STATE_UNASSIGNED, 0xfffe); s=u_skipWhitespace(s+1); } else if(*s=='p') { if(*pFlags!=MBCS_STATE_FLAG_DIRECT) { entry=MBCS_ENTRY_FINAL_SET_ACTION(entry, MBCS_STATE_VALID_16_PAIR); } else { entry=MBCS_ENTRY_FINAL_SET_ACTION(entry, MBCS_STATE_VALID_16); } s=u_skipWhitespace(s+1); } else if(*s=='s') { entry=MBCS_ENTRY_FINAL_SET_ACTION(entry, MBCS_STATE_CHANGE_ONLY); s=u_skipWhitespace(s+1); } else if(*s=='i') { /* illegal set U+ffff */ entry=MBCS_ENTRY_FINAL_SET_ACTION_VALUE(entry, MBCS_STATE_ILLEGAL, 0xffff); s=u_skipWhitespace(s+1); } else { /* default to valid */ entry=MBCS_ENTRY_FINAL_SET_ACTION(entry, MBCS_STATE_VALID_16); } } else { /* this is an intermediate state, nothing to do */ } } /* adjust "final valid" states according to the state flags */ if(MBCS_ENTRY_FINAL_ACTION(entry)==MBCS_STATE_VALID_16) { switch(*pFlags) { case 0: /* no adjustment */ break; case MBCS_STATE_FLAG_DIRECT: /* set the valid-direct code point to "unassigned"==0xfffe */ entry=MBCS_ENTRY_FINAL_SET_ACTION_VALUE(entry, MBCS_STATE_VALID_DIRECT_16, 0xfffe); break; case MBCS_STATE_FLAG_SURROGATES: entry=MBCS_ENTRY_FINAL_SET_ACTION_VALUE(entry, MBCS_STATE_VALID_16_PAIR, 0); break; default: break; } } /* set this entry for the range */ for(i=start; i<=end; ++i) { state[i]=entry; } if(*s==',') { ++s; } else { return *s==0 ? NULL : s; } } }