U_CAPI void U_EXPORT2 flagCB_fromU( const void *context, UConverterFromUnicodeArgs *fromUArgs, const UChar* codeUnits, int32_t length, UChar32 codePoint, UConverterCallbackReason reason, UErrorCode * err) { /* First step - based on the reason code, take action */ if(reason == UCNV_UNASSIGNED) { /* whatever set should be trapped here */ ((FromUFLAGContext*)context)->flag = TRUE; } if(reason == UCNV_CLONE) { /* The following is the recommended way to implement UCNV_CLONE in a callback. */ UConverterFromUCallback saveCallback; const void *saveContext; FromUFLAGContext *old, *cloned; UErrorCode subErr = U_ZERO_ERROR; #if DEBUG_TMI printf("*** FLAGCB: cloning %p ***\n", context); #endif old = (FromUFLAGContext*)context; cloned = flagCB_fromU_openContext(); memcpy(cloned, old, sizeof(FromUFLAGContext)); #if DEBUG_TMI printf("%p: my subcb=%p:%p\n", old, old->subCallback, old->subContext); printf("%p: cloned subcb=%p:%p\n", cloned, cloned->subCallback, cloned->subContext); #endif /* We need to get the sub CB to handle cloning, * so we have to set up the following, temporarily: * * - Set the callback+context to the sub of this (flag) cb * - preserve the current cb+context, it could be anything * * Before: * CNV -> FLAG -> subcb -> ... * * After: * CNV -> subcb -> ... * * The chain from 'something' on is saved, and will be restored * at the end of this block. * */ ucnv_setFromUCallBack(fromUArgs->converter, cloned->subCallback, cloned->subContext, &saveCallback, &saveContext, &subErr); if( cloned->subCallback != NULL ) { /* Now, call the sub callback if present */ cloned->subCallback(cloned->subContext, fromUArgs, codeUnits, length, codePoint, reason, err); } ucnv_setFromUCallBack(fromUArgs->converter, saveCallback, /* Us */ cloned, /* new context */ &cloned->subCallback, /* IMPORTANT! Accept any change in CB or context */ &cloned->subContext, &subErr); if(U_FAILURE(subErr)) { *err = subErr; } } /* process other reasons here if need be */ /* Always call the subCallback if present */ if(((FromUFLAGContext*)context)->subCallback != NULL && reason != UCNV_CLONE) { ((FromUFLAGContext*)context)->subCallback( ((FromUFLAGContext*)context)->subContext, fromUArgs, codeUnits, length, codePoint, reason, err); } /* cleanup - free the memory AFTER calling the sub CB */ if(reason == UCNV_CLOSE) { free((void*)context); } }
UBool convsample_20_didSubstitute(const char *source) { UChar uchars[100]; char bytes[100]; UConverter *conv = NULL; UErrorCode status = U_ZERO_ERROR; uint32_t len, len2; UBool flagVal; FromUFLAGContext * context = NULL; printf("\n\n==============================================\n" "Sample 20: C: Test for substitution using callbacks\n"); /* print out the original source */ printBytes("src", source); printf("\n"); /* First, convert from UTF8 to unicode */ conv = ucnv_open("utf-8", &status); U_ASSERT(status); len = ucnv_toUChars(conv, uchars, 100, source, strlen(source), &status); U_ASSERT(status); printUChars("uch", uchars, len); printf("\n"); /* Now, close the converter */ ucnv_close(conv); /* Now, convert to windows-1252 */ conv = ucnv_open("windows-1252", &status); U_ASSERT(status); /* Converter starts out with the SUBSTITUTE callback set. */ /* initialize our callback */ context = flagCB_fromU_openContext(); /* Set our special callback */ ucnv_setFromUCallBack(conv, flagCB_fromU, context, &(context->subCallback), &(context->subContext), &status); U_ASSERT(status); len2 = ucnv_fromUChars(conv, bytes, 100, uchars, len, &status); U_ASSERT(status); flagVal = context->flag; /* it's about to go away when we close the cnv */ ucnv_close(conv); /* print out the original source */ printBytes("bytes", bytes, len2); return flagVal; /* true if callback was called */ }
UBool convsample_21_didSubstitute(const char *source) { UChar uchars[100]; char bytes[100]; UConverter *conv = NULL, *cloneCnv = NULL; UErrorCode status = U_ZERO_ERROR; uint32_t len, len2; int32_t cloneLen; UBool flagVal = FALSE; UConverterFromUCallback junkCB; FromUFLAGContext *flagCtx = NULL, *cloneFlagCtx = NULL; debugCBContext *debugCtx1 = NULL, *debugCtx2 = NULL, *cloneDebugCtx = NULL; printf("\n\n==============================================\n" "Sample 21: C: Test for substitution w/ callbacks & clones \n"); /* print out the original source */ printBytes("src", source); printf("\n"); /* First, convert from UTF8 to unicode */ conv = ucnv_open("utf-8", &status); U_ASSERT(status); len = ucnv_toUChars(conv, uchars, 100, source, strlen(source), &status); U_ASSERT(status); printUChars("uch", uchars, len); printf("\n"); /* Now, close the converter */ ucnv_close(conv); /* Now, convert to windows-1252 */ conv = ucnv_open("windows-1252", &status); U_ASSERT(status); /* Converter starts out with the SUBSTITUTE callback set. */ /* initialize our callback */ /* from the 'bottom' innermost, out * CNV -> debugCtx1[debug] -> flagCtx[flag] -> debugCtx2[debug] */ #if DEBUG_TMI printf("flagCB_fromU = %p\n", &flagCB_fromU); printf("debugCB_fromU = %p\n", &debugCB_fromU); #endif debugCtx1 = debugCB_openContext(); flagCtx = flagCB_fromU_openContext(); debugCtx2 = debugCB_openContext(); debugCtx1->subCallback = flagCB_fromU; /* debug1 -> flag */ debugCtx1->subContext = flagCtx; flagCtx->subCallback = debugCB_fromU; /* flag -> debug2 */ flagCtx->subContext = debugCtx2; debugCtx2->subCallback = UCNV_FROM_U_CALLBACK_SUBSTITUTE; debugCtx2->subContext = NULL; /* Set our special callback */ ucnv_setFromUCallBack(conv, debugCB_fromU, debugCtx1, &(debugCtx2->subCallback), &(debugCtx2->subContext), &status); U_ASSERT(status); #if DEBUG_TMI printf("Callback chain now: Converter %p -> debug1:%p-> (%p:%p)==flag:%p -> debug2:%p -> cb %p\n", conv, debugCtx1, debugCtx1->subCallback, debugCtx1->subContext, flagCtx, debugCtx2, debugCtx2->subCallback); #endif cloneLen = 1; /* but passing in null so it will clone */ cloneCnv = ucnv_safeClone(conv, NULL, &cloneLen, &status); U_ASSERT(status); #if DEBUG_TMI printf("Cloned converter from %p -> %p. Closing %p.\n", conv, cloneCnv, conv); #endif ucnv_close(conv); #if DEBUG_TMI printf("%p closed.\n", conv); #endif U_ASSERT(status); /* Now, we have to extract the context */ cloneDebugCtx = NULL; cloneFlagCtx = NULL; ucnv_getFromUCallBack(cloneCnv, &junkCB, (const void **)&cloneDebugCtx); if(cloneDebugCtx != NULL) { cloneFlagCtx = (FromUFLAGContext*) cloneDebugCtx -> subContext; } printf("Cloned converter chain: %p -> %p[debug1] -> %p[flag] -> %p[debug2] -> substitute\n", cloneCnv, cloneDebugCtx, cloneFlagCtx, cloneFlagCtx?cloneFlagCtx->subContext:NULL ); len2 = ucnv_fromUChars(cloneCnv, bytes, 100, uchars, len, &status); U_ASSERT(status); if(cloneFlagCtx != NULL) { flagVal = cloneFlagCtx->flag; /* it's about to go away when we close the cnv */ } else { printf("** Warning, couldn't get the subcallback \n"); } ucnv_close(cloneCnv); /* print out the original source */ printBytes("bytes", bytes, len2); return flagVal; /* true if callback was called */ }
UErrorCode convert_to_utf8(const UChar* buffer, int32_t buffer_len, char** converted_buf, int32_t *converted_buf_len, bool force, bool* dropped_bytes) { UErrorCode status = U_ZERO_ERROR; UConverter *conv; int32_t utfConvertedLen = 0; // used to set dropped_bytes flag if force is true FromUFLAGContext * context = NULL; // open UTF8 converter conv = ucnv_open("utf-8", &status); if (U_FAILURE(status)) { ereport(WARNING, (errcode(ERRCODE_EXTERNAL_ROUTINE_EXCEPTION), errmsg("Cannot open utf-8 converter - error: %s.\n", u_errorName(status)))); ucnv_close(conv); return status; } if (force) { // set callback to skip illegal, irregular or unassigned bytes // set converter to use SKIP callback // contecxt will save and call it after calling custom callback ucnv_setFromUCallBack(conv, UCNV_FROM_U_CALLBACK_SKIP, NULL, NULL, NULL, &status); //TODO: refactor warning and error message reporting if (U_FAILURE(status)) { ereport(WARNING, (errcode(ERRCODE_EXTERNAL_ROUTINE_EXCEPTION), errmsg("Cannot set callback on converter - error: %s.\n", u_errorName(status)))); ucnv_close(conv); return status; } // initialize flagging callback context = flagCB_fromU_openContext(); /* Set our special callback */ ucnv_setFromUCallBack(conv, flagCB_fromU, context, &(context->subCallback), &(context->subContext), &status ); if (U_FAILURE(status)) { ereport(WARNING, (errcode(ERRCODE_EXTERNAL_ROUTINE_EXCEPTION), errmsg("Cannot set callback on converter - error: %s.\n", u_errorName(status)))); ucnv_close(conv); return status; } } // convert to UTF8 // input buffer from ucnv_toUChars, which always returns a // NUL-terminated buffer utfConvertedLen = ucnv_fromUChars(conv, *converted_buf, *converted_buf_len, buffer, STRING_IS_NULL_TERMINATED, &status ); if (U_SUCCESS(status)) { *converted_buf_len = utfConvertedLen; ereport(DEBUG1, (errcode(ERRCODE_SUCCESSFUL_COMPLETION), errmsg("Converted string: %s\n", (const char*) *converted_buf))); // see if any bytes where dropped // context struct will go away when converter is closed if (NULL != context) *dropped_bytes = context->flag; else *dropped_bytes = false; } if (U_FAILURE(status)) { ereport(WARNING, (errcode(ERRCODE_EXTERNAL_ROUTINE_EXCEPTION), errmsg("ICU conversion from Unicode to UTF8 failed - error: %s.\n", u_errorName(status)))); } // close the converter ucnv_close(conv); return status; }