Beispiel #1
0
status_t
vmi_convert_str_encoding(
    const unicode_string_t *in,
    unicode_string_t *out,
    const char *outencoding)
{
    iconv_t cd = 0;
    size_t iconv_val = 0;

    if (!in || !out)
        return VMI_FAILURE;

    size_t inlen = in->length;
    size_t outlen = 2 * (inlen + 1);

    char *incurr = (char*)in->contents;

    memset(out, 0, sizeof(*out));
    out->contents = safe_malloc(outlen);
    memset(out->contents, 0, outlen);

    char *outstart = (char*)out->contents;
    char *outcurr = (char*)out->contents;

    out->encoding = outencoding;

    cd = iconv_open(out->encoding, in->encoding);   // outset, inset
    if ((iconv_t) (-1) == cd) { // init failure
        if (EINVAL == errno) {
            dbprint(VMI_DEBUG_READ, "%s: conversion from '%s' to '%s' not supported\n",
                    __FUNCTION__, in->encoding, out->encoding);
        } else {
            dbprint(VMI_DEBUG_READ, "%s: Initializiation failure: %s\n", __FUNCTION__,
                    strerror(errno));
        }   // if-else
        goto fail;
    }   // if

    // init success

    iconv_val = iconv(cd, &incurr, &inlen, &outcurr, &outlen);
    if ((size_t) - 1 == iconv_val) {
        dbprint(VMI_DEBUG_READ, "%s: iconv failed, in string '%s' length %zu, "
                "out string '%s' length %zu\n", __FUNCTION__,
                in->contents, in->length, out->contents, outlen);
        switch (errno) {
            case EILSEQ:
                dbprint(VMI_DEBUG_READ, "invalid multibyte sequence");
                break;
            case EINVAL:
                dbprint(VMI_DEBUG_READ, "incomplete multibyte sequence");
                break;
            case E2BIG:
                dbprint(VMI_DEBUG_READ, "no more room");
                break;
            default:
                dbprint(VMI_DEBUG_READ, "error: %s\n", strerror(errno));
                break;
        }   // switch
        goto fail;
    }   // if failure

    // conversion success
    out->length = (size_t) (outcurr - outstart);
    (void) iconv_close(cd);
    return VMI_SUCCESS;

fail:
    if (out->contents) {
        free(out->contents);
    }
    // make failure really obvious
    memset(out, 0, sizeof(*out));

    if ((iconv_t) (-1) != cd) { // init succeeded
        (void) iconv_close(cd);
    }   // if

    return VMI_FAILURE;
}
Beispiel #2
0
/**************************************************************************
  Convert string from local encoding (8 bit char) to
  display encoding (16 bit unicode) and resut put in pToUniString.
  if pToUniString == NULL then resulting string will be allocate automaticaly.
  'ulength' give real sizeof 'pToUniString' array.

  Function return (Uint16 *) pointer to (new) pToUniString.
**************************************************************************/
Uint16 *convertcopy_to_utf16(Uint16 *pToUniString, size_t ulength,
                             const char *pFromString)
{
  /* Start Parametrs */
  const char *pTocode = get_display_encoding();
  const char *pFromcode = get_internal_encoding();
  const char *pStart = pFromString;
  
  size_t length = strlen(pFromString) + 1;

  Uint16 *pResult = pToUniString;
  /* ===== */

  iconv_t cd = iconv_open(pTocode, pFromcode);

  if (cd == (iconv_t) (-1)) {
    if (errno != EINVAL) {
      return pToUniString;
    }
  }
  
  if (!pResult) {
    /* From 8 bit code to UTF-16 (16 bit code) */
    ulength = length * 2;
    pResult = fc_calloc(1, ulength);
  }

  iconv(cd, NULL, NULL, NULL, NULL);	/* return to the initial state */

  /* Do the conversion for real. */
  {
    const char *pInptr = pStart;
    size_t Insize = length;

    char *pOutptr = (char *)pResult;
    size_t Outsize = ulength;

    while (Insize > 0 && Outsize > 0) {
      size_t Res =
        iconv(cd, (ICONV_CONST char **) &pInptr, &Insize, &pOutptr, &Outsize);
      if (Res == (size_t) (-1)) {
        if (errno == EINVAL) {
          break;
	} else {
          int saved_errno = errno;

          iconv_close(cd);
          errno = saved_errno;
          if (!pToUniString) {
            FC_FREE(pResult);
          }
          return pToUniString;
        }
      }
    }

    {
      size_t Res = iconv(cd, NULL, NULL, &pOutptr, &Outsize);
      if (Res == (size_t) (-1)) {
	int saved_errno = errno;

	iconv_close(cd);
	errno = saved_errno;
	if (!pToUniString) {
	  FC_FREE(pResult);
	}
	return pToUniString;
      }
    }

  }

  iconv_close(cd);

  return (Uint16 *) pResult;
}
Beispiel #3
0
/**
 * iconv_convert : convert a string, using the current codeset
 * return: a malloc'd string with the converted result
 */
char *
iconv_convert (const char *input)
{
//#if HAVE_ICONV
#if 0	// ASUS EXT
  size_t inputsize = strlen (input) + 1;
  size_t dummy = 0;
  size_t length = 0;
  char *result;
  char *inptr, *outptr;
  size_t insize, outsize;

  /* conversion not necessary. save our time. */
  if (!cd)
    return strdup (input);

  /* Determine the length we need. */
  iconv (cd, NULL, NULL, NULL, &dummy);
  {
    static char tmpbuf[BUFSIZ];
    inptr = (char*) input;
    insize = inputsize;
    while (insize > 0)
    {
      outptr = tmpbuf;
      outsize = BUFSIZ;
      if (iconv (cd, &inptr, &insize, &outptr, &outsize) == (size_t) (-1))
      {
        /**
         * if error is EINVAL or EILSEQ, conversion must be stoped,
         * but if it is E2BIG (not enough space in buffer), we just loop again
         */
        if( errno != E2BIG)
        {
          perror ("error iconv");
          return NULL;
        }
      }
      length += outptr - tmpbuf;
    }

    outptr = tmpbuf;
    outsize = BUFSIZ;
    if (iconv (cd, NULL, NULL, &outptr, &outsize) == (size_t) (-1))
    {
      perror ("error iconv");
      return NULL;
    }
    length += outptr - tmpbuf;
  }

  /* length determined, allocate result space */
  if ((result = (char*) malloc (length * sizeof (char))) == NULL)
  {
    perror ("error malloc");
    return NULL;
  }

  /* Do the conversion for real. */
  iconv (cd, NULL, NULL, NULL, &dummy);
  {
    inptr = (char*) input;
    insize = inputsize;
    outptr = result;
    outsize = length;
    while (insize > 0)
    {
      if (iconv (cd, &inptr, &insize, &outptr, &outsize) == (size_t) (-1))
      {
        if (errno != E2BIG)
        {
          perror ("error iconv");
          free (result);
          return NULL;
        }
      }
    }
    if (iconv (cd, NULL, NULL, &outptr, &outsize) == (size_t) (-1))
    {
      perror ("error iconv");
      free (result);
      return NULL;
    }

    if (outsize != 0)
      abort ();
  }

  return result;
#else
  return strdup (input);
#endif
}
Beispiel #4
0
EAPI char *
eina_str_convert(const char *enc_from, const char *enc_to, const char *text)
{
   iconv_t ic;
   char *new_txt, *outp;
   const char *inp;
   size_t inb, outb, outlen, tob, outalloc;

   if (!text)
      return NULL;

   ic = iconv_open(enc_to, enc_from);
   if (ic == (iconv_t)(-1))
      return NULL;

   new_txt = malloc(64);
   inb = strlen(text);
   outb = 64;
   inp = text;
   outp = new_txt;
   outalloc = 64;
   outlen = 0;

   for (;; )
     {
        size_t count;

        tob = outb;
#ifdef __FreeBSD__
        count = iconv(ic, &inp, &inb, &outp, &outb);
#else
        count = iconv(ic, (char **)&inp, &inb, &outp, &outb);
#endif
        outlen += tob - outb;
        if (count == (size_t)(-1))
          {
             if (errno == E2BIG)
               {
                  new_txt = realloc(new_txt, outalloc + 64);
                  outp = new_txt + outlen;
                  outalloc += 64;
                  outb += 64;
               }
             else
               {
                  if (new_txt)
                     free(new_txt);

                  new_txt = NULL;
                  break;
               }
          }

        if (inb == 0)
          {
             if (outalloc == outlen)
                new_txt = realloc(new_txt, outalloc + 1);

             new_txt[outlen] = 0;
             break;
          }
     }
   iconv_close(ic);
   return new_txt;
}
Beispiel #5
0
  bool CharsetConverter::Convert(const std::string& sSource, std::string& sResult)
  {
    sResult.erase();
#ifndef WITH_ICONV
    return false;
#else
    if (m_pIconv == reinterpret_cast<iconv_t>(-1))
    {
      return false;
    }

    const static size_t nOutBuffSize = 256;
    char szOutBuff[nOutBuffSize];
    size_t nOut = nOutBuffSize;

#if defined _LIBICONV_VERSION && _LIBICONV_VERSION >= 0x010D
    const char* szSrc = sSource.data();
    const char* szSrcCurr = NULL;
#else
    char* szSrc = const_cast<char*>(sSource.data());
    char* szSrcCurr = NULL;
#endif
    size_t nSrcSize = sSource.size();
    size_t nSrcPos = 0;
    size_t nSrcCurrSize = 0;
    size_t nSrcCurrSizeRead = 0;

    char* szOutTmp = NULL;
    size_t nRet = static_cast<size_t>(-1);

    while (nSrcPos != nSrcSize)
    {
      nSrcCurrSize = Min(nOutBuffSize / 4, nSrcSize - nSrcPos);
      nSrcCurrSizeRead = nSrcCurrSize;
      szSrcCurr = szSrc + nSrcPos;
      szOutTmp = szOutBuff;

#ifndef sun
      nRet = iconv(m_pIconv, &szSrcCurr, &nSrcCurrSizeRead, &szOutTmp, &nOut);
#else
      nRet = iconv(m_pIconv, const_cast<const char **>(&szSrcCurr), &nSrcCurrSizeRead, &szOutTmp, &nOut);
#endif
      if (nRet == static_cast<size_t>(-1))
      {
        switch (errno)
        {
        case EINVAL:
          break;
        case EILSEQ:
          --nSrcCurrSizeRead;
          break;
        default:
          LogError() << "Error iconv: " << Error::GetLastErrorStr();
          iconv_close(m_pIconv);
          return false;
        }
      }

      sResult.append(szOutBuff, nOutBuffSize - nOut);
      nOut = nOutBuffSize;
      nSrcPos += nSrcCurrSize - nSrcCurrSizeRead;
    }

    return true;
#endif
  }
Beispiel #6
0
/*
 * This function is used in writing text data to an MDB table.
 * If slen is 0, strlen will be used to calculate src's length.
 */
int
mdb_ascii2unicode(MdbHandle *mdb, char *src, size_t slen, char *dest, size_t dlen)
{
        size_t len_in, len_out;
        char *in_ptr, *out_ptr;

	if ((!src) || (!dest) || (!dlen))
		return 0;

        in_ptr = src;
        out_ptr = dest;
        len_in = (slen) ? slen : strlen(in_ptr);
        len_out = dlen;

#ifdef HAVE_ICONV
	iconv(mdb->iconv_out, &in_ptr, &len_in, &out_ptr, &len_out);
	//printf("len_in %d len_out %d\n", len_in, len_out);
	dlen -= len_out;
#else
	if (IS_JET3(mdb)) {
		dlen = MIN(len_in, len_out);
		strncpy(out_ptr, in_ptr, dlen);
	} else {
		unsigned int i;
		slen = MIN(len_in, len_out/2);
		dlen = slen*2;
		for (i=0; i<slen; i++) {
			out_ptr[i*2] = in_ptr[i];
			out_ptr[i*2+1] = 0;
		}
	}
#endif

	/* Unicode Compression */
	if(IS_JET4(mdb) && (dlen>4)) {
		unsigned char *tmp = g_malloc(dlen);
		unsigned int tptr = 0, dptr = 0;
		int comp = 1;

		tmp[tptr++] = 0xff;
		tmp[tptr++] = 0xfe;
		while((dptr < dlen) && (tptr < dlen)) {
			if (((dest[dptr+1]==0) && (comp==0))
			 || ((dest[dptr+1]!=0) && (comp==1))) {
				/* switch encoding mode */
				tmp[tptr++] = 0;
				comp = (comp) ? 0 : 1;
			} else if (dest[dptr]==0) {
				/* this string cannot be compressed */
				tptr = dlen;
			} else if (comp==1) {
				/* encode compressed character */
				tmp[tptr++] = dest[dptr];
				dptr += 2;
			} else if (tptr+1 < dlen) {
				/* encode uncompressed character */
				tmp[tptr++] = dest[dptr];
				tmp[tptr++] = dest[dptr+1];
				dptr += 2;
			} else {
				/* could not encode uncompressed character
				 * into single byte */
				tptr = dlen;
			}
		}
		if (tptr < dlen) {
			memcpy(dest, tmp, tptr);
			dlen = tptr;
		}
		g_free(tmp);
	}

	return dlen;
}
Beispiel #7
0
/* Convert TEXT which is encoded according to LC_CTYPE to UTF-8.  With
   SECURE set to true, use secure memory for the returned buffer.
   Return NULL on error. */
char *
pinentry_local_to_utf8 (char *lc_ctype, char *text, int secure)
{
  char *old_ctype;
  char *source_encoding;
  iconv_t cd;
  const char *input = text;
  size_t input_len = strlen (text) + 1;
  char *output;
  size_t output_len;
  char *output_buf;
  size_t processed;

  /* If no locale setting could be determined, simply copy the
     string.  */
  if (!lc_ctype)
    {
      fprintf (stderr, "%s: no LC_CTYPE known - assuming UTF-8\n",
               this_pgmname);
      output_buf = secure? secmem_malloc (input_len) : malloc (input_len);
      if (output_buf)
        strcpy (output_buf, input);
      return output_buf;
    }

  old_ctype = strdup (setlocale (LC_CTYPE, NULL));
  if (!old_ctype)
    return NULL;
  setlocale (LC_CTYPE, lc_ctype);
  source_encoding = nl_langinfo (CODESET);
  setlocale (LC_CTYPE, old_ctype);
  free (old_ctype);

  /* This is overkill, but simplifies the iconv invocation greatly.  */
  output_len = input_len * MB_LEN_MAX;
  output_buf = output = secure? secmem_malloc (output_len):malloc (output_len);
  if (!output)
    return NULL;

  cd = iconv_open ("UTF-8", source_encoding);
  if (cd == (iconv_t) -1)
    {
      fprintf (stderr, "%s: can't convert from %s to UTF-8: %s\n",
               this_pgmname, source_encoding? source_encoding : "?",
               strerror (errno));
      if (secure)
        secmem_free (output_buf);
      else
        free (output_buf);
      return NULL;
    }
  processed = iconv (cd, &input, &input_len, &output, &output_len);
  iconv_close (cd);
  if (processed == (size_t) -1 || input_len)
    {
      fprintf (stderr, "%s: error converting from %s to UTF-8: %s\n",
               this_pgmname, source_encoding? source_encoding : "?",
               strerror (errno));
      if (secure)
        secmem_free (output_buf);
      else
        free (output_buf);
      return NULL;
    }
  return output_buf;
}
Beispiel #8
0
static intptr_t
file_write_using_iconv(struct OMRPortLibrary *portLibrary, intptr_t fd, const char *buf, intptr_t nbytes)
{
	intptr_t result = 0;
	char stackBuf[512];
	char *bufStart = NULL;
	uintptr_t outBufLen = sizeof(stackBuf);

	iconv_t converter = J9VM_INVALID_ICONV_DESCRIPTOR;
	size_t inbytesleft = 0;
	size_t outbytesleft = 0;
	char *inbuf = NULL;
	char *outbuf = NULL;
	intptr_t bytesToWrite = 0;

#ifdef J9ZOS390
	/* LIR 1280 (z/OS only) - every failed call to iconv_open() is recorded on the operator console, so don't retry */
	if (FALSE == PPG_file_text_iconv_open_failed) {
		/* iconv_get is not an a2e function, so we need to pass it honest-to-goodness EBCDIC strings */
#pragma convlit(suspend)
#endif

#ifndef OMRZTPF
		converter = iconv_get(portLibrary, J9FILETEXT_ICONV_DESCRIPTOR, nl_langinfo(CODESET), "UTF-8");
#else
		converter = iconv_get(portLibrary, J9FILETEXT_ICONV_DESCRIPTOR, "IBM1047", "ISO8859-1" );
#endif

#ifdef J9ZOS390
#pragma convlit(resume)
		if (J9VM_INVALID_ICONV_DESCRIPTOR == converter) {
			PPG_file_text_iconv_open_failed = TRUE;
		}
	}
#endif

	if (J9VM_INVALID_ICONV_DESCRIPTOR == converter) {
		/* no converter available for this code set. Just dump the UTF-8 chars */
		result = portLibrary->file_write(portLibrary, fd, (void *)buf, nbytes);
		return (result == nbytes) ? 0 : result;
	}

	inbuf = (char *)buf; /* for some reason this argument isn't const */
	outbuf = bufStart = stackBuf;
	inbytesleft = nbytes;
	outbytesleft = sizeof(stackBuf);

	while ((size_t)-1 == iconv(converter, &inbuf, &inbytesleft, &outbuf, &outbytesleft)) {
		int tmp_errno = errno;

		if (inbytesleft == 0) {
			break;
		}

		if ((outbytesleft == 0) || (tmp_errno == E2BIG)) {
			/* input conversion stopped due to lack of space in the output buffer */

			if (growBuffer(portLibrary, stackBuf, &bufStart, &outbuf, &outbytesleft, &outBufLen) < 0) {
				/* failed to grow buffer, just output what we've got so far */
				break;
			}

		} else if (tmp_errno == EILSEQ) {
			/* input conversion stopped due to an input byte that does not belong to the input code set */

			const char *unicodeFormat = "\\u%04x";
#define J9FILETEXT_ESCAPE_STR_SIZE 6 /* max size of unicode format */
			char escapedStr[J9FILETEXT_ESCAPE_STR_SIZE];
			char *escapedStrStart = escapedStr;

			uint16_t unicodeC = 0;
			size_t escapedLength = 0;
			size_t utf8Length = decodeUTF8CharN((const uint8_t *)inbuf, &unicodeC, inbytesleft);

			if (utf8Length == 0) {
				/* invalid encoding, including 4-byte UTF-8 */
				utf8Length = 1;
				escapedLength = 1;
				escapedStr[0] = '?';
			} else {
				escapedLength = portLibrary->str_printf(portLibrary, escapedStr, J9FILETEXT_ESCAPE_STR_SIZE, unicodeFormat, (uintptr_t)unicodeC);
			}

			inbytesleft -= utf8Length;
			inbuf += utf8Length;

			if ((size_t)-1 == iconv(converter, &escapedStrStart, &escapedLength, &outbuf, &outbytesleft)) {
				/* not handling EILSEQ here because:
				 *  1. we can't do much if iconv() fails to convert ascii.
				 *  2. inbuf and inbytesleft have been explicitly updated so the while loop will get terminated after converting the rest of the characters.
				 */

				tmp_errno = errno;

				/* if the remaining outbuf is too small, then grow it before storing Unicode string representation */
				if (tmp_errno == E2BIG) {
					if (growBuffer(portLibrary, stackBuf, &bufStart, &outbuf, &outbytesleft, &outBufLen) < 0) {
						/* failed to grow buffer, just output what we've got so far */
						break;
					}
				}
			}
		} else {
			/* input conversion stopped due to an incomplete character or shift sequence at the end of the input buffer */
			break;
		}
	}

	iconv_free(portLibrary, J9FILETEXT_ICONV_DESCRIPTOR, converter);

	/* CMVC 152575 - the converted string is not necessarily the same length in bytes as the original string */
	bytesToWrite = outbuf - bufStart;
	result = portLibrary->file_write(portLibrary, fd, (void *)bufStart, bytesToWrite);

	if (bufStart != stackBuf) {
		portLibrary->mem_free_memory(portLibrary, bufStart);
	}

	return (result == bytesToWrite) ? 0 : result;
}
Beispiel #9
0
char *mdx_make_sjis_to_syscharset(char* str) {
#if 0	
	iconv_t fd = 0;
	size_t len = 0;
	char* result = NULL;
	char* rr = NULL;
	size_t remain=0, oremain=0;
	char* cur = NULL;
	const char* loc = (const char *)"UTF-8";
	unsigned char src[3];
	unsigned char dst[7];
	int ptr = 0;
	size_t sl=0;
	size_t dl=0;
	
	cur = setlocale(LC_CTYPE, "");
	if (!cur) {
		goto error_end;
	}
	if (strstr(cur, "eucJP")) {
		loc = (const char *)"EUC-JP";
	}
	
	fd = iconv_open(loc, "SHIFT-JIS");
	if (fd<0) {
		goto error_end;
	}
	
	/* enough for store utf8 */
	remain = strlen(str);
	if (remain==0) {
		goto error_end;
	}
	oremain = remain*6;
	result = (char *)malloc(sizeof(char)*oremain);
	if (!result) {
		goto error_end;
	}
	memset((void *)result, 0, oremain);
	rr = result;
	
	ptr=0;
	/* to omit X68k specific chars, process charconv one by one */
	do {
		char *sp, *dp;
		unsigned char c = 0;
		
		memset((void *)src, 0, sizeof(src));
		memset((void *)dst, 0, sizeof(dst));
		oremain = 0;
		
		c = src[0] = (unsigned char)str[ptr++];
		sl=1;
		if (!c) break;
		if (c==0x80 || (c>=0x81 && c<=0x84) || (c>=0x88 && c<=0x9f) ||
			(c>=0xe0 && c<=0xea) || (c>=0xf0 && c<=0xf3) || c==0xf6) {
			src[1] = (unsigned char)str[ptr++];
			if (!src[1]) {
				strcat(result, ".");
				break;
			}
			sl++;
		}
		
		sp = (char *)src;
		dp = (char *)dst;
		
		dl = 7;
		len = iconv(fd, (char **)&sp, &sl, (char **)&dp, &dl);
		if (len==(size_t)(-1)) {
			strcat(result, ".");
		} else {
			strcat(result, (char *)dst);
		}
	} while (1);
	
	iconv_close(fd);
	return result;
	
error_end:
	if (result) {
		free(result);
	}
	if (fd>=0) {
		iconv_close(fd);
	}
#endif	
	return strdup(str);
}
Beispiel #10
0
int UTF8ToUCS4(char*inbuf,unsigned int len_src,char*outbuf,unsigned int lenout){iconv_t conv;conv=iconv_open("UCS-4LE","UTF-8");iconv(conv,NULL,NULL,NULL,NULL);int ret;ret=iconv(conv,(const char**)&inbuf,&len_src,&outbuf,&lenout);iconv_close(conv);return ret;}int Draw_string_line(char*_filename,int _font_size,int _y_offset){char*ttf_path="brlcd.ttf";unsigned int ucode_array[50];int array_counter=0;int max_font_height=0;int max_font_width=0;static int _DoinitLib=1;FT_UInt glyph_index;int x_offset=0;int y_offset=_y_offset;const int oled_width=128;const int oled_height=64;char bufout[512];unsigned int len_src=strlen(_filename);unsigned int*out_text;
Beispiel #11
0
/**
 * Transcode one bstr.
 *
 * @param[in] cd
 * @param[in] input
 * @param[in] output
 */
int htp_transcode_bstr(iconv_t cd, bstr *input, bstr **output) {
    // Reset conversion state for every new string
    iconv(cd, NULL, 0, NULL, 0);

    bstr_builder_t *bb = NULL;

    const size_t buflen = 10;
    unsigned char *buf = malloc(buflen);
    if (buf == NULL) {
        return HTP_ERROR;
    }

    const char *inbuf = (const char *)bstr_ptr(input);
    size_t inleft = bstr_len(input);
    char *outbuf = (char *)buf;
    size_t outleft = buflen;

    int loop = 1;
    while (loop) {
        loop = 0;

        if (iconv(cd, (ICONV_CONST char **)&inbuf, &inleft, (char **)&outbuf, &outleft) == (size_t) - 1) {
            if (errno == E2BIG) {
                // Create bstr builder on-demand
                if (bb == NULL) {
                    bb = bstr_builder_create();
                    if (bb == NULL) {
                        free(buf);
                        return HTP_ERROR;
                    }
                }

                // The output buffer is full
                bstr_builder_append_mem(bb, buf, buflen - outleft);

                outbuf = (char *)buf;
                outleft = buflen;

                // Continue in the loop, as there's more work to do
                loop = 1;
            } else {
                // Error
                if (bb != NULL) bstr_builder_destroy(bb);
                free(buf);
                return HTP_ERROR;
            }
        }
    }
    
    if (bb != NULL) {
        bstr_builder_append_mem(bb, buf, buflen - outleft);
        *output = bstr_builder_to_str(bb);
        bstr_builder_destroy(bb);
        if (*output == NULL) {
            free(buf);
            return HTP_ERROR;
        }
    } else {
        *output = bstr_dup_mem(buf, buflen - outleft);
        if (*output == NULL) {
            free(buf);
            return HTP_ERROR;
        }
    }
    
    free(buf);

    return HTP_OK;
}
/*
 * Curl_convert_from_utf8() is an internal function for performing UTF-8
 * conversions on non-ASCII platforms.
 */
CURLcode Curl_convert_from_utf8(struct SessionHandle *data,
                                char *buffer, size_t length)
{
  CURLcode rc;

  if(data->set.convfromutf8) {
    /* use translation callback */
    rc = data->set.convfromutf8(buffer, length);
    if(rc != CURLE_OK) {
      failf(data,
            "CURLOPT_CONV_FROM_UTF8_FUNCTION callback returned %d: %s",
            (int)rc, curl_easy_strerror(rc));
    }
    return rc;
  }
  else {
#ifdef HAVE_ICONV
    /* do the translation ourselves */
    const char *input_ptr;
    char *output_ptr;
    size_t in_bytes, out_bytes, rc;
    int error;

    /* open an iconv conversion descriptor if necessary */
    if(data->utf8_cd == (iconv_t)-1) {
      data->utf8_cd = iconv_open(CURL_ICONV_CODESET_OF_HOST,
                                 CURL_ICONV_CODESET_FOR_UTF8);
      if(data->utf8_cd == (iconv_t)-1) {
        error = ERRNO;
        failf(data,
              "The iconv_open(\"%s\", \"%s\") call failed with errno %i: %s",
              CURL_ICONV_CODESET_OF_HOST,
              CURL_ICONV_CODESET_FOR_UTF8,
              error, strerror(error));
        return CURLE_CONV_FAILED;
      }
    }
    /* call iconv */
    input_ptr = output_ptr = buffer;
    in_bytes = out_bytes = length;
    rc = iconv(data->utf8_cd, &input_ptr, &in_bytes,
               &output_ptr, &out_bytes);
    if((rc == ICONV_ERROR) || (in_bytes != 0)) {
      error = ERRNO;
      failf(data,
            "The Curl_convert_from_utf8 iconv call failed with errno %i: %s",
            error, strerror(error));
      return CURLE_CONV_FAILED;
    }
    if(output_ptr < input_ptr) {
      /* null terminate the now shorter output string */
      *output_ptr = 0x00;
    }
#else
    failf(data, "CURLOPT_CONV_FROM_UTF8_FUNCTION callback required");
    return CURLE_CONV_REQD;
#endif /* HAVE_ICONV */
  }

  return CURLE_OK;
}
int
main (void)
{
  char name[] = "/tmp/widetext.out.XXXXXX";
  char mbbuf[SIZE];
  char mb2buf[SIZE];
  wchar_t wcbuf[SIZE];
  wchar_t wc2buf[SIZE];
  size_t mbsize;
  size_t wcsize;
  int fd;
  FILE *fp;
  size_t n;
  int res;
  int status = 0;
  wchar_t *wcp;

  setlocale (LC_ALL, "de_DE.UTF-8");
  printf ("locale used: %s\n\n", setlocale (LC_ALL, NULL));

  /* Read the file into memory.  */
  mbsize = fread (mbbuf, 1, SIZE, stdin);
  if (mbsize == 0)
    {
      printf ("%Zd: cannot read input file from standard input: %m\n",
	      __LINE__);
      exit (1);
    }

   printf ("INFO: input file has %Zd bytes\n", mbsize);

  /* First convert the text to wide characters.  We use iconv here.  */
  {
    iconv_t cd;
    char *inbuf = mbbuf;
    size_t inleft = mbsize;
    char *outbuf = (char *) wcbuf;
    size_t outleft = sizeof (wcbuf);
    size_t nonr;

    cd = iconv_open ("WCHAR_T", "UTF-8");
    if (cd == (iconv_t) -1)
      {
	printf ("%Zd: cannot get iconv descriptor for conversion to UCS4\n",
		__LINE__);
	exit (1);
      }

    /* We must need only one call and there must be no losses.  */
    nonr = iconv (cd, &inbuf, &inleft, &outbuf, &outleft);
    if (nonr != 0 && nonr != (size_t) -1)
      {
	printf ("%Zd: iconv performed %Zd nonreversible conversions\n",
		__LINE__, nonr);
	exit (1);
      }

    if  ((size_t) nonr == -1 )
      {
	printf ("\
%Zd: iconv returned with %Zd and errno = %m (inleft: %Zd, outleft: %Zd)\n",
		__LINE__, nonr, inleft, outleft);
	exit (1);
      }

    if (inleft != 0)
      {
	printf ("%Zd: iconv didn't convert all input\n", __LINE__);
	exit (1);
      }

    iconv_close (cd);

    if ((sizeof (wcbuf) - outleft) % sizeof (wchar_t) != 0)
      {
	printf ("%Zd: iconv converted not complete wchar_t\n", __LINE__);
	exit (1);
      }

    wcsize = (sizeof (wcbuf) - outleft) / sizeof (wchar_t);
    assert (wcsize + 1 <= SIZE);
  }
static ngx_int_t
ngx_http_do_iconv(ngx_http_request_t *r, ngx_chain_t **c, void *data,
    size_t len, u_char *from, u_char *to, size_t *conved_bytes,
    size_t *rest_bytes)
{
    iconv_t           cd;
    ngx_chain_t      *cl, *chain, **ll;
    ngx_buf_t        *b;
    size_t            cv, rest, rv;

    cv = 0;
    dd("iconv from=%s, to=%s", from, to);
    cd = iconv_open((const char *) to, (const char *) from);

    if (cd == (iconv_t) -1) {
        dd("iconv open error");
        return NGX_ERROR;
    }

    dd("len=%zu, iconv_buf_size=%zu", len, iconv_buf_size);
    ll = &chain;

conv_begin:

    while (len) {
        cl = ngx_alloc_chain_link(r->pool);
        if (cl == NULL) {
            iconv_close(cd);
            return NGX_ERROR;
        }
        /* --- b->temporary--- */
        b = ngx_create_temp_buf(r->pool, iconv_buf_size);
        if (b == NULL) {
            iconv_close(cd);
            return NGX_ERROR;
        }

        cl->buf = b;
        rest = iconv_buf_size;

        do {
            rv = iconv(cd, (void *) &data, &len, (void *) &b->last, &rest);

            if (rv == (size_t) -1) {
                if (errno == EINVAL) {
                    cv += iconv_buf_size - rest;
                    dd("iconv error:EINVAL,len=%d cv=%d rest=%d", (int) len,
                        (int) cv, (int) rest);
                    goto conv_done;
                }

                if (errno == E2BIG) {
                    dd("E2BIG");
                    /* E2BIG error is not considered*/
                    break;
                }

                if (errno == EILSEQ) {
                    ngx_log_error(NGX_LOG_NOTICE, r->connection->log, 0,
                                  "iconv sees invalid character sequence "
                                  "(EILSEQ)");

                    if (len >= 1) {
                        if (rest == 0) {
                            dd("EILSEQ:rest=0");
                            cv += iconv_buf_size - rest;
                            *ll = cl;
                            ll = &cl->next;
                            goto conv_begin;

                        } else {
                            dd("EILSEQ:rest=%d", (int)rest);
                            len--;
                            data = (u_char *)data + 1;
                            /* replace illegal character to '?' */
                            *b->last = '?';
                            b->last = (u_char *)b->last + 1;
                            rest--;
                        }

                    } else {
                        goto conv_done;
                    }

                }
            }

        } while (rv == (size_t) -1 && errno == EILSEQ);

        /* E2BIG error is not considered*/
        /* this code can work but stops when meet illegal encoding */
        /*
        if (rv == (size_t) -1) {
            if (errno == EILSEQ) {
                ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
                        "iconv error:EILSEQ");
                iconv_close(cd);
                return NGX_ERROR;
            }

            if (errno == EINVAL) {
                cv += iconv_buf_size - rest;
                dd("iconv error:EINVAL,len=%d cv=%d rest=%d", (int) len,
                        (int) cv, (int) rest);
                break;
            } else if (errno == E2BIG) {
                dd("iconv error:E2BIG");
            }
        }
        */
        cv += iconv_buf_size - rest;
        *ll = cl;
        ll = &cl->next;
    }

conv_done:

    *ll = NULL;
    if (conved_bytes) {
        *conved_bytes = cv;
    }

    if (rest_bytes) {
        dd("rest bytes:%zu", len);
        *rest_bytes = len;
    }

    if (c) {
        dd("chain: %p", chain);

        /* chain may be null */
        /*
        dd("conv done: chain buf: %.*s",
                (int) (chain->buf->last - chain->buf->pos),
                chain->buf->last);
        */
        *c = chain;
    }

    dd("out");
    iconv_close(cd);
    return NGX_OK;
}
Beispiel #15
0
char *
str_cd_iconv (const char *src, iconv_t cd)
{
  /* For most encodings, a trailing NUL byte in the input will be converted
     to a trailing NUL byte in the output.  But not for UTF-7.  So that this
     function is usable for UTF-7, we have to exclude the NUL byte from the
     conversion and add it by hand afterwards.  */
# if !defined _LIBICONV_VERSION && !defined __GLIBC__
  /* Irix iconv() inserts a NUL byte if it cannot convert.
     NetBSD iconv() inserts a question mark if it cannot convert.
     Only GNU libiconv and GNU libc are known to prefer to fail rather
     than doing a lossy conversion.  For other iconv() implementations,
     we have to look at the number of irreversible conversions returned;
     but this information is lost when iconv() returns for an E2BIG reason.
     Therefore we cannot use the second, faster algorithm.  */

  char *result = NULL;
  size_t length = 0;
  int retval = mem_cd_iconv (src, strlen (src), cd, &result, &length);
  char *final_result;

  if (retval < 0)
    {
      if (result != NULL)
	abort ();
      return NULL;
    }

  /* Add the terminating NUL byte.  */
  final_result =
    (result != NULL ? realloc (result, length + 1) : malloc (length + 1));
  if (final_result == NULL)
    {
      if (result != NULL)
	free (result);
      errno = ENOMEM;
      return NULL;
    }
  final_result[length] = '\0';

  return final_result;

# else
  /* This algorithm is likely faster than the one above.  But it may produce
     iconv() returns for an E2BIG reason, when the output size guess is too
     small.  Therefore it can only be used when we don't need the number of
     irreversible conversions performed.  */
  char *result;
  size_t result_size;
  size_t length;
  const char *inptr = src;
  size_t inbytes_remaining = strlen (src);

  /* Make a guess for the worst-case output size, in order to avoid a
     realloc.  It's OK if the guess is wrong as long as it is not zero and
     doesn't lead to an integer overflow.  */
  result_size = inbytes_remaining;
  {
    size_t approx_sqrt_SIZE_MAX = SIZE_MAX >> (sizeof (size_t) * CHAR_BIT / 2);
    if (result_size <= approx_sqrt_SIZE_MAX / MB_LEN_MAX)
      result_size *= MB_LEN_MAX;
  }
  result_size += 1; /* for the terminating NUL */

  result = (char *) malloc (result_size);
  if (result == NULL)
    {
      errno = ENOMEM;
      return NULL;
    }

  /* Avoid glibc-2.1 bug and Solaris 2.7-2.9 bug.  */
# if defined _LIBICONV_VERSION \
     || !((__GLIBC__ - 0 == 2 && __GLIBC_MINOR__ - 0 <= 1) || defined __sun)
  /* Set to the initial state.  */
  iconv (cd, NULL, NULL, NULL, NULL);
# endif

  /* Do the conversion.  */
  {
    char *outptr = result;
    size_t outbytes_remaining = result_size - 1;

    for (;;)
      {
	/* Here inptr + inbytes_remaining = src + strlen (src),
		outptr + outbytes_remaining = result + result_size - 1.  */
	size_t res = iconv (cd,
			    (ICONV_CONST char **) &inptr, &inbytes_remaining,
			    &outptr, &outbytes_remaining);

	if (res == (size_t)(-1))
	  {
	    if (errno == EINVAL)
	      break;
	    else if (errno == E2BIG)
	      {
		size_t used = outptr - result;
		size_t newsize = result_size * 2;
		char *newresult;

		if (!(newsize > result_size))
		  {
		    errno = ENOMEM;
		    goto failed;
		  }
		newresult = (char *) realloc (result, newsize);
		if (newresult == NULL)
		  {
		    errno = ENOMEM;
		    goto failed;
		  }
		result = newresult;
		result_size = newsize;
		outptr = result + used;
		outbytes_remaining = result_size - 1 - used;
	      }
	    else
	      goto failed;
	  }
	else
	  break;
      }
    /* Avoid glibc-2.1 bug and Solaris 2.7 bug.  */
# if defined _LIBICONV_VERSION \
     || !((__GLIBC__ - 0 == 2 && __GLIBC_MINOR__ - 0 <= 1) || defined __sun)
    for (;;)
      {
	/* Here outptr + outbytes_remaining = result + result_size - 1.  */
	size_t res = iconv (cd, NULL, NULL, &outptr, &outbytes_remaining);

	if (res == (size_t)(-1))
	  {
	    if (errno == E2BIG)
	      {
		size_t used = outptr - result;
		size_t newsize = result_size * 2;
		char *newresult;

		if (!(newsize > result_size))
		  {
		    errno = ENOMEM;
		    goto failed;
		  }
		newresult = (char *) realloc (result, newsize);
		if (newresult == NULL)
		  {
		    errno = ENOMEM;
		    goto failed;
		  }
		result = newresult;
		result_size = newsize;
		outptr = result + used;
		outbytes_remaining = result_size - 1 - used;
	      }
	    else
	      goto failed;
	  }
	else
	  break;
      }
# endif

    /* Add the terminating NUL byte.  */
    *outptr++ = '\0';

    length = outptr - result;
  }

  /* Give away unused memory.  */
  if (length < result_size)
    {
      char *smaller_result = (char *) realloc (result, length);

      if (smaller_result != NULL)
	result = smaller_result;
    }

  return result;

 failed:
  {
    int saved_errno = errno;
    free (result);
    errno = saved_errno;
    return NULL;
  }

# endif
}
Beispiel #16
0
void _converter_( void *from, unsigned long from_len,
			void **to,  unsigned long *to_len )
{
    char          *InBuf;
    size_t        InBytesLeft;
    char          *OutBuf = NULL;
    size_t        OutBytesLeft = 0;
    size_t        _OutBytesLeft = 0;
    size_t        iconv_ret;
    size_t        converted_num = 0;

    *to = NULL;
    *to_len = 0;

    if ( shouldAlloc1 ) {
        /* Obtain work area */
        _i18nwork1 = (size_t *)malloc( WORKSIZE );
        if ( !_i18nwork1 ) {
            _i18nwork1 = NULL;
            return;
        }
        _i18nsize1 = WORKSIZE; 
        shouldAlloc1 = 0;
    }
#ifdef _AIX
    if ( isFirstCall ) {
	if ( ( CD = iconv_open( "IBM-eucJP", "IBM-932" ) ) == (iconv_t)-1 )
	    return; /* no converter */
	amI_932 = !strncasecmp( "IBM-932", nl_langinfo( CODESET ), 7 );
	isFirstCall = 0;
    }
#endif /* _AIX */

    if ( ( !amI_932 ) || ( CD == (iconv_t)-1 ) )
	return;

    InBuf        = (char *)from;
    InBytesLeft  = from_len;
    OutBytesLeft = _i18nsize1;
    OutBuf = (char *)_i18nwork1;

    while( 1 ) {
	/*
	 * InBuf
	 *  v
	 * +----------------------------+
	 * | |                        | |
	 * +----------------------------+
	 *  <-------------------------->
	 *          InBytesLeft
	 *
	 *             |
	 *             | iconv()
	 *             V
	 * (_i18nwork1)
	 * OutBuf
	 *  v
	 * +----------------------------+
	 * | |                        | |
	 * +----------------------------+
	 *  <-------------------------->
	 *          InBytesLeft
	 */

	iconv_ret = iconv( CD, (ICONV_INBUF_TYPE)&InBuf, &InBytesLeft,
                               &OutBuf, &OutBytesLeft );
	if ( iconv_ret == 0 ) {
	    /* iconv done
	     *                             InBuf
	     *                               v
	     * +----------------------------+
	     * |XXXXXXXXXXXXXXXXXXXXXXXXXXXX|
	     * +----------------------------+
	     *                               
	     *                               InBytesLeft=0
	     *
	     * (_i18nwork1)
	     *  |               OutBuf
	     *  V                 v
	     * +----------------------------+
	     * |XXXXXXXXXXXXXXXXX| |      | |
	     * +----------------------------+
	     *  <---------------> <-------->
	     *   converted_num    OutBytesLeft
	     */
	    converted_num = (unsigned long)((char *)OutBuf-(char *)_i18nwork1);
	    *to = (void *)_i18nwork1;
	    *to_len = (unsigned long)converted_num;
	    break;
	} else {
	    if ( errno == E2BIG ) {
		/* Overflow. still data is left.
		 *               InBuf
		 *                 v
		 * +----------------------------+
		 * |XXXXXXXXXXXXXX| |         | |
		 * +----------------------------+
		 *                 <----------->
		 *                  InBytesLeft
		 *
		 * (_i18nwork1)
		 *  |                         OutBuf
		 *  V                          v
		 * +----------------------------+
		 * |XXXXXXXXXXXXXXXXXXXXXXXXXXX |
		 * +----------------------------+
		 *  <-------------------------> 
		 *          converted_num      OutBytesLeft=?
		 */
		void *_p;

		/* Check how many converted already. */

		converted_num =
			(unsigned long)((char *)OutBuf - (char *)_i18nwork1);
		_i18nsize1 += WORKSIZE;
		_p = realloc( _i18nwork1, _i18nsize1 );
		if ( !_p ) {
		    *to = NULL;
		    *to_len = 0;
		    free( _i18nwork1 );
		    _i18nwork1 = NULL;
		    _i18nsize1 = 0;
		    shouldAlloc1 = ~0;
		    break;
		} else {
		    _i18nwork1 = _p;
		    OutBuf = (char *)((char*)_i18nwork1 + converted_num);
		    OutBytesLeft += WORKSIZE;
		}  
	    } else {
		*to = NULL;
		*to_len = 0;
		break;
	    }
	}
    }

    /*
     * NULL terminate
     */
    if ( _i18nsize1 > converted_num ) {
	((char *)_i18nwork1)[converted_num] = '\0';
    } else { /* _i18nsize1 == converted_num */
	void *_p;

	_i18nsize1++;
	_p = realloc( _i18nwork1, _i18nsize1 );

	if ( !_p ) {
	    *to = NULL;
	    *to_len = 0;
	    free( _i18nwork1 );
	    _i18nwork1 = NULL;
	    _i18nsize1 = 0;
	    shouldAlloc1 = ~0;
	} else {
	    _i18nwork1 = _p;
	    ((char *)_i18nwork1)[converted_num] = '\0';
	}
    }
}
Beispiel #17
0
int
mem_cd_iconv (const char *src, size_t srclen, iconv_t cd,
	      char **resultp, size_t *lengthp)
{
# define tmpbufsize 4096
  size_t length;
  char *result;

  /* Avoid glibc-2.1 bug and Solaris 2.7-2.9 bug.  */
# if defined _LIBICONV_VERSION \
     || !((__GLIBC__ - 0 == 2 && __GLIBC_MINOR__ - 0 <= 1) || defined __sun)
  /* Set to the initial state.  */
  iconv (cd, NULL, NULL, NULL, NULL);
# endif

  /* Determine the length we need.  */
  {
    size_t count = 0;
    /* The alignment is needed when converting e.g. to glibc's WCHAR_T or
       libiconv's UCS-4-INTERNAL encoding.  */
    union { unsigned int align; char buf[tmpbufsize]; } tmp;
# define tmpbuf tmp.buf
    const char *inptr = src;
    size_t insize = srclen;

    while (insize > 0)
      {
	char *outptr = tmpbuf;
	size_t outsize = tmpbufsize;
	size_t res = iconv (cd,
			    (ICONV_CONST char **) &inptr, &insize,
			    &outptr, &outsize);

	if (res == (size_t)(-1))
	  {
	    if (errno == E2BIG)
	      ;
	    else if (errno == EINVAL)
	      break;
	    else
	      return -1;
	  }
# if !defined _LIBICONV_VERSION && !defined __GLIBC__
	/* Irix iconv() inserts a NUL byte if it cannot convert.
	   NetBSD iconv() inserts a question mark if it cannot convert.
	   Only GNU libiconv and GNU libc are known to prefer to fail rather
	   than doing a lossy conversion.  */
	else if (res > 0)
	  {
	    errno = EILSEQ;
	    return -1;
	  }
# endif
	count += outptr - tmpbuf;
      }
    /* Avoid glibc-2.1 bug and Solaris 2.7 bug.  */
# if defined _LIBICONV_VERSION \
     || !((__GLIBC__ - 0 == 2 && __GLIBC_MINOR__ - 0 <= 1) || defined __sun)
    {
      char *outptr = tmpbuf;
      size_t outsize = tmpbufsize;
      size_t res = iconv (cd, NULL, NULL, &outptr, &outsize);

      if (res == (size_t)(-1))
	return -1;
      count += outptr - tmpbuf;
    }
# endif
    length = count;
# undef tmpbuf
  }

  if (length == 0)
    {
      *lengthp = 0;
      return 0;
    }
  if (*resultp != NULL && *lengthp >= length)
    result = *resultp;
  else
    {
      result = (char *) malloc (length);
      if (result == NULL)
	{
	  errno = ENOMEM;
	  return -1;
	}
    }

  /* Avoid glibc-2.1 bug and Solaris 2.7-2.9 bug.  */
# if defined _LIBICONV_VERSION \
     || !((__GLIBC__ - 0 == 2 && __GLIBC_MINOR__ - 0 <= 1) || defined __sun)
  /* Return to the initial state.  */
  iconv (cd, NULL, NULL, NULL, NULL);
# endif

  /* Do the conversion for real.  */
  {
    const char *inptr = src;
    size_t insize = srclen;
    char *outptr = result;
    size_t outsize = length;

    while (insize > 0)
      {
	size_t res = iconv (cd,
			    (ICONV_CONST char **) &inptr, &insize,
			    &outptr, &outsize);

	if (res == (size_t)(-1))
	  {
	    if (errno == EINVAL)
	      break;
	    else
	      goto fail;
	  }
# if !defined _LIBICONV_VERSION && !defined __GLIBC__
	/* Irix iconv() inserts a NUL byte if it cannot convert.
	   NetBSD iconv() inserts a question mark if it cannot convert.
	   Only GNU libiconv and GNU libc are known to prefer to fail rather
	   than doing a lossy conversion.  */
	else if (res > 0)
	  {
	    errno = EILSEQ;
	    goto fail;
	  }
# endif
      }
    /* Avoid glibc-2.1 bug and Solaris 2.7 bug.  */
# if defined _LIBICONV_VERSION \
     || !((__GLIBC__ - 0 == 2 && __GLIBC_MINOR__ - 0 <= 1) || defined __sun)
    {
      size_t res = iconv (cd, NULL, NULL, &outptr, &outsize);

      if (res == (size_t)(-1))
	goto fail;
    }
# endif
    if (outsize != 0)
      abort ();
  }

  *resultp = result;
  *lengthp = length;

  return 0;

 fail:
  {
    if (result != *resultp)
      {
	int saved_errno = errno;
	free (result);
	errno = saved_errno;
      }
    return -1;
  }
# undef tmpbufsize
}
Beispiel #18
0
int
main ()
{
#if HAVE_ICONV
  /* Assume that iconv() supports at least the encoding UTF-8.  */

  /* The text is "Japanese (日本語) [\U0001D50D\U0001D51E\U0001D52D]".  */

  /* Test conversion from UTF-8 to UTF-16BE with no errors.  */
  {
    static const char input[] =
      "Japanese (\346\227\245\346\234\254\350\252\236) [\360\235\224\215\360\235\224\236\360\235\224\255]";
    static const char expected[] =
      "\000J\000a\000p\000a\000n\000e\000s\000e\000 \000(\145\345\147\054\212\236\000)\000 \000[\330\065\335\015\330\065\335\036\330\065\335\055\000]";
    iconv_t cd;
    char buf[100];
    const char *inptr;
    size_t inbytesleft;
    char *outptr;
    size_t outbytesleft;
    size_t res;

    cd = iconv_open ("UTF-16BE", "UTF-8");
    ASSERT (cd != (iconv_t)(-1));

    inptr = input;
    inbytesleft = sizeof (input) - 1;
    outptr = buf;
    outbytesleft = sizeof (buf);
    res = iconv (cd,
                 (ICONV_CONST char **) &inptr, &inbytesleft,
                 &outptr, &outbytesleft);
    ASSERT (res == 0 && inbytesleft == 0);
    ASSERT (outptr == buf + (sizeof (expected) - 1));
    ASSERT (memcmp (buf, expected, sizeof (expected) - 1) == 0);

    ASSERT (iconv_close (cd) == 0);
  }

  /* Test conversion from UTF-8 to UTF-16LE with no errors.  */
  {
    static const char input[] =
      "Japanese (\346\227\245\346\234\254\350\252\236) [\360\235\224\215\360\235\224\236\360\235\224\255]";
    static const char expected[] =
      "J\000a\000p\000a\000n\000e\000s\000e\000 \000(\000\345\145\054\147\236\212)\000 \000[\000\065\330\015\335\065\330\036\335\065\330\055\335]\000";
    iconv_t cd;
    char buf[100];
    const char *inptr;
    size_t inbytesleft;
    char *outptr;
    size_t outbytesleft;
    size_t res;

    cd = iconv_open ("UTF-16LE", "UTF-8");
    ASSERT (cd != (iconv_t)(-1));

    inptr = input;
    inbytesleft = sizeof (input) - 1;
    outptr = buf;
    outbytesleft = sizeof (buf);
    res = iconv (cd,
                 (ICONV_CONST char **) &inptr, &inbytesleft,
                 &outptr, &outbytesleft);
    ASSERT (res == 0 && inbytesleft == 0);
    ASSERT (outptr == buf + (sizeof (expected) - 1));
    ASSERT (memcmp (buf, expected, sizeof (expected) - 1) == 0);

    ASSERT (iconv_close (cd) == 0);
  }

  /* Test conversion from UTF-8 to UTF-32BE with no errors.  */
  {
    static const char input[] =
      "Japanese (\346\227\245\346\234\254\350\252\236) [\360\235\224\215\360\235\224\236\360\235\224\255]";
    static const char expected[] =
      "\000\000\000J\000\000\000a\000\000\000p\000\000\000a\000\000\000n\000\000\000e\000\000\000s\000\000\000e\000\000\000 \000\000\000(\000\000\145\345\000\000\147\054\000\000\212\236\000\000\000)\000\000\000 \000\000\000[\000\001\325\015\000\001\325\036\000\001\325\055\000\000\000]";
    iconv_t cd;
    char buf[100];
    const char *inptr;
    size_t inbytesleft;
    char *outptr;
    size_t outbytesleft;
    size_t res;

    cd = iconv_open ("UTF-32BE", "UTF-8");
    ASSERT (cd != (iconv_t)(-1));

    inptr = input;
    inbytesleft = sizeof (input) - 1;
    outptr = buf;
    outbytesleft = sizeof (buf);
    res = iconv (cd,
                 (ICONV_CONST char **) &inptr, &inbytesleft,
                 &outptr, &outbytesleft);
    ASSERT (res == 0 && inbytesleft == 0);
    ASSERT (outptr == buf + (sizeof (expected) - 1));
    ASSERT (memcmp (buf, expected, sizeof (expected) - 1) == 0);

    ASSERT (iconv_close (cd) == 0);
  }

  /* Test conversion from UTF-8 to UTF-32LE with no errors.  */
  {
    static const char input[] =
      "Japanese (\346\227\245\346\234\254\350\252\236) [\360\235\224\215\360\235\224\236\360\235\224\255]";
    static const char expected[] =
      "J\000\000\000a\000\000\000p\000\000\000a\000\000\000n\000\000\000e\000\000\000s\000\000\000e\000\000\000 \000\000\000(\000\000\000\345\145\000\000\054\147\000\000\236\212\000\000)\000\000\000 \000\000\000[\000\000\000\015\325\001\000\036\325\001\000\055\325\001\000]\000\000\000";
    iconv_t cd;
    char buf[100];
    const char *inptr;
    size_t inbytesleft;
    char *outptr;
    size_t outbytesleft;
    size_t res;

    cd = iconv_open ("UTF-32LE", "UTF-8");
    ASSERT (cd != (iconv_t)(-1));

    inptr = input;
    inbytesleft = sizeof (input) - 1;
    outptr = buf;
    outbytesleft = sizeof (buf);
    res = iconv (cd,
                 (ICONV_CONST char **) &inptr, &inbytesleft,
                 &outptr, &outbytesleft);
    ASSERT (res == 0 && inbytesleft == 0);
    ASSERT (outptr == buf + (sizeof (expected) - 1));
    ASSERT (memcmp (buf, expected, sizeof (expected) - 1) == 0);

    ASSERT (iconv_close (cd) == 0);
  }

  /* Test conversion from UTF-16BE to UTF-8 with no errors.  */
  {
    static const char input[] =
      "\000J\000a\000p\000a\000n\000e\000s\000e\000 \000(\145\345\147\054\212\236\000)\000 \000[\330\065\335\015\330\065\335\036\330\065\335\055\000]";
    static const char expected[] =
      "Japanese (\346\227\245\346\234\254\350\252\236) [\360\235\224\215\360\235\224\236\360\235\224\255]";
    iconv_t cd;
    char buf[100];
    const char *inptr;
    size_t inbytesleft;
    char *outptr;
    size_t outbytesleft;
    size_t res;

    cd = iconv_open ("UTF-8", "UTF-16BE");
    ASSERT (cd != (iconv_t)(-1));

    inptr = input;
    inbytesleft = sizeof (input) - 1;
    outptr = buf;
    outbytesleft = sizeof (buf);
    res = iconv (cd,
                 (ICONV_CONST char **) &inptr, &inbytesleft,
                 &outptr, &outbytesleft);
    ASSERT (res == 0 && inbytesleft == 0);
    ASSERT (outptr == buf + (sizeof (expected) - 1));
    ASSERT (memcmp (buf, expected, sizeof (expected) - 1) == 0);

    ASSERT (iconv_close (cd) == 0);
  }

  /* Test conversion from UTF-16LE to UTF-8 with no errors.  */
  {
    static const char input[] =
      "J\000a\000p\000a\000n\000e\000s\000e\000 \000(\000\345\145\054\147\236\212)\000 \000[\000\065\330\015\335\065\330\036\335\065\330\055\335]\000";
    static const char expected[] =
      "Japanese (\346\227\245\346\234\254\350\252\236) [\360\235\224\215\360\235\224\236\360\235\224\255]";
    iconv_t cd;
    char buf[100];
    const char *inptr;
    size_t inbytesleft;
    char *outptr;
    size_t outbytesleft;
    size_t res;

    cd = iconv_open ("UTF-8", "UTF-16LE");
    ASSERT (cd != (iconv_t)(-1));

    inptr = input;
    inbytesleft = sizeof (input) - 1;
    outptr = buf;
    outbytesleft = sizeof (buf);
    res = iconv (cd,
                 (ICONV_CONST char **) &inptr, &inbytesleft,
                 &outptr, &outbytesleft);
    ASSERT (res == 0 && inbytesleft == 0);
    ASSERT (outptr == buf + (sizeof (expected) - 1));
    ASSERT (memcmp (buf, expected, sizeof (expected) - 1) == 0);

    ASSERT (iconv_close (cd) == 0);
  }

  /* Test conversion from UTF-32BE to UTF-8 with no errors.  */
  {
    static const char input[] =
      "\000\000\000J\000\000\000a\000\000\000p\000\000\000a\000\000\000n\000\000\000e\000\000\000s\000\000\000e\000\000\000 \000\000\000(\000\000\145\345\000\000\147\054\000\000\212\236\000\000\000)\000\000\000 \000\000\000[\000\001\325\015\000\001\325\036\000\001\325\055\000\000\000]";
    static const char expected[] =
      "Japanese (\346\227\245\346\234\254\350\252\236) [\360\235\224\215\360\235\224\236\360\235\224\255]";
    iconv_t cd;
    char buf[100];
    const char *inptr;
    size_t inbytesleft;
    char *outptr;
    size_t outbytesleft;
    size_t res;

    cd = iconv_open ("UTF-8", "UTF-32BE");
    ASSERT (cd != (iconv_t)(-1));

    inptr = input;
    inbytesleft = sizeof (input) - 1;
    outptr = buf;
    outbytesleft = sizeof (buf);
    res = iconv (cd,
                 (ICONV_CONST char **) &inptr, &inbytesleft,
                 &outptr, &outbytesleft);
    ASSERT (res == 0 && inbytesleft == 0);
    ASSERT (outptr == buf + (sizeof (expected) - 1));
    ASSERT (memcmp (buf, expected, sizeof (expected) - 1) == 0);

    ASSERT (iconv_close (cd) == 0);
  }

  /* Test conversion from UTF-32LE to UTF-8 with no errors.  */
  {
    static const char input[] =
      "J\000\000\000a\000\000\000p\000\000\000a\000\000\000n\000\000\000e\000\000\000s\000\000\000e\000\000\000 \000\000\000(\000\000\000\345\145\000\000\054\147\000\000\236\212\000\000)\000\000\000 \000\000\000[\000\000\000\015\325\001\000\036\325\001\000\055\325\001\000]\000\000\000";
    static const char expected[] =
      "Japanese (\346\227\245\346\234\254\350\252\236) [\360\235\224\215\360\235\224\236\360\235\224\255]";
    iconv_t cd;
    char buf[100];
    const char *inptr;
    size_t inbytesleft;
    char *outptr;
    size_t outbytesleft;
    size_t res;

    cd = iconv_open ("UTF-8", "UTF-32LE");
    ASSERT (cd != (iconv_t)(-1));

    inptr = input;
    inbytesleft = sizeof (input) - 1;
    outptr = buf;
    outbytesleft = sizeof (buf);
    res = iconv (cd,
                 (ICONV_CONST char **) &inptr, &inbytesleft,
                 &outptr, &outbytesleft);
    ASSERT (res == 0 && inbytesleft == 0);
    ASSERT (outptr == buf + (sizeof (expected) - 1));
    ASSERT (memcmp (buf, expected, sizeof (expected) - 1) == 0);

    ASSERT (iconv_close (cd) == 0);
  }
#endif

  return 0;
}
Beispiel #19
0
/*
 * This function is used in reading text data from an MDB table.
 */
int
mdb_unicode2ascii(MdbHandle *mdb, char *src, size_t slen, char *dest, size_t dlen)
{
	char *tmp = NULL;
	size_t tlen = 0;
	size_t len_in, len_out;
	char *in_ptr, *out_ptr;

	if ((!src) || (!dest) || (!dlen))
		return 0;

	/* Uncompress 'Unicode Compressed' string into tmp */
	if (IS_JET4(mdb) && (slen>=2)
	 && ((src[0]&0xff)==0xff) && ((src[1]&0xff)==0xfe)) {
		unsigned int compress=1;
		src += 2;
		slen -= 2;
		tmp = (char *)g_malloc(slen*2);
		while (slen) {
			if (*src == 0) {
				compress = (compress) ? 0 : 1;
				src++;
				slen--;
			} else if (compress) {
				tmp[tlen++] = *src++;
				tmp[tlen++] = 0;
				slen--;
			} else if (slen >= 2){
				tmp[tlen++] = *src++;
				tmp[tlen++] = *src++;
				slen-=2;
			}
		}
	}

	in_ptr = (tmp) ? tmp : src;
	out_ptr = dest;
	len_in = (tmp) ? tlen : slen;
	len_out = dlen;

#if HAVE_ICONV
	//printf("1 len_in %d len_out %d\n",len_in, len_out);
	while (1) {
		iconv(mdb->iconv_in, &in_ptr, &len_in, &out_ptr, &len_out);
		if ((!len_in) || (errno == E2BIG)) break;
		/* Don't bail if impossible conversion is encountered */
		in_ptr += (IS_JET4(mdb)) ? 2 : 1;
		len_in -= (IS_JET4(mdb)) ? 2 : 1;
		*out_ptr++ = '?';
		len_out--;
	}
	//printf("2 len_in %d len_out %d\n",len_in, len_out);
	dlen -= len_out;
#else
	if (IS_JET3(mdb)) {
		strncpy(out_ptr, in_ptr, len_in);
		dlen = len_in;
	} else {
		/* rough UCS-2LE to ISO-8859-1 conversion */
		unsigned int i;
		for (i=0; i<len_in; i+=2)
			dest[i/2] = (in_ptr[i+1] == 0) ? in_ptr[i] : '?';
		dlen = len_in/2;
	}
#endif

	if (tmp) g_free(tmp);
	dest[dlen]='\0';
	//printf("dest %s\n",dest);
	return dlen;
}
Beispiel #20
0
char *CPLRecodeFromWCharIconv( const wchar_t *pwszSource,
                               const char *pszSrcEncoding,
                               const char *pszDstEncoding )

{
/* -------------------------------------------------------------------- */
/*      What is the source length.                                      */
/* -------------------------------------------------------------------- */
    size_t  nSrcLen = 0;

    while ( pwszSource[nSrcLen] != 0 )
        nSrcLen++;

/* -------------------------------------------------------------------- */
/*      iconv() does not support wchar_t so we need to repack the       */
/*      characters according to the width of a character in the         */
/*      source encoding.  For instance if wchar_t is 4 bytes but our    */
/*      source is UTF16 then we need to pack down into 2 byte           */
/*      characters before passing to iconv().                           */
/* -------------------------------------------------------------------- */
    int nTargetCharWidth = CPLEncodingCharSize( pszSrcEncoding );

    if( nTargetCharWidth < 1 )
    {
        CPLError( CE_Warning, CPLE_AppDefined,
                  "Recode from %s with CPLRecodeFromWChar() failed because"
                  " the width of characters in the encoding are not known.",
                  pszSrcEncoding );
        return CPLStrdup("");
    }

    GByte *pszIconvSrcBuf = (GByte*) CPLCalloc((nSrcLen+1),nTargetCharWidth);

    for( unsigned int iSrc = 0; iSrc <= nSrcLen; iSrc++ )
    {
        if( nTargetCharWidth == 1 )
            pszIconvSrcBuf[iSrc] = (GByte) pwszSource[iSrc];
        else if( nTargetCharWidth == 2 )
            ((short *)pszIconvSrcBuf)[iSrc] = (short) pwszSource[iSrc];
        else if( nTargetCharWidth == 4 )
            ((GInt32 *)pszIconvSrcBuf)[iSrc] = pwszSource[iSrc];
    }

/* -------------------------------------------------------------------- */
/*      Create the iconv() translation object.                          */
/* -------------------------------------------------------------------- */
    iconv_t sConv;

    sConv = iconv_open( pszDstEncoding, pszSrcEncoding );

    if ( sConv == (iconv_t)-1 )
    {
        CPLFree( pszIconvSrcBuf );
        CPLError( CE_Warning, CPLE_AppDefined,
                  "Recode from %s to %s failed with the error: \"%s\".",
                  pszSrcEncoding, pszDstEncoding, strerror(errno) );

        return CPLStrdup( "" );
    }

/* -------------------------------------------------------------------- */
/*      XXX: There is a portability issue: iconv() function could be    */
/*      declared differently on different platforms. The second         */
/*      argument could be declared as char** (as POSIX defines) or      */
/*      as a const char**. Handle it with the ICONV_CPP_CONST macro here.   */
/* -------------------------------------------------------------------- */
    ICONV_CPP_CONST char *pszSrcBuf = (ICONV_CPP_CONST char *) pszIconvSrcBuf;

    /* iconv expects a number of bytes, not characters */
    nSrcLen *= sizeof(wchar_t);

/* -------------------------------------------------------------------- */
/*      Allocate destination buffer.                                    */
/* -------------------------------------------------------------------- */
    size_t  nDstCurLen = MAX(CPL_RECODE_DSTBUF_SIZE, nSrcLen + 1);
    size_t  nDstLen = nDstCurLen;
    char    *pszDestination = (char *)CPLCalloc( nDstCurLen, sizeof(char) );
    char    *pszDstBuf = pszDestination;

    while ( nSrcLen > 0 )
    {
        size_t  nConverted =
            iconv( sConv, &pszSrcBuf, &nSrcLen, &pszDstBuf, &nDstLen );

        if ( nConverted == (size_t)-1 )
        {
            if ( errno == EILSEQ )
            {
                // Skip the invalid sequence in the input string.
                nSrcLen--;
                pszSrcBuf += sizeof(wchar_t);
                if (!bHaveWarned2)
                {
                    bHaveWarned2 = true;
                    CPLError(CE_Warning, CPLE_AppDefined,
                            "One or several characters couldn't be converted correctly from %s to %s.\n"
                            "This warning will not be emitted anymore",
                             pszSrcEncoding, pszDstEncoding);
                }
                continue;
            }

            else if ( errno == E2BIG )
            {
                // We are running out of the output buffer.
                // Dynamically increase the buffer size.
                size_t nTmp = nDstCurLen;
                nDstCurLen *= 2;
                pszDestination =
                    (char *)CPLRealloc( pszDestination, nDstCurLen );
                pszDstBuf = pszDestination + nTmp - nDstLen;
                nDstLen += nDstCurLen - nTmp;
                continue;
            }

            else
                break;
        }
    }

    pszDestination[nDstCurLen - nDstLen] = '\0';

    iconv_close( sConv );

    CPLFree( pszIconvSrcBuf );

    return pszDestination;
}
Beispiel #21
0
	int APP_CC
in_unistr(struct stream* s, char *string, int str_size, int in_len)
{
#ifdef HAVE_ICONV
	size_t ibl = in_len, obl = str_size - 1;
	char *pin = (char *) s->p, *pout = string;
	static iconv_t iconv_h = (iconv_t) - 1;

	if (g_iconv_works)
	{
		if (iconv_h == (iconv_t) - 1)
		{
			if ((iconv_h = iconv_open(g_codepage, WINDOWS_CODEPAGE)) == (iconv_t) - 1)
			{
				log_message(&log_conf, LOG_LEVEL_WARNING, "chansrv[rdp_in_unistr]: iconv_open[%s -> %s] fail %p",
						WINDOWS_CODEPAGE, g_codepage, iconv_h);
				g_iconv_works = False;
				return rdp_in_unistr(s, string, str_size, in_len);
			}
		}

		if (iconv(iconv_h, (ICONV_CONST char **) &pin, &ibl, &pout, &obl) == (size_t) - 1)
		{
			if (errno == E2BIG)
			{
				log_message(&log_conf, LOG_LEVEL_WARNING, "chansrv[rdp_in_unistr]: "
						"server sent an unexpectedly long string, truncating");
			}
			else
			{
				iconv_close(iconv_h);
				iconv_h = (iconv_t) - 1;
				log_message(&log_conf, LOG_LEVEL_WARNING, "chansrv[rdp_in_unistr]: "
						"iconv fail, errno %d\n", errno);
				g_iconv_works = False;
				return rdp_in_unistr(s, string, str_size, in_len);
			}
		}

		/* we must update the location of the current STREAM for future reads of s->p */
		s->p += in_len;
		*pout = 0;
		return pout - string;
	}
	else
#endif
	{
		int i = 0;
		int len = in_len / 2;
		int rem = 0;

		if (len > str_size - 1)
		{
			log_message(&log_conf, LOG_LEVEL_WARNING, "chansrv[rdp_in_unistr]: "
					"server sent an unexpectedly long string, truncating");
			len = str_size - 1;
			rem = in_len - 2 * len;
		}
		while (i < len)
		{
			in_uint8a(s, &string[i++], 1);
			in_uint8s(s, 1);
		}
		in_uint8s(s, rem);
		string[len] = 0;
		return len;
	}
}
Beispiel #22
0
char *CPLRecodeIconv( const char *pszSource,
                      const char *pszSrcEncoding,
                      const char *pszDstEncoding )

{
    iconv_t sConv;

    sConv = iconv_open( pszDstEncoding, pszSrcEncoding );

    if ( sConv == (iconv_t)-1 )
    {
        CPLError( CE_Warning, CPLE_AppDefined,
                  "Recode from %s to %s failed with the error: \"%s\".",
                  pszSrcEncoding, pszDstEncoding, strerror(errno) );

        return CPLStrdup(pszSource);
    }

/* -------------------------------------------------------------------- */
/*      XXX: There is a portability issue: iconv() function could be    */
/*      declared differently on different platforms. The second         */
/*      argument could be declared as char** (as POSIX defines) or      */
/*      as a const char**. Handle it with the ICONV_CPP_CONST macro here.   */
/* -------------------------------------------------------------------- */
    ICONV_CPP_CONST char *pszSrcBuf = (ICONV_CPP_CONST char *)pszSource;
    size_t  nSrcLen = strlen( pszSource );
    size_t  nDstCurLen = MAX(CPL_RECODE_DSTBUF_SIZE, nSrcLen + 1);
    size_t  nDstLen = nDstCurLen;
    char    *pszDestination = (char *)CPLCalloc( nDstCurLen, sizeof(char) );
    char    *pszDstBuf = pszDestination;

    while ( nSrcLen > 0 )
    {
        size_t  nConverted =
            iconv( sConv, &pszSrcBuf, &nSrcLen, &pszDstBuf, &nDstLen );

        if ( nConverted == (size_t)-1 )
        {
            if ( errno == EILSEQ )
            {
                // Skip the invalid sequence in the input string.
                if (!bHaveWarned1)
                {
                    bHaveWarned1 = true;
                    CPLError(CE_Warning, CPLE_AppDefined,
                            "One or several characters couldn't be converted correctly from %s to %s.\n"
                            "This warning will not be emitted anymore",
                             pszSrcEncoding, pszDstEncoding);
                }
                nSrcLen--, pszSrcBuf++;
                continue;
            }

            else if ( errno == E2BIG )
            {
                // We are running out of the output buffer.
                // Dynamically increase the buffer size.
                size_t nTmp = nDstCurLen;
                nDstCurLen *= 2;
                pszDestination =
                    (char *)CPLRealloc( pszDestination, nDstCurLen );
                pszDstBuf = pszDestination + nTmp - nDstLen;
                nDstLen += nDstCurLen - nTmp;
                continue;
            }

            else
                break;
        }
    }

    pszDestination[nDstCurLen - nDstLen] = '\0';

    iconv_close( sConv );

    return pszDestination;
}
Beispiel #23
0
std::string ReaderUtil::Recode(const std::string& str_to_encode,
                               const std::string& src_enc,
                               const std::string& dst_enc) {
	std::string src_enc_str = src_enc;
	std::string dst_enc_str = dst_enc;

	if (src_enc.empty() || dst_enc.empty() || str_to_encode.empty()) {
		return str_to_encode;
	}
	if (atoi(src_enc.c_str()) > 0) {
		src_enc_str = ReaderUtil::CodepageToEncoding(atoi(src_enc.c_str()));
	}
	if (atoi(dst_enc.c_str()) > 0) {
		dst_enc_str = ReaderUtil::CodepageToEncoding(atoi(dst_enc.c_str()));
	}
#ifdef LCF_SUPPORT_ICU
	UErrorCode status = U_ZERO_ERROR;
	int size = str_to_encode.size() * 4;
	UChar* unicode_str = new UChar[size];
	UConverter *conv;
	int length;
	std::string result_str;

	conv = ucnv_open(src_enc_str.c_str(), &status);
	
	if (status != U_ZERO_ERROR && status != U_AMBIGUOUS_ALIAS_WARNING) {
		fprintf(stderr, "liblcf:  ucnv_open() error for source encoding \"%s\": %s\n", src_enc_str.c_str(), u_errorName(status));
		return std::string();
	}
	status = U_ZERO_ERROR;

	length = ucnv_toUChars(conv, unicode_str, size, str_to_encode.c_str(), -1, &status);
	ucnv_close(conv);
	if (status != U_ZERO_ERROR) {
		fprintf(stderr, "liblcf: ucnv_toUChars() error when encoding \"%s\": %s\n", str_to_encode.c_str(), u_errorName(status));
		return std::string();
	}

	char* result = new char[length * 4];

	conv = ucnv_open(dst_enc_str.c_str(), &status);
	if (status != U_ZERO_ERROR && status != U_AMBIGUOUS_ALIAS_WARNING) {
		fprintf(stderr, "liblcf: ucnv_open() error for destination encoding \"%s\": %s\n", dst_enc_str.c_str(), u_errorName(status));
		return std::string();
	}
	status = U_ZERO_ERROR;

	ucnv_fromUChars(conv, result, length * 4, unicode_str, -1, &status);
	ucnv_close(conv);
	if (status != U_ZERO_ERROR) {
		fprintf(stderr, "liblcf: ucnv_fromUChars() error: %s\n", u_errorName(status));
		return std::string();
	}

	result_str = result;

	delete[] unicode_str;
	delete[] result;

	return std::string(result_str);
#else
	iconv_t cd = iconv_open(dst_enc_str.c_str(), src_enc_str.c_str());
	if (cd == (iconv_t)-1)
		return str_to_encode;
	char *src = const_cast<char *>(str_to_encode.c_str());
	size_t src_left = str_to_encode.size();
	size_t dst_size = str_to_encode.size() * 5 + 10;
	char *dst = new char[dst_size];
	size_t dst_left = dst_size;
#    ifdef ICONV_CONST
	char ICONV_CONST *p = src;
#    else
	char *p = src;
#    endif
	char *q = dst;
	size_t status = iconv(cd, &p, &src_left, &q, &dst_left);
	iconv_close(cd);
	if (status == (size_t) -1 || src_left > 0) {
	delete[] dst;
		return std::string();
	}
	*q++ = '\0';
	std::string result(dst);
	delete[] dst;
	return result;
#endif
}
Beispiel #24
0
static int
put_file_func (CameraFilesystem *fs, const char *folder, const char *name, 
	CameraFileType type, CameraFile *file, void *data, GPContext *context)
{
#ifdef HAVE_GD
	Camera *camera = data;
	char *c, *in_name, *out_name, *filedata = NULL;
	int ret, in_width, in_height, in_x, in_y;
	size_t inc, outc;
	double aspect_in, aspect_out;
#ifdef HAVE_ICONV
	char *in, *out;
#endif
	unsigned long filesize = 0;
	gdImagePtr rotated, im_out, im_in = NULL;

	if (strcmp (folder, "/"))
		return GP_ERROR_DIRECTORY_NOT_FOUND;

	inc = strlen (name);
	in_name = strdup (name);
	outc = inc;
	out_name = malloc (outc + 1);
	if (!in_name || !out_name) {
		free (in_name);
		free (out_name);
		return GP_ERROR_NO_MEMORY;
	}

	/* Convert name to ASCII */
#ifdef HAVE_ICONV
	in = in_name;
	out = out_name;
	if (iconv (camera->pl->cd, &in, &inc, &out, &outc) == -1) {
		free (in_name);
		free (out_name);
		gp_log (GP_LOG_ERROR, "iconv",
			"Failed to convert filename to ASCII");
		return GP_ERROR_OS_FAILURE;
	}
	outc = out - out_name;
	out_name[outc] = 0;
#else
	for (i = 0; i < inc; i++) {
		if ((uint8_t)in_name[i] < 0x20 || (uint8_t)in_name[i] > 0x7E)
			out_name[i] = '?';
		else
			out_name[i] = in_name[i];
	}
	out_name[i] = 0;
#endif
	free (in_name);

	/* Remove file extension, and if necessary truncate the name */
	c = strrchr (out_name, '.');
	if (c)
		*c = 0;
	if (outc > ST2205_FILENAME_LENGTH)
		out_name[ST2205_FILENAME_LENGTH] = 0;

	ret = gp_file_get_data_and_size (file, (const char **)&filedata,
					 &filesize);
	if (ret < 0) { free (out_name); return ret; }

	/* Try loading the file using gd, starting with the most often
	   used types first */

	/* gdImageCreateFromJpegPtr is chatty on error, don't call it on
	   non JPEG files */
	if (filesize > 2 &&
	    (uint8_t)filedata[0] == 0xff && (uint8_t)filedata[1] == 0xd8)
		im_in = gdImageCreateFromJpegPtr(filesize, filedata);
	if (im_in == NULL)
		im_in = gdImageCreateFromPngPtr(filesize, filedata);
	if (im_in == NULL)
		im_in = gdImageCreateFromGifPtr(filesize, filedata);
	if (im_in == NULL)
		im_in = gdImageCreateFromWBMPPtr(filesize, filedata);
	if (im_in == NULL) {
		gp_log (GP_LOG_ERROR, "st2205",
			"Unrecognized file format for file: %s%s",
			folder, name);
		free (out_name);
		return GP_ERROR_BAD_PARAMETERS;
	}

	if (needs_rotation (camera)) {
		rotated = gdImageCreateTrueColor (im_in->sy, im_in->sx);
		if (rotated == NULL) {
			gdImageDestroy (im_in);
			free (out_name);
			return GP_ERROR_NO_MEMORY;
		}
		rotate90 (im_in, rotated);
		gdImageDestroy (im_in);
		im_in = rotated;
	}

	im_out = gdImageCreateTrueColor(camera->pl->width, camera->pl->height);
	if (im_out == NULL) {
		gdImageDestroy (im_in);
		free (out_name);
		return GP_ERROR_NO_MEMORY;
	}

	/* Keep aspect */
	aspect_in  = (double)im_in->sx / im_in->sy;
	aspect_out = (double)im_out->sx / im_out->sy;
	if (aspect_in > aspect_out) {
		/* Reduce in width (crop left and right) */
		in_width = (im_in->sx / aspect_in) * aspect_out;
		in_x = (im_in->sx - in_width) / 2;
		in_height = im_in->sy;
		in_y = 0;
	} else {
		/* Reduce in height (crop top and bottom) */
		in_width = im_in->sx;
		in_x = 0;
		in_height = (im_in->sy * aspect_in) / aspect_out;
		in_y = (im_in->sy - in_height) / 2;
	}

	gdImageCopyResampled (im_out, im_in, 0, 0, in_x, in_y,
			      im_out->sx, im_out->sy,
			      in_width, in_height);

	if (im_in->sx != im_out->sx ||
	    im_in->sy != im_out->sy)
		gdImageSharpen(im_out, 100);

	ret = st2205_write_file (camera, out_name, im_out->tpixels);
	if (ret >= 0) {
		/* Add to our filenames list */
		ST2205_SET_FILENAME(camera->pl->filenames[ret], out_name, ret);
		/* And commit the changes to the device */
		ret = st2205_commit(camera);
	}

	gdImageDestroy (im_in);
	gdImageDestroy (im_out);
	free (out_name);
	return ret;
#else
	gp_log(GP_LOG_ERROR,"st2205", "GD compression not supported - no libGD present during build");
	return GP_ERROR_NOT_SUPPORTED;
#endif
}
Beispiel #25
0
/**************************************************************************
  Convert string from display encoding (16 bit unicode) to
  local encoding (8 bit char) and resut put in 'pToString'.
  if 'pToString' == NULL then resulting string will be allocate automaticaly.
  'length' give real sizeof 'pToString' array.

  Function return (char *) pointer to (new) pToString.
**************************************************************************/
char *convertcopy_to_chars(char *pToString, size_t length,
			    const Uint16 * pFromUniString)
{
  /* Start Parametrs */
  const char *pFromcode = get_display_encoding();
  const char *pTocode = get_internal_encoding();
  const char *pStart = (char *) pFromUniString;
  size_t ulength = (unistrlen(pFromUniString) + 1) * 2;

  /* ===== */

  char *pResult;
  iconv_t cd;

  /* ===== */

  if (!pStart) {
    return pToString;
  }

  cd = iconv_open(pTocode, pFromcode);
  if (cd == (iconv_t) (-1)) {
    if (errno != EINVAL) {
      return pToString;
    }
  }

  if (pToString) {
    pResult = pToString;
  } else {
    length = ulength * 2; /* UTF-8: up to 4 bytes per char */
    pResult = fc_calloc(1, length);
  }
  
  iconv(cd, NULL, NULL, NULL, NULL);	/* return to the initial state */

  /* Do the conversion for real. */
  {
    const char *pInptr = pStart;
    size_t Insize = ulength;
    char *pOutptr = pResult;
    size_t Outsize = length;

    while (Insize > 0 && Outsize > 0) {
      size_t Res =
	  iconv(cd, (ICONV_CONST char **) &pInptr, &Insize, &pOutptr, &Outsize);
      if (Res == (size_t) (-1)) {
        log_error("iconv() error: %s", fc_strerror(fc_get_errno()));
	if (errno == EINVAL) {
	  break;
	} else {
	  int saved_errno = errno;
	  iconv_close(cd);
	  errno = saved_errno;
	  if(!pToString) {
	    FC_FREE(pResult);
	  }
	  return pToString;
	}
      }
    }

    {
      size_t Res = iconv(cd, NULL, NULL, &pOutptr, &Outsize);
      if (Res == (size_t) (-1)) {
	int saved_errno = errno;
	iconv_close(cd);
	errno = saved_errno;
	if(!pToString) {
	  FC_FREE(pResult);
	}
	return pToString;
      }
    }

  }

  iconv_close(cd);

  return pResult;
}
static char *make_dict(char *dslfilename, char *outputdir, char *bname)
{
    char *dictfilename;
    FILE *dslfile, *dictfile;
    char inbuff[BUFSIZE+1] = { 0 };
    char outbuff[BUFSIZE+1] = { 0 };
    //FIXME: May be prefer to use ptrdiff_t for second element of replaces array 
    unsigned char replaces[BUFSIZE+1][2] = { {0, 0} };
    unsigned short firstword = 0;
    iconv_t iconv_direction;
    int i;

    if ( !(dslfile = fopen(dslfilename, "r")) ) {
        perror("Error open dslfile");
        exit(errno);
    }

    if ( !(dictfilename = (char *)malloc(strlen(outputdir)+strlen(bname)+strlen(".dict")+1)) ) { 
        fprintf(stderr, "In file %s, line %d ", __FILE__, __LINE__);
        perror("error allocate memory"); 
        exit(errno);
    } 
    sprintf(dictfilename, "%s%s.dict", outputdir, bname);
    if ( !(dictfile = fopen(dictfilename, "w")) ) {
        perror("Error open dictfile");
        exit(errno);
    }
    
    fread(&firstword, sizeof(firstword), 1, dslfile);
    if (UCS_MARK == firstword) {
        if ( 0 > (iconv_direction = iconv_open("UTF-8", "UCS-2")) ) {
            perror("Error conversion direction");
        }
    } else {
        //TODO list.
        fprintf(stderr, "Not Windows UCS-2 format\n");
        exit(1);
    }

    fill_patterns(replaces);

    while ( !(feof(dslfile)) ) {
        size_t inbuff_size;
        size_t outbuff_size = BUFSIZE;
        size_t writems;
        char *inbufptr = (char *) inbuff;
        char *outbufptr = (char *) outbuff;
        long filepos = BUFOVERLAP * UCS_CHARLEN;
        char *buffborder;
        int vector[30] = { 0 };

        inbuff_size = fread(inbuff, sizeof(char), BUFSIZE, dslfile);

        if ( !(feof(dslfile)) ) {
            inbuff_size -= filepos;
            if ( 0 > (iconv(iconv_direction,
                            &inbufptr, &inbuff_size, &outbufptr, &outbuff_size)) ) {
                perror("iconv error");
                exit(errno);
            }
            buffborder = outbufptr;
            inbuff_size += filepos;
            if ( 0 > (iconv(iconv_direction,
                            &inbufptr, &inbuff_size, &outbufptr, &outbuff_size)) ) {
                perror("iconv error");
                exit(errno);
            }
        } else {
            if ( 0 > (iconv(iconv_direction,
                            &inbufptr, &inbuff_size, &outbufptr, &outbuff_size)) ) {
                perror("iconv error");
                exit(errno);
            }
            buffborder = outbufptr;
        }
        *outbufptr = 0;
        
        for (i=1; i < sizeof(patterns)/sizeof(patterns[0]); i++) {
            int match_res =
                pcre_exec(
                    patterns[i].compile, patterns[i].extra,
                    outbuff, outbufptr-outbuff, 0,
                    PCRE_NOTBOL | PCRE_NOTEOL,
                    vector, sizeof(vector)/sizeof(vector[0])
                    );
            switch (match_res) {
            case PCRE_ERROR_NOMATCH: break;
            default:
                fprintf(stderr, "Matching error %d\n", match_res);
                break;
            }
        }
        
        for (i = 0, outbufptr = outbuff; outbuff+i < buffborder; i++) {
            if (replaces[i][0]) {
                writems = fwrite(outbufptr, sizeof(char), outbuff+i-outbufptr, dictfile);
                if ( (outbuff+i-outbufptr) != writems) {
                    fprintf(stderr, "Write error: not enough disk space or another error.\n");
                    exit(1);
                }
                writems = fwrite(
                    patterns[replaces[i][0]].repl,
                    sizeof(char),
                    strlen(patterns[replaces[i][0]].repl),
                    dictfile
                    );
                if ( (strlen(patterns[replaces[i][0]].repl)) != writems) {
                    fprintf(stderr, "Write error: not enough disk space or another error.\n");
                    exit(1);
                }
                outbufptr = outbuff + i + replaces[i][1];
            }
        }

        if (outbufptr >= buffborder) {
            filepos -= (outbufptr - buffborder) * 2;
        } else {
            writems = fwrite(outbufptr, sizeof(char), buffborder-outbufptr, dictfile);
            if ( (buffborder-outbufptr) != writems) {
                fprintf(stderr, "Write error - not enough disk space or another error.\n");
                exit(1);
            }
        }

        memset(replaces, 0, (BUFSIZE+1)*sizeof(replaces[0]));
        if ( !(feof(dslfile)) ) fseek(dslfile, -filepos, SEEK_CUR);
    }

    if ( 0 > (iconv_close(iconv_direction)) ) perror("Error close iconv");
    if (fclose(dictfile)) perror("Error close dictfile");
    if (fclose(dslfile)) perror("Error close dslfile");
    return dictfilename;
}
Beispiel #27
0
void
convert_between_encodings (const char *from, const char *to,
			   const gdb_byte *bytes, unsigned int num_bytes,
			   int width, struct obstack *output,
			   enum transliterations translit)
{
  iconv_t desc;
  struct cleanup *cleanups;
  size_t inleft;
  char *inp;
  unsigned int space_request;

  /* Often, the host and target charsets will be the same.  */
  if (!strcmp (from, to))
    {
      obstack_grow (output, bytes, num_bytes);
      return;
    }

  desc = iconv_open (to, from);
  if (desc == (iconv_t) -1)
    perror_with_name (_("Converting character sets"));
  cleanups = make_cleanup (cleanup_iconv, &desc);

  inleft = num_bytes;
  inp = (char *) bytes;

  space_request = num_bytes;

  while (inleft > 0)
    {
      char *outp;
      size_t outleft, r;
      int old_size;

      old_size = obstack_object_size (output);
      obstack_blank (output, space_request);

      outp = obstack_base (output) + old_size;
      outleft = space_request;

      r = iconv (desc, (ICONV_CONST char **) &inp, &inleft, &outp, &outleft);

      /* Now make sure that the object on the obstack only includes
	 bytes we have converted.  */
      obstack_blank (output, - (int) outleft);

      if (r == (size_t) -1)
	{
	  switch (errno)
	    {
	    case EILSEQ:
	      {
		int i;

		/* Invalid input sequence.  */
		if (translit == translit_none)
		  error (_("Could not convert character "
			   "to `%s' character set"), to);

		/* We emit escape sequence for the bytes, skip them,
		   and try again.  */
		for (i = 0; i < width; ++i)
		  {
		    char octal[5];

		    xsnprintf (octal, sizeof (octal), "\\%.3o", *inp & 0xff);
		    obstack_grow_str (output, octal);

		    ++inp;
		    --inleft;
		  }
	      }
	      break;

	    case E2BIG:
	      /* We ran out of space in the output buffer.  Make it
		 bigger next time around.  */
	      space_request *= 2;
	      break;

	    case EINVAL:
	      /* Incomplete input sequence.  FIXME: ought to report this
		 to the caller somehow.  */
	      inleft = 0;
	      break;

	    default:
	      perror_with_name (_("Internal error while "
				  "converting character sets"));
	    }
	}
    }

  do_cleanups (cleanups);
}
Beispiel #28
0
int main(int argc, char **argv)
{
    FILE *fout = stdout;
    char c;

    boolean alternativeOrder = false;

    while ((c = getopt(argc, argv, "ao:h")) != -1) {
        switch (c) {
        case 'a':
            alternativeOrder = true;
            break;
        case 'o':
            fout = fopen(optarg, "w");

            if (!fout) {
                fprintf(stderr, "Cannot open file: %s\n", optarg);
                return 1;
            }

            break;

        case 'h':

        default:
            usage();

            break;
        }
    }

    if (optind >= argc) {
        usage();
        return 1;
    }

    FILE *fp = fopen(argv[optind], "r");
    if (!fp) {
        fprintf(stderr, "Cannot open file: %s\n", argv[optind]);
        return 1;
    }

    char buf[BUFLEN], bufout[BUFLEN];

    size_t count = fread(buf, 1, HEADER_SIZE, fp);

    if (count < HEADER_SIZE || memcmp(buf, header_str, HEADER_SIZE) != 0) {
        fprintf(stderr, "format error.\n");
        fclose(fp);
        return 1;
    }

    conv = iconv_open("utf-8", "unicode");
    if (conv == (iconv_t) -1) {
        fprintf(stderr, "iconv error.\n");
        return 1;
    }

    fseek(fp, DESC_START, SEEK_SET);
    fread(buf, 1, DESC_LENGTH, fp);
    IconvStr in = buf;
    char *out = bufout;
    size_t inlen = DESC_LENGTH, outlen = BUFLEN;
    iconv(conv, &in, &inlen, &out, &outlen);
    fprintf(stderr, "%s\n", bufout);

    fread(buf, 1, LDESC_LENGTH, fp);
    in = buf;
    out = bufout;
    inlen = LDESC_LENGTH;
    outlen = BUFLEN;
    iconv(conv, &in, &inlen, &out, &outlen);
    fprintf(stderr, "%s\n", bufout);

    fread(buf, 1, NEXT_LENGTH, fp);
    in = buf;
    out = bufout;
    inlen = NEXT_LENGTH;
    outlen = BUFLEN;
    iconv(conv, &in, &inlen, &out, &outlen);
    fprintf(stderr, "%s\n", bufout);

    count = fread(buf, 1, PINYIN_SIZE, fp);

    if (count < PINYIN_SIZE || memcmp(buf, pinyin_str, PINYIN_SIZE) != 0) {
        fprintf(stderr, "format error.\n");
        fclose(fp);
        return 1;
    }

    UT_array pys;
    utarray_init(&pys, &py_icd);

    for (; ;) {
        int16_t index;
        int16_t count;
        fread(&index, 1, sizeof(int16_t), fp);
        fread(&count, 1, sizeof(int16_t), fp);
        ScelPinyin py;
        memset(buf, 0, sizeof(buf));
        memset(&py, 0, sizeof(ScelPinyin));
        fread(buf, count, sizeof(char), fp);

        in = buf;
        out = py.pinyin;
        inlen = count * sizeof(char);
        outlen = 10;
        iconv(conv, &in, &inlen, &out, &outlen);

        utarray_push_back(&pys, &py);

        if (strcmp(py.pinyin, "zuo") == 0)
            break;
    }

    while (!feof(fp)) {
        int16_t symcount;
        int16_t count;
        int16_t wordcount;
        fread(&symcount, 1, sizeof(int16_t), fp);

        if (feof(fp))
            break;

        fread(&count, 1, sizeof(int16_t), fp);

        wordcount = count / 2;
        int16_t* pyindex = malloc(sizeof(int16_t) * wordcount);

        fread(pyindex, wordcount, sizeof(int16_t), fp);

        int s;

        for (s = 0; s < symcount ; s++) {

            memset(buf, 0, sizeof(buf));

            memset(bufout, 0, sizeof(bufout));
            fread(&count, 1, sizeof(int16_t), fp);
            fread(buf, count, sizeof(char), fp);
            in = buf;
            out = bufout;
            inlen = count * sizeof(char);
            outlen = BUFLEN;
            iconv(conv, &in, &inlen, &out, &outlen);

            if (alternativeOrder) {
                fprintf(fout, "%s ", bufout);
            }

            ScelPinyin *py = (ScelPinyin*)utarray_eltptr(
                &pys, (unsigned int)pyindex[0]);
            fprintf(fout, "%s",  py->pinyin);
            int i;

            for (i = 1 ; i < wordcount ; i ++) {
                py = (ScelPinyin*)utarray_eltptr(&pys,
                                                 (unsigned int)pyindex[i]);
                fprintf(fout, "\'%s", py->pinyin);
            }

            if (!alternativeOrder) {
                fprintf(fout, " %s", bufout);
            }
            fprintf(fout, "\n");

            fread(&count, 1, sizeof(int16_t), fp);
            fread(buf, count, sizeof(char), fp);
        }

        free(pyindex);
    }

    iconv_close(conv);

    utarray_done(&pys);
    fclose(fout);
    fclose(fp);
    return 0;
}
Beispiel #29
0
// Use iconv to convert buf to UTF-8.
// Returns buf.start==NULL on error. Returns buf if cp is NULL, or if there is
// obviously no conversion required (e.g. if cp is "UTF-8").
// Returns a newly allocated buffer if conversion is done and succeeds. The
// buffer will be terminated with 0 for convenience (the terminating 0 is not
// included in the returned length).
// Free the returned buffer with talloc_free().
//  buf: input data
//  cp: iconv codepage (or NULL)
//  flags: combination of MP_ICONV_* flags
//  returns: buf (no conversion), .start==NULL (error), or allocated buffer
bstr mp_iconv_to_utf8(bstr buf, const char *cp, int flags)
{
#if HAVE_ICONV
    if (!cp || !cp[0] || mp_charset_is_utf8(cp))
        return buf;

    if (strcasecmp(cp, "ASCII") == 0)
        return buf;

    if (strcasecmp(cp, "UTF-8-BROKEN") == 0)
        return bstr_sanitize_utf8_latin1(NULL, buf);

    iconv_t icdsc;
    if ((icdsc = iconv_open("UTF-8", cp)) == (iconv_t) (-1)) {
        if (flags & MP_ICONV_VERBOSE)
            mp_msg(MSGT_SUBREADER, MSGL_ERR,
                   "Error opening iconv with codepage '%s'\n", cp);
        goto failure;
    }

    size_t size = buf.len;
    size_t osize = size;
    size_t ileft = size;
    size_t oleft = size - 1;

    char *outbuf = talloc_size(NULL, osize);
    char *ip = buf.start;
    char *op = outbuf;

    while (1) {
        int clear = 0;
        size_t rc;
        if (ileft)
            rc = iconv(icdsc, &ip, &ileft, &op, &oleft);
        else {
            clear = 1; // clear the conversion state and leave
            rc = iconv(icdsc, NULL, NULL, &op, &oleft);
        }
        if (rc == (size_t) (-1)) {
            if (errno == E2BIG) {
                size_t offset = op - outbuf;
                outbuf = talloc_realloc_size(NULL, outbuf, osize + size);
                op = outbuf + offset;
                osize += size;
                oleft += size;
            } else {
                if (errno == EINVAL && (flags & MP_ICONV_ALLOW_CUTOFF)) {
                    // This is intended for cases where the input buffer is cut
                    // at a random byte position. If this happens in the middle
                    // of the buffer, it should still be an error. We say it's
                    // fine if the error is within 10 bytes of the end.
                    if (ileft <= 10)
                        break;
                }
                if (flags & MP_ICONV_VERBOSE) {
                    mp_msg(MSGT_SUBREADER, MSGL_ERR,
                           "Error recoding text with codepage '%s'\n", cp);
                }
                talloc_free(outbuf);
                iconv_close(icdsc);
                goto failure;
            }
        } else if (clear)
            break;
    }

    iconv_close(icdsc);

    outbuf[osize - oleft - 1] = 0;
    return (bstr){outbuf, osize - oleft - 1};
#endif

failure:
    return (bstr){0};
}
Beispiel #30
0
extern	Bool
SetValueStringWithLength(
	ValueStruct	*val,
	char		*str,
	size_t		slen,
	char		*codeset)
{
	Bool	rc
	,		fMinus
	,		fPoint;
	size_t	len;
	byte	*p;
	byte	buff[SIZE_NUMBUF+1]
		,	sbuff[SIZE_LONGNAME+1];
	Fixed	from;
	size_t	size;
#ifdef	WITH_I18N
	byte	*q;
	iconv_t	cd;
	size_t	sob
	,		sib;
	char	*istr;
#endif

ENTER_FUNC;
	if		(  val  ==  NULL  ) {
		fprintf(stderr,"no ValueStruct\n");
		rc = FALSE;
	} else
	if		(	(  str   ==  NULL      )
			||	(  *str  ==  CHAR_NIL  ) ) {
		ValueIsNil(val);
		rc = TRUE;
	} else {
		ValueIsNonNil(val);
		switch	(ValueType(val)) {
		  case	GL_TYPE_CHAR:
		  case	GL_TYPE_VARCHAR:
		  case	GL_TYPE_DBCODE:
		  case	GL_TYPE_TEXT:
		  case	GL_TYPE_SYMBOL:
#ifdef	WITH_I18N
			if		(	(  codeset  !=  NULL  )
					&&	(	(  stricmp(codeset,"utf8")   ==  0  )
						||	(  stricmp(codeset,"utf-8")  ==  0  ) ) )	{
				codeset = NULL;
			}
			if		(  codeset  !=  NULL  ) {
				if		(  IS_VALUE_EXPANDABLE(val)  ) {
					len = slen;
				} else {
					len = ValueStringLength(val) < slen ?
						ValueStringLength(val) : slen;
				}
				cd = iconv_open("utf8",codeset);
				while	(TRUE) {
					istr = str;
					sib = len;
					sob = ValueStringSize(val);
					if		(  ( q = ValueString(val) )  !=  NULL  ) {
						memclear(ValueString(val),ValueStringSize(val));
						if		(  iconv(cd,&istr,&sib,(void*)&q,&sob)  ==  0  )	break;
						if		(  errno  ==  E2BIG ) {
							xfree(ValueString(val));
							ValueStringSize(val) *= 2;
						} else
							break;
					} else {
						ValueStringSize(val) = 1;
					}
					ValueString(val) = (byte *)xmalloc(ValueStringSize(val));
				};
				iconv_close(cd);
				*q = 0;
				len = ValueStringSize(val) - sob;
			} else {
#endif
				size = slen + 1;
				len = slen;
				if		(  size  >  ValueStringSize(val)  ) {
					if		(  ValueString(val)  !=  NULL  ) {
						xfree(ValueString(val));
					}
					ValueStringSize(val) = size;
					ValueString(val) = (byte *)xmalloc(size);
				}
				memclear(ValueString(val),ValueStringSize(val));
				strcpy(ValueString(val),str);
#ifdef	WITH_I18N
			}
#endif
			if		(  IS_VALUE_EXPANDABLE(val)  ) {
				ValueStringLength(val) = len;
			}
			rc = TRUE;
			break;
		  case	GL_TYPE_BYTE:
			DecodeStringToBinary(ValueByte(val),ValueByteLength(val),str);
			rc = TRUE;
			break;
		  case	GL_TYPE_BINARY:
#ifdef	BINARY_IS_BASE64
			p = (byte *)xmalloc(strlen(str));
			size = strlen(str);
			size = DecodeBase64(p,size,str,size);
			if		(  ValueByte(val)  !=  NULL  ) {
				xfree(ValueByte(val));
			}
			ValueByteSize(val) = strlen(str);
			ValueByte(val) = p;
#else
			size = 0;
			for	( p = str ; *p != 0 ; ) {
				if		(  *p  ==  '\\'  ) {
					p ++;
					if		(  *p  ==  'u'  ) {
						p += 3;
					} else {
						p ++;
					}
				} else {
					p ++;
				}
				size ++;
			}
			if		(  size  >  ValueByteSize(val)  ) {
				if		(  ValueByte(val)  !=  NULL  ) {
						xfree(ValueByte(val));
				}
				ValueByteSize(val) = size;
				ValueByte(val) = (byte *)xmalloc(size);
			}
			memclear(ValueByte(val),ValueByteSize(val));
			DecodeStringToBinary(ValueByte(val),size,str);
#endif
			ValueByteLength(val) = size;
			rc = TRUE;
			break;
		  case	GL_TYPE_OBJECT:
			ValueObjectId(val) = 0;
			for	( ; slen > 0 ; slen -- )	{
				if		(  isdigit(*str)  )	{
					ValueObjectId(val) = ValueObjectId(val) * 10 + ( *str - '0' );
				}
				str ++;
			}
			if		(  ValueObjectFile(val)  !=  NULL  ) {
				xfree(ValueObjectFile(val));
				ValueObjectFile(val) = NULL;
			}
			rc = TRUE;
			break;
		  case	GL_TYPE_NUMBER:
		  case	GL_TYPE_INT:
		  case	GL_TYPE_FLOAT:
		  case	GL_TYPE_BOOL:
		  case	GL_TYPE_TIMESTAMP:
		  case	GL_TYPE_DATE:
		  case	GL_TYPE_TIME:
#ifdef	WITH_I18N
			if		(  codeset  !=  NULL  ) {
				cd = iconv_open("utf8",codeset);
				istr = str;
				sib = slen;
				sob = SIZE_NUMBUF;
				p = sbuff;
				iconv(cd,&istr,&sib,(void*)&p,&sob);
				iconv_close(cd);
				*p = 0;
				str = sbuff;
			} else {
#endif
				strncpy(sbuff,str,SIZE_NUMBUF);
				str = sbuff;
#ifdef	WITH_I18N
			}
#endif
			switch	(ValueType(val)) {
			  case	GL_TYPE_NUMBER:
				p = buff;
				from.flen = 0;
				from.slen = 0;
				from.sval = buff;
				fPoint = FALSE;
				fMinus = FALSE;
				while	(  *str  !=  0  ) {
					if		(  fPoint  ) {
						from.slen ++;
					}
					if		(  *str  ==  '-'  ) {
						fMinus = TRUE;
					} else
					if	(  isdigit(*str)  ) {
						*p ++ = *str;
						from.flen ++;
					} else
					if		(  *str  ==  '.'  ) {
						fPoint = TRUE;
					}
					str ++;
				}
				*p = 0;
				if		(  fMinus  ) {
					*buff |= 0x40;
				}
				FixedRescale(&ValueFixed(val),&from);
				rc = TRUE;
				break;
			  case	GL_TYPE_INT:
				ValueInteger(val) = StrToInt(str,slen);
				rc = TRUE;
				break;
			  case	GL_TYPE_FLOAT:
				ValueFloat(val) = atof(str);
				rc = TRUE;
				break;
			  case	GL_TYPE_BOOL:
				ValueBool(val) = ( toupper(*str) == 'T' ) ? TRUE : FALSE;
				rc = TRUE;
				break;
			  case	GL_TYPE_TIMESTAMP:
				ValueDateTimeYear(val) = StrToInt(str,4);		str += 4;
				ValueDateTimeMon(val) = StrToInt(str,2) - 1;	str += 2;
				ValueDateTimeMDay(val) = StrToInt(str,2);		str += 2;
				ValueDateTimeHour(val) = StrToInt(str,2);		str += 2;
				ValueDateTimeMin(val) = StrToInt(str,2);		str += 2;
				ValueDateTimeSec(val) = StrToInt(str,2);		str += 2;
				rc = mktime(&ValueDateTime(val)) >= 0 ? TRUE : FALSE;
				break;
			  case	GL_TYPE_DATE:
				ValueDateTimeYear(val) = StrToInt(str,4);		str += 4;
				ValueDateTimeMon(val) = StrToInt(str,2) - 1;	str += 2;
				ValueDateTimeMDay(val) = StrToInt(str,2);		str += 2;
				ValueDateTimeHour(val) = 0;
				ValueDateTimeMin(val) = 0;
				ValueDateTimeSec(val) = 0;
				rc = mktime(&ValueDateTime(val)) >= 0 ? TRUE : FALSE;
				break;
			  case	GL_TYPE_TIME:
				ValueDateTimeYear(val) = 0;
				ValueDateTimeMon(val) = 0;
				ValueDateTimeMDay(val) = 0;
				ValueDateTimeHour(val) = StrToInt(str,2);	str += 2;
				ValueDateTimeMin(val) = StrToInt(str,2);	str += 2;
				ValueDateTimeSec(val) = StrToInt(str,2);	str += 2;
				rc = TRUE;
				break;
			  default:
				rc = TRUE;
				break;
			}
			break;
		  default:
			ValueIsNil(val);
			rc = FALSE;	  
		}
	}
LEAVE_FUNC;
	return	(rc);
}