/** Read settings flags from a list
*
* may call Rf_error
*
* @param opts_fixed list
* @param allow_overlap
* @return flags
*
* @version 0.4-1 (Marek Gagolewski, 2014-12-07)
*
* @version 0.4-1 (Marek Gagolewski, 2014-12-08)
* add `overlap` option
*/
uint32_t StriContainerByteSearch::getByteSearchFlags(SEXP opts_fixed, bool allow_overlap)
{
uint32_t flags = 0;
if (!isNull(opts_fixed) && !Rf_isVectorList(opts_fixed))
Rf_error(MSG__ARG_EXPECTED_LIST, "opts_fixed"); // error() call allowed here
R_len_t narg = isNull(opts_fixed)?0:LENGTH(opts_fixed);
if (narg > 0) {
SEXP names = Rf_getAttrib(opts_fixed, R_NamesSymbol);
if (names == R_NilValue || LENGTH(names) != narg)
Rf_error(MSG__FIXED_CONFIG_FAILED); // error() call allowed here
for (R_len_t i=0; i<narg; ++i) {
if (STRING_ELT(names, i) == NA_STRING)
Rf_error(MSG__FIXED_CONFIG_FAILED); // error() call allowed here
const char* curname = CHAR(STRING_ELT(names, i));
if (!strcmp(curname, "case_insensitive")) {
bool val = stri__prepare_arg_logical_1_notNA(VECTOR_ELT(opts_fixed, i), "case_insensitive");
if (val) flags |= BYTESEARCH_CASE_INSENSITIVE;
} else if (!strcmp(curname, "overlap") && allow_overlap) {
bool val = stri__prepare_arg_logical_1_notNA(VECTOR_ELT(opts_fixed, i), "overlap");
if (val) flags |= BYTESEARCH_OVERLAP;
} else {
Rf_warning(MSG__INCORRECT_FIXED_OPTION, curname);
}
}
}
return flags;
}
/**
* Detect if a character class occurs in a string
*
* @param str character vector
* @param pattern character vector
* @param omit_na single logical value
* @return logical vector
*
* @version 0.3-1 (Bartek Tartanus, 2014-07-25)
*
* @version 0.3-1 (Marek Gagolewski, 2014-10-17)
* using std::vector<int> to avoid mem-leaks
*
* @version 0.3-1 (Marek Gagolewski, 2014-11-04)
* Issue #112: str_prepare_arg* retvals were not PROTECTed from gc
*
* @version 0.4-1 (Marek Gagolewski, 2014-12-04)
* FR #122: omit_na arg added
*
* @version 1.0-3 (Marek Gagolewski, 2016-02-03)
* FR #216: `negate` arg added
*/
SEXP stri_subset_charclass(SEXP str, SEXP pattern, SEXP omit_na, SEXP negate)
{
bool negate_1 = stri__prepare_arg_logical_1_notNA(negate, "negate");
bool omit_na1 = stri__prepare_arg_logical_1_notNA(omit_na, "omit_na");
PROTECT(str = stri_prepare_arg_string(str, "str"));
PROTECT(pattern = stri_prepare_arg_string(pattern, "pattern"));
R_len_t vectorize_length =
stri__recycling_rule(true, 2, LENGTH(str), LENGTH(pattern));
STRI__ERROR_HANDLER_BEGIN(2)
StriContainerUTF8 str_cont(str, vectorize_length);
StriContainerCharClass pattern_cont(pattern, vectorize_length);
// BT: this cannot be done with deque, because pattern is reused so i does not
// go like 0,1,2...n but 0,pat_len,2*pat_len,1,pat_len+1 and so on
// MG: agreed
std::vector<int> which(vectorize_length);
int result_counter = 0;
for (R_len_t i = pattern_cont.vectorize_init();
i != pattern_cont.vectorize_end();
i = pattern_cont.vectorize_next(i))
{
if (str_cont.isNA(i) || pattern_cont.isNA(i)) {
if (omit_na1) which[i] = FALSE;
else {
which[i] = NA_LOGICAL;
result_counter++;
}
continue;
}
const UnicodeSet* pattern_cur = &pattern_cont.get(i);
R_len_t str_cur_n = str_cont.get(i).length();
const char* str_cur_s = str_cont.get(i).c_str();
UChar32 chr = 0;
which[i] = FALSE;
for (R_len_t j=0; j<str_cur_n; ) {
U8_NEXT(str_cur_s, j, str_cur_n, chr);
if (chr < 0) // invalid utf-8 sequence
throw StriException(MSG__INVALID_UTF8);
if (pattern_cur->contains(chr)) {
which[i] = TRUE;
break;
}
}
if (negate_1) which[i] = !which[i];
if (which[i]) result_counter++;
}
SEXP ret;
STRI__PROTECT(ret = stri__subset_by_logical(str_cont, which, result_counter));
STRI__UNPROTECT_ALL
return ret;
STRI__ERROR_HANDLER_END(;/* nothing special to be done on error */)
}
/** Locate all BreakIterator boundaries
*
* @param str character vector
* @param omit_no_match logical
* @param opts_brkiter named list
* @return list
*
* @version 0.2-2 (Marek Gagolewski, 2014-04-22)
*
* @version 0.2-2 (Marek Gagolewski, 2014-04-23)
* removed "title": For Unicode 4.0 and above title boundary
* iteration, please use Word Boundary iterator.
*
* @version 0.2-2 (Marek Gagolewski, 2014-04-25)
* use stri__split_or_locate_boundaries
*
* @version 0.3-1 (Marek Gagolewski, 2014-10-29)
* use opts_brkiter
*
* @version 0.4-1 (Marek Gagolewski, 2014-11-28)
* new args: omit_no_match
*
* @version 0.4-1 (Marek Gagolewski, 2014-12-02)
* use StriRuleBasedBreakIterator
*/
SEXP stri_locate_all_boundaries(SEXP str, SEXP omit_no_match, SEXP opts_brkiter)
{
bool omit_no_match1 = stri__prepare_arg_logical_1_notNA(omit_no_match, "omit_no_match");
PROTECT(str = stri_prepare_arg_string(str, "str"));
StriBrkIterOptions opts_brkiter2(opts_brkiter, "line_break");
STRI__ERROR_HANDLER_BEGIN(1)
R_len_t str_length = LENGTH(str);
StriContainerUTF8_indexable str_cont(str, str_length);
StriRuleBasedBreakIterator brkiter(opts_brkiter2);
SEXP ret;
STRI__PROTECT(ret = Rf_allocVector(VECSXP, str_length));
for (R_len_t i = 0; i < str_length; ++i)
{
if (str_cont.isNA(i)) {
SET_VECTOR_ELT(ret, i, stri__matrix_NA_INTEGER(1, 2));
continue;
}
brkiter.setupMatcher(str_cont.get(i).c_str(), str_cont.get(i).length());
brkiter.first();
deque< pair<R_len_t,R_len_t> > occurrences;
pair<R_len_t,R_len_t> curpair;
while (brkiter.next(curpair))
occurrences.push_back(curpair);
R_len_t noccurrences = (R_len_t)occurrences.size();
if (noccurrences <= 0) {
SET_VECTOR_ELT(ret, i, stri__matrix_NA_INTEGER(omit_no_match1?0:1, 2));
continue;
}
SEXP ans;
STRI__PROTECT(ans = Rf_allocMatrix(INTSXP, noccurrences, 2));
int* ans_tab = INTEGER(ans);
deque< pair<R_len_t, R_len_t> >::iterator iter = occurrences.begin();
for (R_len_t j = 0; iter != occurrences.end(); ++iter, ++j) {
pair<R_len_t, R_len_t> cur_match = *iter;
ans_tab[j] = cur_match.first;
ans_tab[j+noccurrences] = cur_match.second;
}
// Adjust UChar index -> UChar32 index (1-2 byte UTF16 to 1 byte UTF32-code points)
str_cont.UTF8_to_UChar32_index(i, ans_tab,
ans_tab+noccurrences, noccurrences,
1, // 0-based index -> 1-based
0 // end returns position of next character after match
);
SET_VECTOR_ELT(ret, i, ans);
STRI__UNPROTECT(1);
}
stri__locate_set_dimnames_list(ret);
STRI__UNPROTECT_ALL
return ret;
STRI__ERROR_HANDLER_END({ /* nothing special t.b.d. on error */ })
}
/**
* Substitutes vector elements if a pattern occurs in a string
*
* @param str character vector
* @param pattern character vector
* @param value character vector
* @return character vector
*
* @version 1.0-3 (Marek Gagolewski, 2016-02-03)
* FR#124
*
* @version 1.0-3 (Marek Gagolewski, 2016-02-03)
* FR #216: `negate` arg added
*/
SEXP stri_subset_charclass_replacement(SEXP str, SEXP pattern, SEXP negate, SEXP value)
{
bool negate_1 = stri__prepare_arg_logical_1_notNA(negate, "negate");
PROTECT(str = stri_prepare_arg_string(str, "str"));
PROTECT(pattern = stri_prepare_arg_string_1(pattern, "pattern"));
PROTECT(value = stri_prepare_arg_string(value, "value"));
int vectorize_length = LENGTH(str);
int value_length = LENGTH(value);
if (value_length == 0)
Rf_error(MSG__REPLACEMENT_ZERO);
STRI__ERROR_HANDLER_BEGIN(3)
StriContainerUTF8 str_cont(str, vectorize_length);
StriContainerUTF8 value_cont(value, value_length);
StriContainerCharClass pattern_cont(pattern, vectorize_length);
SEXP ret;
STRI__PROTECT(ret = Rf_allocVector(STRSXP, vectorize_length));
R_len_t k = 0;
for (R_len_t i = str_cont.vectorize_init();
i != str_cont.vectorize_end();
i = str_cont.vectorize_next(i))
{
if (str_cont.isNA(i) || pattern_cont.isNA(i)) {
SET_STRING_ELT(ret, i, NA_STRING);
continue;
}
const UnicodeSet* pattern_cur = &pattern_cont.get(i);
R_len_t str_cur_n = str_cont.get(i).length();
const char* str_cur_s = str_cont.get(i).c_str();
UChar32 chr = 0;
bool found = false;
for (R_len_t j=0; j<str_cur_n; ) {
U8_NEXT(str_cur_s, j, str_cur_n, chr);
if (chr < 0) // invalid utf-8 sequence
throw StriException(MSG__INVALID_UTF8);
if (pattern_cur->contains(chr)) {
found = true;
break;
}
}
if ((found && !negate_1) || (!found && negate_1))
SET_STRING_ELT(ret, i, value_cont.toR((k++)%value_length));
else
SET_STRING_ELT(ret, i, str_cont.toR(i));
}
STRI__UNPROTECT_ALL
return ret;
STRI__ERROR_HANDLER_END(;/* nothing special to be done on error */)
}
/**
* Detect if a pattern occurs in a string
*
* @param str R character vector
* @param pattern R character vector containing regular expressions
* @param negate single bool
* @param max_count single int
* @param opts_regex list
*
* @version 0.1-?? (Marcin Bujarski)
*
* @version 0.1-?? (Marek Gagolewski)
* use StriContainerUTF16
*
* @version 0.1-?? (Marek Gagolewski)
* use StriContainerUTF16's vectorization
*
* @version 0.1-?? (Marek Gagolewski, 2013-06-18)
* use StriContainerRegexPattern + opts_regex
*
* @version 0.3-1 (Marek Gagolewski, 2014-11-05)
* Issue #112: str_prepare_arg* retvals were not PROTECTed from gc
*
* @version 1.0-2 (Marek Gagolewski, 2016-01-29)
* Issue #214: allow a regex pattern like `.*` to match an empty string
*
* @version 1.0-3 (Marek Gagolewski, 2016-02-03)
* FR #216: `negate` arg added
*
* @version 1.3.1 (Marek Gagolewski, 2019-02-08)
* #232: `max_count` arg added
*/
SEXP stri_detect_regex(SEXP str, SEXP pattern, SEXP negate,
SEXP max_count, SEXP opts_regex)
{
bool negate_1 = stri__prepare_arg_logical_1_notNA(negate, "negate");
int max_count_1 = stri__prepare_arg_integer_1_notNA(max_count, "max_count");
PROTECT(str = stri_prepare_arg_string(str, "str"));
PROTECT(pattern = stri_prepare_arg_string(pattern, "pattern"));
R_len_t vectorize_length =
stri__recycling_rule(true, 2, LENGTH(str), LENGTH(pattern));
uint32_t pattern_flags = StriContainerRegexPattern::getRegexFlags(opts_regex);
STRI__ERROR_HANDLER_BEGIN(2)
StriContainerUTF16 str_cont(str, vectorize_length);
// StriContainerUTF8 str_cont(str, vectorize_length); // utext_openUTF8, see below
StriContainerRegexPattern pattern_cont(pattern, vectorize_length, pattern_flags);
SEXP ret;
STRI__PROTECT(ret = Rf_allocVector(LGLSXP, vectorize_length));
int* ret_tab = LOGICAL(ret);
for (R_len_t i = pattern_cont.vectorize_init();
i != pattern_cont.vectorize_end();
i = pattern_cont.vectorize_next(i))
{
if (max_count_1 == 0) {
ret_tab[i] = NA_LOGICAL;
continue;
}
STRI__CONTINUE_ON_EMPTY_OR_NA_PATTERN(str_cont,
pattern_cont, ret_tab[i] = NA_LOGICAL)
RegexMatcher *matcher = pattern_cont.getMatcher(i); // will be deleted automatically
matcher->reset(str_cont.get(i));
ret_tab[i] = (int)matcher->find(); // returns UBool
if (negate_1) ret_tab[i] = !ret_tab[i];
if (max_count_1 > 0 && ret_tab[i]) --max_count_1;
// // mbmark-regex-detect1.R: UTF16 0.07171792 s; UText 0.10531605 s
// UText* str_text = NULL;
// UErrorCode status = U_ZERO_ERROR;
// RegexMatcher *matcher = pattern_cont.getMatcher(i); // will be deleted automatically
// str_text = utext_openUTF8(str_text, str_cont.get(i).c_str(), str_cont.get(i).length(), &status);
// STRI__CHECKICUSTATUS_THROW(status, {/* do nothing special on err */})
// matcher->reset(str_text);
// ret_tab[i] = (int)matcher->find(); // returns UBool
// utext_close(str_text);
}
STRI__UNPROTECT_ALL
return ret;
STRI__ERROR_HANDLER_END(;/* nothing special to be done on error */)
}
/**
* Detect if a character class occurs in a string
*
* @param str character vector
* @param pattern character vector
* @param negate single bool
* @param max_count single int
* @return logical vector
*
* @version 0.1-?? (Bartek Tartanus)
*
* @version 0.1-?? (Marek Gagolewski, 2013-06-02)
* Use StrContainerUTF8 and CharClass classes
*
* @version 0.1-?? (Marek Gagolewski, 2013-06-15)
* Use StrContainerCharClass
*
* @version 0.1-?? (Marek Gagolewski, 2013-06-16)
* make StriException-friendly
*
* @version 0.2-1 (Marek Gagolewski, 2014-04-03)
* detects invalid UTF-8 byte stream
*
* @version 0.2-1 (Marek Gagolewski, 2014-04-05)
* StriContainerCharClass now relies on UnicodeSet
*
* @version 0.3-1 (Marek Gagolewski, 2014-11-04)
* Issue #112: str_prepare_arg* retvals were not PROTECTed from gc
*
* @version 1.0-3 (Marek Gagolewski, 2016-02-03)
* FR #216: `negate` arg added
*
* @version 1.3.1 (Marek Gagolewski, 2019-02-08)
* #232: `max_count` arg added
*/
SEXP stri_detect_charclass(SEXP str, SEXP pattern,
SEXP negate, SEXP max_count)
{
bool negate_1 = stri__prepare_arg_logical_1_notNA(negate, "negate");
int max_count_1 = stri__prepare_arg_integer_1_notNA(max_count, "max_count");
PROTECT(str = stri_prepare_arg_string(str, "str"));
PROTECT(pattern = stri_prepare_arg_string(pattern, "pattern"));
R_len_t vectorize_length =
stri__recycling_rule(true, 2, LENGTH(str), LENGTH(pattern));
STRI__ERROR_HANDLER_BEGIN(2)
StriContainerUTF8 str_cont(str, vectorize_length);
StriContainerCharClass pattern_cont(pattern, vectorize_length);
SEXP ret;
STRI__PROTECT(ret = Rf_allocVector(LGLSXP, vectorize_length));
int* ret_tab = LOGICAL(ret);
for (R_len_t i = pattern_cont.vectorize_init();
i != pattern_cont.vectorize_end();
i = pattern_cont.vectorize_next(i))
{
if (max_count_1 == 0 || str_cont.isNA(i) || pattern_cont.isNA(i)) {
ret_tab[i] = NA_LOGICAL;
continue;
}
const UnicodeSet* pattern_cur = &pattern_cont.get(i);
R_len_t str_cur_n = str_cont.get(i).length();
const char* str_cur_s = str_cont.get(i).c_str();
UChar32 chr = 0;
ret_tab[i] = FALSE;
for (R_len_t j=0; j<str_cur_n; ) {
U8_NEXT(str_cur_s, j, str_cur_n, chr);
if (chr < 0) // invalid UTF-8 sequence
throw StriException(MSG__INVALID_UTF8);
if (pattern_cur->contains(chr)) {
ret_tab[i] = TRUE;
break;
}
}
if (negate_1) ret_tab[i] = !ret_tab[i];
if (max_count_1 > 0 && ret_tab[i]) --max_count_1;
}
STRI__UNPROTECT_ALL
return ret;
STRI__ERROR_HANDLER_END(;/* nothing special to be done on error */)
}
/** Convert character vector to UTF-8
*
* @param str character vector
* @param is_unknown_8bit single logical value;
* if TRUE, then in case of ENC_NATIVE or ENC_LATIN1, UTF-8
* REPLACEMENT CHARACTERs (U+FFFD) are
* put for codes > 127
* @return character vector
*
* @version 0.1 (Marek Gagolewski)
* @version 0.2 (Marek Gagolewski, 2013-06-16) make StriException-friendly
*/
SEXP stri_enc_toutf8(SEXP str, SEXP is_unknown_8bit)
{
str = stri_prepare_arg_string(str, "str");
R_len_t n = LENGTH(str);
bool is_unknown_8bit_logical = stri__prepare_arg_logical_1_notNA(is_unknown_8bit, "is_unknown_8bit");
STRI__ERROR_HANDLER_BEGIN
if (is_unknown_8bit_logical) {
SEXP ret;
PROTECT(ret = Rf_allocVector(STRSXP, n));
for (R_len_t i=0; i<n; ++i) {
SEXP curs = STRING_ELT(str, i);
if (curs == NA_STRING) {
SET_STRING_ELT(ret, i, NA_STRING);
continue;
}
else if (IS_ASCII(curs) || IS_UTF8(curs)) {
SET_STRING_ELT(ret, i, curs);
}
else { // some 8-bit encoding
R_len_t curn = LENGTH(curs);
const char* curs_tab = CHAR(curs);
// TODO: buffer reuse....
String8 buf(curn*3+1); // one byte -> either one byte or FFFD, which is 3 bytes in UTF-8
R_len_t k = 0;
for (R_len_t j=0; j<curn; ++j) {
if (U8_IS_SINGLE(curs_tab[j]))
buf.data()[k++] = curs_tab[j];
else { // 0xEF 0xBF 0xBD
buf.data()[k++] = (char)UCHAR_REPLACEMENT_UTF8_BYTE1;
buf.data()[k++] = (char)UCHAR_REPLACEMENT_UTF8_BYTE2;
buf.data()[k++] = (char)UCHAR_REPLACEMENT_UTF8_BYTE3;
}
}
SET_STRING_ELT(ret, i, Rf_mkCharLenCE(buf.data(), k, CE_UTF8));
}
}
UNPROTECT(1);
return ret;
}
else {
// Trivial - everything we need is in StriContainerUTF8 :)
StriContainerUTF8 str_cont(str, n);
return str_cont.toR();
}
STRI__ERROR_HANDLER_END(;/* nothing special to be done on error */)
}
/**
* Convert character vector between given encodings
*
* @param str input character/raw vector or list of raw vectors
* @param from source encoding, \code{NULL} or \code{""} for default enc
* @param to target encoding, \code{NULL} or \code{""} for default enc
* @param to_raw single logical, should list of raw vectors be returned?
* @return a converted character vector or list of raw vectors
*
* @version 0.1-?? (Marek Gagolewski)
*
* @version 0.1-?? (Marek Gagolewski)
* arg to_raw_added, encoding marking
*
* @version 0.1-?? (Marek Gagolewski, 2013-06-16)
* make StriException-friendly
*
* @version 0.1-?? (Marek Gagolewski, 2013-08-08)
* use StriContainerListRaw
*
* @version 0.1-?? (Marek Gagolewski, 2013-11-20)
* BUGFIX call stri_encode_from_marked if necessary
*
* @version 0.2-1 (Marek Gagolewski, 2014-03-28)
* use StriUcnv
*
* @version 0.2-1 (Marek Gagolewski, 2014-04-01)
* estimate required buf size a priori
*
* @version 0.3-1 (Marek Gagolewski, 2014-11-04)
* Issue #112: str_prepare_arg* retvals were not PROTECTed from gc
*/
SEXP stri_encode(SEXP str, SEXP from, SEXP to, SEXP to_raw)
{
const char* selected_from = stri__prepare_arg_enc(from, "from", true); /* this is R_alloc'ed */
if (!selected_from && Rf_isVectorAtomic(str))
return stri_encode_from_marked(str, to, to_raw);
const char* selected_to = stri__prepare_arg_enc(to, "to", true); /* this is R_alloc'ed */
bool to_raw_logical = stri__prepare_arg_logical_1_notNA(to_raw, "to_raw");
// raw vector, character vector, or list of raw vectors:
PROTECT(str = stri_prepare_arg_list_raw(str, "str"));
STRI__ERROR_HANDLER_BEGIN(1)
StriContainerListRaw str_cont(str);
R_len_t str_n = str_cont.get_n();
// get the number of strings to convert; if == 0, then you know what's the result
if (str_n <= 0) {
STRI__UNPROTECT_ALL
return Rf_allocVector(to_raw_logical?VECSXP:STRSXP, 0);
}
/**
* Convert character vector between marked encodings and the encoding provided
*
* @param str input character vector or list of raw vectors
* @param to target encoding, \code{NULL} or \code{""} for default enc
* @param to_raw single logical, should list of raw vectors be returned?
* @return a converted character vector or list of raw vectors
*
* @version 0.1-?? (Marek Gagolewski, 2013-11-12)
*
* @version 0.2-1 (Marek Gagolewski, 2014-03-28)
* use StriUcnv
*
* @version 0.2-1 (Marek Gagolewski, 2014-04-01)
* calc required buf size a priori
*
* @version 0.3-1 (Marek Gagolewski, 2014-11-04)
* Issue #112: str_prepare_arg* retvals were not PROTECTed from gc
*/
SEXP stri_encode_from_marked(SEXP str, SEXP to, SEXP to_raw)
{
PROTECT(str = stri_prepare_arg_string(str, "str"));
const char* selected_to = stri__prepare_arg_enc(to, "to", true); /* this is R_alloc'ed */
bool to_raw_logical = stri__prepare_arg_logical_1_notNA(to_raw, "to_raw");
STRI__ERROR_HANDLER_BEGIN(1)
R_len_t str_n = LENGTH(str);
StriContainerUTF16 str_cont(str, str_n);
// get the number of strings to convert; if == 0, then you know what's the result
if (str_n <= 0) return Rf_allocVector(to_raw_logical?VECSXP:STRSXP, 0);
// Open converters
StriUcnv ucnv(selected_to);
UConverter* uconv_to = ucnv.getConverter(true /*register_callbacks*/);
// Get target encoding mark
cetype_t encmark_to = to_raw_logical?CE_BYTES:ucnv.getCE();
// Prepare out val
SEXP ret;
STRI__PROTECT(ret = Rf_allocVector(to_raw_logical?VECSXP:STRSXP, str_n));
// calculate required buf size
R_len_t bufsize = 0;
for (R_len_t i=0; i<str_n; ++i) {
if (!str_cont.isNA(i) && str_cont.get(i).length() > bufsize)
bufsize = str_cont.get(i).length();
}
bufsize = UCNV_GET_MAX_BYTES_FOR_STRING(bufsize, ucnv_getMaxCharSize(uconv_to));
// "The calculated size is guaranteed to be sufficient for this conversion."
String8buf buf(bufsize);
for (R_len_t i=0; i<str_n; ++i) {
if (str_cont.isNA(i)) {
if (to_raw_logical) SET_VECTOR_ELT(ret, i, R_NilValue);
else SET_STRING_ELT(ret, i, NA_STRING);
continue;
}
R_len_t curn_tmp = str_cont.get(i).length();
const UChar* curs_tmp = str_cont.get(i).getBuffer(); // The buffer content is (probably) not NUL-terminated.
if (!curs_tmp)
throw StriException(MSG__INTERNAL_ERROR);
UErrorCode status = U_ZERO_ERROR;
ucnv_resetFromUnicode(uconv_to);
R_len_t bufneed = ucnv_fromUChars(uconv_to, buf.data(), buf.size(),
curs_tmp, curn_tmp, &status);
if (bufneed <= buf.size()) {
STRI__CHECKICUSTATUS_THROW(status, {/* do nothing special on err */})
}
else {// larger buffer needed [this shouldn't happen?]
buf.resize(bufneed, false/*destroy contents*/);
status = U_ZERO_ERROR;
bufneed = ucnv_fromUChars(uconv_to, buf.data(), buf.size(),
curs_tmp, curn_tmp, &status);
STRI__CHECKICUSTATUS_THROW(status, {/* do nothing special on err */})
}
if (to_raw_logical) {
SEXP outobj;
STRI__PROTECT(outobj = Rf_allocVector(RAWSXP, bufneed));
memcpy(RAW(outobj), buf.data(), (size_t)bufneed);
SET_VECTOR_ELT(ret, i, outobj);
STRI__UNPROTECT(1);
}
else {
SET_STRING_ELT(ret, i,
Rf_mkCharLenCE(buf.data(), bufneed, encmark_to));
}
}
STRI__UNPROTECT_ALL
return ret;
STRI__ERROR_HANDLER_END({/* nothing special on error */})
}
/** Convert character vector to UTF-8
*
* @param str character vector
* @param is_unknown_8bit single logical value;
* if TRUE, then in case of ENC_NATIVE or ENC_LATIN1, UTF-8
* REPLACEMENT CHARACTERs (U+FFFD) are
* put for codes > 127
* @param validate single logical value (or NA)
*
* @return character vector
*
* @version 0.1-XX (Marek Gagolewski)
*
* @version 0.1-XX (Marek Gagolewski, 2013-06-16)
* make StriException-friendly
*
* @version 0.2-1 (Marek Gagolewski, 2014-03-26)
* Use one String8buf;
* is_unknown_8bit_logical and UTF-8 tries now to remove BOMs
*
* @version 0.2-1 (Marek Gagolewksi, 2014-03-30)
* added validate arg
*
* @version 0.3-1 (Marek Gagolewski, 2014-11-04)
* Issue #112: str_prepare_arg* retvals were not PROTECTed from gc
*/
SEXP stri_enc_toutf8(SEXP str, SEXP is_unknown_8bit, SEXP validate)
{
PROTECT(validate = stri_prepare_arg_logical_1(validate, "validate"));
bool is_unknown_8bit_logical =
stri__prepare_arg_logical_1_notNA(is_unknown_8bit, "is_unknown_8bit");
PROTECT(str = stri_prepare_arg_string(str, "str"));
R_len_t n = LENGTH(str);
STRI__ERROR_HANDLER_BEGIN(2)
SEXP ret;
if (!is_unknown_8bit_logical) {
// Trivial - everything we need is in StriContainerUTF8 :)
// which removes BOMs silently
StriContainerUTF8 str_cont(str, n);
STRI__PROTECT(ret = str_cont.toR());
}
else {
// get buf size
R_len_t bufsize = 0;
for (R_len_t i=0; i<n; ++i) {
SEXP curs = STRING_ELT(str, i);
if (curs == NA_STRING || IS_ASCII(curs) || IS_UTF8(curs))
continue;
R_len_t ni = LENGTH(curs);
if (ni > bufsize) bufsize = ni;
}
String8buf buf(bufsize*3); // either 1 byte < 127 or U+FFFD == 3 bytes UTF-8
char* bufdata = buf.data();
STRI__PROTECT(ret = Rf_allocVector(STRSXP, n));
for (R_len_t i=0; i<n; ++i) {
SEXP curs = STRING_ELT(str, i);
if (curs == NA_STRING) {
SET_STRING_ELT(ret, i, NA_STRING);
continue;
}
if (IS_ASCII(curs) || IS_UTF8(curs)) {
R_len_t curs_n = LENGTH(curs);
const char* curs_s = CHAR(curs);
if (curs_n >= 3 &&
(uint8_t)(curs_s[0]) == UTF8_BOM_BYTE1 &&
(uint8_t)(curs_s[1]) == UTF8_BOM_BYTE2 &&
(uint8_t)(curs_s[2]) == UTF8_BOM_BYTE3) {
// has BOM - get rid of it
SET_STRING_ELT(ret, i, Rf_mkCharLenCE(curs_s+3, curs_n-3, CE_UTF8));
}
else
SET_STRING_ELT(ret, i, curs);
continue;
}
// otherwise, we have an 8-bit encoding
R_len_t curn = LENGTH(curs);
const char* curs_tab = CHAR(curs);
R_len_t k = 0;
for (R_len_t j=0; j<curn; ++j) {
if (U8_IS_SINGLE(curs_tab[j]))
bufdata[k++] = curs_tab[j];
else { // 0xEF 0xBF 0xBD
bufdata[k++] = (char)UCHAR_REPLACEMENT_UTF8_BYTE1;
bufdata[k++] = (char)UCHAR_REPLACEMENT_UTF8_BYTE2;
bufdata[k++] = (char)UCHAR_REPLACEMENT_UTF8_BYTE3;
}
}
SET_STRING_ELT(ret, i, Rf_mkCharLenCE(bufdata, k, CE_UTF8));
}
}
// validate utf8 byte stream
if (LOGICAL(validate)[0] != FALSE) { // NA or TRUE
R_len_t ret_n = LENGTH(ret);
for (R_len_t i=0; i<ret_n; ++i) {
SEXP curs = STRING_ELT(ret, i);
if (curs == NA_STRING) continue;
const char* s = CHAR(curs);
R_len_t sn = LENGTH(curs);
R_len_t j = 0;
UChar32 c = 0;
while (c >= 0 && j < sn) {
U8_NEXT(s, j, sn, c);
}
if (c >= 0) continue; // valid, nothing to do
if (LOGICAL(validate)[0] == NA_LOGICAL) {
Rf_warning(MSG__INVALID_CODE_POINT_REPLNA);
SET_STRING_ELT(ret, i, NA_STRING);
}
else {
int bufsize = sn*3; // maximum: 1 byte -> U+FFFD (3 bytes)
String8buf buf(bufsize); // maximum: 1 byte -> U+FFFD (3 bytes)
char* bufdata = buf.data();
j = 0;
R_len_t k = 0;
UBool err = FALSE;
while (!err && j < sn) {
U8_NEXT(s, j, sn, c);
if (c >= 0) {
U8_APPEND((uint8_t*)bufdata, k, bufsize, c, err);
} else {
Rf_warning(MSG__INVALID_CODE_POINT_FIXING);
bufdata[k++] = (char)UCHAR_REPLACEMENT_UTF8_BYTE1;
bufdata[k++] = (char)UCHAR_REPLACEMENT_UTF8_BYTE2;
bufdata[k++] = (char)UCHAR_REPLACEMENT_UTF8_BYTE3;
}
}
if (err) throw StriException(MSG__INTERNAL_ERROR);
SET_STRING_ELT(ret, i, Rf_mkCharLenCE(bufdata, k, CE_UTF8));
}
}
}
STRI__UNPROTECT_ALL
return ret;
STRI__ERROR_HANDLER_END(;/* nothing special to be done on error */)
}
/**
* Replace multiple substrings
*
*
* @param str character vector
* @param from integer vector (possibly with negative indices)
* @param to integer vector (possibly with negative indices) or NULL
* @param length integer vector or NULL
* @param omit_na logical scalar
* @param value character vector replacement
* @return character vector
*
* @version 1.3.2 (Marek Gagolewski, 2019-02-22)
* #30: new function
*
*
* @version 1.4.3 (Marek Gagolewski, 2019-03-12)
* #346: na_omit for `value`
*/
SEXP stri_sub_replacement_all(SEXP str, SEXP from, SEXP to, SEXP length, SEXP omit_na, SEXP value)
{
//PROTECT(str = stri_prepare_arg_string(str, "str"));
PROTECT(str = stri_enc_toutf8(str, Rf_ScalarLogical(FALSE), Rf_ScalarLogical(FALSE)));
PROTECT(from = stri_prepare_arg_list(from, "from"));
PROTECT(to = stri_prepare_arg_list(to, "to"));
PROTECT(length = stri_prepare_arg_list(length, "length"));
PROTECT(value = stri_prepare_arg_list(value, "value"));
bool omit_na_1 = stri__prepare_arg_logical_1_notNA(omit_na, "omit_na");
R_len_t str_len = LENGTH(str);
R_len_t from_len = LENGTH(from);
R_len_t value_len = LENGTH(value);
R_len_t vectorize_len;
if (!isNull(to))
vectorize_len = stri__recycling_rule(true, 4,
str_len, from_len, value_len, LENGTH(to));
else if (!isNull(length))
vectorize_len = stri__recycling_rule(true, 4,
str_len, from_len, value_len, LENGTH(length));
else
vectorize_len = stri__recycling_rule(true, 3, str_len, from_len, value_len);
if (vectorize_len <= 0) {
UNPROTECT(5);
return Rf_allocVector(STRSXP, 0);
}
// no STRI__ERROR_HANDLER_BEGIN block ---- below we can longjmp with Rf_error...
SEXP ret, curs, tmp;
PROTECT(ret = Rf_allocVector(STRSXP, vectorize_len)); // 6
for (R_len_t i = 0; i<vectorize_len; ++i)
{
curs = STRING_ELT(str, i%str_len);
if (curs == NA_STRING) {
SET_STRING_ELT(ret, i, NA_STRING);
continue;
}
if (!isNull(to)) {
PROTECT(tmp = stri__sub_replacement_all_single(curs,
VECTOR_ELT(from, i%from_len),
VECTOR_ELT(to, i%LENGTH(to)), R_NilValue,
omit_na_1, VECTOR_ELT(value, i%value_len)));
}
else if (!isNull(length)) {
PROTECT(tmp = stri__sub_replacement_all_single(curs,
VECTOR_ELT(from, i%from_len),
R_NilValue, VECTOR_ELT(length, i%LENGTH(length)),
omit_na_1, VECTOR_ELT(value, i%value_len)));
}
else {
PROTECT(tmp = stri__sub_replacement_all_single(curs,
VECTOR_ELT(from, i%from_len),
R_NilValue, R_NilValue,
omit_na_1, VECTOR_ELT(value, i%value_len)));
}
SET_STRING_ELT(ret, i, tmp);
UNPROTECT(1); //tmp
}
UNPROTECT(6);
return ret;
}
/**
* Substring replacement function
*
*
* @param str character vector
* @param from integer vector (possibly with negative indices)
* @param to integer vector (possibly with negative indices) or NULL
* @param length integer vector or NULL
* @param omit_na logical scalar
* @param value character vector replacement
* @return character vector
*
* @version 0.1-?? (Bartek Tartanus)
*
* @version 0.1-?? (Marek Gagolewski)
* use StriContainerUTF8 and stri__UChar32_to_UTF8_index
*
* @version 0.1-?? (Marek Gagolewski, 2013-06-01)
* use StriContainerUTF8's UChar32-to-UTF8 index
*
* @version 0.1-?? (Marek Gagolewski, 2013-06-16)
* make StriException-friendly
*
* @version 0.2-1 (Marek Gagolewski, 2014-03-20)
* Use StriContainerUTF8_indexable
*
* @version 0.2-1 (Marek Gagolewski, 2014-04-03)
* Use stri__sub_prepare_from_to_length()
*
* @version 0.3-1 (Marek Gagolewski, 2014-11-04)
* Issue #112: str_prepare_arg* retvals were not PROTECTed from gc
*
* @version 0.5-9003 (Marek Gagolewski, 2015-08-05)
* Bugfix #183: floating point exception when to or length is an empty vector
*
* @version 1.0-2 (Marek Gagolewski, 2016-01-31)
* FR #199: new arg: `omit_na`
* FR #207: allow insertions
*
*
* @version 1.4.3 (Marek Gagolewski, 2019-03-12)
* #346: na_omit for `value`
*/
SEXP stri_sub_replacement(SEXP str, SEXP from, SEXP to, SEXP length, SEXP omit_na, SEXP value)
{
PROTECT(str = stri_prepare_arg_string(str, "str"));
PROTECT(value = stri_prepare_arg_string(value, "value"));
bool omit_na_1 = stri__prepare_arg_logical_1_notNA(omit_na, "omit_na");
R_len_t value_len = LENGTH(value);
R_len_t str_len = LENGTH(str);
R_len_t from_len = 0; // see below
R_len_t to_len = 0; // see below
R_len_t length_len = 0; // see below
int* from_tab = 0; // see below
int* to_tab = 0; // see below
int* length_tab = 0; // see below
R_len_t sub_protected = 2+ /* how many objects to PROTECT on ret? */
stri__sub_prepare_from_to_length(from, to, length,
from_len, to_len, length_len, from_tab, to_tab, length_tab);
R_len_t vectorize_len = stri__recycling_rule(true, 4,
str_len, value_len, from_len, (to_len>length_len)?to_len:length_len);
if (vectorize_len <= 0) {
UNPROTECT(sub_protected);
return Rf_allocVector(STRSXP, 0);
}
STRI__ERROR_HANDLER_BEGIN(sub_protected)
StriContainerUTF8_indexable str_cont(str, vectorize_len);
StriContainerUTF8 value_cont(value, vectorize_len);
SEXP ret;
STRI__PROTECT(ret = Rf_allocVector(STRSXP, vectorize_len));
String8buf buf(0); // @TODO: estimate bufsize a priori
for (R_len_t i = str_cont.vectorize_init();
i != str_cont.vectorize_end();
i = str_cont.vectorize_next(i))
{
R_len_t cur_from = from_tab[i % from_len];
R_len_t cur_to = (to_tab)?to_tab[i % to_len]:length_tab[i % length_len];
if (str_cont.isNA(i)) {
SET_STRING_ELT(ret, i, NA_STRING);
continue;
}
if (cur_from == NA_INTEGER || cur_to == NA_INTEGER || value_cont.isNA(i)) {
if (omit_na_1) {
SET_STRING_ELT(ret, i, str_cont.toR(i));
}
else {
SET_STRING_ELT(ret, i, NA_STRING);
}
continue;
}
if (length_tab) {
if (cur_to <= 0) {
// SET_STRING_ELT(ret, i, R_BlankString);
// continue;
cur_to = 0;
}
else {
cur_to = cur_from + cur_to - 1;
if (cur_from < 0 && cur_to >= 0) cur_to = -1;
}
}
const char* str_cur_s = str_cont.get(i).c_str();
R_len_t str_cur_n = str_cont.get(i).length();
const char* value_cur_s = value_cont.get(i).c_str();
R_len_t value_cur_n = value_cont.get(i).length();
R_len_t cur_from2; // UTF-8 byte indices
R_len_t cur_to2; // UTF-8 byte indices
stri__sub_get_indices(str_cont, i, cur_from, cur_to, cur_from2, cur_to2);
if (cur_to2 < cur_from2) cur_to2 = cur_from2;
R_len_t buflen = str_cur_n-(cur_to2-cur_from2)+value_cur_n;
buf.resize(buflen, false/*destroy contents*/);
memcpy(buf.data(), str_cur_s, (size_t)cur_from2);
memcpy(buf.data()+cur_from2, value_cur_s, (size_t)value_cur_n);
memcpy(buf.data()+cur_from2+value_cur_n, str_cur_s+cur_to2, (size_t)str_cur_n-cur_to2);
SET_STRING_ELT(ret, i, Rf_mkCharLenCE(buf.data(), buflen, CE_UTF8));
}
STRI__UNPROTECT_ALL
return ret;
STRI__ERROR_HANDLER_END(;/* nothing special to be done on error */)
}
/**
* Convert character vector between given encodings
*
* @param str input character vector or list of raw vectors
* @param from source encoding, \code{NULL} or \code{""} for default enc
* @param to target encoding, \code{NULL} or \code{""} for default enc
* @param to_raw single logical, should list of raw vectors be returned?
* @return a converted character vector or list of raw vectors
*
* @version 0.1 (Marek Gagolewski)
* @version 0.2 (Marek Gagolewski) arg to_raw_added, encoding marking
* @version 0.3 (Marek Gagolewski, 2013-06-16) make StriException-friendly
* @version 0.4 (Marek Gagolewski, 2013-08-08) use StriContainerListRaw
* @version 0.5 (Marek Gagolewski, 2013-11-20) BUGFIX call stri_encode_from_marked if necessary
*/
SEXP stri_encode(SEXP str, SEXP from, SEXP to, SEXP to_raw)
{
const char* selected_from = stri__prepare_arg_enc(from, "from", true);
if (!selected_from && Rf_isVectorAtomic(str))
return stri_encode_from_marked(str, to, to_raw);
str = stri_prepare_arg_list_raw(str, "str");
const char* selected_to = stri__prepare_arg_enc(to, "to", true);
bool to_raw_logical = stri__prepare_arg_logical_1_notNA(to_raw, "to_raw");
UConverter* uconv_from = NULL;
UConverter* uconv_to = NULL;
STRI__ERROR_HANDLER_BEGIN
StriContainerListRaw str_cont(str);
R_len_t str_n = str_cont.get_n();
// get the number of strings to convert; if == 0, then you know what's the result
if (str_n <= 0) return Rf_allocVector(to_raw_logical?VECSXP:STRSXP, 0);
// Open converters
uconv_from = stri__ucnv_open(selected_from);
uconv_to = stri__ucnv_open(selected_to);
// Get target encoding mark
UErrorCode err = U_ZERO_ERROR;
const char* uconv_to_name = ucnv_getName(uconv_to, &err);
if (U_FAILURE(err))
throw StriException(err);
cetype_t encmark_to = CE_BYTES; // all other cases than the below ones
// - bytes enc (this is reasonable, isn't it?)
if (!to_raw_logical) { // otherwise not needed
if (!strcmp(uconv_to_name, "US-ASCII") || !strcmp(uconv_to_name, "UTF-8"))
encmark_to = CE_UTF8; // no CE for ASCII, will be auto-detected by mkCharLenCE
else if (!strcmp(uconv_to_name, "ISO-8859-1"))
encmark_to = CE_LATIN1;
else if (!strcmp(uconv_to_name, ucnv_getDefaultName()))
encmark_to = CE_NATIVE;
}
// Prepare out val
SEXP ret;
PROTECT(ret = Rf_allocVector(to_raw_logical?VECSXP:STRSXP, str_n));
String8 buf(0); // will be extended in a moment
for (R_len_t i=0; i<str_n; ++i) {
if (str_cont.isNA(i)) {
if (to_raw_logical) SET_VECTOR_ELT(ret, i, R_NilValue);
else SET_STRING_ELT(ret, i, NA_STRING);
continue;
}
const char* curd = str_cont.get(i).c_str();
R_len_t curn = str_cont.get(i).length();
err = U_ZERO_ERROR;
UnicodeString encs(curd, curn, uconv_from, err); // FROM -> UTF-16 [this is the slow part]
if (U_FAILURE(err))
throw StriException(err);
R_len_t curn_tmp = encs.length();
const UChar* curs_tmp = encs.getBuffer(); // The buffer contents is (probably) not NUL-terminated.
if (!curs_tmp)
throw StriException(MSG__INTERNAL_ERROR);
R_len_t bufneed = UCNV_GET_MAX_BYTES_FOR_STRING(curn_tmp, ucnv_getMaxCharSize(uconv_to));
// "The calculated size is guaranteed to be sufficient for this conversion."
buf.resize(bufneed);
err = U_ZERO_ERROR;
// bufneed = encs.extract(buf.data(), buf.size(), uconv_to, err); // UTF-16 -> TO
ucnv_resetFromUnicode(uconv_to);
bufneed = ucnv_fromUChars(uconv_to, buf.data(), buf.size(), curs_tmp, curn_tmp, &err);
if (bufneed <= buf.size()) {
if (U_FAILURE(err))
throw StriException(err);
}
else {// larger buffer needed [this shouldn't happen?]
// warning("buf extending");
buf.resize(bufneed);
err = U_ZERO_ERROR;
bufneed = ucnv_fromUChars(uconv_to, buf.data(), buf.size(), curs_tmp, curn_tmp, &err);
if (U_FAILURE(err))
throw StriException(err);
if (bufneed > buf.size())
throw StriException(MSG__INTERNAL_ERROR);
}
if (to_raw_logical) {
SEXP outobj = Rf_allocVector(RAWSXP, bufneed);
memcpy(RAW(outobj), buf.data(), (size_t)bufneed);
SET_VECTOR_ELT(ret, i, outobj);
}
else {
SET_STRING_ELT(ret, i, Rf_mkCharLenCE(buf.data(), bufneed, encmark_to));
}
}
if (uconv_from) {
ucnv_close(uconv_from);
uconv_from = NULL;
}
if (uconv_to) {
ucnv_close(uconv_to);
uconv_to = NULL;
}
UNPROTECT(1);
return ret;
STRI__ERROR_HANDLER_END({
if (uconv_from)
ucnv_close(uconv_from);
if (uconv_to)
ucnv_close(uconv_to);
})
}
/**
* Pad a string
*
* vectorized over str, length and pad
* if str or pad or length is NA the result will be NA
*
* @param str character vector
* @param min_length integer vector
* @param side [internal int]
* @param pad character vector
* @param use_length single logical value
* @return character vector
*
* @version 0.1-?? (Bartlomiej Tartanus)
*
* @version 0.2-2 (Marek Gagolewski, 2014-04-20)
* use stri_error_handler, pad should be a single code point, not byte
*
* @version 0.3-1 (Marek Gagolewski, 2014-11-04)
* Issue #112: str_prepare_arg* retvals were not PROTECTed from gc
*
* @version 0.5-1 (Marek Gagolewski, 2015-04-22)
* `use_length` arg added,
* second argument renamed `width`
*/
SEXP stri_pad(SEXP str, SEXP width, SEXP side, SEXP pad, SEXP use_length)
{
// this is an internal arg, check manually, error() allowed here
if (!Rf_isInteger(side) || LENGTH(side) != 1)
Rf_error(MSG__INCORRECT_INTERNAL_ARG);
int _side = INTEGER(side)[0];
if (_side < 0 || _side > 2)
Rf_error(MSG__INCORRECT_INTERNAL_ARG);
bool use_length_val = stri__prepare_arg_logical_1_notNA(use_length, "use_length");
PROTECT(str = stri_prepare_arg_string(str, "str"));
PROTECT(width = stri_prepare_arg_integer(width, "width"));
PROTECT(pad = stri_prepare_arg_string(pad, "pad"));
// side = stri_prepare_arg_string(side, "side");
// const char* side_opts[] = {"left", "right", "both", NULL};
R_len_t str_length = LENGTH(str);
R_len_t width_length = LENGTH(width);
// R_len_t side_length = LENGTH(side);
R_len_t pad_length = LENGTH(pad);
R_len_t vectorize_length = stri__recycling_rule(true, 3,
str_length, width_length, /*side_length, */ pad_length);
STRI__ERROR_HANDLER_BEGIN(3)
StriContainerUTF8 str_cont(str, vectorize_length);
StriContainerInteger width_cont(width, vectorize_length);
// StriContainerUTF8 side_cont(side, vectorize_length);
StriContainerUTF8 pad_cont(pad, vectorize_length);
SEXP ret;
STRI__PROTECT(ret = Rf_allocVector(STRSXP, vectorize_length));
String8buf buf(0); // TODO: prealloc
for (R_len_t i=0; i<vectorize_length; ++i) {
if (str_cont.isNA(i) || pad_cont.isNA(i)
|| /*side_cont.isNA(i) ||*/ width_cont.isNA(i)) {
SET_STRING_ELT(ret, i, NA_STRING);
continue;
}
// get the current string
R_len_t str_cur_n = str_cont.get(i).length();
const char* str_cur_s = str_cont.get(i).c_str();
R_len_t str_cur_width;
// get the width/length of padding code point(s)
R_len_t pad_cur_n = pad_cont.get(i).length();
const char* pad_cur_s = pad_cont.get(i).c_str();
R_len_t pad_cur_width;
if (use_length_val) {
pad_cur_width = 1;
str_cur_width = str_cont.get(i).countCodePoints();
R_len_t k = 0;
UChar32 pad_cur = 0;
U8_NEXT(pad_cur_s, k, pad_cur_n, pad_cur);
if (pad_cur <= 0 || k < pad_cur_n)
throw StriException(MSG__NOT_EQ_N_CODEPOINTS, "pad", 1);
}
else {
pad_cur_width = stri__width_string(pad_cur_s, pad_cur_n);
str_cur_width = stri__width_string(str_cur_s, str_cur_n);
if (pad_cur_width != 1)
throw StriException(MSG__NOT_EQ_N_WIDTH, "pad", 1);
}
// get the minimal width
R_len_t width_cur = width_cont.get(i);
if (str_cur_width >= width_cur) {
// no padding at all
SET_STRING_ELT(ret, i, str_cont.toR(i));
continue;
}
R_len_t padnum = width_cur-str_cur_width;
buf.resize(str_cur_n+padnum*pad_cur_n, false);
char* buftmp = buf.data();
R_len_t k = 0;
switch(_side) {
case 0: // left
for (k=0; k<padnum; ++k) {
memcpy(buftmp, pad_cur_s, pad_cur_n);
buftmp += pad_cur_n;
}
memcpy(buftmp, str_cur_s, str_cur_n);
buftmp += str_cur_n;
break;
case 1: // right
memcpy(buftmp, str_cur_s, str_cur_n);
buftmp += str_cur_n;
for (k=0; k<padnum; ++k) {
memcpy(buftmp, pad_cur_s, pad_cur_n);
buftmp += pad_cur_n;
}
break;
case 2: // both
for (k=0; k<padnum/2; ++k) {
memcpy(buftmp, pad_cur_s, pad_cur_n);
buftmp += pad_cur_n;
}
memcpy(buftmp, str_cur_s, str_cur_n);
buftmp += str_cur_n;
for (; k<padnum; ++k) {
memcpy(buftmp, pad_cur_s, pad_cur_n);
buftmp += pad_cur_n;
}
break;
}
SET_STRING_ELT(ret, i, Rf_mkCharLenCE(buf.data(), (int)(buftmp-buf.data()), CE_UTF8));
}
STRI__UNPROTECT_ALL
return ret;
STRI__ERROR_HANDLER_END(;/* nothing special to be done on error */)
}
/** Word wrap text
*
* @param str character vector
* @param width single integer
* @param cost_exponent single double
* @param indent single integer
* @param exdent single integer
* @param prefix single string
* @param initial single string
* @param locale locale identifier or NULL for default locale
* @param use_length single logical value
*
* @return list
*
* @version 0.1-?? (Bartek Tartanus)
*
* @version 0.2-2 (Marek Gagolewski, 2014-04-27)
* single function for wrap_greedy and wrap_dynamic
* (dispatch inside);
* use BreakIterator
*
* @version 0.3-1 (Marek Gagolewski, 2014-11-04)
* Issue #112: str_prepare_arg* retvals were not PROTECTed from gc
*
* @version 0.4-1 (Marek Gagolewski, 2014-12-06)
* new args: indent, exdent, prefix, initial
*
* @version 0.5-1 (Marek Gagolewski, 2014-12-19)
* #133 allow width <= 0
*
* @version 0.5-1 (Marek Gagolewski, 2015-02-28)
* don't trim so many white spaces at the end of each word (normalize arg does that)
* #139: allow a "whitespace" break iterator
*
* @version 0.5-1 (Marek Gagolewski, 2015-04-23)
* `use_length` arg added
*
*
* @version 0.5-1 (Marek Gagolewski, 2015-06-09)
* BIGSKIP: no more CHARSXP on out on "" input
*/
SEXP stri_wrap(SEXP str, SEXP width, SEXP cost_exponent,
SEXP indent, SEXP exdent, SEXP prefix, SEXP initial, SEXP whitespace_only,
SEXP use_length, SEXP locale)
{
bool use_length_val = stri__prepare_arg_logical_1_notNA(use_length, "use_length");
double exponent_val = stri__prepare_arg_double_1_notNA(cost_exponent, "cost_exponent");
bool whitespace_only_val = stri__prepare_arg_logical_1_notNA(whitespace_only, "whitespace_only");
int width_val = stri__prepare_arg_integer_1_notNA(width, "width");
if (width_val <= 0) width_val = 0;
int indent_val = stri__prepare_arg_integer_1_notNA(indent, "indent");
if (indent_val < 0) Rf_error(MSG__EXPECTED_POSITIVE, "indent");
int exdent_val = stri__prepare_arg_integer_1_notNA(exdent, "exdent");
if (exdent_val < 0) Rf_error(MSG__EXPECTED_POSITIVE, "exdent");
const char* qloc = stri__prepare_arg_locale(locale, "locale", true); /* this is R_alloc'ed */
Locale loc = Locale::createFromName(qloc);
PROTECT(str = stri_prepare_arg_string(str, "str"));
PROTECT(prefix = stri_prepare_arg_string_1(prefix, "prefix"));
PROTECT(initial = stri_prepare_arg_string_1(initial, "initial"));
BreakIterator* briter = NULL;
UText* str_text = NULL;
STRI__ERROR_HANDLER_BEGIN(3)
UErrorCode status = U_ZERO_ERROR;
briter = BreakIterator::createLineInstance(loc, status);
STRI__CHECKICUSTATUS_THROW(status, {/* do nothing special on err */})
R_len_t str_length = LENGTH(str);
StriContainerUTF8_indexable str_cont(str, str_length);
StriContainerUTF8 prefix_cont(prefix, 1);
StriContainerUTF8 initial_cont(initial, 1);
// prepare indent/exdent/prefix/initial stuff:
// 1st line, 1st para (i==0, u==0): initial+indent
// nth line, 1st para (i==0, u> 0): prefix +exdent
// 1st line, nth para (i> 0, u==0): prefix +indent
// nth line, nth para (i> 0, u> 0): prefix +exdent
StriWrapLineStart ii(initial_cont.get(0), indent_val);
StriWrapLineStart pi(prefix_cont.get(0), indent_val);
StriWrapLineStart pe(prefix_cont.get(0), exdent_val);
status = U_ZERO_ERROR;
//Unicode Newline Guidelines - Unicode Technical Report #13
UnicodeSet uset_linebreaks(UnicodeString::fromUTF8("[\\u000A-\\u000D\\u0085\\u2028\\u2029]"), status);
STRI__CHECKICUSTATUS_THROW(status, {/* do nothing special on err */})
uset_linebreaks.freeze();
status = U_ZERO_ERROR;
UnicodeSet uset_whitespaces(UnicodeString::fromUTF8("\\p{White_space}"), status);
STRI__CHECKICUSTATUS_THROW(status, {/* do nothing special on err */})
uset_whitespaces.freeze();
SEXP ret;
STRI__PROTECT(ret = Rf_allocVector(VECSXP, str_length));
for (R_len_t i = 0; i < str_length; ++i)
{
if (str_cont.isNA(i) || prefix_cont.isNA(0) || initial_cont.isNA(0)) {
SET_VECTOR_ELT(ret, i, stri__vector_NA_strings(1));
continue;
}
status = U_ZERO_ERROR;
const char* str_cur_s = str_cont.get(i).c_str();
R_len_t str_cur_n = str_cont.get(i).length();
str_text = utext_openUTF8(str_text, str_cur_s, str_cont.get(i).length(), &status);
STRI__CHECKICUSTATUS_THROW(status, {/* do nothing special on err */})
status = U_ZERO_ERROR;
briter->setText(str_text, status);
STRI__CHECKICUSTATUS_THROW(status, {/* do nothing special on err */})
// all right, first let's generate a list of places at which we may do line breaks
deque< R_len_t > occurrences_list; // this could be an R_len_t queue
R_len_t match = briter->first();
while (match != BreakIterator::DONE) {
if (!whitespace_only_val)
occurrences_list.push_back(match);
else {
if (match > 0 && match < str_cur_n) {
UChar32 c;
U8_GET((const uint8_t*)str_cur_s, 0, match-1, str_cur_n, c);
if (uset_whitespaces.contains(c))
occurrences_list.push_back(match);
}
else
occurrences_list.push_back(match);
}
match = briter->next();
}
R_len_t noccurrences = (R_len_t)occurrences_list.size(); // number of boundaries
if (noccurrences <= 1) { // no match (1 boundary == 0)
SET_VECTOR_ELT(ret, i, Rf_ScalarString(str_cont.toR(i)));
continue;
}
// the number of "words" is:
R_len_t nwords = noccurrences - 1;
// convert occurrences_list to a vector
// in order to obtain end positions (in a string) of each "words",
// noting that occurrences_list.at(0) == 0
#ifndef NDEBUG
if (occurrences_list.at(0) != 0)
throw StriException("NDEBUG: stri_wrap: (occurrences_list.at(0) != 0)");
#endif
std::vector<R_len_t> end_pos_orig(nwords);
deque<R_len_t>::iterator iter = ++(occurrences_list.begin());
for (R_len_t j = 0; iter != occurrences_list.end(); ++iter, ++j) {
end_pos_orig[j] = (*iter); // this is a UTF-8 index
}
// now:
// we'll get the total widths/number of code points in each "word"
std::vector<R_len_t> widths_orig(nwords);
// we'll get the total widths/number of code points without trailing whitespaces
std::vector<R_len_t> widths_trim(nwords);
// we'll get the end positions without trailing whitespaces
std::vector<R_len_t> end_pos_trim(nwords);
// detect line endings (fail on a match)
UChar32 c = 0;
R_len_t j = 0;
R_len_t cur_block = 0;
R_len_t cur_width_orig = 0;
R_len_t cur_width_trim = 0;
R_len_t cur_count_orig = 0;
R_len_t cur_count_trim = 0;
R_len_t cur_end_pos_trim = 0;
while (j < str_cur_n) {
R_len_t jlast = j;
U8_NEXT(str_cur_s, j, str_cur_n, c);
if (c < 0) // invalid utf-8 sequence
throw StriException(MSG__INVALID_UTF8);
if (uset_linebreaks.contains(c))
throw StriException(MSG__NEWLINE_FOUND);
cur_width_orig += stri__width_char(c);
++cur_count_orig;
if (uset_whitespaces.contains(c)) {
// OLD: trim all white spaces from the end:
// ++cur_count_trim;
// [we have the normalize arg for that]
// NEW: trim just one white space at the end:
cur_width_trim = stri__width_char(c);
cur_count_trim = 1;
cur_end_pos_trim = jlast;
}
else {
cur_width_trim = 0;
cur_count_trim = 0;
cur_end_pos_trim = j;
}
if (j >= str_cur_n || end_pos_orig[cur_block] <= j) {
// we'll start a new block in a moment
if (use_length_val) {
widths_orig[cur_block] = cur_count_orig;
widths_trim[cur_block] = cur_count_orig-cur_count_trim;
}
else {
widths_orig[cur_block] = cur_width_orig;
widths_trim[cur_block] = cur_width_orig-cur_width_trim;
}
end_pos_trim[cur_block] = cur_end_pos_trim;
cur_block++;
cur_width_orig = 0;
cur_width_trim = 0;
cur_count_orig = 0;
cur_count_trim = 0;
cur_end_pos_trim = j;
}
}
// do wrap
std::deque<R_len_t> wrap_after; // wrap line after which word in {0..nwords-1}?
if (exponent_val <= 0.0) {
stri__wrap_greedy(wrap_after, nwords, width_val,
widths_orig, widths_trim,
(use_length_val)?((i==0)?ii.count:pi.count):((i==0)?ii.width:pi.width),
(use_length_val)?pe.count:pe.width);
}
else {
stri__wrap_dynamic(wrap_after, nwords, width_val, exponent_val,
widths_orig, widths_trim,
(use_length_val)?((i==0)?ii.count:pi.count):((i==0)?ii.width:pi.width),
(use_length_val)?pe.count:pe.width);
}
// wrap_after.size() line breaks => wrap_after.size()+1 lines
R_len_t nlines = (R_len_t)wrap_after.size()+1;
R_len_t last_pos = 0;
SEXP ans;
STRI__PROTECT(ans = Rf_allocVector(STRSXP, nlines));
deque<R_len_t>::iterator iter_wrap = wrap_after.begin();
for (R_len_t u = 0; iter_wrap != wrap_after.end(); ++iter_wrap, ++u) {
R_len_t wrap_after_cur = *iter_wrap;
R_len_t cur_pos = end_pos_trim[wrap_after_cur];
std::string cs;
if (i == 0 && u == 0) cs = ii.str;
else if (i > 0 && u == 0) cs = pi.str;
else cs = pe.str;
cs.append(str_cur_s+last_pos, cur_pos-last_pos);
SET_STRING_ELT(ans, u, Rf_mkCharLenCE(cs.c_str(), cs.size(), CE_UTF8));
last_pos = end_pos_orig[wrap_after_cur];
}
// last line goes here:
std::string cs;
if (i == 0 && nlines-1 == 0) cs = ii.str;
else if (i > 0 && nlines-1 == 0) cs = pi.str;
else cs = pe.str;
cs.append(str_cur_s+last_pos, end_pos_trim[nwords-1]-last_pos);
SET_STRING_ELT(ans, nlines-1, Rf_mkCharLenCE(cs.c_str(), cs.size(), CE_UTF8));
SET_VECTOR_ELT(ret, i, ans);
STRI__UNPROTECT(1);
}
if (briter) { delete briter; briter = NULL; }
if (str_text) { utext_close(str_text); str_text = NULL; }
STRI__UNPROTECT_ALL
return ret;
STRI__ERROR_HANDLER_END({
if (briter) { delete briter; briter = NULL; }
if (str_text) { utext_close(str_text); str_text = NULL; }
})
}
/**
* Parse date-time objects
*
* @param str
* @param format
* @param tz
* @param lenient
* @param locale
*
* @return character vector
*
* @version 0.5-1 (Marek Gagolewski, 2015-01-08)
* @version 0.5-1 (Marek Gagolewski, 2015-01-11) lenient arg added
* @version 0.5-1 (Marek Gagolewski, 2015-02-22) use tz
* @version 0.5-1 (Marek Gagolewski, 2015-03-01) set tzone attrib on retval
*/
SEXP stri_datetime_parse(SEXP str, SEXP format, SEXP lenient, SEXP tz, SEXP locale) {
PROTECT(str = stri_prepare_arg_string(str, "str"));
const char* locale_val = stri__prepare_arg_locale(locale, "locale", true);
const char* format_val = stri__prepare_arg_string_1_notNA(format, "format");
bool lenient_val = stri__prepare_arg_logical_1_notNA(lenient, "lenient");
if (!isNull(tz)) PROTECT(tz = stri_prepare_arg_string_1(tz, "tz"));
else PROTECT(tz); /* needed to set tzone attrib */
// "format" may be one of:
const char* format_opts[] = {
"date_full", "date_long", "date_medium", "date_short",
"date_relative_full", "date_relative_long", "date_relative_medium", "date_relative_short",
"time_full", "time_long", "time_medium", "time_short",
"time_relative_full", "time_relative_long", "time_relative_medium", "time_relative_short",
"datetime_full", "datetime_long", "datetime_medium", "datetime_short",
"datetime_relative_full", "datetime_relative_long", "datetime_relative_medium", "datetime_relative_short",
NULL};
int format_cur = stri__match_arg(format_val, format_opts);
TimeZone* tz_val = stri__prepare_arg_timezone(tz, "tz", true/*allowdefault*/);
Calendar* cal = NULL;
DateFormat* fmt = NULL;
STRI__ERROR_HANDLER_BEGIN(2)
R_len_t vectorize_length = LENGTH(str);
StriContainerUTF16 str_cont(str, vectorize_length);
UnicodeString format_str(format_val);
UErrorCode status = U_ZERO_ERROR;
if (format_cur >= 0) {
DateFormat::EStyle style = DateFormat::kNone;
switch (format_cur % 8) {
case 0: style = DateFormat::kFull; break;
case 1: style = DateFormat::kLong; break;
case 2: style = DateFormat::kMedium; break;
case 3: style = DateFormat::kShort; break;
case 4: style = DateFormat::kFullRelative; break;
case 5: style = DateFormat::kLongRelative; break;
case 6: style = DateFormat::kMediumRelative; break;
case 7: style = DateFormat::kShortRelative; break;
default: style = DateFormat::kNone; break;
}
/* ICU 54.1: Relative time styles are not currently supported. */
switch (format_cur / 8) {
case 0:
fmt = DateFormat::createDateInstance(style, Locale::createFromName(locale_val));
break;
case 1:
fmt = DateFormat::createTimeInstance((DateFormat::EStyle)(style & ~DateFormat::kRelative),
Locale::createFromName(locale_val));
break;
case 2:
fmt = DateFormat::createDateTimeInstance(style,
(DateFormat::EStyle)(style & ~DateFormat::kRelative),
Locale::createFromName(locale_val));
break;
default:
fmt = NULL;
break;
}
}
else
fmt = new SimpleDateFormat(format_str, Locale::createFromName(locale_val), status);
STRI__CHECKICUSTATUS_THROW(status, {/* do nothing special on err */})
status = U_ZERO_ERROR;
cal = Calendar::createInstance(locale_val, status);
STRI__CHECKICUSTATUS_THROW(status, {/* do nothing special on err */})
cal->adoptTimeZone(tz_val);
tz_val = NULL; /* The Calendar takes ownership of the TimeZone. */
cal->setLenient(lenient_val);
SEXP ret;
STRI__PROTECT(ret = Rf_allocVector(REALSXP, vectorize_length));
for (R_len_t i=0; i<vectorize_length; ++i) {
if (str_cont.isNA(i)) {
REAL(ret)[i] = NA_REAL;
continue;
}
status = U_ZERO_ERROR;
ParsePosition pos;
fmt->parse(str_cont.get(i), *cal, pos);
if (pos.getErrorIndex() >= 0)
REAL(ret)[i] = NA_REAL;
else {
status = U_ZERO_ERROR;
REAL(ret)[i] = ((double)cal->getTime(status))/1000.0;
if (U_FAILURE(status)) REAL(ret)[i] = NA_REAL;
}
}
if (!isNull(tz)) Rf_setAttrib(ret, Rf_ScalarString(Rf_mkChar("tzone")), tz);
stri__set_class_POSIXct(ret);
if (tz_val) { delete tz_val; tz_val = NULL; }
if (fmt) { delete fmt; fmt = NULL; }
if (cal) { delete cal; cal = NULL; }
STRI__UNPROTECT_ALL
return ret;
STRI__ERROR_HANDLER_END({
if (tz_val) { delete tz_val; tz_val = NULL; }
if (fmt) { delete fmt; fmt = NULL; }
if (cal) { delete cal; cal = NULL; }
})
}
