double RootMeanSquaredError::calculate_performance(void) const
{
// Control sentence
#ifdef __OPENNN_DEBUG__
check();
#endif
// Neural network stuff
const MultilayerPerceptron* multilayer_perceptron_pointer = neural_network_pointer->get_multilayer_perceptron_pointer();
const size_t inputs_number = multilayer_perceptron_pointer->get_inputs_number();
const size_t outputs_number = multilayer_perceptron_pointer->get_outputs_number();
// Data set stuff
const Matrix<double>& data = data_set_pointer->get_data();
const Instances& instances = data_set_pointer->get_instances();
const size_t training_instances_number = instances.count_training_instances_number();
const Vector<size_t> training_indices = instances.arrange_training_indices();
size_t training_index;
const Variables& variables = data_set_pointer->get_variables();
const Vector<size_t> inputs_indices = variables.arrange_inputs_indices();
const Vector<size_t> targets_indices = variables.arrange_targets_indices();
// Root mean squared error
Vector<double> inputs(inputs_number);
Vector<double> outputs(outputs_number);
Vector<double> targets(outputs_number);
double sum_squared_error = 0.0;
int i = 0;
#pragma omp parallel for private(i, training_index, inputs, outputs, targets) reduction(+:sum_squared_error)
for(i = 0; i < (int)training_instances_number; i++)
{
training_index = training_indices[i];
// Input vector
inputs = data_set_pointer->get_instance(training_index, inputs_indices);
// Output vector
outputs = multilayer_perceptron_pointer->calculate_outputs(inputs);
// Target vector
// targets = data_set_pointer->get_instance(training_index, targets_indices);
// Sum squaresd error
// sum_squared_error += outputs.calculate_sum_squared_error(targets);
sum_squared_error += outputs.calculate_sum_squared_error(data, training_index, targets_indices);
}
return(sqrt(sum_squared_error/(double)training_instances_number));
}
void LH_ITERATE_PATCHED_WORDS_NAME(const unichar *buf, size_t bufsize, lh_word_check_t *check, lh_word_patch_t *patch, lh_word_callback_t *callback, void *userdata)
{
size_t pos = 0;
size_t word_start;
size_t word_length;
unichar patch_buf[WORD_MAX_CHARS];
const unichar *arg_begin;
size_t arg_length;
utf8char word_buf[BUFSIZEOF__UTF8_WORD];
utf8char *hugeword_buf = NULL;
size_t hugeword_buf_size = 0;
utf8char *word_end;
int prop;
#ifdef LH_ITERATOR_DEBUG
int wordctr = 0, wordcount = LH_COUNT_WORDS_NAME (buf, bufsize, check);
#define wordctr_INC1 wordctr++
#else
#define wordctr_INC1
#endif
while (pos < bufsize)
{
prop = UNICHAR_GETPROPS_EXPN(buf,bufsize,pos);
if (prop & UCP_ALPHA)
{
word_start = pos;
do pos++; while ((pos < bufsize) && (UNICHAR_GETPROPS_EXPN(buf,bufsize,pos) & UCP_ALPHA));
word_length = pos - word_start;
if (WORD_MAX_CHARS < word_length)
continue;
if (NULL!=check && 0 == check(buf+word_start, word_length))
{
DBG_PRINTF_NOISE_WORD(word_start,word_length);
continue;
}
if (NULL != patch)
{ /* word should be patched */
if (0 == patch (buf+word_start, word_length, patch_buf, &arg_length))
{
DBG_PRINTF_PATCH_FAILED(word_start,word_length);
continue;
}
arg_begin = patch_buf;
}
else
{ /* argument should be taken right from \c buf */
arg_begin = buf+word_start;
arg_length = word_length;
}
word_end = (utf8char *)eh_encode_buffer__UTF8 (arg_begin, arg_begin+arg_length, (char *)(word_buf), (char *)(word_buf+BUFSIZEOF__UTF8_WORD));
if (NULL != word_end)
{
callback (word_buf, word_end-word_buf, userdata);
wordctr_INC1;
continue;
}
if (hugeword_buf_size<(word_length*MAX_UTF8_CHAR))
{ /* overflow danger detected */
if (hugeword_buf_size)
dk_free (hugeword_buf, hugeword_buf_size);
hugeword_buf_size = word_length*MAX_UTF8_CHAR;
hugeword_buf = (utf8char *) dk_alloc (hugeword_buf_size);
}
word_end = (utf8char *)eh_encode_buffer__UTF8 (arg_begin, arg_begin+arg_length, (char *)(hugeword_buf), (char *)(hugeword_buf+hugeword_buf_size));
callback (hugeword_buf, word_end-hugeword_buf, userdata);
wordctr_INC1;
continue;
}
if (prop & UCP_IDEO)
{
word_start = pos;
pos++;
word_length = pos - word_start;
if (NULL!=check && 0 == check(buf+word_start, word_length))
{
DBG_PRINTF_NOISE_IDEO(word_start,word_length);
continue;
}
if (NULL != patch)
{ /* word should be patched */
if (0 == patch (buf+word_start, word_length, patch_buf, &arg_length))
{
DBG_PRINTF_IDEO_PATCH_FAILED(word_start,word_length);
continue;
}
arg_begin = patch_buf;
}
else
{ /* argument should be taken right from \c buf */
arg_begin = buf+word_start;
arg_length = word_length;
}
word_end = (utf8char *)eh_encode_buffer__UTF8 (arg_begin, arg_begin+arg_length, (char *)(word_buf), (char *)(word_buf+BUFSIZEOF__UTF8_WORD));
callback (word_buf, word_end-word_buf, userdata);
wordctr_INC1;
continue;
}
pos++;
}
if (hugeword_buf_size)
dk_free (hugeword_buf, hugeword_buf_size);
#ifdef LH_ITERATOR_DEBUG
if (wordctr != wordcount)
GPF_T;
#endif
}
static int
test1 (void)
{
const char * in =
"{\n"
" \"headers\": {\n"
" \"type\": \"request\",\n"
" \"tag\": 666\n"
" },\n"
" \"body\": {\n"
" \"name\": \"torrent-info\",\n"
" \"arguments\": {\n"
" \"ids\": [ 7, 10 ]\n"
" }\n"
" }\n"
"}\n";
tr_variant top, *headers, *body, *args, *ids;
const char * str;
int64_t i;
const int err = tr_variantFromJson (&top, in, strlen(in));
check (!err);
check (tr_variantIsDict (&top));
check ((headers = tr_variantDictFind (&top, tr_quark_new("headers",7))));
check (tr_variantIsDict (headers));
check (tr_variantDictFindStr (headers, tr_quark_new("type",4), &str, NULL));
check_streq ("request", str);
check (tr_variantDictFindInt (headers, TR_KEY_tag, &i));
check_int_eq (666, i);
check ((body = tr_variantDictFind (&top, tr_quark_new("body",4))));
check (tr_variantDictFindStr (body, TR_KEY_name, &str, NULL));
check_streq ("torrent-info", str);
check ((args = tr_variantDictFind (body, tr_quark_new("arguments",9))));
check (tr_variantIsDict (args));
check ((ids = tr_variantDictFind (args, TR_KEY_ids)));
check (tr_variantIsList (ids));
check_int_eq (2, tr_variantListSize (ids));
check (tr_variantGetInt (tr_variantListChild (ids, 0), &i));
check_int_eq (7, i);
check (tr_variantGetInt (tr_variantListChild (ids, 1), &i));
check_int_eq (10, i);
tr_variantFree (&top);
return 0;
}
void AWalkerAIController::Possess(APawn *aPawn)
{
Super::Possess(aPawn);
check(aPawn != nullptr);
aPawn->OnTakeAnyDamage.AddDynamic(this, &AWalkerAIController::OnPawnTookDamage);
}
void FontRendererImpl::DrawText(const std::string& text, const Rect& rect, const RGBA& color, float fontSize, float fontScale, const std::string& fontRef)
{
// wait for a swap to complete
FrpSeqAllocatorWaitForSwap();
m_mutex.lock();
// create or find a text format
ComPtr<IDWriteTextFormat> textFormat;
auto formatKey = std::make_pair(fontRef, fontSize);
auto formatIter = m_textFormatCache.find(formatKey);
if (formatIter != m_textFormatCache.end())
{
textFormat = formatIter->second;
}
else
{
m_dwFactory->CreateTextFormat(ToWide(fontRef).c_str(), nullptr, DWRITE_FONT_WEIGHT_NORMAL, DWRITE_FONT_STYLE_NORMAL, DWRITE_FONT_STRETCH_NORMAL, fontSize, L"en-us", textFormat.GetAddressOf());
m_textFormatCache[formatKey] = textFormat;
}
// create or find a cached text layout
ComPtr<IDWriteTextLayout> textLayout;
auto layoutKey = std::make_pair(textFormat.Get(), std::make_pair(color.AsARGB(), text));
auto layoutIter = m_textLayoutCache.find(layoutKey);
if (layoutIter != m_textLayoutCache.end())
{
textLayout = layoutIter->second;
}
else
{
std::wstring wideText = ToWide(text);
// parse colors and the lot
std::wstring noColorTextString;
std::vector<DWRITE_TEXT_RANGE> textRanges;
std::vector<RGBA> textColors;
{
std::wstringstream noColorText;
int count = 0;
static const RGBA colors[] = {
RGBA(0, 0, 0),
RGBA(255, 0, 0),
RGBA(0, 255, 0),
RGBA(255, 255, 0),
RGBA(0, 0, 255),
RGBA(0, 255, 255),
RGBA(255, 0, 255),
RGBA(255, 255, 255),
RGBA(100, 0, 0),
RGBA(0, 0, 100)
};
textRanges.reserve(50);
textColors.reserve(50);
textRanges.push_back({ 0, UINT32_MAX });
textColors.push_back(color);
for (int i = 0; i < wideText.length(); i++)
{
if (wideText[i] == '^' && (i + 1) < wideText.length() && isdigit(wideText[i + 1]))
{
textRanges.back().length = count - textRanges.back().startPosition;
textRanges.push_back({ (UINT32)count, UINT32_MAX });
textColors.push_back(colors[wideText[i + 1] - '0']);
++i;
continue;
}
noColorText << wideText[i];
++count;
}
textRanges.back().length = count - textRanges.back().startPosition;
noColorTextString = noColorText.str();
}
m_dwFactory->CreateTextLayout(noColorTextString.c_str(), static_cast<UINT32>(noColorTextString.length()), textFormat.Get(), rect.Width(), rect.Height(), textLayout.GetAddressOf());
m_textLayoutCache[layoutKey] = textLayout;
// set effect
for (size_t i : irange(textRanges.size()))
{
DWRITE_TEXT_RANGE effectRange = textRanges[i];
RGBA color = textColors[i];
static thread_local std::map<uint32_t, ComPtr<FrDrawingEffect>> effects;
auto it = effects.find(color.AsARGB());
if (it == effects.end())
{
ComPtr<FrDrawingEffect> effect = Make<FrDrawingEffect>();
effect->SetColor(textColors[i]);
it = effects.insert({ color.AsARGB(), effect }).first;
}
check(SUCCEEDED(textLayout->SetDrawingEffect((IUnknown*)it->second.Get(), effectRange)));
}
}
// draw
auto drawingContext = new FrDrawingContext();
textLayout->Draw(drawingContext, m_textRenderer.Get(), rect.Left(), rect.Top());
auto numRuns = drawingContext->glyphRuns.size();
if (numRuns)
{
for (auto& run : drawingContext->glyphRuns)
{
m_queuedRenderables.push_back(std::make_unique<FrGlyphRunRenderable>(run));
//m_queuedGlyphRuns.push_back(run);
}
}
delete drawingContext;
m_mutex.unlock();
}
void SInlineEditableTextBlock::Construct( const FArguments& InArgs )
{
check(InArgs._Style);
OnBeginTextEditDelegate = InArgs._OnBeginTextEdit;
OnTextCommittedDelegate = InArgs._OnTextCommitted;
IsSelected = InArgs._IsSelected;
OnVerifyTextChanged= InArgs._OnVerifyTextChanged;
Text = InArgs._Text;
bIsReadOnly = InArgs._IsReadOnly;
bIsMultiLine = InArgs._MultiLine;
DoubleSelectDelay = 0.0f;
ChildSlot
[
SAssignNew(HorizontalBox, SHorizontalBox)
+SHorizontalBox::Slot()
.VAlign(VAlign_Center)
[
SAssignNew(TextBlock, STextBlock)
.Text(Text)
.TextStyle( &InArgs._Style->TextStyle )
.Font(InArgs._Font)
.ColorAndOpacity( InArgs._ColorAndOpacity )
.ShadowColorAndOpacity( InArgs._ShadowColorAndOpacity )
.ShadowOffset( InArgs._ShadowOffset )
.HighlightText( InArgs._HighlightText )
.ToolTipText( InArgs._ToolTipText )
.WrapTextAt( InArgs._WrapTextAt )
.Justification( InArgs._Justification )
.LineBreakPolicy( InArgs._LineBreakPolicy )
]
];
if( bIsMultiLine )
{
SAssignNew(MultiLineTextBox, SMultiLineEditableTextBox)
.Text(InArgs._Text)
.Style(&InArgs._Style->EditableTextBoxStyle)
.Font(InArgs._Font)
.ToolTipText( InArgs._ToolTipText )
.OnTextChanged(this, &SInlineEditableTextBlock::OnTextChanged)
.OnTextCommitted(this, &SInlineEditableTextBlock::OnTextBoxCommitted)
.SelectAllTextWhenFocused(true)
.ClearKeyboardFocusOnCommit(true)
.RevertTextOnEscape(true)
.ModiferKeyForNewLine(InArgs._ModiferKeyForNewLine);
}
else
{
SAssignNew(TextBox, SEditableTextBox)
.Text(InArgs._Text)
.Style(&InArgs._Style->EditableTextBoxStyle)
.Font(InArgs._Font)
.ToolTipText( InArgs._ToolTipText )
.OnTextChanged( this, &SInlineEditableTextBlock::OnTextChanged )
.OnTextCommitted(this, &SInlineEditableTextBlock::OnTextBoxCommitted)
.SelectAllTextWhenFocused(true)
.ClearKeyboardFocusOnCommit(false);
}
}
uint32 FSoundFileDecoder::Run()
{
while (bIsDecoding)
{
for (int32 DataIndex = 0; DataIndex < Settings.NumDecodeBuffers; ++DataIndex)
{
FSoundFileDecodeData& Data = DecodeData[DataIndex];
// If the data is not active, continue
if (!Data.bIsActive)
{
continue;
}
int32 CurrentWriteBufferIndex = Data.CurrentWriteBufferIndex.GetValue();
while (CurrentWriteBufferIndex != Data.CurrentReadBufferIndex.GetValue())
{
TArray<float>& DecodeBuffer = Data.DecodedBuffers[CurrentWriteBufferIndex];
SoundFileCount RequestedNumSamples = (SoundFileCount)DecodeBuffer.Num();
SoundFileCount NumSamplesActuallyRead = 0;
// If the data is looping, then we need to keep reading samples from the file until we've reached the number of requested
// samples. For *very* short duration sounds (or for long-duration decode buffers), we may have an edge case where
// the sound file needs to seek from the beginning multiple times to fulfill the decode buffer size request
if (Data.bIsLooping)
{
// Keep looping until we've read everything we requested
while (true)
{
ESoundFileError::Type Result = Data.SoundFileReader->ReadSamples(DecodeBuffer.GetData(), RequestedNumSamples, NumSamplesActuallyRead);
check(Result == ESoundFileError::NONE);
// If we've read all the samples we wanted, then we're good
if (RequestedNumSamples == NumSamplesActuallyRead)
{
break;
}
// ... otherwise, we need to seek back to the beginning of the file and try to read samples that are left
else
{
// Seek to beginning of file
SoundFileCount OutFrameOffset = 0;
Data.SoundFileReader->SeekFrames(0, ESoundFileSeekMode::FROM_START, OutFrameOffset);
// Calculate samples left we need to read
RequestedNumSamples -= NumSamplesActuallyRead;
}
}
}
else
{
ESoundFileError::Type Result = Data.SoundFileReader->ReadSamples(DecodeBuffer.GetData(), RequestedNumSamples, NumSamplesActuallyRead);
check(Result == ESoundFileError::NONE);
// if the number of samples we read was not the number we requested, then we've reached the end of the file
if (RequestedNumSamples != NumSamplesActuallyRead)
{
// tell the data that this entry is no longer active
Data.bIsActive = false;
// and what our last sample index is
Data.LastSampleIndex = NumSamplesActuallyRead;
}
}
// Increment the buffer index
CurrentWriteBufferIndex = CurrentWriteBufferIndex % Settings.NumDecodeBuffers;
// Publish results by writing setting buffer index so other thread can read it
Data.CurrentWriteBufferIndex.Set(CurrentWriteBufferIndex);
}
}
// Once we're done decoding everything, then put this thread to sleep
// Any new requests (or any shutdowns) will trigger this thread to wake up.
ThreadDecodeEvent->Wait((uint32)-1);
}
return true;
}
bool check(const php::Program& p) {
for (DEBUG_ONLY auto& u : p.units) assert(check(*u));
return true;
}
void AudioStream_Ogg::open(const std::string &path, int startTime, int inLoops, const SoundTransform &inTransform) {
if (openal_is_shutdown) return;
int result;
mPath = std::string(path.c_str());
mStartTime = startTime;
mLoops = inLoops;
mIsValid = true;
mSuspend = false;
#ifdef ANDROID
mInfo = AndroidGetAssetFD(path.c_str());
oggFile = fdopen(mInfo.fd, "rb");
fseek(oggFile, mInfo.offset, 0);
ov_callbacks callbacks;
callbacks.read_func = &nme::AudioStream_Ogg::read_func;
callbacks.seek_func = &nme::AudioStream_Ogg::seek_func;
callbacks.close_func = &nme::AudioStream_Ogg::close_func;
callbacks.tell_func = &nme::AudioStream_Ogg::tell_func;
#else
oggFile = fopen(path.c_str(), "rb");
#endif
if(!oggFile) {
//throw std::string("Could not open Ogg file.");
LOG_SOUND("Could not open Ogg file.");
mIsValid = false;
return;
}
oggStream = new OggVorbis_File();
#ifdef ANDROID
result = ov_open_callbacks(this, oggStream, NULL, 0, callbacks);
#else
result = ov_open(oggFile, oggStream, NULL, 0);
#endif
if(result < 0) {
fclose(oggFile);
oggFile = 0;
//throw std::string("Could not open Ogg stream. ") + errorString(result);
LOG_SOUND("Could not open Ogg stream.");
//LOG_SOUND(errorString(result).c_str());
mIsValid = false;
return;
}
vorbisInfo = ov_info(oggStream, -1);
vorbisComment = ov_comment(oggStream, -1);
if(vorbisInfo->channels == 1) {
format = AL_FORMAT_MONO16;
} else {
format = AL_FORMAT_STEREO16;
}
if (startTime != 0)
{
double seek = startTime * 0.001;
ov_time_seek(oggStream, seek);
}
alGenBuffers(2, buffers);
check();
alGenSources(1, &source);
check();
alSource3f(source, AL_POSITION, 0.0, 0.0, 0.0);
alSource3f(source, AL_VELOCITY, 0.0, 0.0, 0.0);
alSource3f(source, AL_DIRECTION, 0.0, 0.0, 0.0);
alSourcef (source, AL_ROLLOFF_FACTOR, 0.0 );
alSourcei (source, AL_SOURCE_RELATIVE, AL_TRUE );
setTransform(inTransform);
} //open
void ReturnToPool(FEvent* Event)
{
check(Event);
Pool.Push(Event);
}
bool check(const php::Unit& u) {
assert(check(*u.pseudomain));
for (DEBUG_ONLY auto& c : u.classes) assert(check(*c));
for (DEBUG_ONLY auto& f : u.funcs) assert(check(*f));
return true;
}
double RootMeanSquaredError::calculate_selection_performance(void) const
{
// Control sentence (if debug)
#ifdef __OPENNN_DEBUG__
check();
#endif
// Neural network staff
const MultilayerPerceptron* multilayer_perceptron_pointer = neural_network_pointer->get_multilayer_perceptron_pointer();
const size_t inputs_number = multilayer_perceptron_pointer->get_inputs_number();
const size_t outputs_number = multilayer_perceptron_pointer->get_outputs_number();
// Data set stuff
const Instances& instances = data_set_pointer->get_instances();
const size_t selection_instances_number = instances.count_selection_instances_number();
if(selection_instances_number == 0)
{
return(0.0);
}
const Vector<size_t> selection_indices = instances.arrange_selection_indices();
size_t selection_index;
const Variables& variables = data_set_pointer->get_variables();
const Vector<size_t> inputs_indices = variables.arrange_inputs_indices();
const Vector<size_t> targets_indices = variables.arrange_targets_indices();
Vector<double> inputs(inputs_number);
Vector<double> outputs(outputs_number);
Vector<double> targets(outputs_number);
int i = 0;
double selection_performance = 0.0;
#pragma omp parallel for private(i, selection_index, inputs, outputs, targets) reduction(+ : selection_performance)
for(i = 0; i < (int)selection_instances_number; i++)
{
selection_index = selection_indices[i];
// Input vector
inputs = data_set_pointer->get_instance(selection_index, inputs_indices);
// Output vector
outputs = multilayer_perceptron_pointer->calculate_outputs(inputs);
// Target vector
targets = data_set_pointer->get_instance(selection_index, targets_indices);
// Sum of squares error
selection_performance += outputs.calculate_sum_squared_error(targets);
}
return(sqrt(selection_performance/(double)selection_instances_number));
}
Vector<double> RootMeanSquaredError::calculate_gradient(void) const
{
// Control sentence
#ifdef __OPENNN_DEBUG__
check();
#endif
// Neural network stuff
const MultilayerPerceptron* multilayer_perceptron_pointer = neural_network_pointer->get_multilayer_perceptron_pointer();
const size_t inputs_number = multilayer_perceptron_pointer->get_inputs_number();
const size_t outputs_number = multilayer_perceptron_pointer->get_outputs_number();
const size_t layers_number = multilayer_perceptron_pointer->get_layers_number();
const size_t parameters_number = multilayer_perceptron_pointer->count_parameters_number();
// Data set stuff
Vector< Vector< Vector<double> > > first_order_forward_propagation(2);
const bool has_conditions_layer = neural_network_pointer->has_conditions_layer();
const ConditionsLayer* conditions_layer_pointer = has_conditions_layer ? neural_network_pointer->get_conditions_layer_pointer() : NULL;
Vector<double> particular_solution;
Vector<double> homogeneous_solution;
// Data set stuff
const Instances& instances = data_set_pointer->get_instances();
const size_t training_instances_number = instances.count_training_instances_number();
const Vector<size_t> training_indices = instances.arrange_training_indices();
size_t training_index;
const Variables& variables = data_set_pointer->get_variables();
const Vector<size_t> inputs_indices = variables.arrange_inputs_indices();
const Vector<size_t> targets_indices = variables.arrange_targets_indices();
Vector<double> inputs(inputs_number);
Vector<double> targets(outputs_number);
// Performance functional stuff
const double performance = calculate_performance();
Vector< Vector<double> > layers_delta;
Vector<double> output_gradient(outputs_number);
Vector<double> point_gradient(parameters_number, 0.0);
// Main loop
Vector<double> gradient(parameters_number, 0.0);
int i = 0;
#pragma omp parallel for private(i, training_index, inputs, targets, first_order_forward_propagation, output_gradient, \
layers_delta, particular_solution, homogeneous_solution, point_gradient)
for(i = 0; i < (int)training_instances_number; i++)
{
training_index = training_indices[i];
inputs = data_set_pointer->get_instance(training_index, inputs_indices);
targets = data_set_pointer->get_instance(training_index, targets_indices);
first_order_forward_propagation = multilayer_perceptron_pointer->calculate_first_order_forward_propagation(inputs);
const Vector< Vector<double> >& layers_activation = first_order_forward_propagation[0];
const Vector< Vector<double> >& layers_activation_derivative = first_order_forward_propagation[1];
if(!has_conditions_layer)
{
output_gradient = (layers_activation[layers_number-1]-targets)/(training_instances_number*performance);
layers_delta = calculate_layers_delta(layers_activation_derivative, output_gradient);
}
else
{
particular_solution = conditions_layer_pointer->calculate_particular_solution(inputs);
homogeneous_solution = conditions_layer_pointer->calculate_homogeneous_solution(inputs);
output_gradient = (particular_solution+homogeneous_solution*layers_activation[layers_number-1] - targets)/(training_instances_number*performance);
layers_delta = calculate_layers_delta(layers_activation_derivative, homogeneous_solution, output_gradient);
}
point_gradient = calculate_point_gradient(inputs, layers_activation, layers_delta);
#pragma omp critical
gradient += point_gradient;
}
return(gradient);
}
double RootMeanSquaredError::calculate_performance(const Vector<double>& parameters) const
{
// Control sentence (if debug)
#ifdef __OPENNN_DEBUG__
check();
#endif
#ifdef __OPENNN_DEBUG__
std::ostringstream buffer;
const size_t size = parameters.size();
const size_t parameters_number = neural_network_pointer->count_parameters_number();
if(size != parameters_number)
{
buffer << "OpenNN Exception: RootMeanSquaredError class.\n"
<< "double calculate_performance(const Vector<double>&) const method.\n"
<< "Size (" << size << ") must be equal to number of parameters (" << parameters_number << ").\n";
throw std::logic_error(buffer.str());
}
#endif
// Neural network stuff
const MultilayerPerceptron* multilayer_perceptron_pointer = neural_network_pointer->get_multilayer_perceptron_pointer();
const size_t inputs_number = multilayer_perceptron_pointer->get_inputs_number();
const size_t outputs_number = multilayer_perceptron_pointer->get_outputs_number();
// Data set stuff
const Instances& instances = data_set_pointer->get_instances();
const size_t training_instances_number = instances.count_training_instances_number();
const Vector<size_t> training_indices = instances.arrange_training_indices();
size_t training_index;
const Variables& variables = data_set_pointer->get_variables();
const Vector<size_t> inputs_indices = variables.arrange_inputs_indices();
const Vector<size_t> targets_indices = variables.arrange_targets_indices();
// Root mean squared error
Vector<double> inputs(inputs_number);
Vector<double> outputs(outputs_number);
Vector<double> targets(outputs_number);
double sum_squared_error = 0.0;
int i = 0;
#pragma omp parallel for private(i, training_index, inputs, outputs, targets) reduction(+:sum_squared_error)
for(i = 0; i < (int)training_instances_number; i++)
{
training_index = training_indices[i];
// Input vector
inputs = data_set_pointer->get_instance(training_index, inputs_indices);
// Output vector
outputs = multilayer_perceptron_pointer->calculate_outputs(inputs, parameters);
// Target vector
targets = data_set_pointer->get_instance(training_index, targets_indices);
// Sum squaresd error
sum_squared_error += outputs.calculate_sum_squared_error(targets);
}
return(sqrt(sum_squared_error/(double)training_instances_number));
}
BufferType* FD3D12Adapter::CreateRHIBuffer(FRHICommandListImmediate* RHICmdList,
const D3D12_RESOURCE_DESC& InDesc,
uint32 Alignment,
uint32 Stride,
uint32 Size,
uint32 InUsage,
FRHIResourceCreateInfo& CreateInfo,
bool SkipCreate)
{
const bool bIsDynamic = (InUsage & BUF_AnyDynamic) ? true : false;
BufferType* BufferOut = CreateLinkedObject<BufferType>([&](FD3D12Device* Device)
{
BufferType* NewBuffer = new BufferType(Device, Stride, Size, InUsage);
NewBuffer->BufferAlignment = Alignment;
if (SkipCreate == false)
{
AllocateBuffer(Device, InDesc, Size, InUsage, CreateInfo, Alignment, NewBuffer->ResourceLocation);
}
return NewBuffer;
});
if (CreateInfo.ResourceArray)
{
if (bIsDynamic == false && BufferOut->ResourceLocation.IsValid())
{
check(Size == CreateInfo.ResourceArray->GetResourceDataSize());
// Get an upload heap and initialize data
FD3D12ResourceLocation SrcResourceLoc(BufferOut->GetParentDevice());
void* pData = SrcResourceLoc.GetParentDevice()->GetDefaultFastAllocator().Allocate<FD3D12ScopeLock>(Size, 4UL, &SrcResourceLoc);
check(pData);
FMemory::Memcpy(pData, CreateInfo.ResourceArray->GetResourceData(), Size);
const auto& pfnUpdateBuffer = [&]()
{
BufferType* CurrentBuffer = BufferOut;
while (CurrentBuffer != nullptr)
{
FD3D12Resource* Destination = CurrentBuffer->ResourceLocation.GetResource();
FD3D12Device* Device = Destination->GetParentDevice();
FD3D12CommandListHandle& hCommandList = Device->GetDefaultCommandContext().CommandListHandle;
// Copy from the temporary upload heap to the default resource
{
// Writable structured bufferes are sometimes initialized with inital data which means they sometimes need tracking.
FConditionalScopeResourceBarrier ConditionalScopeResourceBarrier(hCommandList, Destination, D3D12_RESOURCE_STATE_COPY_DEST, 0);
Device->GetDefaultCommandContext().numCopies++;
hCommandList->CopyBufferRegion(
Destination->GetResource(),
CurrentBuffer->ResourceLocation.GetOffsetFromBaseOfResource(),
SrcResourceLoc.GetResource()->GetResource(),
SrcResourceLoc.GetOffsetFromBaseOfResource(), Size);
hCommandList.UpdateResidency(Destination);
}
CurrentBuffer = CurrentBuffer->GetNextObject();
}
};
//TODO: This should be a deferred op like the buffer lock/unlocks
// We only need to synchronize when creating default resource buffers (because we need a command list to initialize them)
if (RHICmdList)
{
FScopedRHIThreadStaller StallRHIThread(*RHICmdList);
pfnUpdateBuffer();
}
else
{
pfnUpdateBuffer();
}
}
// Discard the resource array's contents.
CreateInfo.ResourceArray->Discard();
}
return BufferOut;
}
static void
rm_tree(char **argv)
{
FTS *fts;
FTSENT *p;
int needstat;
int flags;
int rval;
/*
* Remove a file hierarchy. If forcing removal (-f), or interactive
* (-i) or can't ask anyway (stdin_ok), don't stat the file.
*/
needstat = !uid || (!fflag && !iflag && stdin_ok);
/*
* If the -i option is specified, the user can skip on the pre-order
* visit. The fts_number field flags skipped directories.
*/
#define SKIPPED 1
flags = FTS_PHYSICAL;
if (!needstat)
flags |= FTS_NOSTAT;
if (Wflag)
flags |= FTS_WHITEOUT;
if (xflag)
flags |= FTS_XDEV;
if (!(fts = fts_open(argv, flags, NULL))) {
if (fflag && errno == ENOENT)
return;
err(1, "fts_open");
}
while ((p = fts_read(fts)) != NULL) {
switch (p->fts_info) {
case FTS_DNR:
if (!fflag || p->fts_errno != ENOENT) {
warnx("%s: %s",
p->fts_path, strerror(p->fts_errno));
eval = 1;
}
continue;
case FTS_ERR:
errx(1, "%s: %s", p->fts_path, strerror(p->fts_errno));
case FTS_NS:
/*
* Assume that since fts_read() couldn't stat the
* file, it can't be unlinked.
*/
if (!needstat)
break;
if (!fflag || p->fts_errno != ENOENT) {
warnx("%s: %s",
p->fts_path, strerror(p->fts_errno));
eval = 1;
}
continue;
case FTS_D:
/* Pre-order: give user chance to skip. */
if (!fflag && !check(p->fts_path, p->fts_accpath,
p->fts_statp)) {
(void)fts_set(fts, p, FTS_SKIP);
p->fts_number = SKIPPED;
}
else if (!uid &&
(p->fts_statp->st_flags & (UF_APPEND|UF_IMMUTABLE)) &&
!(p->fts_statp->st_flags & (SF_APPEND|SF_IMMUTABLE)) &&
lchflags(p->fts_accpath,
p->fts_statp->st_flags &= ~(UF_APPEND|UF_IMMUTABLE)) < 0)
goto err;
continue;
case FTS_DP:
/* Post-order: see if user skipped. */
if (p->fts_number == SKIPPED)
continue;
break;
default:
if (!fflag &&
!check(p->fts_path, p->fts_accpath, p->fts_statp))
continue;
}
rval = 0;
if (!uid &&
(p->fts_statp->st_flags & (UF_APPEND|UF_IMMUTABLE)) &&
!(p->fts_statp->st_flags & (SF_APPEND|SF_IMMUTABLE)))
rval = lchflags(p->fts_accpath,
p->fts_statp->st_flags &= ~(UF_APPEND|UF_IMMUTABLE));
if (rval == 0) {
/*
* If we can't read or search the directory, may still be
* able to remove it. Don't print out the un{read,search}able
* message unless the remove fails.
*/
switch (p->fts_info) {
case FTS_DP:
case FTS_DNR:
rval = rmdir(p->fts_accpath);
if (rval == 0 || (fflag && errno == ENOENT)) {
if (rval == 0 && vflag)
(void)printf("%s\n",
p->fts_path);
if (rval == 0 && info) {
info = 0;
(void)printf("%s\n",
p->fts_path);
}
continue;
}
break;
case FTS_W:
rval = undelete(p->fts_accpath);
if (rval == 0 && (fflag && errno == ENOENT)) {
if (vflag)
(void)printf("%s\n",
p->fts_path);
if (info) {
info = 0;
(void)printf("%s\n",
p->fts_path);
}
continue;
}
break;
case FTS_NS:
/*
* Assume that since fts_read() couldn't stat
* the file, it can't be unlinked.
*/
if (fflag)
continue;
/* FALLTHROUGH */
case FTS_F:
case FTS_NSOK:
if (Pflag)
if (!rm_overwrite(p->fts_accpath, p->fts_info ==
FTS_NSOK ? NULL : p->fts_statp))
continue;
/* FALLTHROUGH */
default:
rval = unlink(p->fts_accpath);
if (rval == 0 || (fflag && errno == ENOENT)) {
if (rval == 0 && vflag)
(void)printf("%s\n",
p->fts_path);
if (rval == 0 && info) {
info = 0;
(void)printf("%s\n",
p->fts_path);
}
continue;
}
}
}
err:
warn("%s", p->fts_path);
eval = 1;
}
if (!fflag && errno)
err(1, "fts_read");
fts_close(fts);
}
void* FD3D12DynamicRHI::LockBuffer(FRHICommandListImmediate* RHICmdList, BufferType* Buffer, uint32 Offset, uint32 Size, EResourceLockMode LockMode)
{
#if STATS
LockBufferCalls++;
SCOPE_CYCLE_COUNTER(STAT_D3D12LockBufferTime);
INC_DWORD_STAT_BY(STAT_D3D12LockBufferCalls, LockBufferCalls);
#endif
FD3D12LockedResource& LockedData = Buffer->LockedData;
check(LockedData.bLocked == false);
FD3D12Device* Device = GetRHIDevice();
FD3D12Adapter& Adapter = GetAdapter();
// Determine whether the buffer is dynamic or not.
const bool bIsDynamic = (Buffer->GetUsage() & BUF_AnyDynamic) ? true : false;
void* Data = nullptr;
if (bIsDynamic)
{
check(LockMode == RLM_WriteOnly);
BufferType* CurrentBuffer = Buffer;
// Update all of the resources in the LDA chain
while (CurrentBuffer)
{
// Allocate a new resource
// If on the RenderThread, queue up a command on the RHIThread to rename this buffer at the correct time
if (ShouldDeferBufferLockOperation(RHICmdList))
{
FRHICommandRenameUploadBuffer<BufferType>* Command = new (RHICmdList->AllocCommand<FRHICommandRenameUploadBuffer<BufferType>>()) FRHICommandRenameUploadBuffer<BufferType>(CurrentBuffer, Device);
Data = Adapter.GetUploadHeapAllocator().AllocUploadResource(Buffer->GetSize(), Buffer->BufferAlignment, Command->NewResource);
}
else
{
FD3D12ResourceLocation Location(CurrentBuffer->GetParentDevice());
Data = Adapter.GetUploadHeapAllocator().AllocUploadResource(Buffer->GetSize(), Buffer->BufferAlignment, Location);
CurrentBuffer->Rename(Location);
}
CurrentBuffer = CurrentBuffer->GetNextObject();
}
}
else
{
FD3D12Resource* pResource = Buffer->ResourceLocation.GetResource();
// Locking for read must occur immediately so we can't queue up the operations later.
if (LockMode == RLM_ReadOnly)
{
LockedData.bLockedForReadOnly = true;
// If the static buffer is being locked for reading, create a staging buffer.
FD3D12Resource* StagingBuffer = nullptr;
const GPUNodeMask Node = Device->GetNodeMask();
VERIFYD3D12RESULT(Adapter.CreateBuffer(D3D12_HEAP_TYPE_READBACK, Node, Node, Offset + Size, &StagingBuffer));
// Copy the contents of the buffer to the staging buffer.
{
const auto& pfnCopyContents = [&]()
{
FD3D12CommandContext& DefaultContext = Device->GetDefaultCommandContext();
FD3D12CommandListHandle& hCommandList = DefaultContext.CommandListHandle;
FScopeResourceBarrier ScopeResourceBarrierSource(hCommandList, pResource, pResource->GetDefaultResourceState(), D3D12_RESOURCE_STATE_COPY_SOURCE, 0);
// Don't need to transition upload heaps
DefaultContext.numCopies++;
hCommandList->CopyBufferRegion(
StagingBuffer->GetResource(),
0,
pResource->GetResource(),
Offset, Size);
hCommandList.UpdateResidency(StagingBuffer);
hCommandList.UpdateResidency(pResource);
DefaultContext.FlushCommands(true);
};
if (ShouldDeferBufferLockOperation(RHICmdList))
{
// Sync when in the render thread implementation
check(IsInRHIThread() == false);
RHICmdList->ImmediateFlush(EImmediateFlushType::FlushRHIThread);
pfnCopyContents();
}
else
{
check(IsInRHIThread());
pfnCopyContents();
}
}
LockedData.ResourceLocation.AsStandAlone(StagingBuffer, Size);
Data = LockedData.ResourceLocation.GetMappedBaseAddress();
}
else
{
// If the static buffer is being locked for writing, allocate memory for the contents to be written to.
Data = Device->GetDefaultFastAllocator().Allocate<FD3D12ScopeLock>(Size, D3D12_TEXTURE_DATA_PLACEMENT_ALIGNMENT, &LockedData.ResourceLocation);
}
}
LockedData.LockedOffset = Offset;
LockedData.LockedPitch = Size;
LockedData.bLocked = true;
// Return the offset pointer
check(Data != nullptr);
return Data;
}
static void
rm_file(char **argv)
{
struct stat sb;
int rval;
char *f;
/*
* Remove a file. POSIX 1003.2 states that, by default, attempting
* to remove a directory is an error, so must always stat the file.
*/
while ((f = *argv++) != NULL) {
/* Assume if can't stat the file, can't unlink it. */
if (lstat(f, &sb)) {
if (Wflag) {
sb.st_mode = S_IFWHT|S_IWUSR|S_IRUSR;
} else {
if (!fflag || errno != ENOENT) {
warn("%s", f);
eval = 1;
}
continue;
}
} else if (Wflag) {
warnx("%s: %s", f, strerror(EEXIST));
eval = 1;
continue;
}
if (S_ISDIR(sb.st_mode) && !dflag) {
warnx("%s: is a directory", f);
eval = 1;
continue;
}
if (!fflag && !S_ISWHT(sb.st_mode) && !check(f, f, &sb))
continue;
rval = 0;
if (!uid && !S_ISWHT(sb.st_mode) &&
(sb.st_flags & (UF_APPEND|UF_IMMUTABLE)) &&
!(sb.st_flags & (SF_APPEND|SF_IMMUTABLE)))
rval = lchflags(f, sb.st_flags & ~(UF_APPEND|UF_IMMUTABLE));
if (rval == 0) {
if (S_ISWHT(sb.st_mode))
rval = undelete(f);
else if (S_ISDIR(sb.st_mode))
rval = rmdir(f);
else {
if (Pflag)
if (!rm_overwrite(f, &sb))
continue;
rval = unlink(f);
}
}
if (rval && (!fflag || errno != ENOENT)) {
warn("%s", f);
eval = 1;
}
if (vflag && rval == 0)
(void)printf("%s\n", f);
if (info && rval == 0) {
info = 0;
(void)printf("%s\n", f);
}
}
}
void FSoundFileDecoder::Signal()
{
check(ThreadDecodeEvent != nullptr);
ThreadDecodeEvent->Trigger();
}
void FSlateOpenGLVS::Create( const FString& Filename )
{
check( ShaderID==0 );
// Compile the vertex shader
CompileShader( Filename, GL_VERTEX_SHADER );
}
int main(int argc, char* argv[])
{
// check for correct number of args
if (argc != 2 && argc != 3)
{
printf("Usage: speller [dictionary] text\n");
return 1;
}
// structs for timing data
struct rusage before, after;
// benchmarks
double time_load = 0.0, time_check = 0.0, time_size = 0.0, time_unload = 0.0;
// determine dictionary to use
char* dictionary = (argc == 3) ? argv[1] : DICTIONARY;
// load dictionary
getrusage(RUSAGE_SELF, &before);
bool loaded = load(dictionary);
getrusage(RUSAGE_SELF, &after);
// abort if dictionary not loaded
if (!loaded)
{
printf("Could not load %s.\n", dictionary);
return 1;
}
// calculate time to load dictionary
time_load = calculate(&before, &after);
// try to open text
char* text = (argc == 3) ? argv[2] : argv[1];
FILE* fp = fopen(text, "r");
if (fp == NULL)
{
printf("Could not open %s.\n", text);
unload();
return 1;
}
// prepare to report misspellings
printf("\nMISSPELLED WORDS\n\n");
// prepare to spell-check
int index = 0, misspellings = 0, words = 0;
char word[LENGTH+1];
// spell-check each word in text
for (int c = fgetc(fp); c != EOF; c = fgetc(fp))
{
// allow only alphabetical characters and apostrophes
if (isalpha(c) || (c == '\'' && index > 0))
{
// append character to word
word[index] = c;
index++;
// ignore alphabetical strings too long to be words
if (index > LENGTH)
{
// consume remainder of alphabetical string
while ((c = fgetc(fp)) != EOF && isalpha(c));
// prepare for new word
index = 0;
}
}
// ignore words with numbers (like MS Word can)
else if (isdigit(c))
{
// consume remainder of alphanumeric string
while ((c = fgetc(fp)) != EOF && isalnum(c));
// prepare for new word
index = 0;
}
// we must have found a whole word
else if (index > 0)
{
// terminate current word
word[index] = '\0';
// update counter
words++;
// check word's spelling
getrusage(RUSAGE_SELF, &before);
bool misspelled = !check(word);
getrusage(RUSAGE_SELF, &after);
// update benchmark
time_check += calculate(&before, &after);
// print word if misspelled
if (misspelled)
{
printf("%s\n", word);
misspellings++;
}
// prepare for next word
index = 0;
}
}
// check whether there was an error
if (ferror(fp))
{
fclose(fp);
printf("Error reading %s.\n", text);
unload();
return 1;
}
// close text
fclose(fp);
// determine dictionary's size
getrusage(RUSAGE_SELF, &before);
unsigned int n = size();
getrusage(RUSAGE_SELF, &after);
// calculate time to determine dictionary's size
time_size = calculate(&before, &after);
// unload dictionary
getrusage(RUSAGE_SELF, &before);
bool unloaded = unload();
getrusage(RUSAGE_SELF, &after);
// abort if dictionary not unloaded
if (!unloaded)
{
printf("Could not unload %s.\n", dictionary);
return 1;
}
// calculate time to unload dictionary
time_unload = calculate(&before, &after);
// report benchmarks
printf("\nWORDS MISSPELLED: %d\n", misspellings);
printf("WORDS IN DICTIONARY: %d\n", n);
printf("WORDS IN TEXT: %d\n", words);
printf("TIME IN load: %.2f\n", time_load);
printf("TIME IN check: %.2f\n", time_check);
printf("TIME IN size: %.2f\n", time_size);
printf("TIME IN unload: %.2f\n", time_unload);
printf("TIME IN TOTAL: %.2f\n\n",
time_load + time_check + time_size + time_unload);
// that's all folks
return 0;
}
void FSlateOpenGLPS::Create( const FString& Filename )
{
check( ShaderID==0 );
// Compile the pixel shader
CompileShader( Filename, GL_FRAGMENT_SHADER );
}
static float GetForwardSpeed(const AWheeledVehicle &Vehicle)
{
const auto *MovementComponent = Vehicle.GetVehicleMovementComponent();
check(nullptr != MovementComponent);
return MovementComponent->GetForwardSpeed();
}
void FSlateOpenGLShader::CompileShader( const FString& Filename, GLenum ShaderType )
{
// Create a new shader ID.
ShaderID = glCreateShader( ShaderType );
GLint CompileStatus = GL_FALSE;
check( ShaderID );
// Load the file to a string
FString Source;
bool bFileFound = FFileHelper::LoadFileToString( Source, *Filename );
check(bFileFound);
FString Header;
// pass the #define along to the shader
#if PLATFORM_USES_ES2
Header.Append("#define PLATFORM_USES_ES2 1\n");
#elif PLATFORM_LINUX
#if LINUX_USE_OPENGL_3_2
Header.Append("#version 150\n#define PLATFORM_USES_ES2 0\n");
#else
Header.Append("#version 120\n#define PLATFORM_USES_ES2 0\n");
#endif // LINUX_USE_OPENGL_3_2
#else
Header.Append("#version 120\n#define PLATFORM_USES_ES2 0\n");
#endif
#if PLATFORM_LINUX
Header.Append("#define PLATFORM_LINUX 1\n");
#else
Header.Append("#define PLATFORM_LINUX 0\n");
#endif
#if PLATFORM_MAC
Header.Append("#define PLATFORM_MAC 1\n");
#else
Header.Append("#define PLATFORM_MAC 0\n");
#endif
#if USE_709
Header.Append("#define USE_709 1\n");
#else
Header.Append("#define USE_709 0\n");
#endif
// Allocate a buffer big enough to store the string in ascii format
ANSICHAR* Chars[2] = {0};
Chars[0] = new ANSICHAR[Header.Len()+1];
FCStringAnsi::Strcpy(Chars[0], Header.Len() + 1, TCHAR_TO_ANSI(*Header));
Chars[1] = new ANSICHAR[Source.Len()+1];
FCStringAnsi::Strcpy(Chars[1], Source.Len() + 1, TCHAR_TO_ANSI(*Source));
// give opengl the source code for the shader
glShaderSource( ShaderID, 2, (const ANSICHAR**)Chars, NULL );
delete[] Chars[0];
delete[] Chars[1];
// Compile the shader and check for success
glCompileShader( ShaderID );
glGetShaderiv( ShaderID, GL_COMPILE_STATUS, &CompileStatus );
if( CompileStatus == GL_FALSE )
{
// The shader did not compile. Display why it failed.
FString Log = GetGLSLShaderLog( ShaderID );
checkf(false, TEXT("Failed to compile shader: %s\n%s"), *Filename, *Log );
// Delete the shader since it failed.
glDeleteShader( ShaderID );
ShaderID = 0;
}
}
Distance Distance::operator -(const Distance& _distance) const
{
/// @todo
check(false,"note yet implemented") ;
}
FBlendedCurve::FBlendedCurve(const class UAnimInstance* AnimInstance)
{
check(AnimInstance);
InitFrom(AnimInstance->CurrentSkeleton);
}
void LH_ITERATE_WORDS_NAME(const unichar *buf, size_t bufsize, lh_word_check_t *check, lh_word_callback_t *callback, void *userdata)
{
size_t pos = 0;
size_t word_start;
size_t word_length;
utf8char word_buf[BUFSIZEOF__UTF8_WORD];
utf8char *hugeword_buf = NULL;
size_t hugeword_buf_size = 0;
utf8char *word_end;
int prop;
#ifdef LH_ITERATOR_DEBUG
int wordctr = 0, wordcount = LH_COUNT_WORDS_NAME (buf, bufsize, check);
#define wordctr_INC1 wordctr++
#else
#define wordctr_INC1
#endif
while (pos < bufsize)
{
prop = UNICHAR_GETPROPS_EXPN (buf, bufsize, pos);
if (prop & UCP_ALPHA)
{
word_start = pos;
do pos++; while ((pos < bufsize) && (UNICHAR_GETPROPS_EXPN (buf, bufsize, pos) & UCP_ALPHA));
word_length = pos - word_start;
if (WORD_MAX_CHARS < word_length)
continue;
if (NULL!=check && 0 == check(buf+word_start, word_length))
continue;
word_end = (utf8char *)eh_encode_buffer__UTF8 (buf+word_start, buf+pos, (char *)word_buf, (char *)(word_buf+BUFSIZEOF__UTF8_WORD));
if (NULL != word_end)
{
callback (word_buf, word_end-word_buf, userdata);
wordctr_INC1;
continue;
}
if (hugeword_buf_size<(word_length*MAX_UTF8_CHAR))
{
if (hugeword_buf_size)
dk_free (hugeword_buf, hugeword_buf_size);
hugeword_buf_size = word_length*MAX_UTF8_CHAR;
hugeword_buf = (utf8char *) dk_alloc (hugeword_buf_size);
}
word_end = (utf8char *)eh_encode_buffer__UTF8 (buf+word_start, buf+pos, (char *)hugeword_buf, (char *)(hugeword_buf+hugeword_buf_size));
callback (hugeword_buf, word_end-hugeword_buf, userdata);
wordctr_INC1;
continue;
}
if (prop & UCP_IDEO)
{
word_start = pos;
pos++;
if (NULL!=check && 0 == check(buf+pos-1, 1))
continue;
word_end = (utf8char *)eh_encode_buffer__UTF8 (buf+word_start, buf+pos, (char *)(word_buf), (char *)(word_buf+BUFSIZEOF__UTF8_WORD));
callback (word_buf, word_end-word_buf, userdata);
wordctr_INC1;
continue;
}
pos++;
}
if (hugeword_buf_size)
dk_free (hugeword_buf, hugeword_buf_size);
#ifdef LH_ITERATOR_DEBUG
if (wordctr != wordcount)
GPF_T;
#endif
}
static void get_spc_xid6( byte const* begin, long size, track_info_t* out )
{
// header
byte const* end = begin + size;
if ( size < 8 || memcmp( begin, "xid6", 4 ) )
{
check( false );
return;
}
long info_size = get_le32( begin + 4 );
byte const* in = begin + 8;
if ( end - in > info_size )
{
debug_printf( "Extra data after SPC xid6 info\n" );
end = in + info_size;
}
int year = 0;
char copyright [256 + 5];
int copyright_len = 0;
int const year_len = 5;
while ( end - in >= 4 )
{
// header
int id = in [0];
int data = in [3] * 0x100 + in [2];
int type = in [1];
int len = type ? data : 0;
in += 4;
if ( len > end - in )
{
check( false );
break; // block goes past end of data
}
// handle specific block types
char* field = 0;
switch ( id )
{
case 0x01: field = out->song; break;
case 0x02: field = out->game; break;
case 0x03: field = out->author; break;
case 0x04: field = out->dumper; break;
case 0x07: field = out->comment; break;
case 0x14: year = data; break;
//case 0x30: // intro length
// Many SPCs have intro length set wrong for looped tracks, making it useless
/*
case 0x30:
check( len == 4 );
if ( len >= 4 )
{
out->intro_length = get_le32( in ) / 64;
if ( out->length > 0 )
{
long loop = out->length - out->intro_length;
if ( loop >= 2000 )
out->loop_length = loop;
}
}
break;
*/
case 0x13:
copyright_len = min( len, (int) sizeof copyright - year_len );
memcpy( ©right [year_len], in, copyright_len );
break;
default:
if ( id < 0x01 || (id > 0x07 && id < 0x10) ||
(id > 0x14 && id < 0x30) || id > 0x36 )
debug_printf( "Unknown SPC xid6 block: %X\n", (int) id );
break;
}
if ( field )
{
check( type == 1 );
Gme_File::copy_field_( field, (char const*) in, len );
}
// skip to next block
in += len;
// blocks are supposed to be 4-byte aligned with zero-padding...
byte const* unaligned = in;
while ( (in - begin) & 3 && in < end )
{
if ( *in++ != 0 )
{
// ...but some files have no padding
in = unaligned;
debug_printf( "SPC info tag wasn't properly padded to align\n" );
break;
}
}
}
char* p = ©right [year_len];
if ( year )
{
*--p = ' ';
for ( int n = 4; n--; )
{
*--p = char (year % 10 + '0');
year /= 10;
}
copyright_len += year_len;
}
if ( copyright_len )
Gme_File::copy_field_( out->copyright, p, copyright_len );
check( in == end );
}
static int
test_utf8 (void)
{
const char * in = "{ \"key\": \"Letöltések\" }";
tr_variant top;
const char * str;
char * json;
int err;
const tr_quark key = tr_quark_new ("key", 3);
err = tr_variantFromJson (&top, in, strlen(in));
check (!err);
check (tr_variantIsDict (&top));
check (tr_variantDictFindStr (&top, key, &str, NULL));
check_streq ("Letöltések", str);
if (!err)
tr_variantFree (&top);
in = "{ \"key\": \"\\u005C\" }";
err = tr_variantFromJson (&top, in, strlen(in));
check (!err);
check (tr_variantIsDict (&top));
check (tr_variantDictFindStr (&top, key, &str, NULL));
check_streq ("\\", str);
if (!err)
tr_variantFree (&top);
/**
* 1. Feed it JSON-escaped nonascii to the JSON decoder.
* 2. Confirm that the result is UTF-8.
* 3. Feed the same UTF-8 back into the JSON encoder.
* 4. Confirm that the result is JSON-escaped.
* 5. Dogfood that result back into the parser.
* 6. Confirm that the result is UTF-8.
*/
in = "{ \"key\": \"Let\\u00f6lt\\u00e9sek\" }";
err = tr_variantFromJson (&top, in, strlen(in));
check (!err);
check (tr_variantIsDict (&top));
check (tr_variantDictFindStr (&top, key, &str, NULL));
check_streq ("Letöltések", str);
json = tr_variantToStr (&top, TR_VARIANT_FMT_JSON, NULL);
if (!err)
tr_variantFree (&top);
check (json);
check (strstr (json, "\\u00f6") != NULL);
check (strstr (json, "\\u00e9") != NULL);
err = tr_variantFromJson (&top, json, strlen(json));
check (!err);
check (tr_variantIsDict (&top));
check (tr_variantDictFindStr (&top, key, &str, NULL));
check_streq ("Letöltések", str);
if (!err)
tr_variantFree (&top);
tr_free (json);
return 0;
}
// The stack-unwinding loop.
static inline void unwind_loop(ExcInfo* exc_data) {
// NB. https://monoinfinito.wordpress.com/series/exception-handling-in-c/ is a very useful resource
// as are http://www.airs.com/blog/archives/460 and http://www.airs.com/blog/archives/464
unw_cursor_t cursor;
unw_context_t uc; // exists only to initialize cursor
#ifndef NDEBUG
// poison stack memory. have had problems with these structures being insufficiently initialized.
memset(&uc, 0xef, sizeof uc);
memset(&cursor, 0xef, sizeof cursor);
#endif
unw_getcontext(&uc);
unw_init_local(&cursor, &uc);
auto unwind_session = getActivePythonUnwindSession();
while (unw_step(&cursor) > 0) {
unw_proc_info_t pip;
static StatCounter frames_unwound("num_frames_unwound_cxx");
frames_unwound.log();
// NB. unw_get_proc_info is slow; a significant chunk of all time spent unwinding is spent here.
check(unw_get_proc_info(&cursor, &pip));
assert((pip.lsda == 0) == (pip.handler == 0));
assert(pip.flags == 0);
if (VERBOSITY("cxx_unwind") >= 4) {
print_frame(&cursor, &pip);
}
// let the PythonUnwindSession know that we're in a new frame,
// giving it a chance to possibly add a traceback entry for
// it.
unwindingThroughFrame(unwind_session, &cursor);
// Skip frames without handlers
if (pip.handler == 0) {
continue;
}
RELEASE_ASSERT(pip.handler == (uintptr_t)__gxx_personality_v0,
"personality function other than __gxx_personality_v0; "
"don't know how to unwind through non-C++ functions");
// Don't call __gxx_personality_v0; we perform dispatch ourselves.
// 1. parse LSDA header
lsda_info_t info;
parse_lsda_header(&pip, &info);
call_site_entry_t entry;
{
// 2. Find our current IP in the call site table.
unw_word_t ip;
unw_get_reg(&cursor, UNW_REG_IP, &ip);
// ip points to the instruction *after* the instruction that caused the error - which is generally (always?)
// a call instruction - UNLESS we're in a signal frame, in which case it points at the instruction that
// caused the error. For now, we assume we're never in a signal frame. So, we decrement it by one.
//
// TODO: double-check that we never hit a signal frame.
--ip;
bool found = find_call_site_entry(&info, (const uint8_t*)ip, &entry);
// If we didn't find an entry, an exception happened somewhere exceptions should never happen; terminate
// immediately.
if (!found) {
panic();
}
}
// 3. Figure out what to do based on the call site entry.
if (!entry.landing_pad) {
// No landing pad means no exception handling or cleanup; keep unwinding!
continue;
}
// After this point we are guaranteed to resume something rather than unwinding further.
if (VERBOSITY("cxx_unwind") >= 4) {
print_lsda(&info);
}
int64_t switch_value = determine_action(&info, &entry);
if (switch_value != CLEANUP_ACTION) {
// we're transfering control to a non-cleanup landing pad.
// i.e. a catch block. thus ends our unwind session.
endPythonUnwindSession(unwind_session);
#if STAT_TIMERS
pyston::StatTimer::finishOverride();
#endif
pyston::is_unwinding = false;
}
static_assert(THREADING_USE_GIL, "have to make the unwind session usage in this file thread safe!");
// there is a python unwinding implementation detail leaked
// here - that the unwind session can be ended but its
// exception storage is still around.
//
// this manifests itself as this short window here where we've
// (possibly) ended the unwind session above but we still need
// to pass exc_data (which is the exceptionStorage for this
// unwind session) to resume().
//
// the only way this could bite us is if we somehow clobber
// the PythonUnwindSession's storage, or cause a GC to occur, before
// transfering control to the landing pad in resume().
//
resume(&cursor, entry.landing_pad, switch_value, exc_data);
}
// Hit end of stack! return & let unwindException determine what to do.
}