bool test_google_float_workload_cpu_images_classification::run() { bool run_ok = true; test_measurement_result run_result; run_result.description = "RUN SUMMARY: " + test_description; C_time_control run_timer; std::cout << "-> Testing: " << test_description << std::endl; try { if(!init()) throw std::runtime_error("init() returns false so can't run test"); run_timer.tick(); //start time measurement run_result << std::string("run test with " + current_tested_device->get_device_description()); for(uint32_t batch :{1,8,48}) { C_time_control loop_timer; // compiling workload nn_workload_t *workload = nullptr; NN_WORKLOAD_DATA_TYPE input_format = NN_WORKLOAD_DATA_TYPE_F32_ZXY_BATCH; NN_WORKLOAD_DATA_TYPE output_format = NN_WORKLOAD_DATA_TYPE_F32_1D_BATCH; auto status = di->workflow_compile_function(&workload, di->device, workflow, &input_format, &output_format, batch); if(!workload) throw std::runtime_error("workload compilation failed for batch = " + std::to_string(batch) + " status: " + std::to_string(status)); test_measurement_result local_result; local_result.description = "RUN PART: (batch " + std::to_string(batch)+") execution of " + test_description; bool local_ok = true; auto images_list_iterator = images_list.begin(); auto images_list_end = images_list.end(); while(images_list_iterator != images_list_end) { auto diff_itr = images_list_end - images_list_iterator < batch ? images_list_end - images_list_iterator : batch; std::vector< std::string > batch_images(images_list_iterator,images_list_iterator + diff_itr); images_list_iterator += diff_itr; nn::data< float,4 > *images = nullptr; images = nn_data_load_from_image_list(&batch_images, img_size, image_process, batch, RGB_order); if(images) { nn_data_t *input_array[1] ={images}; nn::data<float, 2> *workload_output = new nn::data<float, 2>(1000, batch); if(workload_output == nullptr) throw std::runtime_error("unable to create workload_output for batch = " +std::to_string(batch)); nn::data<float> *output_array_cmpl[1] ={nn::data_cast<float,0>(workload_output)}; di->workload_execute_function(workload,reinterpret_cast<void**>(input_array),reinterpret_cast<void**>(output_array_cmpl),&status); float *value_cmpl = reinterpret_cast<float *>(workload_output->buffer); for(auto &image_filename : batch_images) { std::ifstream reference_output_file(image_filename + ".txt", std::ifstream::in); // Comparison with the reference output workload float difference = 0; for(int index = 0; index < 1000; ++index) { std::string reference_value_str; std::getline(reference_output_file,reference_value_str); float reference_value = std::stof(reference_value_str); float delta = value_cmpl[index]-reference_value; difference += abs(delta); } if(difference < threshold_to_pass_test) local_result << std::string("note: " + image_filename + " difference = " + std::to_string(difference)); else { local_result << std::string("error: image file: " + image_filename +" the difference exceeded the allowable threshold for compliance: " + std::to_string(difference) + " > " + std::to_string(threshold_to_pass_test)); local_ok = false; run_ok = false; } reference_output_file.close(); value_cmpl += 1000; } batch_images.clear(); if(images != nullptr) delete images; if(workload_output != nullptr) delete workload_output; } } // batch loop summary: local_result.passed = local_ok; loop_timer.tock(); local_result.time_consumed = loop_timer.get_time_diff(); local_result.clocks_consumed = loop_timer.get_clocks_diff(); tests_results << local_result; if(workload != nullptr) di->workload_delete_function(workload); } // END: for(uint32_t batch :{1,8,48}) } catch(std::runtime_error &error) { run_result << "error: " + std::string(error.what()); run_ok = false; } catch(...) { run_result << "error: unknown"; run_ok = false; } run_timer.tock(); run_result.time_consumed = run_timer.get_time_diff(); run_result.clocks_consumed = run_timer.get_clocks_diff(); run_result.passed = run_ok; tests_results << run_result; if (!done()) run_ok=false; std::cout << "<- Test " << (run_ok ? "passed" : "failed") << std::endl;; return run_ok; }
bool test_caffe_float_workload_cpu_time::run() { bool run_ok = true; test_measurement_result run_result; run_result.description = "RUN SUMMARY: " + test_description; C_time_control run_timer; std::cout << "-> Testing: " << test_description << std::endl; try { if(!init()) throw std::runtime_error("error: init() returns false so can't run test"); run_timer.tick(); //start time measurement run_result << std::string("run test with " + current_tested_device->get_device_description()); // --------------------------------------------------------------------------------------------------------- // TODO: here test code //{ // BKM pattern of test with time measuring: // bool local_ok=true; // test_measurement_result local_result; // local_result.description = "RUN PART: (name part) of " + test_description; // C_time_control local_timer; // // begin local test // // end of local test // // summary: // local_timer.tock(); // local_result.time_consumed = local_timer.time_diff_string(); // local_result.clocks_consumed = local_timer.get_clocks_diff(); // tests_results << local_result; //} // The pattern, of complex instruction above, can be multiplied for(uint16_t batch :{1,8,48}) { std::vector<uint64_t> time_diffs; std::vector<uint64_t> clock_diffs; nn::data<float,4> *images = new nn::data<float,4>(img_size,img_size,3,batch); nn_data_populate(nn::data_cast<float,0>(images),0.0f,255.0f); nn_data_t *input_array[1] ={images}; auto workload_output = new nn::data<float, 2>(1000, batch); nn::data<float> *output_array_cmpl[1] ={ nn::data_cast<float, 0>(workload_output) }; nn_workload_t *workload = nullptr; // compiling workload NN_WORKLOAD_DATA_TYPE input_format = NN_WORKLOAD_DATA_TYPE_F32_ZXY_BATCH; NN_WORKLOAD_DATA_TYPE output_format = NN_WORKLOAD_DATA_TYPE_F32_1D_BATCH; auto status = di->workflow_compile_function(&workload,di->device,workflow,&input_format,&output_format,batch); if(!workload) throw std::runtime_error("workload compilation failed for batch = " + std::to_string(batch) + " status: " + std::to_string(status)); test_measurement_result local_result; local_result.description = "RUN PART: (batch " + std::to_string(batch)+") execution of " + test_description; local_result.loops = loops; // begin local test for(auto i = 0; i< loops; ++i) { NN_API_STATUS status; C_time_control loop_timer; di->workload_execute_function(workload,reinterpret_cast<void**>(input_array),reinterpret_cast<void**>(output_array_cmpl),&status); loop_timer.tock(); time_diffs.push_back(loop_timer.get_time_diff()/batch); clock_diffs.push_back(loop_timer.get_clocks_diff()/batch); } // end of local test // summary: uint64_t min_value = *std::min_element(time_diffs.begin(),time_diffs.end()); local_result.time_consumed = std::accumulate(time_diffs.begin(),time_diffs.end(),0.0)/time_diffs.size(); local_result.time_consumed_min = min_value; local_result.time_consumed_max = *std::max_element(time_diffs.begin(),time_diffs.end()); local_result << std::string("note: The shortest time for one image obtained from the chrono: " + C_time_control::time_diff_string(min_value)); local_result << std::string("note: Values of time's and clock's were divided by current value of batch: "+std::to_string(batch)); local_result.clocks_consumed = std::accumulate(clock_diffs.begin(),clock_diffs.end(),0.0)/clock_diffs.size(); local_result.clocks_consumed_min = *std::min_element(clock_diffs.begin(),clock_diffs.end()); local_result.clocks_consumed_max = *std::max_element(clock_diffs.begin(),clock_diffs.end()); tests_results << local_result; if(images != nullptr) delete images; if(workload_output != nullptr) delete workload_output; if(workload != nullptr) di->workload_delete_function(workload); } // --------------------------------------------------------------------------------------------------------- run_ok = true; } catch(std::runtime_error &error) { run_result << "error: " + std::string(error.what()); run_ok = false; } catch(...) { run_result << "error: unknown"; run_ok = false; } run_timer.tock(); run_result.time_consumed = run_timer.get_time_diff(); run_result.clocks_consumed = run_timer.get_clocks_diff(); run_result.passed = run_ok; tests_results << run_result; if (!done()) run_ok=false; std::cout << "<- Test " << (run_ok ? "passed" : "failed") << std::endl;; return run_ok; }
bool test_convolution_float_cpu_random::run() { bool run_ok = true; test_measurement_result run_result; run_result.description = "RUN SUMMARY: " + test_description; std::cout << "-> Testing: " << test_description << std::endl; try { if(!init()) throw std::runtime_error( "init() returns false so can't run test" ); NN_WORKLOAD_DATA_TYPE input_format = NN_WORKLOAD_DATA_TYPE_F32_3D_BATCH; NN_WORKLOAD_DATA_TYPE output_format = NN_WORKLOAD_DATA_TYPE_F32_3D_BATCH; for(uint32_t batch : { 1, 8, 48 }) { bool local_ok = true; test_measurement_result local_result; local_result.description = "RUN PART: (batch " + std::to_string( batch ) + ") execution of " + test_description; C_time_control local_timer; // begin local test uint32_t z = 2, img_size = 227, num_features_map = 8; nn::data<float, 4> *images = new nn::data<float, 4>( img_size, img_size, z, batch ); if(images == nullptr) throw std::runtime_error("Cant't create images nn::data"); nn_data_populate( nn::data_cast<float, 0>(images), 0.0f, 255.0f ); nn::data<float, 4> *images_with_padding = new nn::data<float, 4>( img_size + 2, img_size + 2, z, batch ); if(images_with_padding == nullptr) { delete images; throw std::runtime_error("Cant't create images_with_padding nn::data"); } { // padding for input for naive method nn_data_populate( nn::data_cast<float, 0>(images_with_padding), 0.0f ); for(uint32_t tmp_batch = 0; tmp_batch < batch; ++tmp_batch) for(uint32_t tmp_z = 0; tmp_z < z; ++tmp_z) for(uint32_t y = 0; y < img_size; ++y) for(uint32_t x = 0; x < img_size; ++x) images_with_padding->at( x, y, tmp_z, tmp_batch ) = images->at( x, y, tmp_z, tmp_batch ); } nn_workload_t *workload = nullptr; nn_data_t *input_array[1] = { images }; auto workload_output = new nn::data<float, 4>( img_size, img_size, num_features_map, batch ); if(workload_output==nullptr) { delete images; delete images_with_padding; throw std::runtime_error("unable to create workload_output for batch = " +std::to_string(batch)); } nn::data<float> *output_array_cmpl[1] = { nn::data_cast<float, 0>(workload_output) }; auto naive_output = new nn::data<float, 4>( img_size, img_size, num_features_map, batch ); if(naive_output==nullptr) { delete images; delete images_with_padding; delete workload_output; throw std::runtime_error("unable to create naive_output for batch = " +std::to_string(batch)); } auto status = di->workflow_compile_function( &workload, di->device, workflow, &input_format, &output_format, batch ); if(!workload) throw std::runtime_error( "workload compilation failed for batch = " + std::to_string( batch ) + " status: " + std::to_string( status ) ); test_measurement_result run_result; run_result.description = "RUN PART: (batch " + std::to_string( batch ) + ") execution of " + test_description; // changing order needed //di->workload_execute_function( workload, reinterpret_cast<void**>(input_array), reinterpret_cast<void**>(output_array_cmpl), &status ); float* biases = nullptr; float* weights = nullptr; { // read biases and weights if(NN_WORK_ITEM_TYPE_CONVOLUTION == workflow->input[0]->use[0].item->type) { auto tmp = reinterpret_cast<nn_arguments_forward_convolution_t*>(&workflow->input[0]->use[0].item->arguments); biases = reinterpret_cast<float*>(tmp->biases->buffer); weights = reinterpret_cast<float*>(tmp->weights->buffer); } } if(nullptr == biases || nullptr == weights) throw std::runtime_error( "reading weight or biases for naive version failed for batch = " + std::to_string( batch ) ); naive_convolv_float_implementation( reinterpret_cast<float*>(images_with_padding->buffer), reinterpret_cast<float*>(naive_output->buffer), biases, weights, batch, num_features_map, z, img_size, img_size, img_size + 2, img_size + 2, 3, 3, 1, 1, NN_ACTIVATION_FUNCTION_RELU ); //local_ok = compare_4d_data( workload_output, naive_output ); local_ok = true; // BLIND TEST // end of local test // summary: local_timer.tock(); local_result.time_consumed = local_timer.get_time_diff(); local_result.clocks_consumed = local_timer.get_clocks_diff(); local_result.passed = local_ok; tests_results << local_result; run_ok = run_ok && local_ok; if(workload_output) delete workload_output; if(naive_output) delete naive_output; if(images) delete images; if(images_with_padding) delete images_with_padding; } } catch(std::runtime_error &error) { tests_results << run_result; std::cout << "error: " << error.what() << std::endl; } catch(std::exception &error) { tests_results << run_result; std::cout << "error: " << error.what() << std::endl; } catch(...) { tests_results << run_result; std::cout << "error: unknown" << std::endl; } if(!done()) run_ok = false; std::cout << "<- Test " << (run_ok ? "passed" : "failed") << std::endl;; return run_ok; }
bool test_softmax_float_cpu_random::run() { bool run_ok = true; test_measurement_result run_result; run_result.description = "RUN SUMMARY: " + test_description; C_time_control run_timer; std::cout << "-> Testing: " << test_description << std::endl; try { if( !init() ) throw std::runtime_error( "init() returns false so can't run test" ); run_timer.tick(); //start time measurement run_result << std::string( "run test with " + current_tested_device->get_device_description() ); NN_WORKLOAD_DATA_TYPE input_format = NN_WORKLOAD_DATA_TYPE_F32_1D_BATCH; NN_WORKLOAD_DATA_TYPE output_format = NN_WORKLOAD_DATA_TYPE_F32_1D_BATCH; const int softmax_size = 1000; for( auto batch : { 1, 8, 48 } ) { // --------------------------------------------------------------------------------------------------------- { // simple sample pattern of test with time measuring: bool local_ok = true; test_measurement_result local_result; local_result.description = "RUN PART: (batch " + std::to_string( batch ) + ") execution of " + test_description; C_time_control local_timer; // begin local test auto input = new nn::data<float>( softmax_size, batch ); if(input == nullptr) throw std::runtime_error("unable to create input for batch = " +std::to_string(batch)); auto workload_output = new nn::data<float>( softmax_size, batch ); if(workload_output == nullptr) throw std::runtime_error("unable to create workload_output for batch = " +std::to_string(batch)); nn_data_populate( workload_output, 0.0f ); nn_data_populate( input, 0.0f, 20.0f ); nn_workload_t *workload = nullptr; nn_data_t *input_array[1] = { input }; nn::data<float> *output_array_cmpl[1] = { nn::data_cast<float, 0>(workload_output) }; auto status = di->workflow_compile_function( &workload, di->device, workflow, &input_format, &output_format, batch ); if( !workload ) throw std::runtime_error( "workload compilation failed for batch = " + std::to_string( batch ) + " status: " + std::to_string( status ) ); di->workload_execute_function( workload, reinterpret_cast<void**>(input_array), reinterpret_cast<void**>(output_array_cmpl), &status ); auto naive_output = cpu_layer_softmax( input ); local_ok = compare_data(workload_output, naive_output); // end of local test // summary: local_timer.tock(); local_result.time_consumed = local_timer.get_time_diff(); local_result.clocks_consumed = local_timer.get_clocks_diff(); local_result.passed = local_ok; tests_results << local_result; run_ok = run_ok && local_ok; if( input ) delete input; if( workload_output ) delete workload_output; if( naive_output ) delete naive_output; if( workload ) delete workload; } // The pattern, of complex instruction above, can be multiplied // END of run tests // --------------------------------------------------------------------------------------------------------- } } catch( std::runtime_error &error ) { run_result << "error: " + std::string( error.what() ); run_ok = false; } catch( std::exception &error ) { run_result << "error: " + std::string( error.what() ); run_ok = false; } catch( ... ) { run_result << "unknown error"; run_ok = false; } run_timer.tock(); run_result.time_consumed = run_timer.get_time_diff(); run_result.clocks_consumed = run_timer.get_clocks_diff(); run_result.passed = run_ok; tests_results << run_result; if( !done() ) run_ok = false; std::cout << "<- Test " << (run_ok ? "passed" : "failed") << std::endl;; return run_ok; }