void memcpy ( gpu_data& dest, size_t dest_offset, const gpu_data& src, size_t src_offset, size_t num ) { DLIB_CASSERT(dest_offset + num <= dest.size()); DLIB_CASSERT(src_offset + num <= src.size()); if (num == 0) return; // if there is aliasing if (&dest == &src && std::max(dest_offset, src_offset) < std::min(dest_offset,src_offset)+num) { // if they perfectly alias each other then there is nothing to do if (dest_offset == src_offset) return; else std::memmove(dest.host()+dest_offset, src.host()+src_offset, sizeof(float)*num); } else { // if we write to the entire thing then we can use device_write_only() if (dest_offset == 0 && num == dest.size()) { // copy the memory efficiently based on which copy is current in each object. if (src.device_ready()) CHECK_CUDA(cudaMemcpy(dest.device_write_only(), src.device()+src_offset, num*sizeof(float), cudaMemcpyDeviceToDevice)); else CHECK_CUDA(cudaMemcpy(dest.device_write_only(), src.host()+src_offset, num*sizeof(float), cudaMemcpyHostToDevice)); } else { // copy the memory efficiently based on which copy is current in each object. if (dest.device_ready() && src.device_ready()) CHECK_CUDA(cudaMemcpy(dest.device()+dest_offset, src.device()+src_offset, num*sizeof(float), cudaMemcpyDeviceToDevice)); else if (!dest.device_ready() && src.device_ready()) CHECK_CUDA(cudaMemcpy(dest.host()+dest_offset, src.device()+src_offset, num*sizeof(float), cudaMemcpyDeviceToHost)); else if (dest.device_ready() && !src.device_ready()) CHECK_CUDA(cudaMemcpy(dest.device()+dest_offset, src.host()+src_offset, num*sizeof(float), cudaMemcpyHostToDevice)); else CHECK_CUDA(cudaMemcpy(dest.host()+dest_offset, src.host()+src_offset, num*sizeof(float), cudaMemcpyHostToHost)); } } }
void binary_relation_detector_trainer:: set_beta ( double new_beta ) { DLIB_CASSERT(new_beta >= 0, "Invalid beta"); beta = new_beta; }
named_entity_extractor:: named_entity_extractor( const std::vector<std::string>& tag_name_strings_, const total_word_feature_extractor& fe_, const dlib::sequence_segmenter<ner_feature_extractor>& segmenter_, const dlib::multiclass_linear_decision_function<dlib::sparse_linear_kernel<ner_sample_type>,unsigned long>& df_ ) : tag_name_strings(tag_name_strings_), fe(fe_), segmenter(segmenter_), df(df_) { // make sure the requirements are not violated. DLIB_CASSERT(df.number_of_classes() >= tag_name_strings.size(),"invalid inputs"); DLIB_CASSERT(segmenter.get_feature_extractor().num_features() == fe.get_num_dimensions(),"invalid inputs"); std::set<unsigned long> df_tags(df.get_labels().begin(), df.get_labels().end()); for (unsigned long i = 0; i < tag_name_strings.size(); ++i) { DLIB_CASSERT(df_tags.count(i) == 1, "The classifier must be capable of predicting each possible tag as output."); } compute_fingerprint(); }
void scale_rows ( tensor& out, const tensor& m, const tensor& v ) { DLIB_CASSERT(have_same_dimensions(out,m)); DLIB_CASSERT(is_vector(v)); if (m.size() == 0 && v.size() == 0) return; DLIB_CASSERT(m.size() != 0); DLIB_CASSERT(m.num_samples() == v.size()); #ifdef DLIB_USE_CUDA cuda::scale_rows(out, m, v); #else out = scale_rows(mat(m), mat(v)); #endif }
void binary_relation_detector_trainer:: add_negative_binary_relation ( const std::vector<std::string>& tokens, unsigned long arg1_start, unsigned long arg1_length, unsigned long arg2_start, unsigned long arg2_length ) { DLIB_CASSERT(arg1_length > 0 && arg2_length > 0, "Invalid Inputs"); DLIB_CASSERT(mitie_entities_overlap(arg1_start,arg1_length,arg2_start,arg2_length) == 0, "Binary relation arguments can't overlap."); DLIB_CASSERT(arg1_start+arg1_length <= tokens.size() && arg2_start+arg2_length <= tokens.size(), "Invalid Inputs"); neg_sentences.push_back(tokens); neg_arg1s.push_back(std::make_pair(arg1_start, arg1_start+arg1_length)); neg_arg2s.push_back(std::make_pair(arg2_start, arg2_start+arg2_length)); }
void memcpy ( gpu_data& dest, const gpu_data& src ) { DLIB_CASSERT(dest.size() == src.size()); if (src.size() == 0 || &dest == &src) return; memcpy(dest,0, src, 0, src.size()); }
void scale_columns ( tensor& out, const tensor& m, const tensor& v ) { DLIB_CASSERT(have_same_dimensions(out,m)); DLIB_CASSERT(is_vector(v)); if (m.size() == 0 && v.size() == 0) return; DLIB_CASSERT(m.size() != 0); DLIB_CASSERT(m.size()/m.num_samples() == v.size()); #ifdef DLIB_USE_CUDA cuda::scale_columns(out, m, v); #else DLIB_CASSERT(false, "shouldn't be called right now"); out = scale_columns(mat(m), mat(v)); #endif }
void log10 ( tensor& dest, const tensor& src ) { DLIB_CASSERT(dest.size() == src.size()); #ifdef DLIB_USE_CUDA cuda::log10(dest,src); #else dest = log10(mat(src)); #endif }
void scale_rows2 ( float beta, tensor& out, const tensor& m1, const tensor& m2, const tensor& v1, const tensor& v2 ) { DLIB_CASSERT(have_same_dimensions(out,m1)); DLIB_CASSERT(have_same_dimensions(out,m2)); DLIB_CASSERT(have_same_dimensions(v1,v2)); DLIB_CASSERT(is_vector(mat(v1))); DLIB_CASSERT(v1.size() == m1.num_samples()); #ifdef DLIB_USE_CUDA cuda::scale_rows2(beta, out, m1, m2, v1, v2); #else if (beta == 0) out = scale_rows(mat(m1) - scale_rows(mat(m2),mat(v1)), mat(v2)); else out = beta*mat(out) + scale_rows(mat(m1) - scale_rows(mat(m2),mat(v1)), mat(v2)); #endif }
void server:: start ( ) { // make sure requires clause is not broken DLIB_CASSERT( this->is_running() == false, "\tvoid server::start" << "\n\tis_running() == " << this->is_running() << "\n\tthis: " << this ); start_accepting_connections(); }
void server:: set_max_connections ( int max ) { // make sure requires clause is not broken DLIB_CASSERT( max >= 0 , "\tvoid server::set_max_connections" << "\n\tmax == " << max << "\n\tthis: " << this ); max_connections_mutex.lock(); max_connections = max; max_connections_mutex.unlock(); }
void memcpy ( gpu_data& dest, const gpu_data& src ) { DLIB_CASSERT(dest.size() == src.size(), ""); if (src.size() == 0) return; // copy the memory efficiently based on which copy is current in each object. if (dest.device_ready() && src.device_ready()) CHECK_CUDA(cudaMemcpy(dest.device(), src.device(), src.size()*sizeof(float), cudaMemcpyDeviceToDevice)); else if (!dest.device_ready() && src.device_ready()) CHECK_CUDA(cudaMemcpy(dest.host_write_only(), src.device(), src.size()*sizeof(float), cudaMemcpyDeviceToHost)); else if (dest.device_ready() && !src.device_ready()) CHECK_CUDA(cudaMemcpy(dest.device(), src.host(), src.size()*sizeof(float), cudaMemcpyHostToDevice)); else CHECK_CUDA(cudaMemcpy(dest.host_write_only(), src.host(), src.size()*sizeof(float), cudaMemcpyHostToHost)); }
void server:: set_listening_ip ( const std::string& ip ) { // make sure requires clause is not broken DLIB_CASSERT( ( ( is_ip_address(ip) || ip == "" ) && this->is_running() == false ), "\tvoid server::set_listening_ip" << "\n\tip == " << ip << "\n\tis_running() == " << this->is_running() << "\n\tthis: " << this ); listening_ip_mutex.lock(); listening_ip = ip; listening_ip_mutex.unlock(); }
void server:: set_listening_port ( int port ) { // make sure requires clause is not broken DLIB_CASSERT( ( port >= 0 && this->is_running() == false ), "\tvoid server::set_listening_port" << "\n\tport == " << port << "\n\tis_running() == " << this->is_running() << "\n\tthis: " << this ); listening_port_mutex.lock(); listening_port = port; listening_port_mutex.unlock(); }
void linker:: link ( connection& a, connection& b ) { // make sure requires clause is not broken DLIB_CASSERT( this->is_running() == false , "\tvoid linker::link" << "\n\tis_running() == " << this->is_running() << "\n\tthis: " << this ); running_mutex.lock(); running = true; running_mutex.unlock(); cons_mutex.lock(); A = &a; B = &b; cons_mutex.unlock(); service_connection_running_mutex.lock(); service_connection_running = true; service_connection_running_mutex.unlock(); service_connection_error_mutex.lock(); service_connection_error = false; service_connection_error_mutex.unlock(); // if we fail to make the thread if (!create_new_thread(service_connection,this)) { a.shutdown(); b.shutdown(); service_connection_running_mutex.lock(); service_connection_running = false; service_connection_running_mutex.unlock(); cons_mutex.lock(); A = 0; B = 0; cons_mutex.unlock(); running_mutex.lock(); running = false; running_mutex.unlock(); throw dlib::thread_error ( ECREATE_THREAD, "failed to make new thread in linker::link()" ); } // forward data from a to b char buf[200]; int status; bool error = false; // becomes true if one of the connections returns an error while (true) { status = a.read(buf,sizeof(buf)); // if there was an error reading from the socket if (status == OTHER_ERROR) { error = true; break; } else if (status == SHUTDOWN) { b.shutdown(); } if (status <= 0) { // if a has closed normally if (status == 0) b.shutdown_outgoing(); break; } status = b.write(buf,status); // if there was an error writing to the socket then break if (status == OTHER_ERROR) { error = true; break; } if (status <= 0) break; } // if there was an error then shutdown both connections if (error) { a.shutdown(); b.shutdown(); } // wait for the other thread to end service_connection_running_mutex.lock(); while(service_connection_running) { service_connection_running_signaler.wait(); } service_connection_running_mutex.unlock(); // make sure connections are shutdown a.shutdown(); b.shutdown(); // both threads have ended so the connections are no longer needed cons_mutex.lock(); A = 0; B = 0; cons_mutex.unlock(); // if service_connection terminated due to an error then set error to true service_connection_error_mutex.lock(); if (service_connection_error) error = true; service_connection_error_mutex.unlock(); // if we are ending because of an error if (error) { // signal that the link() function is ending running_mutex.lock(); running = false; running_signaler.broadcast(); running_mutex.unlock(); // throw the exception for this error throw dlib::socket_error ( ECONNECTION, "a connection returned an error in linker::link()" ); } // signal that the link() function is ending running_mutex.lock(); running = false; running_signaler.broadcast(); running_mutex.unlock(); }
binary_relation_detector binary_relation_detector_trainer:: train ( ) const { DLIB_CASSERT(num_positive_examples() > 0, "Not enough training data given."); DLIB_CASSERT(num_negative_examples() > 0, "Not enough training data given."); std::vector<sparse_vector_type> samples; std::vector<double> labels; for (unsigned long i = 0; i < pos_sentences.size(); ++i) { samples.push_back(extract_binary_relation(pos_sentences[i], pos_arg1s[i], pos_arg2s[i], tfe).feats); labels.push_back(+1); } for (unsigned long i = 0; i < neg_sentences.size(); ++i) { samples.push_back(extract_binary_relation(neg_sentences[i], neg_arg1s[i], neg_arg2s[i], tfe).feats); labels.push_back(-1); } randomize_samples(samples, labels); const int cv_folds = 6; brdt_cv_objective obj(num_threads, cv_folds, beta, samples, labels); matrix<double,2,1> params; params = 5000.0/samples.size(), 5000.0/samples.size(); // We do the parameter search in log space. params = log(params); // can't do the parameter search if we don't have enough data. So if we don't // have much data then just use the default parameters. if (pos_sentences.size() > (unsigned)cv_folds) { matrix<double,2,1> lower_params, upper_params; lower_params = 1.0/samples.size(), 1.0/samples.size(); upper_params = 100000.0/samples.size(), 100000.0/samples.size(); lower_params = log(lower_params); upper_params = log(upper_params); const double rho_begin = min(upper_params-lower_params)*0.15; const double rho_end = log(1.2/samples.size()) - log(1.0/samples.size()); find_max_bobyqa(obj, params, params.size()*2+1, lower_params, upper_params, rho_begin, rho_end, 200); } // Note that we rescale the parameters to account for the fact that the cross // validation was done on a dataset slightly smaller than the one we ultimately train // on and the C parameters of this trainer are not normalized by the number of training // samples. params = exp(params) * (cv_folds-1.0)/cv_folds; svm_c_linear_dcd_trainer<sparse_linear_kernel<sparse_vector_type> > trainer; trainer.set_c_class1(params(0)); trainer.set_c_class2(params(1)); cout << "using parameters of: " << trans(params); cout << "now doing training..." << endl; binary_relation_detector bd; bd.df = trainer.train(samples, labels); bd.relation_type = relation_name; bd.total_word_feature_extractor_fingerprint = tfe.get_fingerprint(); cout << "test on train: " << test_binary_decision_function(bd.df, samples, labels) << endl; return bd; }