static int _wipe_signature(struct device *dev, const char *type, const char *name,
int wipe_len, int yes, force_t force, int *wiped,
int (*signature_detection_fn)(struct device *dev, uint64_t *offset_found))
{
int wipe;
uint64_t offset_found;
wipe = signature_detection_fn(dev, &offset_found);
if (wipe == -1) {
log_error("Fatal error while trying to detect %s on %s.",
type, name);
return 0;
}
if (wipe == 0)
return 1;
/* Specifying --yes => do not ask. */
if (!yes && (force == PROMPT) &&
yes_no_prompt("WARNING: %s detected on %s. Wipe it? [y/n]: ",
type, name) == 'n') {
log_error("Aborted wiping of %s.", type);
return 0;
}
log_print_unless_silent("Wiping %s on %s.", type, name);
if (!dev_set(dev, offset_found, wipe_len, 0)) {
log_error("Failed to wipe %s on %s.", type, name);
return 0;
}
(*wiped)++;
return 1;
}
int run_vanilla_nolemma(unsigned epochs, float alpha){
//Data set (Ratnaparkhi's 94 RRR data set)
vector<string> ydict;
vector<string> xdict;
PPADataEncoder data_set("PPAttachData/training");
data_set.add_data("PPAttachData/devset");
data_set.add_data("PPAttachData/test");
data_set.getYdictionary(ydict);
data_set.getXdictionary(xdict);
//External Word vectors
Word2vec w2v;
vector<string> wvdict;
af::array w2v_embeddings;
w2v.load_dictionary("PPAttachData/embeddings/deps.words");
w2v.filter(xdict);
//training set
data_set.clear();
data_set.add_data("PPAttachData/training");
data_set.getXdictionary(xdict);
//build network
cerr << "building network"<<endl;
SymbolicFeedForwardNetwork<string,string> net;
net.set_output_layer("loss",new SoftMaxLoss<string>(ydict));
net.add_layer("top",new LinearLayer());
net.add_layer("hidden",new ReLUActivation(400));
net.add_layer("A",new LinearLayer());
net.add_input_layer("lookupA",new LinearLookup<string>(w2v.get_keys(),w2v.get_values(),data_set.x_vocab_size(),true));
net.connect_layers("loss","top");
net.connect_layers("top","hidden");
net.connect_layers("hidden","A");
net.connect_layers("A","lookupA");
vector<string> ydata;
vector<vector<string>> xdata(1,vector<string>());
data_set.getYdata(ydata);
data_set.getXdata(xdata[0]);
net.set_batch_data(ydata,xdata);
net.train_all(ydata,xdata,epochs,100,alpha,true,epochs/2);//10 epochs, batch size= 100,alpha=0.01, Adagrad=On, start averaging at 50th epoch
PPADataEncoder dev_set("PPAttachData/devset");
dev_set.getYdata(ydata);
dev_set.getXdata(xdata[0]);
float dev_acc = net.eval_avg(ydata,xdata);
cout << "dev acc = " << dev_acc << endl;
PPADataEncoder test_set("PPAttachData/test");
test_set.getYdata(ydata);
test_set.getXdata(xdata[0]);
float test_acc = net.eval_avg(ydata,xdata);
cout << "test acc = " << test_acc << endl;
return 0;
}
/**
* Main function. Initializes the USB-device, parses commandline-parameters and
* calls the functions that communicate with the device.
* \param argc Number of arguments.
* \param argv Arguments.
* \return Error code.
*/
int main(int argc, char **argv)
{
usb_dev_handle *handle = NULL;
if (argc < 2) {
usage(argv[0]);
exit(1);
}
usb_init();
if (usbOpenDevice (&handle, USBDEV_SHARED_VENDOR, "www.schatenseite.de", USBDEV_SHARED_PRODUCT, "USB-LED-Fader") != 0) {
fprintf(stderr, "Could not find USB device \"USB-LED-Fader\" with vid=0x%x pid=0x%x\n", USBDEV_SHARED_VENDOR, USBDEV_SHARED_PRODUCT);
exit(1);
}
/* We have searched all devices on all busses for our USB device above. Now
* try to open it and perform the vendor specific control operations for the
* function requested by the user.
*/
if (strcmp(argv[1], "test") == 0) {
dev_test(handle, argc, argv);
} else if (strcmp(argv[1], "set") == 0) {
dev_set(handle, argc, argv);
} else if (strcmp(argv[1], "clear") == 0) {
dev_clear(handle, argc, argv);
} else if (strcmp(argv[1], "status") == 0) {
dev_status(handle, argc, argv);
} else if (strcmp(argv[1], "reset") == 0) {
dev_reset(handle, argc, argv);
} else if (strcmp(argv[1], "show") == 0) {
dev_show(handle, argc, argv);
} else {
usage(argv[0]);
exit(1);
}
usb_close(handle);
return 0;
}
static int _blkid_wipe(blkid_probe probe, struct device *dev, const char *name,
uint32_t types_to_exclude, uint32_t types_no_prompt,
int yes, force_t force)
{
static const char _msg_failed_offset[] = "Failed to get offset of the %s signature on %s.";
static const char _msg_failed_length[] = "Failed to get length of the %s signature on %s.";
static const char _msg_wiping[] = "Wiping %s signature on %s.";
const char *offset = NULL, *type = NULL, *magic = NULL,
*usage = NULL, *label = NULL, *uuid = NULL;
loff_t offset_value;
size_t len;
if (!blkid_probe_lookup_value(probe, "TYPE", &type, NULL)) {
if (_type_in_flag_list(type, types_to_exclude))
return 2;
if (blkid_probe_lookup_value(probe, "SBMAGIC_OFFSET", &offset, NULL)) {
log_error(_msg_failed_offset, type, name);
return 0;
}
if (blkid_probe_lookup_value(probe, "SBMAGIC", &magic, &len)) {
log_error(_msg_failed_length, type, name);
return 0;
}
} else if (!blkid_probe_lookup_value(probe, "PTTYPE", &type, NULL)) {
if (blkid_probe_lookup_value(probe, "PTMAGIC_OFFSET", &offset, NULL)) {
log_error(_msg_failed_offset, type, name);
return 0;
}
if (blkid_probe_lookup_value(probe, "PTMAGIC", &magic, &len)) {
log_error(_msg_failed_length, type, name);
return 0;
}
usage = "partition table";
} else
return_0;
offset_value = strtoll(offset, NULL, 10);
if (!usage)
(void) blkid_probe_lookup_value(probe, "USAGE", &usage, NULL);
(void) blkid_probe_lookup_value(probe, "LABEL", &label, NULL);
(void) blkid_probe_lookup_value(probe, "UUID", &uuid, NULL);
/* Return values ignored here, in the worst case we print NULL */
log_verbose("Found existing signature on %s at offset %s: LABEL=\"%s\" "
"UUID=\"%s\" TYPE=\"%s\" USAGE=\"%s\"",
name, offset, label, uuid, type, usage);
if (!_type_in_flag_list(type, types_no_prompt)) {
if (!yes && (force == PROMPT) &&
yes_no_prompt("WARNING: %s signature detected on %s at offset %s. "
"Wipe it? [y/n]: ", type, name, offset) == 'n') {
log_error("Aborted wiping of %s.", type);
return 0;
}
log_print_unless_silent(_msg_wiping, type, name);
} else
log_verbose(_msg_wiping, type, name);
if (!dev_set(dev, offset_value, len, 0)) {
log_error("Failed to wipe %s signature on %s.", type, name);
return 0;
}
return 1;
}
/*
* See if we may pvcreate on this device.
* 0 indicates we may not.
*/
static int pvcreate_check(struct cmd_context *cmd, const char *name)
{
struct physical_volume *pv;
struct device *dev;
uint64_t md_superblock;
/* is the partition type set correctly ? */
if ((arg_count(cmd, force_ARG) < 1) && !is_lvm_partition(name)) {
log_error("%s: Not LVM partition type: use -f to override",
name);
return 0;
}
/* Is there a pv here already? */
/* FIXME Use partial mode here? */
pv = pv_read(cmd, name, NULL, NULL, 0);
/* Allow partial & exported VGs to be destroyed. */
/* We must have -ff to overwrite a non orphan */
if (pv && !is_orphan(pv) && arg_count(cmd, force_ARG) != 2) {
log_error("Can't initialize physical volume \"%s\" of "
"volume group \"%s\" without -ff", name, pv_vg_name(pv));
return 0;
}
/* prompt */
if (pv && !is_orphan(pv) && !arg_count(cmd, yes_ARG) &&
yes_no_prompt(_really_init, name, pv_vg_name(pv)) == 'n') {
log_print("%s: physical volume not initialized", name);
return 0;
}
if (sigint_caught())
return 0;
dev = dev_cache_get(name, cmd->filter);
/* Is there an md superblock here? */
if (!dev && md_filtering()) {
unlock_vg(cmd, ORPHAN);
persistent_filter_wipe(cmd->filter);
lvmcache_destroy();
init_md_filtering(0);
if (!lock_vol(cmd, ORPHAN, LCK_VG_WRITE)) {
log_error("Can't get lock for orphan PVs");
init_md_filtering(1);
return 0;
}
dev = dev_cache_get(name, cmd->filter);
init_md_filtering(1);
}
if (!dev) {
log_error("Device %s not found (or ignored by filtering).", name);
return 0;
}
if (!dev_test_excl(dev)) {
log_error("Can't open %s exclusively. Mounted filesystem?",
name);
return 0;
}
/* Wipe superblock? */
if (dev_is_md(dev, &md_superblock) &&
((!arg_count(cmd, uuidstr_ARG) &&
!arg_count(cmd, restorefile_ARG)) ||
arg_count(cmd, yes_ARG) ||
(yes_no_prompt("Software RAID md superblock "
"detected on %s. Wipe it? [y/n] ", name) == 'y'))) {
log_print("Wiping software RAID md superblock on %s", name);
if (!dev_set(dev, md_superblock, 4, 0)) {
log_error("Failed to wipe RAID md superblock on %s",
name);
return 0;
}
}
if (sigint_caught())
return 0;
if (pv && !is_orphan(pv) && arg_count(cmd, force_ARG)) {
log_warn("WARNING: Forcing physical volume creation on "
"%s%s%s%s", name,
!is_orphan(pv) ? " of volume group \"" : "",
!is_orphan(pv) ? pv_vg_name(pv) : "",
!is_orphan(pv) ? "\"" : "");
}
return 1;
}
static int pvcreate_single(struct cmd_context *cmd, const char *pv_name,
void *handle)
{
struct pvcreate_params *pp = (struct pvcreate_params *) handle;
void *pv;
void *existing_pv;
struct id id, *idp = NULL;
const char *uuid = NULL;
uint64_t size = 0;
struct device *dev;
struct list mdas;
int pvmetadatacopies;
uint64_t pvmetadatasize;
struct volume_group *vg;
const char *restorefile;
uint64_t pe_start = 0;
uint32_t extent_count = 0, extent_size = 0;
if (arg_count(cmd, uuidstr_ARG)) {
uuid = arg_str_value(cmd, uuidstr_ARG, "");
if (!id_read_format(&id, uuid))
return EINVALID_CMD_LINE;
if ((dev = device_from_pvid(cmd, &id)) &&
(dev != dev_cache_get(pv_name, cmd->filter))) {
log_error("uuid %s already in use on \"%s\"", uuid,
dev_name(dev));
return ECMD_FAILED;
}
idp = &id;
}
if (arg_count(cmd, restorefile_ARG)) {
restorefile = arg_str_value(cmd, restorefile_ARG, "");
/* The uuid won't already exist */
init_partial(1);
if (!(vg = backup_read_vg(cmd, NULL, restorefile))) {
log_error("Unable to read volume group from %s",
restorefile);
return ECMD_FAILED;
}
init_partial(0);
if (!(existing_pv = find_pv_in_vg_by_uuid(vg, idp))) {
log_error("Can't find uuid %s in backup file %s",
uuid, restorefile);
return ECMD_FAILED;
}
pe_start = pv_pe_start(existing_pv);
extent_size = pv_pe_size(existing_pv);
extent_count = pv_pe_count(existing_pv);
}
if (!lock_vol(cmd, ORPHAN, LCK_VG_WRITE)) {
log_error("Can't get lock for orphan PVs");
return ECMD_FAILED;
}
if (!pvcreate_check(cmd, pv_name))
goto error;
if (sigint_caught())
goto error;
if (arg_sign_value(cmd, physicalvolumesize_ARG, 0) == SIGN_MINUS) {
log_error("Physical volume size may not be negative");
goto error;
}
size = arg_uint64_value(cmd, physicalvolumesize_ARG, UINT64_C(0)) * 2;
if (arg_sign_value(cmd, metadatasize_ARG, 0) == SIGN_MINUS) {
log_error("Metadata size may not be negative");
goto error;
}
pvmetadatasize = arg_uint64_value(cmd, metadatasize_ARG, UINT64_C(0))
* 2;
if (!pvmetadatasize)
pvmetadatasize = find_config_tree_int(cmd,
"metadata/pvmetadatasize",
DEFAULT_PVMETADATASIZE);
pvmetadatacopies = arg_int_value(cmd, metadatacopies_ARG, -1);
if (pvmetadatacopies < 0)
pvmetadatacopies = find_config_tree_int(cmd,
"metadata/pvmetadatacopies",
DEFAULT_PVMETADATACOPIES);
if (!(dev = dev_cache_get(pv_name, cmd->filter))) {
log_error("%s: Couldn't find device. Check your filters?",
pv_name);
goto error;
}
list_init(&mdas);
if (!(pv = pv_create(cmd->fmt, dev, idp, size, pe_start,
extent_count, extent_size,
pvmetadatacopies, pvmetadatasize, &mdas))) {
log_error("Failed to setup physical volume \"%s\"", pv_name);
goto error;
}
log_verbose("Set up physical volume for \"%s\" with %" PRIu64
" available sectors", pv_name, pv_size(pv));
/* Wipe existing label first */
if (!label_remove(pv_dev(pv))) {
log_error("Failed to wipe existing label on %s", pv_name);
goto error;
}
if (pp->zero) {
log_verbose("Zeroing start of device %s", pv_name);
if (!dev_open_quiet(dev)) {
log_error("%s not opened: device not zeroed", pv_name);
goto error;
}
if (!dev_set(dev, UINT64_C(0), (size_t) 2048, 0)) {
log_error("%s not wiped: aborting", pv_name);
dev_close(dev);
goto error;
}
dev_close(dev);
}
log_very_verbose("Writing physical volume data to disk \"%s\"",
pv_name);
if (!(pv_write(cmd, (struct physical_volume *)pv, &mdas,
arg_int64_value(cmd, labelsector_ARG,
DEFAULT_LABELSECTOR)))) {
log_error("Failed to write physical volume \"%s\"", pv_name);
goto error;
}
log_print("Physical volume \"%s\" successfully created", pv_name);
unlock_vg(cmd, ORPHAN);
return ECMD_PROCESSED;
error:
unlock_vg(cmd, ORPHAN);
return ECMD_FAILED;
}
int run_sampler(unsigned epochs,float alpha,unsigned batch_size){
//load sampler
string training_path = "PPAttachData/training.lemma";
string param_path = "PPAttachData/wordsketches/";
string vpath = param_path + string("vdistrib");
string x1vpath = param_path + string("x1givenv");
string pvpath = param_path + string("pgivenv");
string x2vppath = param_path + string("x2givenvp");
string px1path = param_path + string("pgivenx1");
string x2x1ppath = param_path + string("x2givenx1p");
DataSampler samp(training_path.c_str(),
vpath.c_str(),
x1vpath.c_str(),
pvpath.c_str(),
x2vppath.c_str(),
px1path.c_str(),
x2x1ppath.c_str());
//load dev and test
PPADataEncoder dev_set("PPAttachData/devset.lemma");
PPADataEncoder test_set("PPAttachData/test.lemma");
//load Word vectors
Word2vec w2v;
vector<string> wvdict;
af::array w2v_embeddings;
w2v.load_dictionary("PPAttachData/embeddings/deps.words.lemmatized");
//w2v.filter(xdict);
//make network
vector<string> ydict;
samp.getYdictionary(ydict);
SymbolicFeedForwardNetwork<string,string> net;
net.set_output_layer("loss",new SoftMaxLoss<string>(ydict));
net.add_layer("top",new LinearLayer());
net.add_layer("hidden",new ReLUActivation(400));
net.add_layer("A",new LinearLayer());
net.add_input_layer("lookupA",new LinearLookup<string>(w2v.get_keys(),w2v.get_values(),4,false));
net.connect_layers("loss","top");
net.connect_layers("top","hidden");
net.connect_layers("hidden","A");
net.connect_layers("A","lookupA");
for(int E = 0; E < epochs;++E){
vector<string> ydata;
vector<vector<string>> xdata;
//af::timer start1 = af::timer::start();
samp.generate_sample(ydata,xdata,batch_size);
//printf("elapsed seconds (sampling): %g\n", af::timer::stop(start1));
PPADataEncoder sampdata(ydata,xdata);
vector<string> enc_ydata;
vector<vector<string>> enc_xdata(1,vector<string>());
sampdata.getYdata(enc_ydata);
sampdata.getXdata(enc_xdata[0]);
//af::timer start2 = af::timer::start();
net.set_batch_data(enc_ydata,enc_xdata);
float loss = net.train_one(alpha,true,true);
//printf("elapsed seconds (backprop): %g\n", af::timer::stop(start2));
if (E % 20 == 0){
vector<string> devy;
vector<vector<string>> devx(1,vector<string>());
dev_set.getYdata(devy);
dev_set.getXdata(devx[0]);
float acc = net.eval_avg(devy,devx); //auto-eval on dev data
cout << "epoch " << E << ", loss= " << loss << ", eval (dev) = " << acc << endl;
}else {
cout << "epoch" << E <<endl;
}
}
vector<string> testy;
vector<vector<string>> testx(1,vector<string>());
test_set.getYdata(testy);
test_set.getXdata(testx[0]);
float acc = net.eval_avg(testy,testx);
cout << "final eval (test) = " << acc << endl;
return 0;
}
