void KV_CCIN2P3_GE::SetBatchSystemParameters(const KVNameValueList& nl)
{
// Use the parameters in the list to set all relevant parameters for batch system.
KVBatchSystem::SetBatchSystemParameters(nl);
SetJobTime(nl.GetStringValue("JobTime"));
SetJobMemory(nl.GetStringValue("JobMemory"));
SetJobDisk(nl.GetStringValue("JobDisk"));
SetMultiJobsMode(nl.GetBoolValue("MultiJobsMode"));
SetRunsPerJob(nl.GetIntValue("RunsPerJob"));
if (nl.GetTStringValue("EMailAddress") != "") {
if (nl.GetBoolValue("EMailOnStart")) SetSendMailOnJobStart();
if (nl.GetBoolValue("EMailOnEnd")) SetSendMailOnJobEnd();
SetSendMailAddress(nl.GetStringValue("EMailAddress"));
}
}
void KVAvailableRunsFile::Update(Bool_t no_existing_file)
{
// Examine the contents of the repository directory corresponding to this datatype
// for parent dataset fDataSet.
// For each file found which was not already in the list of available runs and
// which corresponds to a run in the database gDataBase,
// we add an entry to the available runlist file:
// [run number]|[date of modification]|[name of file]
// For "old" runs we keep the existing informations (including KV version & username)
//
// When no_existing_file=kTRUE we are making an available runs file
// for the first time. There is no pre-existing file.
TString runlist;
AssignAndDelete(runlist,
gSystem->ConcatFileName(GetFilePath(),
GetFileName()));
if(!no_existing_file){
// read all existing informations
ReadFile();
//use "lockfile" to make sure nobody else tries to modify available_runs file
//while we are working on it
if(!runlist_lock.Lock(runlist.Data())) return;
}
//open temporary file
TString tmp_file_path(GetFileName());
ofstream tmp_file;
KVBase::OpenTempFile(tmp_file_path, tmp_file);
KVDataRepository *repository = fDataSet->GetRepository();
cout << endl << "Updating runlist : " << flush;
//get directory listing from repository
KVUniqueNameList *dir_list =
repository->GetDirectoryListing(fDataSet, GetDataType());
if (!dir_list)
return;
TIter next(dir_list);
KVBase *objs;
//progress bar
Int_t ntot = dir_list->GetSize();
Int_t n5pc = TMath::Max(ntot / 20, 1);
Int_t ndone = 0;
KVDBTable *run_table = 0;
KVDataBase* db = fDataSet->GetDataBase();
if (!db){
db = new KVDataBase();
db->AddTable("Runs","List of Runs");
}
run_table = db->GetTable("Runs");
while ((objs = (KVBase *) next())) { // loop over all entries in directory
Int_t run_num;
//is this the correct name of a run in the repository ?
Info("Update","%s %d",objs->GetName(),IsRunFileName(objs->GetName()));
if ((run_num = IsRunFileName(objs->GetName()))) {
KVDBRun *run = (KVDBRun *) run_table->GetRecord(run_num);
if (run) {
FileStat_t fs;
//get file modification date
if (repository->
GetFileInfo(fDataSet, GetDataType(),
objs->GetName(), fs)) {
//runfile exists in repository
TDatime modt(fs.fMtime);
if(!no_existing_file){
// was there already an entry for exactly the same file in the previous file ?
Int_t occIdx=0;
KVNameValueList* prevEntry = RunHasFileWithDateAndName(run->GetNumber(), objs->GetName(), modt, occIdx);
if(prevEntry){
// copy infos of previous entry
tmp_file << run->GetNumber() << '|' << modt.AsSQLString() << '|' << objs->GetName();
if(prevEntry->HasParameter(Form("KVVersion[%d]",occIdx))){
tmp_file <<"|"<< prevEntry->GetStringValue(Form("KVVersion[%d]",occIdx)) <<"|"<<prevEntry->GetStringValue(Form("Username[%d]",occIdx));
}
tmp_file << endl;
}
else
{
// New Entry - write in temporary runlist file '[run number]|[date of modification]|[name of file]
tmp_file << run->GetNumber() << '|' << modt.AsSQLString() << '|' << objs->GetName() << endl;
}
}
else // no previous existing file
{
// New Entry in a new file - write in temporary runlist file '[run number]|[date of modification]|[name of file]
tmp_file << run->GetNumber() << '|' << modt.AsSQLString() << '|' << objs->GetName() << endl;
}
}
}
else{
Info("Update","the current run [%s] is not in database",objs->GetName());
FileStat_t fs;
if (repository->GetFileInfo(fDataSet, GetDataType(),objs->GetName(), fs))
{
TDatime modt(fs.fMtime);
// New Entry in a new file - write in temporary runlist file '[run number]|[date of modification]|[name of file]
tmp_file << run_num << '|' << modt.AsSQLString() << '|' << objs->GetName() << endl;
}
else{
Warning("Update","%s GetFileInfo return kFALSE",objs->GetName());
}
}
}
ndone++;
if (!(ndone % n5pc))
cout << '>' << flush;
}
cout << " DONE" << endl;
delete dir_list;
//close temp file
tmp_file.close();
if(no_existing_file){
//use "lockfile" to make sure nobody else tries to modify available_runs file
//while we are working on it
if(!runlist_lock.Lock(runlist.Data())) return;
}
//copy temporary file to KVFiles directory, overwrite previous
gSystem->CopyFile(tmp_file_path, runlist, kTRUE);
//set access permissions to 664
gSystem->Chmod(runlist.Data(), CHMODE(6,6,4));
//remove lockfile
runlist_lock.Release();
//delete temp file
gSystem->Unlink(tmp_file_path);
}
void KVAvailableRunsFile::ReadFile()
{
// Read all infos in available runs file and store as KVNameValueList objects in fAvailableRuns.
// For each run in the file we add a KVNameValueList with the following fields:
//
// Name = run number
// Occurs = number of times run appears in file
// Filename[0] = name of first file for run
// Filename[1] =
// ...
// Filename[Occurs-1] = name of last file for run
// Date[0] = date & time of generation of first file etc.
// KVVersion[0] = name of KaliVeda version used to generate first file etc. (if known)
// Username[0] = name of user who generated first file etc. (if known)
//does runlist exist ?
if (!OpenAvailableRunsFile()) {
Error("ReadFile", "Cannot open available runs file");
return;
}
if(fAvailableRuns) delete fAvailableRuns;
fAvailableRuns = new KVHashList;
fAvailableRuns->SetOwner(kTRUE);
TString fLine;
Int_t line_number=1;
fLine.ReadLine(fRunlist);
KVNumberList duplicate_lines;
Int_t fRunNumber;
while (fRunlist.good()) {
TObjArray *toks = fLine.Tokenize('|'); // split into fields
// number of fields can vary
// nfields = 2: run number, date
// nfields = 3: run number, date, filename
// nfields = 5: run number, date, filename, KaliVeda version, username
Int_t nfields = toks->GetEntries();
KVString kvs(((TObjString *) toks->At(0))->GetString());
fRunNumber = kvs.Atoi();
if(nfields<2){
Warning("ReadFile", "Less than 2 fields in entry for run %d (line:%d)???",fRunNumber,line_number);
toks->ls();
continue;
}
//get date string
KVString datestring(((TObjString *) toks->At(1))->GetString());
// is run already in list ?
KVNameValueList* NVL = (KVNameValueList*)fAvailableRuns->FindObject(kvs);
Int_t Occurs = (NVL ? NVL->GetIntValue("Occurs") : 0);
if(!NVL) {
NVL = new KVNameValueList(kvs);
fAvailableRuns->Add(NVL);
}
else
{
// check date for run is different to any others
Bool_t ok = kTRUE;
for(Int_t ii=0;ii<Occurs;ii++){
KVString olddate = NVL->GetStringValue(Form("Date[%d]",ii));
if(olddate==datestring){
ok=kFALSE;
duplicate_lines.Add(line_number);
break;
}
}
if(!ok){
delete toks;
line_number++;
fLine.ReadLine(fRunlist);
continue;
}
}
Occurs++;
NVL->SetValue("Occurs", Occurs);
NVL->SetValue(Form("Date[%d]",Occurs-1), datestring.Data());
//backwards compatibility
//an old available_runs file will not have the filename field
//in this case we assume that the name of the file is given by the
//dataset's base file name (i.e. with no date/time suffix)
KVString filename;
if (nfields > 2) {
filename = ((TObjString *) toks->At(2))->GetString();
} else {
filename = fDataSet->GetBaseFileName(GetDataType(), fRunNumber);
}
NVL->SetValue(Form("Filename[%d]",Occurs-1), filename.Data());
KVString kvversion,username;
if (nfields > 4) {
kvversion = ((TObjString *) toks->At(3))->GetString();
username = ((TObjString *) toks->At(4))->GetString();
NVL->SetValue(Form("KVVersion[%d]",Occurs-1), kvversion.Data());
NVL->SetValue(Form("Username[%d]",Occurs-1), username.Data());
}
delete toks;
line_number++;
fLine.ReadLine(fRunlist);
}
CloseAvailableRunsFile();
if(duplicate_lines.GetNValues()){
Info("ReadFile", "There were %d duplicate entries in available runs file, they will be removed", duplicate_lines.GetNValues());
RemoveDuplicateLines(duplicate_lines);
}
}
