KaliVeda
Toolkit for HIC analysis
KVAvailableRunsFile.cpp
1 /*
2 $Id: KVAvailableRunsFile.cpp,v 1.18 2009/03/12 14:01:02 franklan Exp $
3 $Revision: 1.18 $
4 $Date: 2009/03/12 14:01:02 $
5 */
6 
7 //Created by KVClassFactory on Fri May 5 10:46:40 2006
8 //Author: franklan
9 
10 #include "KVAvailableRunsFile.h"
11 #include "KVDBRun.h"
12 #include "TObjArray.h"
13 #include "TObjString.h"
14 #include "KVString.h"
15 #include "KVList.h"
16 #include "KVDataRepository.h"
17 #include "KVRunFile.h"
18 
20 
22 
23 
24 
33 std::optional<run_index_t> KVAvailableRunsFile::IsRunFileName(const Char_t* filename)
34 {
35  // This method tests the string given as 'filename' to see if it could be the name of a runfile
36  // of the datatype of this available runs file (GetDataType()), for its parent dataset, fDataSet.
37  // Any protocol and/or path information in the filename is first removed.
38  //
39  // The format for a given dataset is defined by variable
40  //
41  // [dataset].DataSet.RunFileName.[datatype]: [format]
42 
43  TString fmt = fDataSet->GetDataSetEnv(Form("DataSet.RunFileName.%s", GetDataType()), ""); //get format string for current dataset
44  if (fmt == "") {
45  Error("IsRunFileName", "No default format set for datatype: %s",
46  GetDataType());
47  return {};
48  }
49  return IsRunFileName(fmt, filename);
50 }
51 
52 
53 
99 
100 std::optional<run_index_t> KVAvailableRunsFile::IsRunFileName(const KVString& fmt, const KVString& filename, const KVString& separators)
101 {
102  // This method tests the string given as 'filename' to see if it could be the name of a runfile.
103  // Any protocol and/or path information in the filename is first removed.
104  // The decision as to whether the filename is valid or not is based on whether it corresponds
105  // to the given format string, e.g. run_%04d.dat
106  //
107  // If good, we return the run number & eventual index as a std::optional<run_index_t>
108  // (if no index is defined for the file i.e. if %I does not appear in the format,
109  // this part of the run_index_t has no value).
110  //
111  // If not, the std::optional has no value. This can be tested as:
112  //
113  //~~~{.cpp}
114  // if(toto.IsRunFileName(...))
115  // {
116  // ...ok, filename is good & run number was retrieved
117  // }
118  //~~~
119  //
120  // KNOWN CASES:
121  //~~~
122  // dataset example filename format
123  // ======= ================ =====================
124  // INDRA_camp1 run127.raw run%R.raw
125  // INDRA_camp2 run127.raw run%R.raw
126  // INDRA_camp4 run127.raw run%R.raw
127  // INDRA_camp5 run127.raw run%R.raw
128  // INDRA_e416a Run356_06-Oct-05_12_44_12.dat Run%R_%D.dat
129  // INDRA_e475s run_0085.dat.13-Mar-06.12:46:24 run_%R.dat.%D1.%D2
130  // INDRA_e494s run_0026.dat.04-May-07.20:33:00 run_%R.dat.%D1.%D2
131  // INDRA_e503 run_0026.dat.04-May-07.20:33:00 run_%R.dat.%D1.%D2
132  // INDRA_e613 run_0022.dat.08Sep11_00h54m47s run_%R.dat.%D
133  // INDRAFAZIA.E789 run_0017.dat.04-04-19_08h25m57s.1 run_%R.dat.%D.%I
134  // FAZIA* run000345 run%R
135  //
136  // %R : run number
137  // %D : date and time
138  // %D1.%D2 : date and time separated by '.'
139  // %I : file index number. WARNING: the first file has NO index, the
140  // second file has index '1', etc. The index returned for the first
141  // file is zero.
142  //~~~
143  //
144  // The filename will be broken up according to the separators (default: ".")
145  // Then the different parts will be analysed according to the given format string
146  KVString _file(gSystem->BaseName(filename)); //Remove protocol and/or path
147 
148  // the filename should contain the same number of "."-separated parts as the format
149  // string, unless the format contains "%I", in which case it can have one less part
150  // (case of first file for run with no index)
151  int np_fmt = fmt.GetNValues(separators);
152  bool with_index = fmt.Contains("%I");
153  bool two_part_date = false;
154  bool got_date = false;
155  int np_fn = _file.GetNValues(separators);
157  KVString _fmt = fmt;
158  if (np_fn == np_fmt + 1) {
159  // if there is 1 more segment after splitting of the filename than in the format string,
160  // try adding an index at the end
161  _fmt.Append(".%I");
162  with_index = true;
163  ++np_fmt;
164  }
165  if (np_fmt == np_fn || (with_index && np_fn == np_fmt - 1)) {
166  _file.Begin(separators);
167  _fmt.Begin(separators);
168  int index(-1), run(0);
169  char date1[100], date2[100];
170  if (fmt.Contains("%D1") && fmt.Contains("%D2")) two_part_date = true;
171 
172  while (!_file.End()) {
173  KVString run_part = _file.Next();
174  KVString fmt_part = _fmt.Next();
175 
176  if (fmt_part.Contains("%R")) {
177  fmt_part.ReplaceAll("%R", "%d");
178  if (fmt_part.Contains("%D")) {
179  fmt_part.ReplaceAll("%D", "%s");
180  sscanf(run_part.Data(), fmt_part.Data(), &run, date1);
181  got_date = true;
182  }
183  else
184  sscanf(run_part.Data(), fmt_part.Data(), &run);
185  }
186  else if (fmt_part.Contains("%I")) {
187  fmt_part.ReplaceAll("%I", "%d");
188  sscanf(run_part.Data(), fmt_part.Data(), &index);
189  }
190  else if (fmt_part.Contains("%D1")) {
191  fmt_part.ReplaceAll("%D1", "%s");
192  sscanf(run_part.Data(), fmt_part.Data(), date1);
193  got_date = true;
194  }
195  else if (fmt_part.Contains("%D2")) {
196  fmt_part.ReplaceAll("%D2", "%s");
197  sscanf(run_part.Data(), fmt_part.Data(), date2);
198  got_date = true;
199  }
200  else if (fmt_part.Contains("%D")) {
201  fmt_part.ReplaceAll("%D", "%s");
202  sscanf(run_part.Data(), fmt_part.Data(), date1);
203  got_date = true;
204  }
205  }
206  if (got_date) {
207  if (two_part_date) date_read_from_filename.Form("%s.%s", date1, date2);
208  else date_read_from_filename = date1;
209  }
210  if (with_index) return run_index_t{run, index};
211  else return run_index_t{run, std::nullopt};
212  }
213  return {};
214 }
215 
216 
217 
223 
225 {
226  // We assume that 'name' is the name of a runfile according to given format 'fmt'.
227  // (see KVAvailableRunsFile::IsRunFileName for accepted formats for runfile names with dates).
228  // We attempt several methods to try to extract a date from 'name'.
229  // If successful, we return kTRUE and 'date' contains the result.
230 
231  auto run = IsRunFileName(fmt, name);
232  if (!run) return kFALSE;
233  // if runfile name contains a date, it is now in variable date_read_from_filename
234  if (date_read_from_filename == "") return kFALSE;
239  return kTRUE;
240  }
241  else {
245  return kTRUE;
246  }
249  return kTRUE;
250  }
253  return kTRUE;
254  }
255  }
256  return kFALSE;
257 }
258 
259 
260 
262 
264 {
265  FileStat_t fs;
266  if (repository->GetFileInfo(fDataSet, GetDataType(), objs->GetName(), fs)) {
267  //runfile exists in repository
268  modt = KVDatime(fs.fMtime);
269  return kTRUE;
270  }
271  return kFALSE;
272 }
273 
274 
275 
277 
279  const KVBase* objs,
280  DataBase* RunListTable,
281  const run_index_t& run_num,
282  KVDBRun* run
283  , bool with_version_and_username)
284 {
285  KVString filename = objs->GetName();
286  KVDatime modt;
287  if (get_file_modification_date(repository, objs, modt)) {
288  insert_runfile_into_database(repository, filename, modt, RunListTable, run_num, run, with_version_and_username);
289  return true;
290  }
291  return false;
292 }
293 
294 
295 
297 
298 void KVAvailableRunsFile::insert_runfile_into_database(KVDataRepository* repository, const KVString& filename, const KVDatime& modt, DataBase* RunListTable,
299  const run_index_t& run_num, KVDBRun* run, bool with_version_and_username)
300 {
301 #ifdef WITH_RSQLITE
302  auto& runs_table = RunListTable->get_table("Available_Runs");
303  runs_table.prepare_insert_single_row();
304  runs_table["run"] = run_num.run();
305  if (run_num.has_index())
306  runs_table["index"] = run_num.index();
307  else
308  runs_table["index"].set_null();
309  runs_table["filename"] = filename;
310  runs_table["date"] = modt.AsSQLString();
311  int sys_id = 1;
312  // is the run associated with a system which is already in the db?
313  if (run && run->GetSystem()) {
314  auto& sys_table = RunListTable->get_table("Systems");
315  if (!RunListTable->count("Systems", "*", Form("system = '%s'", run->GetSystemName()))) {
316  // system not in table; need to add it
317  sys_table.prepare_insert_single_row();
318  sys_table["system"] = run->GetSystemName();
319  sys_table.insert_single_row();
320  }
321  // now retrieve system id for foreign key cross-reference
322  RunListTable->select_data("Systems", "id", Form("system = '%s'", run->GetSystemName()));
323  while (RunListTable->get_next_result())
324  sys_id = sys_table["id"].get_data<int>();
325  }
326  runs_table["sys_id"] = sys_id;
327  if (with_version_and_username) {
328  std::unique_ptr<UserGroup_t> userinfo(gSystem->GetUserInfo());
329  runs_table["username"] = userinfo->fUser;
330  runs_table["version"] = GetKVVersion();
331  }
332  runs_table.insert_single_row();
333 #endif
334 }
335 
336 
337 
347 
348 void KVAvailableRunsFile::Update(const std::unique_ptr<DataBase>& db_handle)
349 {
350  // Examine the contents of the repository directory corresponding to this datatype
351  // for parent dataset fDataSet.
352  //
353  // For each file found which was not already in the list of available runs and
354  // we add an entry to the available runlist database file.
355  //
356  // This may be called by OpenAvailableRunsFile() in order to initially create
357  // the new db file: in this case db_handle contains the pointer to the (already open) db.
358 
359 #ifdef WITH_RSQLITE
360  KVSQLite::database* RunListTable;
361  std::unique_ptr<KVSQLite::database> _db;
362  if (db_handle) {
363  // we have been called by OpenAvailableRunsFile() to create a new db file
364  RunListTable = db_handle.get();
365  }
366  else {
367  // we have been called to either create or update an existing db file
368  _db = OpenAvailableRunsFile(true);
369  RunListTable = _db.get();
370  }
371 
372  auto& runs_table = RunListTable->get_table("Available_Runs");
373  auto& sys_table = RunListTable->get_table("Systems");
374 
375  KVDataRepository* repository = fDataSet->GetRepository();
376 
377  std::cout << std::endl << "Updating runlist : " << std::flush;
378  //get directory listing from repository
379  auto dir_list = repository->GetDirectoryListing(fDataSet, GetDataType());
380  if (!dir_list)
381  return;
382 
383  TIter next(dir_list.get());
384  KVBase* objs;
385  //progress bar
386  Int_t ntot = dir_list->GetSize();
387  Int_t n5pc = TMath::Max(ntot / 20, 1);
388  Int_t ndone = 0;
389  std::unique_ptr<KVExpDB> db_garbage;
390  KVExpDB* db = fDataSet->GetDataBase();
391  if (!db) {
392  db = new KVExpDB();
393  db_garbage.reset(db);//clean up
394  }
395  while ((objs = (KVBase*) next())) { // loop over all files in directory
396 
397  auto run_num = IsRunFileName(objs->GetName());
398 
399  if (run_num) { // file has correct name format for runfile
400 
401  KVDBRun* run = db->GetDBRun(run_num.value().run());
402 
403  if (!run) {
404  // if dataset has an index multiplier, this may be a recon file with an artificial run number
405  if (db->HasIndexMultiplier()) {
406  // try to decompose run number into run + index
407  auto r = run_num.value().run() / db->GetIndexMultiplier();
408  auto i = run_num.value().run() % db->GetIndexMultiplier();
409  run_num->clear();
410  run_num->set_run(r);
411  if (i) run_num->set_index(i);
412  run = db->GetDBRun(r);
413  }
414  }
415 
416  if (run && run->HasRunFile(run_num.value())) { // the run number & index is included in the database of runs
417 
418  bool add_to_db = true;
419  if (!making_new_db) {
420  // is there an entry for the same run/index & filename in the database?
421  int occurences = 0;
422  auto selection = SelectFileName(objs->GetName()) + " AND " + SelectRunAndOrIndex(run_num.value());
423  occurences = RunListTable->count("Available_Runs", "*", selection);
424  add_to_db = (occurences == 0);
425  }
426 
427  if (add_to_db) {
428  insert_runfile_into_database(repository, objs, RunListTable, run_num.value(), run);
429  }
430  }
431  else { // run not included in database of runs
432  Info("Update", "the current run [%s] is not in database", objs->GetName());
433  if (!insert_runfile_into_database(repository, objs, RunListTable, run_num.value(), nullptr))
434  Warning("Update", "%s GetFileInfo return kFALSE", objs->GetName());
435  }
436  }
437 
438  ndone++;
439  if (!(ndone % n5pc))
440  std::cout << '>' << std::flush;
441  }
442 
443  std::cout << " DONE" << std::endl;
444  making_new_db = false;
445 #endif
446 }
447 
448 
449 
450 
457 
458 Bool_t KVAvailableRunsFile::GetRunInfo(const run_index_t& run, TDatime& modtime, TString& filename)
459 {
460  // Look for a given run/index number in the file, and read the file's modification date/time and filename
461  // If run not found, returns kFALSE
462  // If available runs file does not exist, Update() is called to create it.
463  // If there are multiple versions of the same run (abnormal situation),
464  // we print a warning and give info on the most recent file.
465 
466  KVList filenames, dates;
467  //get infos for all versions of run
468  GetRunInfos(run, &dates, &filenames);
469  if (filenames.GetEntries() == 1) { //only one version
470  filename = ((TObjString*) filenames.First())->String();
471  modtime = ((TObjString*) dates.First())->String().Data();
472  return kTRUE;
473  }
474  else if (filenames.GetEntries() > 1) { //several versions
475  Warning("GetRunInfo",
476  "Multiple versions of this runfile exist in the repository. Infos for most recent file will be returned.");
477  Warning("GetRunInfo",
478  "You should clean the repository using KVDataSet::CleanMultiRunfiles.");
479  //find most recent version
480  TDatime most_recent("1998-12-25 00:00:00");
481  Int_t i_most_recent = 0;
482  for (int i = 0; i < dates.GetEntries(); i++) {
483  //check if run is most recent
484  TDatime rundate(((TObjString*) dates.At(i))->String().Data());
485  if (rundate > most_recent) {
486  most_recent = rundate;
487  i_most_recent = i;
488  }
489  }
490  filename = ((TObjString*) filenames.At(i_most_recent))->String();
491  modtime = ((TObjString*) dates.At(i_most_recent))->String().Data();
492  return kTRUE;
493  }
494  return kFALSE;
495 }
496 
497 
498 
499 
509 
511 {
512  //Look for a given run/index in the database, and store the file's modification date/time and filename
513  //in the two KVList as TObjString objects (these objects belong to the
514  //lists and will be deleted by them).
515  //
516  //We do not stop at the first run found, but add informations for every occurence of the run in the database
517  //(however, we do ignore any spurious duplicate entries if they exist)
518  //
519  //If available runs db does not exist, it will be created
520 
521 #ifdef WITH_RSQLITE
522  auto RunDB = OpenAvailableRunsFile();
523 
524  //clear lists - delete objects
525  dates->Delete();
526  files->Delete();
527 
528  auto& runtable = RunDB->get_table("Available_Runs");
529  if (RunDB->select_data("Available_Runs", "filename,date", SelectRunAndOrIndex(run), kTRUE)) {
530  while (RunDB->get_next_result()) {
531  dates->Add(new TObjString(runtable["date"].data().GetString()));
532  files->Add(new TObjString(runtable["filename"].data().GetString()));
533  }
534  }
535  else {
536  Error("GetRunInfos", "DB query failure? selection=%s", SelectRunAndOrIndex(run).Data());
537  }
538 #endif
539 }
540 
541 
542 
551 
552 std::unique_ptr<TList> KVAvailableRunsFile::GetListOfAvailableSystems(const KVDBSystem* systol)
553 {
554  //Create and fill a sorted list of available systems based on the runs in the available runs db.
555  //If systol!=0 then create and fill a list of available files (KVRunFile objects) for the given system.
556  //
557  //For each system in the list we set the number of available runs : this number
558  //can be retrieved with KVDBSystem::GetNumberRuns()
559  //
560  // Note that 'bad' runfiles are excluded
561 
562  auto sys_list = std::make_unique<TList>();
563 
564  KVExpDB* db = fDataSet->GetDataBase();
565  std::unique_ptr<KVExpDB> garbage_db;
566  if (!db) {
567  db = new KVExpDB;
568  garbage_db.reset(db);//clean up
569  }
570 #ifdef WITH_RSQLITE
571  auto runDB = OpenAvailableRunsFile();
572 
573  if (systol) {
574  // list of runfiles for given system
575  sys_list->SetOwner(kTRUE);
576  runDB->select_data("Available_Runs,Systems", "run,index,filename,date,version,username", Form("system = '%s'", systol->GetName()), true);
577  while (runDB->get_next_result()) {
578  auto a_run = db->GetDBRun((*runDB)["Available_Runs"]["run"].get_data<int>());
579  auto run = a_run->GetNumber();
580  std::optional<int> index = ((*runDB)["Available_Runs"]["index"].is_null() ?
581  std::nullopt : std::optional<int>((*runDB)["Available_Runs"]["index"].get_data<int>()));
582  KVDatime fDatime((*runDB)["Available_Runs"]["date"].get_data<TString>(), KVDatime::kSQL);
583  KVString kvversion = (*runDB)["Available_Runs"]["version"].get_data<KVString>();
584  KVString username = (*runDB)["Available_Runs"]["username"].get_data<KVString>();
585  KVString filename = (*runDB)["Available_Runs"]["filename"].get_data<KVString>();
586  if (IsCalled("raw")) {
587  // for raw data files, use the actual date/time of the start of data taking in the database
588  TString start_date = a_run->GetRunFile({run, index}).GetStartDate();
589  if (start_date != "") fDatime.Set(start_date);
590  }
591  if (!db->GetDBRunFile({run, index}).IsBad())
592  sys_list->Add(new KVRunFile(a_run, {run, index}, filename, fDatime, kvversion, username));
593  }
594  }
595  else {
596  // list of systems for available runs
597  sys_list->SetOwner(kFALSE);
598  runDB->select_data("Systems", "system", "id > 1");
599  while (runDB->get_next_result()) {
600  auto sys = db->GetSystem((*runDB)["Systems"]["system"].get_data<TString>());
601  if (sys && sys->GetNGoodRunFiles()) sys_list->Add(sys);
602  }
603  // set number of runs & events associated with each system
604  TIter it_sys(sys_list.get());
605  while (auto sys = (KVDBSystem*)it_sys()) {
606  runDB->select_data("Available_Runs,Systems", "run,index", Form("system = '%s'", sys->GetName()), true);
607  std::map<int, int> nruns;
608  ULong64_t events = 0;
609  while (runDB->get_next_result()) {
610  auto run = (*runDB)["Available_Runs"]["run"].get_data<int>();
611  auto index = (*runDB)["Available_Runs"]["index"].is_null() ? -1 : (*runDB)["Available_Runs"]["index"].get_data<int>();
612  if (!db->GetDBRunFile({run, index}).IsBad()) {
613  ++nruns[run];
614  events += db->GetDBRunFile({run, index}).GetEvents();
615  }
616  }
617  sys->SetNumberRuns(nruns.size());
618  sys->SetEvents(events);
619  }
620  }
621 
622  //sort list of systems in order of increasing run number
623  //sort list of runs in order of increasing run number
624  if (sys_list && sys_list->GetSize() > 1)
625  sys_list->Sort();
626 #endif
627  return sys_list;
628 }
629 
630 
631 
636 
637 void KVAvailableRunsFile::UpdateInfos(const run_index_t& run, const KVString& filename, const KVString& kvversion, const KVString& username)
638 {
639  // Call this method to update informations on the file "filename" corresponding to run,
640  // by adding/replacing the KV version and username read from the file itself (not necessarily
641  // corresponding to current KV version and username)
642 #ifdef WITH_RSQLITE
643  auto runDB = OpenAvailableRunsFile();
644 
645  auto selection = SelectRunAndOrIndex(run) + " AND " + SelectFileName(filename);
646 
647  (*runDB)["Available_Runs"]["version"] = kvversion;
648  (*runDB)["Available_Runs"]["username"] = username;
649  runDB->update("Available_Runs", "version,username", selection);
650 #endif
651 }
652 
653 
654 
657 
658 void KVAvailableRunsFile::Remove(const run_index_t& run, const KVString& filename)
659 {
660  //Remove from the db ALL entries corresponding to the given run/index (and filename if given)
661 
662 #ifdef WITH_RSQLITE
663  auto runDB = OpenAvailableRunsFile();
664 
665  auto selection = SelectRunAndOrIndex(run);
666  if (!filename.IsNull()) selection += " AND " + SelectFileName(filename);
667 
668  runDB->delete_data("Available_Runs", selection);
669 #endif
670 }
671 
672 
673 
674 
678 
679 void KVAvailableRunsFile::Add(const run_index_t& run, const KVString& filename)
680 {
681  //Add to the file an entry corresponding to this run, assumed to be present in the repository
682  //with the given filename.
683 
684 #ifdef WITH_RSQLITE
685  auto runDB = OpenAvailableRunsFile();
686 
688 #endif
689 }
690 
691 
692 
695 
696 std::forward_list<run_index_t> KVAvailableRunsFile::CheckMultiRunfiles()
697 {
698  //Returns a list with all run/indexes for which more than one file is in the available runs db
699 
700  std::forward_list<run_index_t> multiruns;
701 #ifdef WITH_RSQLITE
702  auto runDB = OpenAvailableRunsFile();
703 
704  runDB->select_data("Available_Runs", "run,index,COUNT(*)", "", false, "GROUP BY run, index HAVING COUNT(*)>1");
705 
706  while (runDB->get_next_result()) {
707  multiruns.push_front({(*runDB)["Available_Runs"]["run"].get_data<int>(),
708  ((*runDB)["Available_Runs"]["index"].is_null() ?
709  std::nullopt : std::optional<int>((*runDB)["Available_Runs"]["index"].get_data<int>()))});
710  }
711 #endif
712  return multiruns;
713 }
714 
715 
716 
717 
725 
726 run_index_list KVAvailableRunsFile::GetRunList(const KVDBSystem* sys)
727 {
728  //Returns list of available run/indexes for this data type.
729  //
730  //If 'sys' gives the address of a valid database reaction system, only files
731  //corresponding to the system will be included.
732  //
733  //Note that runfiles which are considered to be 'bad' are excluded
734 
735  TString selection, tables("Available_Runs");
736  if (sys) {
737  selection = Form("system = '%s'", sys->GetName());
738  tables += ",Systems";
739  }
740 
741  KVExpDB* db = fDataSet->GetDataBase();
742  std::unique_ptr<KVExpDB> garbage_db;
743  if (!db) {
744  db = new KVExpDB;
745  garbage_db.reset(db);//clean up
746  }
747  run_index_list l;
748 #ifdef WITH_RSQLITE
749  auto runDB = OpenAvailableRunsFile();
750 
751  runDB->select_data(tables, "run,index", selection);
752  while (runDB->get_next_result()) {
753  auto run = (*runDB)["Available_Runs"]["run"].get_data<int>();
754  auto index = (*runDB)["Available_Runs"]["index"].is_null() ? -1 : (*runDB)["Available_Runs"]["index"].get_data<int>();
755  if (!db->GetDBRunFile({run, index}).IsBad())
756  l.Add({run, index});
757  }
758 #endif
759  return l;
760 }
761 
762 
763 
770 
772 {
773  // Filename of file containing information on available runs
774  // i.e. [repository].available_runs.[dataset subdir].[dattype subdir].sqlite
775  //
776  // If [dataset subdir] contains "/" (i.e. if data is grouped into subdirectories)
777  // we replace them by "_"
778 
780  if (!fDataSet) {
781  Error("GetFileName", "Dataset has not been set for this file.");
782  filename = "";
783  }
784  else {
785  KVString datapath = fDataSet->GetDataPathSubdir();
786  datapath.ReplaceAll("/", "_");
787  filename.Form("%s.available_runs.%s.%s.sqlite", fDataSet->GetRepository()->GetName(), datapath.Data(),
789  }
790  return filename;
791 }
792 
793 
794 
809 
811 {
812  // Returns the full path to the directory where the available runs file is stored.
813  // This is by default the same directory where the dataset files are stored.
814  // However, if KaliVeda was installed using a GNU-style directory layout
815  // (possibly in the system directories, e.g. Ubuntu packages)
816  // then the path will be the user's working directory + dataset name
817  //
818  // Alternatively, by setting the config variable
819  // KVAvailableRunsFile.RunFileDirectory: /some/path
820  //
821  // files will be stored/looked for in
822  // /some/path/[dataset]
823  //
824  // You can use environment variables - $(VAR) - in the path
825 
826  KVString filepath;
827  if (!fDataSet) {
828  Error("GetFilePath", "Dataset has not been set for this file.");
829  filepath = "";
830  }
831  else {
832  TString p = gEnv->GetValue("KVAvailableRunsFile.RunFileDirectory", "");
833  if (p == "") {
834  // no user-defined run file directory
835  if (is_gnuinstall()) {
836  // GNU-style install: use working directory
837  filepath = GetWORKDIRFilePath(fDataSet->GetName());
838  }
839  else
840  filepath = fDataSet->GetDataSetDir();
841  }
842  else {
844  AssignAndDelete(filepath, gSystem->ConcatFileName(p.Data(), fDataSet->GetName()));
845  }
846  }
847  return filepath;
848 }
849 
850 
851 
855 
857 {
858  // Return the full path on the local machine to the file
859  // used to store information on available runs
860 
861  KVString path;
863  return path;
864 }
865 
866 
867 
872 
874 {
875  // Check that the directory given by GetFilePath() exists and is writable
876  // If it does not exist, it will be created if possible
877  // If it exists but is not writable, or cannot be created, returns kFALSE.
878 
879  if (gSystem->AccessPathName(GetFilePath(), kFileExists)) { // directory does not exist
880  if (gSystem->mkdir(GetFilePath(), kTRUE) == -1) {
881  Error("CheckDirectoryForAvailableRunsFile", "cannot create directory %s - check access permissions", GetFilePath().Data());
882  return kFALSE;
883  }
884  // set access permissions on created directory
885  gSystem->Chmod(GetFilePath(), 0775);
886  }
887  if (gSystem->AccessPathName(GetFilePath(), kWritePermission)) { // directory is not writable
888  Error("CheckDirectoryForAvailableRunsFile", "directory %s is not writable - check access permissions", GetFilePath().Data());
889  return kFALSE;
890  }
891  return kTRUE;
892 }
893 
894 
895 
897 
898 std::unique_ptr<DataBase> KVAvailableRunsFile::OpenAvailableRunsFile(bool called_by_update)
899 {
900 #ifdef WITH_RSQLITE
903  auto db = std::make_unique<KVSQLite::database>(GetFullPathToAvailableRunsFile());
904  if (!db->good()) {
905  Error("OpenAvailableRunsFile", "Failed to open/create sqlite database file %s", GetFullPathToAvailableRunsFile().Data());
906  db.reset(nullptr);
907  return db;
908  }
909  if (making_new_db) {
910  // set access permissions on database file
912 
913  // create systems table
914  KVSQLite::table sys_tab("Systems");
916  sys_tab.primary_key("id");
917  sys_tab.add_column("system", KVSQLite::column_type::TEXT).NOT_NULL().UNIQUE();
918  auto& t = db->add_table(sys_tab);
919  // add default 'unknown' system (which will have id=1)
920  t.prepare_insert_single_row();
921  t["system"] = "unknown";
922  t.insert_single_row();
923 
924  // create runs table
925  KVSQLite::table run_tab("Available_Runs");
927  run_tab.primary_key("id");
929  run_tab.add_column("index", KVSQLite::column_type::INTEGER);
930  run_tab.add_column("filename", KVSQLite::column_type::TEXT).NOT_NULL();
932  run_tab.add_column("version", KVSQLite::column_type::TEXT);
933  run_tab.add_column("username", KVSQLite::column_type::TEXT);
935  run_tab.foreign_key("sys_id", "Systems", "id");
936 
937  db->add_table(run_tab);
938 
939  if (!called_by_update) Update(db);
940  }
941  return db;
942 #else
943  return {};
944 #endif
945 }
946 
947 
948 
954 
956 {
957  // return kTRUE if the given file for this runfile is lacking some information
958  // e.g. the KV version and username
959  //
960  // N.B.: if no file is known for this run, we return kFALSE
961 #ifdef WITH_RSQLITE
962  auto runDB = OpenAvailableRunsFile();
963  auto selection = SelectRunAndOrIndex(run) + " AND " + SelectFileName(filename);
964  runDB->select_data("Available_Runs", "run,version", selection);
965  while (runDB->get_next_result()) {
966  if ((*runDB)["Available_Runs"]["version"].is_null()) return true;
967  }
968 #endif
969  return false;
970 }
971 
972 
int Int_t
ROOT::R::TRInterface & r
bool Bool_t
char Char_t
constexpr Bool_t kFALSE
constexpr Bool_t kTRUE
R__EXTERN TEnv * gEnv
winID h TVirtualViewer3D TVirtualGLPainter p
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void data
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char filename
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t index
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize fs
char name[80]
char * Form(const char *fmt,...)
void AssignAndDelete(TString &target, char *tobedeleted)
kFileExists
kWritePermission
R__EXTERN TSystem * gSystem
Handles lists of available runs for different datasets and types of data.
Bool_t CheckDirectoryForAvailableRunsFile()
virtual std::unique_ptr< DataBase > OpenAvailableRunsFile(bool called_by_update=false)
Bool_t InfosNeedUpdate(const run_index_t &run, const KVString &filename)
bool insert_runfile_into_database(KVDataRepository *repository, const KVBase *objs, DataBase *RunListTable, const run_index_t &run_num, KVDBRun *run, bool with_version_and_username=false)
virtual void Remove(const run_index_t &run, const KVString &filename="")
Remove from the db ALL entries corresponding to the given run/index (and filename if given)
void UpdateInfos(const run_index_t &run, const KVString &filename, const KVString &kvversion, const KVString &username)
Bool_t ExtractDateFromFileName(const Char_t *name, KVDatime &date)
const Char_t * GetDataType() const
TString SelectRunAndOrIndex(const run_index_t &run)
dataset to which this file belongs
std::forward_list< run_index_t > CheckMultiRunfiles()
Returns a list with all run/indexes for which more than one file is in the available runs db.
virtual void Update(const std::unique_ptr< DataBase > &db_handle={})
const KVDataSet * fDataSet
void GetRunInfos(const run_index_t &run, KVList *dates, KVList *names)
KVString GetFileName() const
KVString GetFullPathToAvailableRunsFile() const
run_index_list GetRunList(const KVDBSystem *system=0)
std::optional< run_index_t > IsRunFileName(const Char_t *filename)
TString SelectFileName(const KVString &filename)
KVString GetFilePath() const
static KVString date_read_from_filename
virtual bool get_file_modification_date(KVDataRepository *repository, const KVBase *objs, KVDatime &modt)
virtual void Add(const run_index_t &run, const KVString &filename)
std::unique_ptr< TList > GetListOfAvailableSystems(const KVDBSystem *systol=0)
Base class for KaliVeda framework.
Definition: KVBase.h:139
void Error(const char *method, const char *msgfmt,...) const override
Definition: KVBase.cpp:1650
static const Char_t * GetWORKDIRFilePath(const Char_t *namefile="")
Definition: KVBase.cpp:121
static bool is_gnuinstall()
Definition: KVBase.h:274
virtual Bool_t IsCalled(const Char_t *name) const
Definition: KVBase.h:189
static const Char_t * GetKVVersion()
Returns KaliVeda version string.
Definition: KVBase.cpp:853
void Warning(const char *method, const char *msgfmt,...) const override
Definition: KVBase.cpp:1637
virtual Int_t GetNumber() const
Definition: KVDBRecord.h:73
void SetEvents(ULong64_t evt_number)
Definition: KVDBRunFile.h:183
Description of an experimental run in database ,,.
Definition: KVDBRun.h:41
const Char_t * GetSystemName() const
Definition: KVDBRun.h:260
KVDBSystem * GetSystem() const
Definition: KVDBRun.cpp:239
Bool_t HasRunFile(const run_index_t &run_index) const
Definition: KVDBRun.h:70
Database class used to store information on different colliding systems studied during an experiment....
Definition: KVDBSystem.h:52
Base class for managing repositories of experimental data.
virtual std::unique_ptr< KVUniqueNameList > GetDirectoryListing(const KVDataSet *dataset, const Char_t *datatype="", const Char_t *subdir="")
virtual Bool_t GetFileInfo(const KVDataSet *dataset, const Char_t *datatype, const Char_t *runfile, FileStat_t &fs) const
KVDataRepository * GetRepository() const
Get pointer to data repository in which dataset is stored.
Definition: KVDataSet.cpp:1310
const Char_t * GetDataSetDir() const
Definition: KVDataSet.cpp:670
KVString GetDataTypeSubdir(const Char_t *type) const
Definition: KVDataSet.h:190
virtual const Char_t * GetDataPathSubdir() const
Definition: KVDataSet.h:185
KVExpDB * GetDataBase(Option_t *opt="") const
Definition: KVDataSet.cpp:285
Extension of TDatime to handle various useful date formats.
Definition: KVDatime.h:33
@ kSQL
Definition: KVDatime.h:44
static Bool_t IsGANACQFormat(const Char_t *date)
Definition: KVDatime.cpp:433
static Bool_t IsSQLFormat(const Char_t *date)
Definition: KVDatime.cpp:496
static Bool_t IsGANACQ2010Format(const Char_t *date)
Definition: KVDatime.cpp:455
void SetSQLDate(const Char_t *SQLDateString)
Definition: KVDatime.cpp:204
void SetGanacqNarvalDate(const Char_t *GanacqDateString)
Definition: KVDatime.cpp:297
static Bool_t IsGANACQNarvalFormat(const Char_t *date)
Definition: KVDatime.cpp:476
void SetGanacq2010Date(const Char_t *GanacqDateString)
Definition: KVDatime.cpp:273
void SetGanacqDate(const Char_t *GanacqDateString)
Definition: KVDatime.cpp:323
Base class to describe database of an experiment ,,.
Definition: KVExpDB.h:20
virtual KVDBSystem * GetSystem(const Char_t *system) const
Definition: KVExpDB.h:110
KVDBRun * GetDBRun(Int_t number) const
Definition: KVExpDB.h:90
const KVDBRunFile & GetDBRunFile(const run_index_t &r) const
Definition: KVExpDB.h:94
Int_t GetIndexMultiplier() const
Definition: KVExpDB.h:181
Bool_t HasIndexMultiplier() const
Definition: KVExpDB.h:180
Extended TList class which owns its objects by default.
Definition: KVList.h:28
Description of an individual data file in an experimental dataset.
Definition: KVRunFile.h:19
column & UNIQUE()
Definition: SQLiteDB.h:190
column & NOT_NULL()
Definition: SQLiteDB.h:197
Interface to ROOT SQLite database backend.
Definition: SQLiteDB.h:477
bool select_data(const TString &tables, const TString &columns="*", const TString &selection="", bool distinct=false, const TString &anything_else="") const
Definition: SQLiteDB.cpp:835
bool get_next_result() const
Definition: SQLiteDB.cpp:954
KVSQLite::table & get_table(const TString &name)
Definition: SQLiteDB.h:571
int count(const TString &tables, const TString &column="*", const TString &selection="", bool distinct=false, const TString &anything_else="") const
Definition: SQLiteDB.cpp:1124
void primary_key(const TString &cols)
Definition: SQLiteDB.h:432
void foreign_key(const TString &child_key, const TString &parent_table, const TString &parent_key)
Definition: SQLiteDB.cpp:1674
column & add_column(const KVSQLite::column &c)
Definition: SQLiteDB.cpp:1623
void prepare_insert_single_row()
Definition: SQLiteDB.cpp:1810
void Add(TObject *obj) override
void Delete(Option_t *option="") override
Extension of ROOT TString class which allows backwards compatibility with ROOT v3....
Definition: KVString.h:73
void Begin(TString delim) const
Definition: KVString.cpp:565
Bool_t End() const
Definition: KVString.cpp:634
KVString Next(Bool_t strip_whitespace=kFALSE) const
Definition: KVString.cpp:695
Int_t GetNValues(TString delim) const
Definition: KVString.cpp:886
const char * AsSQLString() const
void Set()
virtual const char * GetValue(const char *name, const char *dflt) const
virtual void Add(TObject *obj)
const char * GetName() const override
virtual void Info(const char *method, const char *msgfmt,...) const
const char * Data() const
void ToUpper()
TString & Append(char c, Ssiz_t rep=1)
void Form(const char *fmt,...)
Bool_t Contains(const char *pat, ECaseCompare cmp=kExact) const
TString & ReplaceAll(const char *s1, const char *s2)
virtual int Chmod(const char *file, UInt_t mode)
virtual char * ConcatFileName(const char *dir, const char *name)
virtual int mkdir(const char *name, Bool_t recursive=kFALSE)
virtual UserGroup_t * GetUserInfo(const char *user=nullptr)
virtual Bool_t AccessPathName(const char *path, EAccessMode mode=kFileExists)
virtual const char * BaseName(const char *pathname)
virtual char * ExpandPathName(const char *path)
Specifies a runfile according to run number and file index ,.
Definition: run_index.h:31
int run() const
Definition: run_index.h:50
int index(int no_index=-1) const
Definition: run_index.h:55
bool has_index() const
Definition: run_index.h:59
unsigned long long ULong64_t
BinData::ErrorType GetDataType(const TGraph *gr, DataOptions &fitOpt)
void Error(const char *location, const char *fmt,...)
Double_t Max(Double_t a, Double_t b)
const char * String
TLine l
ClassImp(TPyArg)