KaliVeda
Toolkit for HIC analysis
Loading...
Searching...
No Matches
KVDataRepository.cpp
1/*
2$Id: KVDataRepository.cpp,v 1.21 2009/01/16 14:55:20 franklan Exp $
3$Revision: 1.21 $
4$Date: 2009/01/16 14:55:20 $
5*/
6
7//Created by KVClassFactory on Wed Apr 26 16:17:37 2006
8//Author: franklan
9
10#include "KVDataRepository.h"
11#include "KVDataSetManager.h"
12#include "KVBase.h"
13#include "KVList.h"
14#include "TError.h"
15#include "KVDataSet.h"
16#include "Riostream.h"
17#include "TObjString.h"
18#include "TFile.h"
19#include "TOrdCollection.h"
20#include "TRegexp.h"
21#include "TPluginManager.h"
22#include "TNetFile.h"
23#include "TROOT.h"
24#include "KVConfig.h"
25
26#include <TClass.h>
27#include <TMethodCall.h>
28
29//macro converting octal filemode to decimal value
30//to convert e.g. 664 (=u+rw, g+rw, o+r) use CHMODE(6,6,4)
31#define CHMODE(u,g,o) ((u << 6) + (g << 3) + o)
32
33using namespace std;
34
36
37KVDataRepository* gDataRepository;
38
39//___________________________________________________________________________
40
41
46
48{
49 //Make this the 'active' or 'default' data repository,
50 //i.e. gDataRepository points to this and gDataSetManager points to the
51 //associated data set manager.
52 gDataRepository = this;
53 gDataSetManager = fDSM;
54}
55
56
57
58
68
70{
71 //Initialises data repository based on information in .kvrootrc file.
72 //Each line beginning "name.DataRepository" is used for this data repository,
73 //where "name" is the name of this repository.
74 //
75 //Then we create and initialise the data set manager for this repository.
76 //
77 //returns kTRUE if all goes well.
78 //returns kFALSE if not (i.e. data set manager cannot be initialised, no available datasets)
79
81 gEnv->GetValue(Form("%s.DataRepository.RootDir", GetName()), "");
82 if (fLocalrootdir == "") {
83 Error("Init", "Top-level directory for repository %s is not defined. Set %s.DataRepository.RootDir variable in .kvrootrc",
84 GetName(), GetName());
85 return kFALSE;
86 }
88 gEnv->GetValue(Form("%s.DataRepository.AccessProtocol", GetName()),
89 "local");
91 gEnv->GetValue(Form("%s.DataRepository.ReadProtocol", GetName()),
92 "local");
93 //can we create and write directly new files in the repository ?
94 //for local repositories, default is yes, unless 'DataRepository.CanWrite' = 'NO'
95 //for remote repositories, default is no, unless 'DataRepository.CanWrite' = 'YES'
96 fCanWrite = (Bool_t)gEnv->GetValue(Form("%s.DataRepository.CanWrite", GetName()), (Int_t)(!IsRemote()));
97 //xrootd protocol
99 gEnv->GetValue(Form("%s.DataRepository.XRDServer", GetName()), "");
101 gEnv->GetValue(Form("%s.DataRepository.XRDRootDir", GetName()), "");
102 //access to xrootd via ssh tunnel ?
103 fXRDtunnel = (fXRDtunPort = (Int_t)gEnv->GetValue(Form("%s.DataRepository.XRDTunnel.port", GetName()), 0));
104 if (fXRDtunnel) {
106 }
107 //rfio protocol
109 gEnv->GetValue(Form("%s.DataRepository.RFIOServer", GetName()), "");
111 gEnv->GetValue(Form("%s.DataRepository.RFIORootDir", GetName()),
112 "");
113 //file transfer protocol
115 gEnv->
116 GetValue(Form("%s.DataRepository.FileTransfer.type", GetName()),
117 "sftp");
118 if (fTransfertype.Contains("bbftp")) {
120 fTransfertype = "bbftp";
122 Error("Init", "Executable for bbftp client not found. Check %s.DataRepository.FileTransfer.type",
123 GetName());
124 }
125 }
126 else if (fTransfertype.Contains("sftp")) {
128 fTransfertype = "sftp";
130 fTransferExec = "";
131 }
132 }
133 else if (fTransfertype.Contains("root")) {
134 fTransferExec = "root";
135 fTransfertype = "xrd";
136 if (!fXRDtunnel) fTransferExec = ""; // must have viable SSH tunnel description
137 }
139 gEnv->
140 GetValue(Form("%s.DataRepository.FileTransfer.server", GetName()),
141 "");
142 fTransferuser = gEnv->GetValue(Form("%s.DataRepository.FileTransfer.user", GetName()), ""); //if not given, we use current username
143 if (fTransferuser == "") {
145 fTransferuser = user->fUser;
146 }
147 //set full paths to be used for checking existence of files and for opening files
150 //data set manager
151 if (fDSM) {
152 delete fDSM;
153 }
155
156 //return status of dataset manager
157 return fDSM->Init(this);
158}
159
160
161
162
165
167{
168 //Print info on repository
169 Info("Print", "configuration of repository is as follows - ");
170 cout << "\tname = " << GetName() << endl;
171 cout << "\tRootDir = " << fLocalrootdir.Data() << endl;
172 cout << "\tAccessProtocol = " << fAccessprotocol.Data() << endl;
173 cout << "\tAccessRootDir = " << fAccessroot.Data() << endl;
174 cout << "\tReadProtocol = " << fReadprotocol.Data() << endl;
175 cout << "\tReadRootDir = " << fReadroot.Data() << endl;
176 cout << "\tCanWrite = " << fCanWrite << endl;
177 cout << "\tXRDServer = " << fXrootdserver.Data() << endl;
178 cout << "\tXRDRootDir = " << fXrootdrootdir.Data() << endl;
179 if (fXRDtunnel) {
180 cout << "\tXRDTunnel.host = " << fXRDtunHost.Data() << endl;
181 cout << "\tXRDTunnel.port = " << fXRDtunPort << endl;
182 cout << "\tXRDTunnel.user = " << fXRDtunUser.Data() << endl;
183 cout << "\tXRDTunnel.retry = " << fXRDtunRetry << endl;
184 }
185 cout << "\tRFIOServer = " << fRfioserver.Data() << endl;
186 cout << "\tRFIORootDir = " << fRfiorootdir.Data() << endl;
187 cout << "\tTransferType = " << fTransfertype.Data() << endl;
188 cout << "\tTransferServer = " << fTransferserver.Data() << endl;
189 cout << "\tTransferUser = " << fTransferuser.Data() << endl;
190}
191
192
193
194
197
199{
200 //Default constructor
201 fDSM = 0;
202 fHelpers = 0;
203 fCommitDataSet = 0;
204 SetType("local");
205}
206
207
208
209
212
214{
215 //Destructor
216 if (fDSM)
217 delete fDSM;
218 fDSM = 0;
219
220 if (fHelpers) {
221 fHelpers->Delete();
223 }
224
225}
226
227
228
229
233
235{
236 //Returns kTRUE if the following path is valid
237 // /root_of_data_repository/dir/[subdir]
238
239 TString _dirname, tmp;
241 if (subdir)
242 AssignAndDelete(_dirname,
243 gSystem->ConcatFileName(tmp.Data(), subdir));
244 else
245 _dirname = tmp;
246 //resolve any environment variables in path
247 gSystem->ExpandPathName(_dirname);
248 //if AccessPathName() returns 0, the subdir exists
249 return (!gSystem->AccessPathName(_dirname.Data()));
250}
251
252
253
254
261
262void KVDataRepository::SetFullPath(TString& path, const Char_t* protocol)
263{
264 //Will replace contents of 'path' with the full base path needed for the given protocol.
265 //
266 //protocol = "local" : path = fLocalrootdir
267 //protocol = "root" : path = "root://" + fXrootdserver + "/" + fXrootdrootdir
268 //protocol = "rfio" : path = "rfio:" + fRfioserver + ":" + fRfiorootdir
269
270 TString tmp(protocol);
271 if (tmp == "local")
272 path = fLocalrootdir;
273 else if (tmp == "root")
274 path.Form("root://%s/%s", fXrootdserver.Data(),
276 else if (tmp == "rfio")
277 path.Form("rfio:%s:%s", fRfioserver.Data(), fRfiorootdir.Data());
278 else
279 Warning("SetFullPath",
280 "Unknown protocol in call to SetFullPath : %s", protocol);
281}
282
283
284
285
294
296 const Char_t* datatype,
297 const Char_t* runfile,
298 FileStat_t& fs)
299{
300 //Checks if the run file of given type is physically present in dataset subdirectory,
301 //i.e. (schematically), if
302 //
303 // /root_of_data_repository/[datasetdir]/[datatypedir]/[runfile]
304 //
305 //exists. If it does, the returned value is kTRUE (=1), in which case the FileStat_t object
306 //contains information about the file.
307
308 TString path, tmp;
309 AssignAndDelete(path,
311 AssignAndDelete(tmp, gSystem->ConcatFileName(path.Data(), dataset->GetDataTypeSubdir(datatype)));
312 AssignAndDelete(path, gSystem->ConcatFileName(tmp.Data(), runfile));
313 return !gSystem->GetPathInfo(path.Data(), fs);
314}
315
316
317
318
326
328 const Char_t* datatype,
329 const Char_t* runfile)
330{
331 //Checks if the run file of given type is physically present in dataset subdirectory,
332 //i.e. (schematically), if
333 //
334 // /root_of_data_repository/[datasetdir]/[datatypedir]/[runfile]
335 //
336 //exists. If it does, the returned value is kTRUE (=1).
337
338 TString path, tmp;
339 AssignAndDelete(path,
341 AssignAndDelete(tmp, gSystem->ConcatFileName(path.Data(), dataset->GetDataTypeSubdir(datatype)));
342 AssignAndDelete(path, gSystem->ConcatFileName(tmp.Data(), runfile));
343 return !gSystem->AccessPathName(path.Data());
344}
345
346
347
348
364
366 const Char_t*
367 datatype,
368 const Char_t*
369 runfile)
370{
371 //Returns the full path (including protocol specification) needed by TFile::Open or TChain::Add
372 //to open a runfile belonging to the given dataset.
373 //The protocol depends on the value of fReadprotocol.
374 //If a special protocol has been defined for the given dataset and/or datatype, as in this example
375 //for "raw" data:
376 //
377 //ccali.DataRepository.ReadProtocol.raw: rfio
378 //
379 //it will be used instead of the default (fReadprotocol).
380 //If a given data type is not accessible for a repository, use "none":
381 //
382 //ccali.DataRepository.ReadProtocol.raw: none
383 //
384 //If this is the case, an error message will be printed and an empty string returned.
385
386 static TString path;
387
388 TString read_proto = GetReadProtocol(dataset->GetName(), datatype);
389 if (read_proto == "none") {
390 Error("GetFullPathToOpenFile", "Datatype \"%s\" can not be read from the repository \"%s\"",
391 datatype, GetName());
392 path = "";
393 return path.Data();
394 }
395 TString read_root;
396 if (read_proto != fReadprotocol) {
397 SetFullPath(read_root, read_proto.Data());
398 }
399 else {
400 read_root = fReadroot;
401 }
402 TString tmp, datasetdir = dataset->GetDataPathSubdir();
403 AssignAndDelete(path,
404 gSystem->ConcatFileName(read_root.Data(),
405 datasetdir.Data()));
406 AssignAndDelete(tmp, gSystem->ConcatFileName(path.Data(), dataset->GetDataTypeSubdir(datatype)));
407 AssignAndDelete(path, gSystem->ConcatFileName(tmp.Data(), runfile));
408 return path.Data();
409}
410
411
412
413
420
422 const Char_t*
423 datatype,
424 const Char_t*
425 runfile)
426{
427 //Used by KVDataTransfer.
428 //Returns the full path needed to transfer a runfile belonging to the given dataset
429 //either from or to the repository, using sftp or bbftp etc.
430 //This is just a concatenation of the repository root directory with the dataset
431 //subdirectories and filename.
432
433 static TString path;
434 TString tmp, datasetdir = dataset->GetDataPathSubdir();
435 AssignAndDelete(path,
437 datasetdir.Data()));
438 AssignAndDelete(tmp, gSystem->ConcatFileName(path.Data(), dataset->GetDataTypeSubdir(datatype)));
439 AssignAndDelete(path, gSystem->ConcatFileName(tmp.Data(), runfile));
440 return path.Data();
441}
442
443
444
445
457
459 const Char_t* datatype)
460{
461 //Returns string containing protocol for reading files of the given datatype
462 //belonging to the dataset whose name is given.
463 //Protocol = "local", "root", "rfio" , "none"
464 //(if "none", the data can not be read)
465 //If a specific protocol for the dataset and/or datatype is not defined in $KVROOT/KVFiles/.kvrootrc,
466 //the default protocol (=fReadprotocol) is returned.
467 //Format for defining specific protocols:
468 // name_of_repository.DataRepository.ReadProtocol.dataset_name.data_type: protocol
469 // name_of_repository.DataRepository.ReadProtocol.dataset_name: protocol
470 // name_of_repository.DataRepository.ReadProtocol.data_type: protocol
471
472 static TString prot;
473 prot =
474 gEnv->
475 GetValue(Form
476 ("%s.DataRepository.ReadProtocol.%s.%s", GetName(),
477 dataset, datatype), "");
478 if (prot != "")
479 return prot.Data();
480 prot =
481 gEnv->
482 GetValue(Form
483 ("%s.DataRepository.ReadProtocol.%s", GetName(), dataset),
484 "");
485 if (prot != "")
486 return prot.Data();
487 prot =
488 gEnv->
489 GetValue(Form
490 ("%s.DataRepository.ReadProtocol.%s", GetName(), datatype),
491 "");
492 if (prot != "")
493 return prot.Data();
494 return fReadprotocol;
495}
496
497
498
499
503
505 const Char_t* datatype,
506 const Char_t* filename,
507 const Char_t* destination)
508{
509 //Copy file [datasetdir]/[datatypedir]/[filename] from the repository to [destination]
510 //We check if the file to copy exists.
511
512 if (CheckFileStatus(dataset, datatype, filename)) {
513 TString path, tmp;
514 AssignAndDelete(path,
516 AssignAndDelete(tmp, gSystem->ConcatFileName(path.Data(), dataset->GetDataTypeSubdir(datatype)));
518 //copy file
519 CopyFile(path.Data(), destination);
520 }
521}
522
523
524
525
530
532 const KVDataSet* dataset,
533 const Char_t* datatype,
534 const Char_t* filename)
535{
536 //Copy file [source] to [datasetdir]/[datatypedir]/[filename] in the repository
537 //The file access permissions are set to '664 (u:rw, g:rw, o:r)
538 //Returns status of file transfer command
539
540 TString path, tmp;
541 AssignAndDelete(path,
543 AssignAndDelete(tmp, gSystem->ConcatFileName(path.Data(), dataset->GetDataTypeSubdir(datatype)));
545
546 //copy file
547 int status = CopyFile(source, path.Data());
548 if (status == 0) {
549 //change file access permissions to 664
550 Chmod(path.Data(), CHMODE(6, 6, 4));
551 }
552 else {
553 Error("CopyFileToRepository", "Problem copying file %s to repository (%d)",
554 source, status);
555 }
556 return status;
557}
558
559
560
561
568
570 const Char_t* datatype)
571{
572 //Create a new subdirectory in the repository:
573 // /root_of_data_repository/[datasetdir]
574 //or with 'datatype' given:
575 // /root_of_data_repository/[datasetdir]/[datatypedir]
576 //Set access permissions to 775 (u:rwx, g:rwx, o:rx)
577
578 TString path, tmp;
579 AssignAndDelete(tmp,
581 dataset->GetDataPathSubdir()));
582 if (strcmp(datatype, ""))
583 AssignAndDelete(path, gSystem->ConcatFileName(tmp.Data(), dataset->GetDataTypeSubdir(datatype)));
584 else
585 path = tmp;
586 cout << "Creating new repository directory: " << path.Data() << endl;
587 gSystem->mkdir(path.Data(), kTRUE); // create all parent directories as needed
588 //change file access permissions to 775
589 Chmod(path.Data(), CHMODE(7, 7, 5));
590}
591
592
593
594
606
608 const Char_t* datatype, const Char_t* subdir)
609{
610 //Use the access protocol defined by DataRepository.AccessProtocol (=local by default)
611 //in order to open the directory
612 //
613 // /root_of_data_repository/[datasetdir]/[datatype]/[subdir]
614 // /root_of_data_repository/[datasetdir]/[datatype] (if subdir="", default value)
615 // /root_of_data_repository/[datasetdir] (if datatype="", default value)
616 //
617 //and fill a TList with one KVBase object for each entry in the directory,
618 //excluding "." and ".."
619 //User must delete the KVUniqueNameList after use (list will delete its members)
620
621 TString path, tmp;
622 AssignAndDelete(path,
624 dataset->GetDataPathSubdir()));
625 if (strcmp(datatype, "")) {
626 AssignAndDelete(tmp, gSystem->ConcatFileName(path.Data(), dataset->GetDataTypeSubdir(datatype)));
627 path = tmp;
628 if (strcmp(subdir, "")) {
629 AssignAndDelete(tmp, gSystem->ConcatFileName(path.Data(), subdir));
630 path = tmp;
631 }
632 }
633 //open directory
634 void* dirp = gSystem->OpenDirectory(path.Data());
635 if (!dirp) {
636 Error("KVDataRepository::GetDirectoryListing", "Cannot open %s",
637 path.Data());
638 return 0;
639 }
640
642 dirlist->SetOwner(kTRUE);
643
644 TObjString* direntry = new TObjString(gSystem->GetDirEntry(dirp));
645 while (direntry->GetString() != "") { //loop over all entries in directory
646 if (direntry->GetString() == "." || direntry->GetString() == "..") {
647 //skip "." and ".."
648 delete direntry;
649 }
650 else {
651 dirlist->Add(new KVBase(direntry->GetString().Data()));
652 delete direntry;
653 }
654 //get next entry
655 direntry = new TObjString(gSystem->GetDirEntry(dirp));
656 }
657 //delete last TObjString, which should contain ""
658 delete direntry;
659 //close directory
660 gSystem->FreeDirectory(dirp);
661 return dirlist;
662}
663
664
665
666
676
678 const Char_t* datatype,
679 const Char_t* filename)
680{
681 //This will create and open a new ROOT runfile for the dataset in the repository,
682 //ready to be written.
683 //
684 //If the subdirectory for 'datatype' does not exist, it will be created.
685 //
686 //In fact, if the repository is not one in which files can be created and written
687 //directly (i.e. if CanWrite() = kFALSE), the file will be created in the local working
688 //directory, and only copied into the repository when CommitFile is called.
689
690 cout << "CreateNewFile : " << filename << endl;
691 if (!CanWrite()) {
692 //files cannot be created and written directly in the repository
693 //store path info for subsequent CommitFile
694 /*
695 fCommitDataSet = dataset;
696 fCommitDataType = datatype;
697 fCommitFileName = filename;
698 */
699 //create local file
700 return new TFile(filename, "recreate");
701 }
702 //create file in local repository - make sure subdirectory exists!
703 if (!CheckSubdirExists(dataset->GetDataPathSubdir(), dataset->GetDataTypeSubdir(datatype))) {
704 //create subdirectory
705 MakeSubdirectory(dataset, datatype);
706 }
707 return new TFile(GetFullPathToOpenFile(dataset, datatype, filename),
708 "recreate");
709}
710
711
712
713
729
730void KVDataRepository::CommitFile(TFile* file, const Char_t* datatype, const KVDataSet* dataset)
731{
732 //Add this file (previously created by a call to CreateNewFile) to the repository.
733 //Any objects should be written to the file before calling this method, either by
734 //calling the Write() method of each object, or by calling file->Write().
735 //No file->Write() is done here: we only close (delete) the TFile.
736 //
737 //For repositories in which files can be created and written directly (i.e. if CanWrite() = kTRUE),
738 //we just have to close the file.
739 //For repositories where CanWrite() = kFALSE, we close the file, then copy it to the correct place in the repository,
740 //using the previously recorded values of fCommitDataSetDir, fCommitDataType,
741 //and fCommitFileName. Then we remove the local copy from disk.
742 //
743 //NB: after calling this method, the TFile pointer 'file' must not be used!!!
744
745 //close ROOT file
746 //if (fCommitFileName == "") {
747 fCommitFileName = file->GetName();
748 /*
749 Info("KVDataRepository::CommitFile",
750 "Recover name of the file ==> %s",fCommitFileName.Data());
751 }
752 */
753 delete file;
754
755 if (CanWrite()) { //all we have to do for repositories in which files can be written directly
756 return;
757 }
758
759 fCommitDataSet = dataset;
760 fCommitDataType = datatype;
761
762 //create file in local repository - make sure subdirectory exists!
764
765 cout << endl << "Copying file " << fCommitFileName << " to repository"
766 << endl;
767 TString s;
773 //delete local file
775 /*
776 fCommitDataSet = 0;
777 fCommitDataType = fCommitFileName = "";
778 */
779}
780
781
782
783
789
791 const Char_t* datatype,
792 const Char_t* filename, Bool_t confirm)
793{
794 //Delete repository file [datasetdir]/[datatype]/[filename]
795 //
796 //By default (confirm=kTRUE) we ask for confirmation before deleting.
797 //Set confirm=kFALSE to delete without confirmation (DANGER!!!)
798
799 TString path, tmp;
800 AssignAndDelete(path,
802 AssignAndDelete(tmp, gSystem->ConcatFileName(path.Data(), dataset->GetDataTypeSubdir(datatype)));
804 TString cmd;
805 cout << "Deleting file from repository: " << filename << endl;
806 if (confirm) {
807 cout <<
808 "Are you sure you want to delete this file permanently ? (y/n)"
809 << endl;
810 TString answer;
811 cin >> answer;
812 answer.ToUpper();
813 if (!answer.BeginsWith("Y")) {
814 cout << "File not deleted" << endl;
815 return;
816 }
817 }
818 gSystem->Unlink(path.Data());
819}
820
821
822
823
826
828{
829 //Return pointer to data set manager for this repository
830 return fDSM;
831}
832
833
834
835
838
840{
841 //Create and return pointer to new data set manager
842 return (new KVDataSetManager);
843}
844
845
846
847
851
852TSystem* KVDataRepository::FindHelper(const char* path, void* dirptr)
853{
854 // Create helper TSystem to handle file and directory operations that
855 // might be special for remote file access, like via rfiod or rootd.
856
857 if (!fHelpers)
859
861 TSystem* helper = 0;
862 TUrl url(path, kTRUE);
863
864 // look for existing helpers
865 TIter next(fHelpers);
866 while ((helper = (TSystem*) next()))
867 if (HelperIsConsistentWith(helper, path, dirptr))
868 return helper;
869
870 if (!path)
871 return 0;
872
873 // create new helper
874 TRegexp re("^root.*:"); // also roots, rootk, etc
875 TString pname = path;
876 if (pname.Index(re) != kNPOS) {
877 // rootd daemon ...
878 if ((h = gROOT->GetPluginManager()->FindHandler("TSystem", path)) &&
879 h->LoadPlugin() == 0)
880 helper = (TSystem*) h->ExecPlugin(2, path, kFALSE);
881 else
882#ifdef __WITHOUT_TNETSYSTEM_CTOR_BOOL_T
883 helper = new TNetSystem(path);
884#else
885 helper = new TNetSystem(path, kFALSE);
886#endif
887 }
888 else if (!strcmp(url.GetProtocol(), "http") &&
889 pname.BeginsWith("http")) {
890 // http ...
891 if ((h = gROOT->GetPluginManager()->FindHandler("TSystem", path)) &&
892 h->LoadPlugin() == 0)
893 helper = (TSystem*) h->ExecPlugin(0);
894 else {
895 ; // no default helper yet
896 }
897 }
898 else if ((h = gROOT->GetPluginManager()->FindHandler("TSystem", path))) {
899 if (h->LoadPlugin() == -1)
900 return 0;
901 helper = (TSystem*) h->ExecPlugin(0);
902 }
903
904 if (helper)
905 fHelpers->Add(helper);
906
907 return helper;
908}
909
910
911
912
916
917int KVDataRepository::Chmod(const char* file, UInt_t mode)
918{
919 //Used to change file access permissions in the repository
920
921 //do we need a special helper for this filesystem ?
922 TSystem* helper = FindHelper(file);
923 return (helper ? helper->Chmod(file, mode) : gSystem->Chmod(file, mode));
924}
925
926
927
928
935
936int KVDataRepository::CopyFile(const char* f, const char* t, Bool_t overwrite)
937{
938 //Method to copy any file from anywhere to anywhere
939 //(if the necessary TFile and TSystem plugins are available).
940 // Copy a file. If overwrite is true and file already exists the
941 // file will be overwritten. Returns 0 when successful, -1 in case
942 // of failure, -2 in case the file already exists and overwrite was false.
943
944 if (!gSystem->AccessPathName(t) && !overwrite)
945 return -2;
946
947 //save current directory
948 TDirectory* dir_sav = gDirectory;
949
950 TUrl path(f, kTRUE);
951 TUrl path2(t, kTRUE);
952
953 path.SetOptions("filetype=raw");
954 TFile* from = TFile::Open(path.GetUrl(), "READ");
955 if (!from || from->IsZombie()) {
956 dir_sav->cd();
957 return -1;
958 }
959// from->ls();
960
961 path2.SetOptions("filetype=raw");
962 TString open_option("CREATE");
963 if (overwrite) open_option.Prepend("RE");
964 TFile* to = TFile::Open(path2.GetUrl(), open_option.Data());
965 if (!to || to->IsZombie()) {
966 dir_sav->cd();
967 return -2;
968 }
969// to->ls();
970
971 //set buffer size depending on size of file to copy
972 //as a default use 1MB buffer (good for large files and rfio-hpss system);
973 //use a 1KB buffer for files smaller than 1MB
974 int bufsize = 1024 * 1024;
975 Long64_t filesize = from->GetSize();
976 if (filesize > 0 && filesize < 1024 * 1024) bufsize = 1024;
977 char* buf = new char[bufsize];
978
979 int ret = 0;
980 Long64_t bytes_read, bytes_wrote, bytes_left, last_bytes_read, last_bytes_wrote;
981 bytes_read = bytes_wrote = bytes_left = last_bytes_read = last_bytes_wrote = 0;
982
983// printf("Filesize=%lld Buffersize=%d\n",filesize,bufsize);
984
985// int op = 1;
986
987 while (bufsize && !ret && !from->ReadBuffer(buf, bufsize)) {
988
989 bytes_read = from->GetBytesRead() - last_bytes_read;
990 last_bytes_read = from->GetBytesRead();
991 ret = (int)to->WriteBuffer(buf, bytes_read);
992 bytes_wrote = to->GetBytesWritten() - last_bytes_wrote;
993 last_bytes_wrote = to->GetBytesWritten();
994 if (bytes_wrote != bytes_read)
995 ret = -3;
996 //adjust buffer size
997 bytes_left = filesize - last_bytes_read;
998 if (bytes_left < bufsize) bufsize = (int)bytes_left;
999
1000// printf("----------------------------------%d-----------------------------------\n",op++);
1001// printf("bytes_read=%lld last_bytes_read=%lld total_bytes_read=%lld\n",
1002// bytes_read, last_bytes_read, from->GetBytesRead());
1003// printf("bytes_wrote=%lld last_bytes_wrote=%lld total_bytes_wrote=%lld\n",
1004// bytes_wrote, last_bytes_wrote, to->GetBytesWritten());
1005// printf("bytes_left=%lld bufsize=%d\n",
1006// bytes_left, bufsize);
1007 }
1008
1009// printf("----------------------------------END-----------------------------------\n");
1010// printf("bytes_read=%lld last_bytes_read=%lld total_bytes_read=%lld\n",
1011// bytes_read, last_bytes_read, from->GetBytesRead());
1012// printf("bytes_wrote=%lld last_bytes_wrote=%lld total_bytes_wrote=%lld\n",
1013// bytes_wrote, last_bytes_wrote, to->GetBytesWritten());
1014// printf("bytes_left=%lld bufsize=%d\n",
1015// bytes_left, bufsize);
1016
1017 delete from;
1018 delete to;
1019 delete[] buf;
1020
1021 //reset working directory
1022 dir_sav->cd();
1023
1024 return ret;
1025}
1026
1027
1028
1029
1034
1035Bool_t KVDataRepository::HelperIsConsistentWith(TSystem* helper, const char* path, void* dirptr)
1036{
1037 // Copy of (protected) method TSystem::ConsistentWith
1038 // Check consistency of this helper with the one required
1039 // by 'path' or 'dirptr'
1040
1041 Bool_t checkproto = kFALSE;
1042 if (path) {
1043 if (!helper->GetDirPtr()) {
1044 TUrl url(path, kTRUE);
1045 if (!strncmp(url.GetProtocol(), helper->GetName(), strlen(helper->GetName())))
1046 checkproto = kTRUE;
1047 }
1048 }
1049
1050 Bool_t checkdir = kFALSE;
1051 if (helper->GetDirPtr() && helper->GetDirPtr() == dirptr)
1052 checkdir = kTRUE;
1053
1054 return (checkproto || checkdir);
1055}
1056
1057
1058
1059
1080
1082{
1083 //PRIVATE method, called by constructor.
1084 //Called if an SSH tunnel is to be used to connect to an xrootd server, e.g. if the repository
1085 //configuration includes the following lines (or similar):
1086 //
1087 // #ccali.DataRepository.XRDTunnel.host: ccali.in2p3.fr
1088 // #ccali.DataRepository.XRDTunnel.port: 10000
1089 // #ccali.DataRepository.XRDTunnel.retry: 5
1090 // #ccali.DataRepository.XRDTunnel.user:
1091 //
1092 //In this case we need to replace the real name & port of the xrootd server:
1093 //
1094 //#ccali.DataRepository.XRDServer: ccxrdsn012:1999
1095 //
1096 //with the name & port of the SSH tunnel
1097 //
1098 //#ccali.DataRepository.XRDServer: localhost:10000
1099 //
1100 //which will be used for path names in e.g. GetFullPathToOpenFile.
1101 //The tunnel is not created here; it should be opened when needed by calling IsConnected()
1102
1103 fXRDtunHost = gEnv->GetValue(Form("%s.DataRepository.XRDTunnel.host", GetName()), "");
1104 if (fXRDtunHost == "") {
1105 Error("PrepareXRDTunnel", "Give host through which to tunnel : %s.DataRepository.XRDTunnel.host",
1106 GetName());
1108 return;
1109 }
1110 fXRDtunUser = gEnv->GetValue(Form("%s.DataRepository.XRDTunnel.user", GetName()), "");
1111 if (fXRDtunUser == "") {//if not given, use local user name
1112 UserGroup_t* user = gSystem->GetUserInfo();
1113 fXRDtunUser = user->fUser;
1114 }
1115 //spec for tunnel connection
1117 //max. number of retries before giving up when trying to open tunnel
1118 fXRDtunRetry = (Int_t)gEnv->GetValue(Form("%s.DataRepository.XRDTunnel.retry", GetName()), 30);
1119 //now change xrootd server/port to use for pathnames
1120 fXrootdserver.Form("localhost:%d", fXRDtunPort);
1121}
1122
1123
1124
1125
1131
1133{
1134 // Create new instance of class derived from KVDataRepository.
1135 // Actual class of object depends on 'type' which is used to select one of the
1136 // Plugin.KVDataRepository's defined in .kvrootrc.
1137
1138 //check and load plugin library
1139 TPluginHandler* ph = KVBase::LoadPlugin("KVDataRepository", type);
1140 if (!ph)
1141 return new KVDataRepository();
1142
1143 //execute constructor/macro for plugin
1144 return ((KVDataRepository*) ph->ExecPlugin(0));
1145}
1146
1147
1148
1149
1156
1158{
1159 // Create new instance of class derived from KVAvailableRunsFile.
1160 // Actual class of object depends on the type of the repository,
1161 // which is used to select one of the
1162 // Plugin.KVDataAvailableRunsFile's defined in .kvrootrc.
1163
1164 //check and load plugin library
1165 TPluginHandler* ph = KVBase::LoadPlugin("KVAvailableRunsFile", GetType());
1166 if (!ph)
1167 return new KVAvailableRunsFile(data_type, ds);
1168
1169 //execute constructor/macro for plugin
1170 return ((KVAvailableRunsFile*) ph->ExecPlugin(2, data_type, ds));
1171}
1172
1173
1174
1175
1203
1205{
1206 //Open a file using plugin defined in $KVROOT/KVFiles/.kvrootrc
1207 //The default base classes for each type are defined as in this example:
1208 //
1209 // # Default base classes for reading runfiles of different types
1210 // DataSet.RunFileClass.raw: KVRawDataReader
1211 // DataSet.RunFileClass.recon: TFile
1212 // DataSet.RunFileClass.ident: TFile
1213 // DataSet.RunFileClass.root: TFile
1214 //
1215 //A different base class can be defined for a specific dataset/datatype
1216 //by adding a line to your $HOME/.kvrootrc like this:
1217 //
1218 //name_of_dataset.DataSet.RunFileClass.data_type: BaseClassName
1219 //
1220 //The actual class to be used is then defined by plugins in $KVROOT/KVFiles/.kvrootrc,
1221 //for example
1222 //
1223 //Plugin.KVRawDataReader: raw.INDRA* KVINDRARawDataReader KVIndra "KVINDRARawDataReader()"
1224 //
1225 //which defines the plugin for raw data for all datasets whose name begins with "INDRA"
1226 //If no plugin is found for the base class defined by DataSet.RunFileClass, the base class is used.
1227 //
1228 //To actually open the file, each base class & plugin must define a method
1229 // static BaseClass* Open(const Char_t* path, Option_t* opt="", ...)
1230 //which takes the full path to the file as argument (any other arguments taking default options)
1231 //and returns a pointer of the BaseClass type to the created object which can be used to read the file.
1232
1233 TString fname = ds->GetFullPathToRunfile(type, run);
1234 if (fname == "") return nullptr; //file not found
1235
1236 return OpenDataSetFile(ds, type, fname, opt);
1237}
1238
1239
1240
1241
1275
1276TObject* KVDataRepository::OpenDataSetFile(const KVDataSet* ds, const Char_t* type, const TString& fname, Option_t* opt)
1277{
1278 //Open a file using plugin defined in $KVROOT/KVFiles/.kvrootrc
1279 //fname is the full path required to open the file
1280 //The default base classes for each type are defined as in this example:
1281 //
1282 // # Default base classes for reading runfiles of different types
1283 // DataSet.RunFileClass.raw: KVRawDataReader
1284 // DataSet.RunFileClass.recon: TFile
1285 // DataSet.RunFileClass.ident: TFile
1286 // DataSet.RunFileClass.root: TFile
1287 //
1288 //A different base class can be defined for a specific dataset/datatype
1289 //by adding a line to your $HOME/.kvrootrc like this:
1290 //
1291 //name_of_dataset.DataSet.RunFileClass.data_type: BaseClassName
1292 //
1293 //The actual class to be used is then defined by plugins in $KVROOT/KVFiles/.kvrootrc, for example
1294 //
1295 //name_of_dataset.DataSet.RunFileClass.data_type: plugin_name
1296 //
1297 //where 'plugin_name' is one of the known plugins for the default base class for the given data_type,
1298 //i.e. if data_type="raw" we could use one of either "GANIL" or "MFM" as plugin_name:
1299 //
1300 //Plugin.KVRawDataReader: GANIL KVGANILDataReader KVMultiDetexp_events "KVGANILDataReader()"
1301 //+Plugin.KVRawDataReader: MFM KVMFMDataFileReader KVMultiDetdaq_cec "KVMFMDataFileReader()"
1302 //
1303 //If no plugin is found for the base class defined by DataSet.RunFileClass, the base class is used.
1304 //
1305 //To actually open the file, each base class & plugin must define a method
1306 // static BaseClass* Open(const Char_t* path, Option_t* opt="", ...)
1307 //which takes the full path to the file as argument (any other arguments taking default options)
1308 //and returns a pointer of the BaseClass type to the created object which can be used to read the file.
1309
1310 // check connection to repository (e.g. SSH tunnel) in case of remote repository
1311 if (!IsConnected()) return nullptr;
1312
1313 //get class for dataset & type
1314 KVString plugin_class = ds->GetDataSetEnv(Form("DataSet.RunFileClass.%s", type));
1315
1316 TPluginHandler* ph = nullptr;
1317 KVString base_class;
1318 if (IsThisAPlugin(plugin_class, base_class)) {
1319
1320 //look for plugin specific to dataset & type
1321 ph = LoadPlugin(base_class, plugin_class);
1322
1323 }
1324 else
1325 base_class = plugin_class;
1326 TClass* cl;
1327 if (!ph) {
1328 //no plugin - use base class
1329 cl = TClass::GetClass(base_class.Data());
1330 }
1331 else {
1332 cl = TClass::GetClass(ph->GetClass());
1333 }
1334
1335 //set up call to static Open method
1336 TMethodCall* methcall;
1337 if (strcmp(opt, "")) {
1338 //Open with option
1339 methcall = new TMethodCall(cl, "Open", Form("\"%s\", \"%s\"", fname.Data(), opt));
1340 }
1341 else {
1342 //Open without option
1343 methcall = new TMethodCall(cl, "Open", Form("\"%s\"", fname.Data()));
1344 }
1345
1346 if (!methcall->IsValid()) {
1347 if (ph) Error("OpenDataSetFile", "Open method for class %s is not valid", ph->GetClass());
1348 else Error("OpenDataSetFile", "Open method for class %s is not valid", base_class.Data());
1349 delete methcall;
1350 return nullptr;
1351 }
1352
1353 //open the file
1354 Long_t retval;
1355 methcall->Execute(retval);
1356 delete methcall;
1357 return ((TObject*)(retval));
1358}
1359
1360
1361
1365
1367{
1368 // Ensure that all required subdirectories exist and any missing ones are created
1369 // in order to store a runfile of given DataType for the given DataSet
1370
1371 if (!CheckSubdirExists(DataSet->GetDataPathSubdir(), DataSet->GetDataTypeSubdir(DataType))) {
1372 //does dataset directory exist ?
1373 if (!CheckSubdirExists(DataSet->GetDataPathSubdir())) {
1374 //create dataset directory
1375 MakeSubdirectory(DataSet);
1376 }
1377 //create subdirectory
1378 MakeSubdirectory(DataSet, DataType);
1379 }
1380}
1381
1382
1383
1384#ifdef __CCIN2P3_RFIO
1385
1386ClassImp(KVRFIOSystem)
1387
1388
1389//This class corrects a bug in TRFIOSystem::Unlink
1390//and adds a fully-functioning Chmod method and
1391//CopyFile which can handle either or both of source
1392//and target file being accessed via rfiod (the other is
1393//assumed to be on the local filesystem).
1394
1395
1396
1397
1398extern "C" {
1399 FILE* rfio_fopen(const char* path, const char* mode);
1400 int rfio_fclose(FILE* s);
1401 int rfio_fread(void* buffer, size_t size, size_t count, FILE* stream);
1402 int rfio_fwrite(const void* buffer, size_t size, size_t count, FILE* stream);
1403 int rfio_feof(FILE* stream);
1404 int rfio_stat(const char* path, struct stat* statbuf);
1405 int rfio_rmdir(const char* path);
1406 int rfio_unlink(const char* path);
1407 int rfio_chmod(const char* path, mode_t mode);
1408};
1409
1410
1411
1412
1416
1417Int_t KVRFIOSystem::Unlink(const char* path)
1418{
1419 // Unlink, i.e. remove, a file or directory. Returns 0 when succesfull,
1420 // -1 in case of failure.
1421
1422 TUrl url(path);
1423 struct stat finfo;
1424 if (rfio_stat(url.GetFile(), &finfo) < 0)
1425 return -1;
1426
1427 if (R_ISDIR(finfo.st_mode))
1428 return rfio_rmdir(url.GetFile());
1429 else
1430 return rfio_unlink(url.GetFile());
1431}
1432
1433
1434
1435
1438
1439int KVRFIOSystem::Chmod(const char* file, UInt_t mode)
1440{
1441 // Set the file permission bits. Returns -1 in case or error, 0 otherwise.
1442
1443 TUrl url(file);
1444 return rfio_chmod(url.GetFile(), mode);
1445}
1446
1447
1448#endif
1449
int Int_t
unsigned int UInt_t
long Long_t
#define SafeDelete(p)
#define f(i)
bool Bool_t
char Char_t
constexpr Bool_t kFALSE
constexpr Ssiz_t kNPOS
constexpr Bool_t kTRUE
const char Option_t
#define gDirectory
R__EXTERN TEnv * gEnv
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char filename
Option_t Option_t TPoint TPoint const char mode
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize fs
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t Atom_t Time_t type
#define gROOT
char * Form(const char *fmt,...)
void AssignAndDelete(TString &target, char *tobedeleted)
Bool_t R_ISDIR(Int_t mode)
R__EXTERN TSystem * gSystem
Handles lists of available runs for different datasets and types of data.
static Bool_t FindExecutable(TString &exec, const Char_t *path="$(PATH)")
Definition KVBase.cpp:1001
KVBase()
Default constructor.
Definition KVBase.cpp:317
virtual void SetType(const Char_t *str)
Definition KVBase.h:173
static Bool_t IsThisAPlugin(const TString &uri, TString &base)
Definition KVBase.cpp:1357
virtual const Char_t * GetType() const
Definition KVBase.h:177
static TPluginHandler * LoadPlugin(const Char_t *base, const Char_t *uri="0")
Definition KVBase.cpp:793
Base class for managing repositories of experimental data.
virtual KVDataSetManager * GetDataSetManager() const
Return pointer to data set manager for this repository.
virtual int CopyFileToRepository(const Char_t *source, const KVDataSet *dataset, const Char_t *datatype, const Char_t *filename)
virtual int Chmod(const char *file, UInt_t mode)
virtual TObject * OpenDataSetRunFile(const KVDataSet *ds, const Char_t *type, Int_t run, Option_t *opt="")
virtual Bool_t IsRemote() const
Returns kTRUE for remote repositories, kFALSE for local repositories.
virtual void CopyFileFromRepository(const KVDataSet *dataset, const Char_t *datatype, const Char_t *filename, const Char_t *destination)
virtual const Char_t * GetReadProtocol(const Char_t *dataset, const Char_t *datatype)
static KVDataRepository * NewRepository(const Char_t *type)
virtual void Print(Option_t *opt="") const
Print info on repository.
virtual int CopyFile(const char *f, const char *t, Bool_t overwrite=kFALSE)
virtual ~KVDataRepository()
Destructor.
virtual KVUniqueNameList * GetDirectoryListing(const KVDataSet *dataset, const Char_t *datatype="", const Char_t *subdir="")
virtual Bool_t CheckFileStatus(const KVDataSet *dataset, const Char_t *datatype, const Char_t *runfile)
KVDataSetManager * fDSM
handles datasets in repository
virtual Bool_t GetFileInfo(const KVDataSet *dataset, const Char_t *datatype, const Char_t *runfile, FileStat_t &fs)
virtual Bool_t CheckSubdirExists(const Char_t *dir, const Char_t *subdir=0)
virtual Bool_t Init()
virtual void PrepareXRDTunnel()
void SetFullPath(TString &path, const Char_t *protocol)
virtual KVAvailableRunsFile * NewAvailableRunsFile(const Char_t *, const KVDataSet *)
TSeqCollection * fHelpers
List of helper classes for alternative file/directory access.
void CreateAllNeededSubdirectories(const KVDataSet *DataSet, const Char_t *DataType)
virtual KVDataSetManager * NewDataSetManager()
Create and return pointer to new data set manager.
virtual Bool_t CanWrite() const
Bool_t HelperIsConsistentWith(TSystem *helper, const char *path, void *dirptr=0)
TSystem * FindHelper(const char *path, void *dirptr=0)
virtual void CommitFile(TFile *file, const Char_t *datatype, const KVDataSet *dataset)
virtual void DeleteFile(const KVDataSet *dataset, const Char_t *datatype, const Char_t *filename, Bool_t confirm=kTRUE)
virtual TFile * CreateNewFile(const KVDataSet *dataset, const Char_t *datatype, const Char_t *filename)
KVDataRepository()
Default constructor.
virtual const Char_t * GetFullPathToTransferFile(const KVDataSet *dataset, const Char_t *datatype, const Char_t *runfile)
virtual Bool_t IsConnected()
Always returns kTRUE for local repositories.
const KVDataSet * fCommitDataSet
used by CreateNewFile and CommitFile
virtual void MakeSubdirectory(const KVDataSet *dataset, const Char_t *datatype="")
virtual const Char_t * GetFullPathToOpenFile(const KVDataSet *dataset, const Char_t *datatype, const Char_t *runfile)
TObject * OpenDataSetFile(const KVDataSet *ds, const Char_t *type, const TString &fname, Option_t *opt="")
Manage all datasets contained in a given data repository.
virtual Bool_t Init(KVDataRepository *=0)
Manage an experimental dataset corresponding to a given experiment or campaign.
Definition KVDataSet.h:35
virtual const Char_t * GetDataPathSubdir() const
Returns name of top-level directory in data repository used to store data files for this dataset.
Definition KVDataSet.h:95
const Char_t * GetDataSetEnv(const Char_t *type, const Char_t *defval="") const
TString GetFullPathToRunfile(const Char_t *type, Int_t run) const
const Char_t * GetDataTypeSubdir(const Char_t *type) const
Definition KVDataSet.h:99
virtual void SetOwner(Bool_t enable=kTRUE)
Extension of ROOT TString class which allows backwards compatibility with ROOT v3....
Definition KVString.h:73
Optimised list in which named objects can only be placed once.
virtual void Add(TObject *obj)
void Delete(Option_t *option="") override=0
virtual Bool_t cd()
virtual const char * GetValue(const char *name, const char *dflt) const
virtual Long64_t GetSize() const
virtual Long64_t GetBytesRead() const
virtual Bool_t WriteBuffer(const char *buf, Int_t len)
virtual Long64_t GetBytesWritten() const
virtual Bool_t ReadBuffer(char *buf, Int_t len)
Bool_t IsValid() const
void Execute()
const char * GetName() const override
const TString & GetString() const
virtual void Warning(const char *method, const char *msgfmt,...) const
R__ALWAYS_INLINE Bool_t IsZombie() const
virtual void Error(const char *method, const char *msgfmt,...) const
virtual void Info(const char *method, const char *msgfmt,...) const
const char * GetClass() const
Longptr_t ExecPlugin(int nargs)
void Add(TObject *obj) override
const char * Data() const
void ToUpper()
Bool_t BeginsWith(const char *s, ECaseCompare cmp=kExact) const
TString & Prepend(char c, Ssiz_t rep=1)
void Form(const char *fmt,...)
Bool_t Contains(const char *pat, ECaseCompare cmp=kExact) const
Ssiz_t Index(const char *pat, Ssiz_t i=0, ECaseCompare cmp=kExact) const
virtual void * GetDirPtr() const
const char * pwd()
virtual int Chmod(const char *file, UInt_t mode)
virtual void FreeDirectory(void *dirp)
virtual void * OpenDirectory(const char *name)
virtual char * ConcatFileName(const char *dir, const char *name)
virtual int mkdir(const char *name, Bool_t recursive=kFALSE)
virtual int GetPathInfo(const char *path, FileStat_t &buf)
virtual UserGroup_t * GetUserInfo(const char *user=nullptr)
virtual Bool_t AccessPathName(const char *path, EAccessMode mode=kFileExists)
virtual const char * GetDirEntry(void *dirp)
virtual char * ExpandPathName(const char *path)
virtual int Unlink(const char *name)
const char * GetUrl(Bool_t withDeflt=kFALSE) const
void SetOptions(const char *opt)
const char * GetProtocol() const
long long Long64_t
TH1 * h
TString fUser
ClassImp(TPyArg)