52 pPrintSourceCheckSum(false), pPrintTargetCheckSum(false),
53 pPrintAdditionalCheckSum(false)
65 if( pPrintProgressBar )
69 std::cerr <<
"Job: " << jobNum <<
"/" << jobTotal << std::endl;
70 std::cerr <<
"Source: " << source->
GetURL() << std::endl;
71 std::cerr <<
"Target: " << destination->
GetURL() << std::endl;
79 d.target = destination;
80 pOngoingJobs[jobNum] = d;
90 std::map<uint16_t, JobData>::iterator it = pOngoingJobs.find( jobNum );
91 if( it == pOngoingJobs.end() )
94 JobData &d = it->second;
99 JobProgress( jobNum, d.bytesProcessed, d.bytesTotal );
101 if( pPrintProgressBar )
103 if( pOngoingJobs.size() > 1 )
104 std::cerr <<
"\r" << std::string(70,
' ') <<
"\r";
106 std::cerr << std::endl;
110 results->
Get(
"status", st );
113 pOngoingJobs.erase(it);
117 std::string checkSum;
119 results->
Get(
"size", size );
120 if( pPrintSourceCheckSum )
122 results->
Get(
"sourceCheckSum", checkSum );
126 if( pPrintTargetCheckSum )
128 results->
Get(
"targetCheckSum", checkSum );
132 if( pPrintAdditionalCheckSum )
134 std::vector<std::string> addcksums;
135 results->
Get(
"additionalCkeckSum", addcksums );
136 for(
auto &cks : addcksums )
140 pOngoingJobs.erase(it);
148 JobData &d = pOngoingJobs.begin()->second;
152 speed = d.bytesProcessed/(now-d.started);
154 speed = d.bytesProcessed;
162 prog = (int)((
double)d.bytesProcessed/d.bytesTotal*50);
163 proc = (int)((
double)d.bytesProcessed/d.bytesTotal*100);
170 bar.append( prog,
'=' );
174 std::ostringstream o;
177 o <<
"[" << std::setw(3) << std::right << proc <<
"%]";
178 o <<
"[" << std::setw(50) << std::left;
190 std::map<uint16_t, JobData>::iterator it;
191 std::ostringstream o;
193 for( it = pOngoingJobs.begin(); it != pOngoingJobs.end(); ++it )
195 JobData &d = it->second;
196 uint16_t jobNum = it->first;
200 speed = d.bytesProcessed/(now-d.started);
204 proc = (int)((
double)d.bytesProcessed/d.bytesTotal*100);
208 o <<
"[#" << jobNum <<
": ";
220 uint64_t bytesProcessed,
221 uint64_t bytesTotal )
225 if( pPrintProgressBar )
227 time_t now = time(0);
228 if( (now - pPrevious < 1) && (bytesProcessed != bytesTotal) )
232 std::map<uint16_t, JobData>::iterator it = pOngoingJobs.find( jobNum );
233 if( it == pOngoingJobs.end() )
236 JobData &d = it->second;
238 d.bytesProcessed = bytesProcessed;
239 d.bytesTotal = bytesTotal;
241 std::string progress;
242 if( pOngoingJobs.size() == 1 )
247 std::cerr <<
"\r" << progress << std::flush;
255 const std::string &checkSum,
258 if( checkSum.empty() )
260 std::string::size_type i = checkSum.find(
':' );
261 std::cerr << checkSum.substr( 0, i+1 ) <<
" ";
262 std::cerr << checkSum.substr( i+1, checkSum.length()-i ) <<
" ";
265 std::cerr << url->
GetPath() <<
" ";
269 std::cerr << url->
GetPath() <<
" ";
273 std::cerr << std::endl;
287 JobData(): bytesProcessed(0), bytesTotal(0),
288 started(0), source(0), target(0) {}
289 uint64_t bytesProcessed;
297 bool pPrintProgressBar;
298 bool pPrintSourceCheckSum;
299 bool pPrintTargetCheckSum;
300 bool pPrintAdditionalCheckSum;
301 std::map<uint16_t, JobData> pOngoingJobs;
312 std::cerr <<
"SOCKS Proxies are not yet supported" << std::endl;
324 if( !newCGI || !(*newCGI) )
330 if( url.find(
'?' ) == std::string::npos )
333 if( url.find(
'&' ) == std::string::npos )
350 cursor = cursor->
Next;
357 cursor = cursor->
Next;
374 default:
return "other";
384 for( count = 0; file; file = file->
Next, ++count ) {};
397 if( file->
Doff == 0 )
399 char *slash = file->
Path;
400 for( ; *slash; ++slash ) {};
401 for( ; *slash !=
'/' && slash > file->
Path; --slash ) {};
410 std::string basePath,
413 using namespace XrdCl;
415 Log *log = DefaultEnv::GetLog();
416 log->
Debug( AppMsg,
"Indexing %s", basePath.c_str() );
420 | DirListFlags::Locate | DirListFlags::Merge, dirList );
423 log->
Info( AppMsg,
"Failed to get directory listing for %s: %s",
432 for(
auto itr = dirList->
Begin(); itr != dirList->
End(); ++itr )
437 std::string path = basePath +
'/' + e->
GetName();
438 current =
new XrdCpFile( path.c_str(), badUrl );
441 log->
Error( AppMsg,
"Bad URL: %s", current->
Path );
446 current->
Doff = dirOffset;
461 std::vector<XrdCl::PropertyList *>::iterator it;
462 for( it = results.begin(); it != results.end(); ++it )
469int main(
int argc,
char **argv )
471 using namespace XrdCl;
486 Log *log = DefaultEnv::GetLog();
490 else if( config.
Dlvl == 2 ) log->
SetLevel( Log::DebugMsg );
491 else if( config.
Dlvl == 3 ) log->
SetLevel( Log::DumpMsg );
501 bool makedir =
false;
503 bool delegate =
false;
504 bool preserveXAttr =
false;
505 bool rmOnBadCksum =
false;
506 bool continue_ =
false;
507 bool recurse =
false;
508 bool zipappend =
false;
509 bool doserver =
false;
510 std::string thirdParty =
"none";
525 DlgEnv::Instance().Enable();
529 DlgEnv::Instance().Disable();
542 if( force && continue_ )
544 std::cerr <<
"Invalid argument combination: continue + force." << std::endl;
551 std::string checkSumType;
552 std::string checkSumPreset;
553 std::string checkSumMode =
"none";
556 checkSumMode =
"end2end";
557 std::vector<std::string> ckSumParams;
558 Utils::splitString( ckSumParams, config.
CksVal,
":" );
559 if( ckSumParams.size() > 1 )
561 if( ckSumParams[1] ==
"print" )
563 checkSumMode =
"target";
567 checkSumPreset = ckSumParams[1];
569 checkSumType = ckSumParams[0];
574 checkSumMode =
"source";
575 std::vector<std::string> ckSumParams;
576 Utils::splitString( ckSumParams, config.
CksVal,
":" );
577 if( ckSumParams.size() == 2 )
579 checkSumMode =
"source";
580 checkSumType = ckSumParams[0];
585 std::cerr <<
"Invalid parameter: " << config.
CksVal << std::endl;
611 nbSources = config.
nSrcs;
625 if( config.
nStrm != 0 )
626 env->
PutInt(
"SubStreamsPerChannel", config.
nStrm + 1 );
628 if( config.
Retry != -1 )
635 env->
PutInt(
"NoTlsOK", 1 );
638 env->
PutInt(
"TlsNoData", 1 );
641 env->
PutInt(
"TlsMetalink", 1 );
644 env->
PutInt(
"ZipMtlnCksum", 1 );
646 int chunkSize = DefaultCPChunkSize;
647 env->
GetInt(
"CPChunkSize", chunkSize );
649 int blockSize = DefaultXCpBlockSize;
650 env->
GetInt(
"XCpBlockSize", blockSize );
652 int parallelChunks = DefaultCPParallelChunks;
653 env->
GetInt(
"CPParallelChunks", parallelChunks );
654 if( parallelChunks < 1 ||
655 parallelChunks > std::numeric_limits<uint8_t>::max() )
657 std::cerr <<
"Can only handle between 1 and ";
658 std::cerr << (int)std::numeric_limits<uint8_t>::max();
659 std::cerr <<
" chunks in parallel. You asked for " << parallelChunks;
660 std::cerr <<
"." << std::endl;
666 int val = DefaultPreserveXAttrs;
667 env->
GetInt(
"PreserveXAttrs", val );
668 if( val ) preserveXAttr =
true;
671 log->
Dump( AppMsg,
"Chunk size: %d, parallel chunks %d, streams: %d",
672 chunkSize, parallelChunks, config.
nStrm + 1 );
677 std::vector<XrdCl::PropertyList*> resultVect;
688 char buf[FILENAME_MAX];
689 char *cwd = getcwd( buf, FILENAME_MAX );
709 bool targetIsDir =
false;
710 bool targetExists =
false;
722 if( statInfo->
TestFlags( StatInfo::IsDir ) )
734 log->
Error( AppMsg,
"%s (destination)", st.
ToString().c_str() );
735 std::cerr << st.
ToStr() << std::endl;
742 if( !targetIsDir && targetExists && !force && !recurse && !zipappend )
746 log->
Error( AppMsg,
"%s (destination)", st.
ToString().c_str() );
747 std::cerr <<
"Run: " << st.
ToStr() << std::endl;
758 std::cerr <<
"Multiple sources were given but target is not a directory.";
759 std::cerr << std::endl;
767 bool remoteSrcIsDir =
false;
778 remoteSrcIsDir =
true;
783 std::string url = source.
GetURL();
787 std::cerr <<
"Error indexing remote directory.";
809 std::string source = sourceFile->
Path;
813 if( source[0] ==
'/' )
814 source =
"file://" + source;
817 char buf[FILENAME_MAX];
818 char *cwd = getcwd( buf, FILENAME_MAX );
825 source =
"file://" + std::string( cwd ) +
'/' + source;
831 log->
Dump( AppMsg,
"Processing source entry: %s, type %s, target file: %s",
838 std::string target = dest;
841 bool srcIsDir =
false;
843 if( strncmp( sourceFile->
ProtName,
"file", 4 ) == 0 )
844 srcIsDir = std::string( sourceFile->
Path ).size() == size_t( sourceFile->
Doff + sourceFile->
Dlen );
847 srcIsDir = remoteSrcIsDir;
852 std::string srcDir( sourceFile->
Path, sourceFile->
Doff );
854 if( srcDir[srcDir.size() - 1] ==
'/' )
855 srcDir = srcDir.substr( 0, srcDir.size() - 1 );
856 size_t diroff = srcDir.rfind(
'/' );
858 if( diroff == std::string::npos ) diroff = 0;
860 target += sourceFile->
Path + diroff;
862 target = target.substr( 0 , target.rfind(
'/') );
866 properties.
Set(
"source", source );
867 properties.
Set(
"target", target );
868 properties.
Set(
"force", force );
869 properties.
Set(
"posc", posc );
870 properties.
Set(
"coerce", coerce );
871 properties.
Set(
"makeDir", makedir );
872 properties.
Set(
"dynamicSource", dynSrc );
873 properties.
Set(
"thirdParty", thirdParty );
874 properties.
Set(
"checkSumMode", checkSumMode );
875 properties.
Set(
"checkSumType", checkSumType );
876 properties.
Set(
"checkSumPreset", checkSumPreset );
877 properties.
Set(
"chunkSize", chunkSize );
878 properties.
Set(
"parallelChunks", parallelChunks );
879 properties.
Set(
"zipArchive", zip );
880 properties.
Set(
"xcp", xcp );
881 properties.
Set(
"xcpBlockSize", blockSize );
882 properties.
Set(
"delegate", delegate );
883 properties.
Set(
"targetIsDir", targetIsDir );
884 properties.
Set(
"preserveXAttr", preserveXAttr );
885 properties.
Set(
"xrate", config.
xRate );
887 properties.
Set(
"rmOnBadCksum", rmOnBadCksum );
888 properties.
Set(
"continue", continue_ );
889 properties.
Set(
"zipAppend", zipappend );
891 properties.
Set(
"doServer", doserver );
894 properties.
Set(
"zipSource", zipFile );
897 properties.
Set(
"nbXcpSources", nbSources );
903 std::cerr <<
"AddJob " << source <<
" -> " << target <<
": ";
904 std::cerr << st.
ToStr() << std::endl;
906 resultVect.push_back( results );
907 sourceFile = sourceFile->
Next;
914 processConfig.
Set(
"jobType",
"configuration" );
916 process.
AddJob( processConfig, 0 );
925 std::cerr <<
"Prepare: " << st.
ToStr() << std::endl;
929 st = process.
Run( &progress );
932 if( resultVect.size() == 1 )
933 std::cerr <<
"Run: " << st.
ToStr() << std::endl;
936 std::vector<XrdCl::PropertyList*>::iterator it;
938 uint16_t jobsRun = 0;
940 for( it = resultVect.begin(); it != resultVect.end(); ++it, ++i )
942 if( !(*it)->HasProperty(
"status" ) )
948 std::cerr <<
"Job #" << i <<
": " << st.
ToStr();
953 std::cerr <<
"Jobs total: " << resultVect.size();
954 std::cerr <<
", run: " << jobsRun;
955 std::cerr <<
", errors: " << errors << std::endl;
bool AllOptionsSupported(XrdCpConfig *config)
int main(int argc, char **argv)
const char * FileType2String(XrdCpFile::PType type)
void ProcessCommandLineEnv(XrdCpConfig *config)
void CleanUpResults(std::vector< XrdCl::PropertyList * > &results)
XrdCpFile * IndexRemote(XrdCl::FileSystem *fs, std::string basePath, long dirOffset)
void AdjustFileInfo(XrdCpFile *file)
uint32_t CountSources(XrdCpFile *file)
void AppendCGI(std::string &url, const char *newCGI)
void PrintAdditionalCheckSum(bool print)
void PrintSourceCheckSum(bool print)
virtual void EndJob(uint16_t jobNum, const XrdCl::PropertyList *results)
End job.
void PrintProgressBar(bool print)
void PrintCheckSum(const XrdCl::URL *url, const std::string &checkSum, uint64_t size)
Print the checksum.
virtual void JobProgress(uint16_t jobNum, uint64_t bytesProcessed, uint64_t bytesTotal)
Job progress.
std::string GetProgressBar(time_t now)
Get progress bar.
virtual void BeginJob(uint16_t jobNum, uint16_t jobTotal, const XrdCl::URL *source, const XrdCl::URL *destination)
Begin job.
std::string GetSummaryBar(time_t now)
Get sumary bar.
ProgressDisplay()
Constructor.
void PrintTargetCheckSum(bool print)
static int mapError(int rc)
Copy the data from one point to another.
XRootDStatus Run(CopyProgressHandler *handler)
Run the copy jobs.
XRootDStatus AddJob(const PropertyList &properties, PropertyList *results)
Interface for copy progress notification.
static PostMaster * GetPostMaster()
Get default post master.
static Env * GetEnv()
Get default client environment.
const std::string & GetName() const
Get file name.
StatInfo * GetStatInfo()
Get the stat info object.
Iterator End()
Get the end iterator.
Iterator Begin()
Get the begin iterator.
bool PutInt(const std::string &key, int value)
bool PutString(const std::string &key, const std::string &value)
bool GetInt(const std::string &key, int &value)
Send file/filesystem queries to an XRootD cluster.
XRootDStatus DirList(const std::string &path, DirListFlags::Flags flags, ResponseHandler *handler, uint16_t timeout=0) XRD_WARN_UNUSED_RESULT
XRootDStatus Stat(const std::string &path, ResponseHandler *handler, uint16_t timeout=0) XRD_WARN_UNUSED_RESULT
void SetLevel(LogLevel level)
Set the level of the messages that should be sent to the destination.
void Error(uint64_t topic, const char *format,...)
Report an error.
void Dump(uint64_t topic, const char *format,...)
Print a dump message.
void Info(uint64_t topic, const char *format,...)
Print an info.
void Debug(uint64_t topic, const char *format,...)
Print a debug message.
bool Stop()
Stop the postmaster.
A key-value pair map storing both keys and values as strings.
void Set(const std::string &name, const Item &value)
bool Get(const std::string &name, Item &item) const
bool TestFlags(uint32_t flags) const
Test flags.
const std::string & GetPath() const
Get the path.
std::string GetHostId() const
Get the host part of the URL (user:password@host:port)
std::string GetPathWithParams() const
Get the path with params.
std::string GetURL() const
Get the URL.
const std::string & GetProtocol() const
Get the protocol.
static std::string BytesToString(uint64_t bytes)
Convert bytes to a human readable string.
const std::string & GetErrorMessage() const
Get error message.
std::string ToStr() const
Convert to string.
void Config(int argc, char **argv, int Opts=0)
std::vector< std::string > AddCksVal
static const uint64_t DoZipMtlnCksum
static const uint64_t DoNoPbar
static const uint64_t DoCoerce
static const uint64_t DoForce
static const uint64_t DoRmOnBadCksum
static const uint64_t DoNoTlsOK
static const uint64_t DoTpc
static const uint64_t DoCksum
static const uint64_t DoCksrc
static const uint64_t DoTpcDlgt
static const uint64_t DoZip
static const uint64_t DoContinue
static const uint64_t DoRecurse
static const uint64_t DoZipAppend
static const uint64_t DoDynaSrc
static const uint64_t DoSources
static const uint64_t DoXAttr
static const uint64_t DoTlsMLF
static const int optRmtRec
static const uint64_t DoPath
static const uint64_t DoPosc
static const uint64_t DoTpcOnly
static const uint64_t DoTlsNoData
static const uint64_t DoServer
bool IsOK() const
We're fine.
std::string ToString() const
Create a string representation.
uint32_t errNo
Errno, if any.
int GetShellCode() const
Get the status code that may be returned to the shell.