diff --git a/README.md b/README.md index 89bfec1..2e2ee46 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Grive2 0.5.1-dev -14 Jan 2016, Vitaliy Filippov +28 Sep 2016, Vitaliy Filippov http://yourcmc.ru/wiki/Grive2 @@ -91,6 +91,7 @@ Grive uses cmake to build. Basic install sequence is - added options to limit upload and download speed - faster upload of new and changed files. now Grive uploads files without first calculating md5 checksum when file is created locally or when its size changes. +- added -P/--progress-bar option to print ASCII progress bar for each processed file (pull request by @svartkanin) ### Grive2 v0.5 diff --git a/grive/doc/grive.1 b/grive/doc/grive.1 index 3b0b56a..cf0bc26 100644 --- a/grive/doc/grive.1 +++ b/grive/doc/grive.1 @@ -75,6 +75,9 @@ subdirectory. Internally converted to an ignore regexp, remembered for next runs \fB\-v\fR, \fB\-\-version\fR Displays program version .TP +\fB\-P\fR, \fB\-\-progress-bar\fR +Print ASCII progress bar for each downloaded/uploaded file. +.TP \fB\-V\fR, \fB\-\-verbose\fR Verbose mode. Enables more messages than usual. diff --git a/grive/src/main.cc b/grive/src/main.cc index eef90d3..c87a854 100644 --- a/grive/src/main.cc +++ b/grive/src/main.cc @@ -18,6 +18,7 @@ */ #include "util/Config.hh" +#include "util/ProgressBar.hh" #include "base/Drive.hh" #include "drive2/Syncer2.hh" @@ -126,6 +127,7 @@ int Main( int argc, char **argv ) ( "ignore", po::value(), "Perl RegExp to ignore files (matched against relative paths)." ) ( "upload-speed,U", po::value(), "Limit upload speed in kbytes per second" ) ( "download-speed,D", po::value(), "Limit download speed in kbytes per second" ) + ( "progress-bar,P", "Enable progress bar for upload/download of files") ; po::variables_map vm; @@ -156,6 +158,13 @@ int Main( int argc, char **argv ) if ( vm.count( "log-http" ) ) http->SetLog( new http::ResponseLog( vm["log-http"].as(), ".txt" ) ); + std::unique_ptr pb; + if ( vm.count( "progress-bar" ) ) + { + pb.reset( new ProgressBar() ); + http->SetProgressReporter( pb.get() ); + } + if ( vm.count( "auth" ) ) { OAuth2 token( http.get(), client_id, client_secret ) ; @@ -208,7 +217,13 @@ int Main( int argc, char **argv ) if ( vm.count( "dry-run" ) == 0 ) { + // The progress bar should just be enabled when actual file transfers take place + if ( pb ) + pb->setShowProgressBar( true ) ; drive.Update() ; + if ( pb ) + pb->setShowProgressBar( false ) ; + drive.SaveState() ; } else diff --git a/libgrive/src/base/Resource.cc b/libgrive/src/base/Resource.cc index b03aa5c..23ce335 100644 --- a/libgrive/src/base/Resource.cc +++ b/libgrive/src/base/Resource.cc @@ -187,7 +187,7 @@ void Resource::FromRemoteFile( const Entry& remote ) { Log( "file %1% is created in remote (change %2%)", path, remote.ChangeStamp(), log::verbose ) ; - + m_size = remote.Size(); m_state = remote_new ; } else @@ -214,6 +214,7 @@ void Resource::FromRemoteFile( const Entry& remote ) if ( remote.MTime().Sec() > m_mtime.Sec() ) { Log( "file %1% is changed in remote", path, log::verbose ) ; + m_size = remote.Size(); m_state = remote_changed ; } diff --git a/libgrive/src/base/Syncer.cc b/libgrive/src/base/Syncer.cc index 2cc312c..f68d309 100644 --- a/libgrive/src/base/Syncer.cc +++ b/libgrive/src/base/Syncer.cc @@ -41,7 +41,7 @@ http::Agent* Syncer::Agent() const void Syncer::Download( Resource *res, const fs::path& file ) { http::Download dl( file.string(), http::Download::NoChecksum() ) ; - long r = m_http->Get( res->ContentSrc(), &dl, http::Header() ) ; + long r = m_http->Get( res->ContentSrc(), &dl, http::Header(), res->Size() ) ; if ( r <= 400 ) { if ( res->ServerTime() != DateTime() ) diff --git a/libgrive/src/drive2/Feed2.cc b/libgrive/src/drive2/Feed2.cc index 8a3244e..8f3d234 100644 --- a/libgrive/src/drive2/Feed2.cc +++ b/libgrive/src/drive2/Feed2.cc @@ -42,7 +42,7 @@ bool Feed2::GetNext( http::Agent *http ) return false ; http::ValResponse out ; - http->Get( m_next, &out, http::Header() ) ; + http->Get( m_next, &out, http::Header(), 0 ) ; Val m_content = out.Response() ; Val::Array items = m_content["items"].AsArray() ; diff --git a/libgrive/src/drive2/Syncer2.cc b/libgrive/src/drive2/Syncer2.cc index f7c21d2..fa5e05d 100644 --- a/libgrive/src/drive2/Syncer2.cc +++ b/libgrive/src/drive2/Syncer2.cc @@ -235,7 +235,7 @@ std::unique_ptr Syncer2::GetChanges( long min_cstamp ) long Syncer2::GetChangeStamp( long min_cstamp ) { http::ValResponse res ; - m_http->Get( ChangesFeed( min_cstamp, 1 ), &res, http::Header() ) ; + m_http->Get( ChangesFeed( min_cstamp, 1 ), &res, http::Header(), 0 ) ; return std::atoi( res.Response()["largestChangeId"].Str().c_str() ); } diff --git a/libgrive/src/http/Agent.cc b/libgrive/src/http/Agent.cc index 39276ac..7cfc99f 100644 --- a/libgrive/src/http/Agent.cc +++ b/libgrive/src/http/Agent.cc @@ -52,9 +52,10 @@ long Agent::Put( long Agent::Get( const std::string& url, DataStream *dest, - const Header& hdr ) + const Header& hdr, + u64_t downloadFileBytes ) { - return Request( "GET", url, NULL, dest, hdr ); + return Request( "GET", url, NULL, dest, hdr, downloadFileBytes ); } long Agent::Post( diff --git a/libgrive/src/http/Agent.hh b/libgrive/src/http/Agent.hh index 8268e19..3ca7c11 100644 --- a/libgrive/src/http/Agent.hh +++ b/libgrive/src/http/Agent.hh @@ -22,6 +22,7 @@ #include #include "ResponseLog.hh" #include "util/Types.hh" +#include "util/Progress.hh" namespace gr { @@ -59,7 +60,8 @@ public : virtual long Get( const std::string& url, DataStream *dest, - const Header& hdr ) ; + const Header& hdr, + u64_t downloadFileBytes = 0 ) ; virtual long Post( const std::string& url, @@ -72,7 +74,8 @@ public : const std::string& url, SeekStream *in, DataStream *dest, - const Header& hdr ) = 0 ; + const Header& hdr, + u64_t downloadFileBytes = 0 ) = 0 ; virtual void SetUploadSpeed( unsigned kbytes ) ; virtual void SetDownloadSpeed( unsigned kbytes ) ; @@ -84,6 +87,8 @@ public : virtual std::string Escape( const std::string& str ) = 0 ; virtual std::string Unescape( const std::string& str ) = 0 ; + + virtual void SetProgressReporter( Progress* ) = 0; } ; } } // end of namespace diff --git a/libgrive/src/http/CurlAgent.cc b/libgrive/src/http/CurlAgent.cc index 3bfdfca..122822d 100644 --- a/libgrive/src/http/CurlAgent.cc +++ b/libgrive/src/http/CurlAgent.cc @@ -28,9 +28,6 @@ #include -// dependent libraries -#include - #include #include #include @@ -68,6 +65,7 @@ struct CurlAgent::Impl std::string error_headers ; std::string error_data ; DataStream *dest ; + u64_t total_download, total_upload ; } ; static struct curl_slist* SetHeader( CURL* handle, const Header& hdr ); @@ -94,6 +92,7 @@ void CurlAgent::Init() m_pimpl->error_headers = ""; m_pimpl->error_data = ""; m_pimpl->dest = NULL; + m_pimpl->total_download = m_pimpl->total_upload = 0; } CurlAgent::~CurlAgent() @@ -111,6 +110,11 @@ void CurlAgent::SetLog(ResponseLog *log) m_log.reset( log ); } +void CurlAgent::SetProgressReporter(Progress *progress) +{ + m_pb = progress; +} + std::size_t CurlAgent::HeaderCallback( void *ptr, size_t size, size_t nmemb, CurlAgent *pthis ) { char *str = static_cast(ptr) ; @@ -131,7 +135,7 @@ std::size_t CurlAgent::HeaderCallback( void *ptr, size_t size, size_t nmemb, Cur if ( pos != line.npos ) { std::size_t end_pos = line.find( "\r\n", pos ) ; - pthis->m_pimpl->location = line.substr( loc.size(), end_pos - loc.size() ) ; + pthis->m_pimpl->location = line.substr( pos+loc.size(), end_pos - loc.size() ) ; } return size*nmemb ; @@ -142,6 +146,7 @@ std::size_t CurlAgent::Receive( void* ptr, size_t size, size_t nmemb, CurlAgent assert( pthis != 0 ) ; if ( pthis->m_log.get() ) pthis->m_log->Write( (const char*)ptr, size*nmemb ); + if ( pthis->m_pimpl->error && pthis->m_pimpl->error_data.size() < 65536 ) { // Do not feed error responses to destination stream @@ -151,6 +156,19 @@ std::size_t CurlAgent::Receive( void* ptr, size_t size, size_t nmemb, CurlAgent return pthis->m_pimpl->dest->Write( static_cast(ptr), size * nmemb ) ; } +int CurlAgent::progress_callback( CurlAgent *pthis, curl_off_t totalDownload, curl_off_t finishedDownload, curl_off_t totalUpload, curl_off_t finishedUpload ) +{ + // Only report download progress when set explicitly + totalDownload = pthis->m_pimpl->total_download; + if ( !totalUpload ) + totalUpload = pthis->m_pimpl->total_upload; + pthis->m_pb->reportProgress( + totalDownload > 0 ? totalDownload : totalUpload, + totalDownload > 0 ? finishedDownload : finishedUpload + ); + return 0; +} + long CurlAgent::ExecCurl( const std::string& url, DataStream *dest, @@ -168,6 +186,10 @@ long CurlAgent::ExecCurl( struct curl_slist *slist = SetHeader( m_pimpl->curl, hdr ) ; + curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L); + curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, progress_callback); + curl_easy_setopt(curl, CURLOPT_XFERINFODATA, this); + CURLcode curl_code = ::curl_easy_perform(curl); curl_slist_free_all(slist); @@ -202,11 +224,13 @@ long CurlAgent::Request( const std::string& url, SeekStream *in, DataStream *dest, - const Header& hdr ) + const Header& hdr, + u64_t downloadFileBytes ) { Trace("HTTP %1% \"%2%\"", method, url ) ; Init() ; + m_pimpl->total_download = downloadFileBytes ; CURL *curl = m_pimpl->curl ; // set common options diff --git a/libgrive/src/http/CurlAgent.hh b/libgrive/src/http/CurlAgent.hh index c25fa1a..b659f50 100644 --- a/libgrive/src/http/CurlAgent.hh +++ b/libgrive/src/http/CurlAgent.hh @@ -24,6 +24,8 @@ #include #include +#include + namespace gr { class DataStream ; @@ -43,13 +45,15 @@ public : ResponseLog* GetLog() const ; void SetLog( ResponseLog *log ) ; + void SetProgressReporter( Progress *progress ) ; long Request( const std::string& method, const std::string& url, SeekStream *in, DataStream *dest, - const Header& hdr ) ; + const Header& hdr, + u64_t downloadFileBytes = 0 ) ; std::string LastError() const ; std::string LastErrorHeaders() const ; @@ -59,6 +63,8 @@ public : std::string Escape( const std::string& str ) ; std::string Unescape( const std::string& str ) ; + static int progress_callback( CurlAgent *pthis, curl_off_t totalDownload, curl_off_t finishedDownload, curl_off_t totalUpload, curl_off_t finishedUpload ); + private : static std::size_t HeaderCallback( void *ptr, size_t size, size_t nmemb, CurlAgent *pthis ) ; static std::size_t Receive( void* ptr, size_t size, size_t nmemb, CurlAgent *pthis ) ; @@ -72,8 +78,9 @@ private : private : struct Impl ; - std::unique_ptr m_pimpl ; - std::unique_ptr m_log ; + std::unique_ptr m_pimpl ; + std::unique_ptr m_log ; + Progress* m_pb ; } ; } } // end of namespace diff --git a/libgrive/src/protocol/AuthAgent.cc b/libgrive/src/protocol/AuthAgent.cc index f2f59d6..6baf113 100644 --- a/libgrive/src/protocol/AuthAgent.cc +++ b/libgrive/src/protocol/AuthAgent.cc @@ -48,6 +48,11 @@ void AuthAgent::SetLog( http::ResponseLog *log ) return m_agent->SetLog( log ); } +void AuthAgent::SetProgressReporter( Progress *progress ) +{ + m_agent->SetProgressReporter( progress ); +} + void AuthAgent::SetUploadSpeed( unsigned kbytes ) { m_agent->SetUploadSpeed( kbytes ); @@ -71,7 +76,8 @@ long AuthAgent::Request( const std::string& url, SeekStream *in, DataStream *dest, - const http::Header& hdr ) + const http::Header& hdr, + u64_t downloadFileBytes ) { long response; Header auth; @@ -80,7 +86,7 @@ long AuthAgent::Request( auth = AppendHeader( hdr ); if ( in ) in->Seek( 0, 0 ); - response = m_agent->Request( method, url, in, dest, auth ); + response = m_agent->Request( method, url, in, dest, auth, downloadFileBytes ); } while ( CheckRetry( response ) ); return CheckHttpResponse( response, url, auth ); } diff --git a/libgrive/src/protocol/AuthAgent.hh b/libgrive/src/protocol/AuthAgent.hh index 7f16dc1..328e8cf 100644 --- a/libgrive/src/protocol/AuthAgent.hh +++ b/libgrive/src/protocol/AuthAgent.hh @@ -44,7 +44,8 @@ public : const std::string& url, SeekStream *in, DataStream *dest, - const http::Header& hdr ) ; + const http::Header& hdr, + u64_t downloadFileBytes = 0 ) ; std::string LastError() const ; std::string LastErrorHeaders() const ; @@ -57,6 +58,8 @@ public : void SetUploadSpeed( unsigned kbytes ) ; void SetDownloadSpeed( unsigned kbytes ) ; + void SetProgressReporter( Progress *progress ) ; + private : http::Header AppendHeader( const http::Header& hdr ) const ; bool CheckRetry( long response ) ; diff --git a/libgrive/src/util/Progress.hh b/libgrive/src/util/Progress.hh new file mode 100644 index 0000000..9e087f9 --- /dev/null +++ b/libgrive/src/util/Progress.hh @@ -0,0 +1,14 @@ +#pragma once + +#include "util/Types.hh" + +namespace gr { + +class Progress +{ +public: + virtual void reportProgress(u64_t total, u64_t processed) = 0; +}; + +} +; diff --git a/libgrive/src/util/ProgressBar.cc b/libgrive/src/util/ProgressBar.cc new file mode 100644 index 0000000..9ad8de7 --- /dev/null +++ b/libgrive/src/util/ProgressBar.cc @@ -0,0 +1,86 @@ +#include +#include +#include +#include + +#include "ProgressBar.hh" + +namespace gr +{ + +ProgressBar::ProgressBar(): showProgressBar(false), last(1000) +{ +} + +ProgressBar::~ProgressBar() +{ +} + +void ProgressBar::setShowProgressBar(bool showProgressBar) +{ + this->showProgressBar = showProgressBar; +} + +unsigned short int ProgressBar::determineTerminalSize() +{ + struct winsize w; + ioctl(STDOUT_FILENO, TIOCGWINSZ, &w); + return w.ws_col; +} + +void ProgressBar::printBytes(u64_t bytes) +{ + if (bytes >= 1024*1024*1024) + printf("%.3f GB", (double)bytes/1024/1024/1024); + else if (bytes >= 1024*1024) + printf("%.3f MB", (double)bytes/1024/1024); + else + printf("%.3f KB", (double)bytes/1024); +} + +void ProgressBar::reportProgress(u64_t total, u64_t processed) +{ + if (showProgressBar && total) + { + // libcurl seems to process more bytes then the actual file size :) + if (processed > total) + processed = total; + double fraction = (double)processed/total; + + int point = round(fraction*1000); + if (point != this->last) + { + // only print progress after >= 0.1% change + this->last = point; + + // 10 for prefix of percent and 22 for suffix of file size + int availableSize = determineTerminalSize() - 32; + int totalDots; + if (availableSize > 100) + totalDots = 100; + else if (availableSize < 0) + totalDots = 10; + else + totalDots = availableSize; + + int dotz = round(fraction * totalDots); + int count = 0; + // delete previous output line + printf("\r\33[2K [%3.0f%%] [", fraction * 100); + for (; count < dotz - 1; count++) + putchar('='); + putchar('>'); + for (; count < totalDots - 1; count++) + putchar(' '); + printf("] "); + printBytes(processed); + putchar('/'); + printBytes(total); + if (point == 1000) + putchar('\n'); + fflush(stdout); + } + } +} + +} diff --git a/libgrive/src/util/ProgressBar.hh b/libgrive/src/util/ProgressBar.hh new file mode 100644 index 0000000..5e10eee --- /dev/null +++ b/libgrive/src/util/ProgressBar.hh @@ -0,0 +1,25 @@ +#pragma once + +#include "util/Progress.hh" + +namespace gr { + +class ProgressBar: public Progress +{ +public: + ProgressBar(); + virtual ~ProgressBar(); + + void reportProgress(u64_t total, u64_t processed); + void setShowProgressBar(bool showProgressBar); + +private: + static void printBytes(u64_t bytes); + static unsigned short int determineTerminalSize(); + + bool showProgressBar; + int last; +}; + +} +;