Implement progressbar (by @svartkanin adjusted by @vitalif)

README.md

@@ -1,6 +1,6 @@
 # Grive2 0.5.1-dev
 
-14 Jan 2016, Vitaliy Filippov
+28 Sep 2016, Vitaliy Filippov
 
 http://yourcmc.ru/wiki/Grive2
 
@@ -91,6 +91,7 @@ Grive uses cmake to build. Basic install sequence is
 - added options to limit upload and download speed
 - faster upload of new and changed files. now Grive uploads files without first calculating
   md5 checksum when file is created locally or when its size changes.
+- added -P/--progress-bar option to print ASCII progress bar for each processed file (pull request by @svartkanin)
 
 ### Grive2 v0.5
 

grive/doc/grive.1

@@ -75,6 +75,9 @@ subdirectory. Internally converted to an ignore regexp, remembered for next runs
 \fB\-v\fR, \fB\-\-version\fR
 Displays program version
 .TP
+\fB\-P\fR, \fB\-\-progress-bar\fR
+Print ASCII progress bar for each downloaded/uploaded file.
+.TP
 \fB\-V\fR, \fB\-\-verbose\fR
 Verbose mode. Enables more messages than usual.
 

grive/src/main.cc

@@ -18,6 +18,7 @@
 */
 
 #include "util/Config.hh"
+#include "util/ProgressBar.hh"
 
 #include "base/Drive.hh"
 #include "drive2/Syncer2.hh"
@@ -126,6 +127,7 @@ int Main( int argc, char **argv )
 		( "ignore",		po::value<std::string>(), "Perl RegExp to ignore files (matched against relative paths)." )
 		( "upload-speed,U", po::value<unsigned>(), "Limit upload speed in kbytes per second" )
 		( "download-speed,D", po::value<unsigned>(), "Limit download speed in kbytes per second" )
+		( "progress-bar,P", "Enable progress bar for upload/download of files")
 	;
 	
 	po::variables_map vm;
@@ -156,6 +158,13 @@ int Main( int argc, char **argv )
 	if ( vm.count( "log-http" ) )
 		http->SetLog( new http::ResponseLog( vm["log-http"].as<std::string>(), ".txt" ) );
 
+	std::unique_ptr<ProgressBar> pb;
+	if ( vm.count( "progress-bar" ) )
+	{
+		pb.reset( new ProgressBar() );
+		http->SetProgressReporter( pb.get() );
+	}
+
 	if ( vm.count( "auth" ) )
 	{
 		OAuth2 token( http.get(), client_id, client_secret ) ;
@@ -208,7 +217,13 @@ int Main( int argc, char **argv )
 
 	if ( vm.count( "dry-run" ) == 0 )
 	{
+		// The progress bar should just be enabled when actual file transfers take place
+		if ( pb )
+			pb->setShowProgressBar( true ) ;
 		drive.Update() ;
+		if ( pb )
+			pb->setShowProgressBar( false ) ;
+
 		drive.SaveState() ;
 	}
 	else

libgrive/src/base/Resource.cc

@@ -187,7 +187,7 @@ void Resource::FromRemoteFile( const Entry& remote )
 		{
 			Log( "file %1% is created in remote (change %2%)", path,
 				remote.ChangeStamp(), log::verbose ) ;
-			
+			m_size = remote.Size();
 			m_state = remote_new ;
 		}
 		else
@@ -214,6 +214,7 @@ void Resource::FromRemoteFile( const Entry& remote )
 		if ( remote.MTime().Sec() > m_mtime.Sec() )
 		{
 			Log( "file %1% is changed in remote", path, log::verbose ) ;
+			m_size = remote.Size();
 			m_state = remote_changed ;
 		}
 		

libgrive/src/base/Syncer.cc

@@ -41,7 +41,7 @@ http::Agent* Syncer::Agent() const
 void Syncer::Download( Resource *res, const fs::path& file )
 {
 	http::Download dl( file.string(), http::Download::NoChecksum() ) ;
-	long r = m_http->Get( res->ContentSrc(), &dl, http::Header() ) ;
+	long r = m_http->Get( res->ContentSrc(), &dl, http::Header(), res->Size() ) ;
 	if ( r <= 400 )
 	{
 		if ( res->ServerTime() != DateTime() )

libgrive/src/drive2/Feed2.cc

@@ -42,7 +42,7 @@ bool Feed2::GetNext( http::Agent *http )
 		return false ;
 	
 	http::ValResponse out ;
-	http->Get( m_next, &out, http::Header() ) ;
+	http->Get( m_next, &out, http::Header(), 0 ) ;
 	Val m_content = out.Response() ;
 	
 	Val::Array items = m_content["items"].AsArray() ;

libgrive/src/drive2/Syncer2.cc

@@ -235,7 +235,7 @@ std::unique_ptr<Feed> Syncer2::GetChanges( long min_cstamp )
 long Syncer2::GetChangeStamp( long min_cstamp )
 {
 	http::ValResponse res ;
-	m_http->Get( ChangesFeed( min_cstamp, 1 ), &res, http::Header() ) ;
+	m_http->Get( ChangesFeed( min_cstamp, 1 ), &res, http::Header(), 0 ) ;
 
 	return std::atoi( res.Response()["largestChangeId"].Str().c_str() );
 }

libgrive/src/http/Agent.cc

@@ -52,9 +52,10 @@ long Agent::Put(
 long Agent::Get(
 	const std::string& 	url,
 	DataStream			*dest,
-	const Header&		hdr )
+	const Header&		hdr,
+	u64_t			downloadFileBytes )
 {
-	return Request( "GET", url, NULL, dest, hdr );
+	return Request( "GET", url, NULL, dest, hdr, downloadFileBytes );
 }
 
 long Agent::Post(

libgrive/src/http/Agent.hh

@@ -22,6 +22,7 @@
 #include <string>
 #include "ResponseLog.hh"
 #include "util/Types.hh"
+#include "util/Progress.hh"
 
 namespace gr {
 
@@ -59,7 +60,8 @@ public :
 	virtual long Get(
 		const std::string& 	url,
 		DataStream			*dest,
-		const Header&		hdr ) ;
+		const Header&		hdr,
+		u64_t			downloadFileBytes = 0 ) ;
 	
 	virtual long Post(
 		const std::string& 	url,
@@ -72,7 +74,8 @@ public :
 		const std::string&	url,
 		SeekStream			*in,
 		DataStream			*dest,
-		const Header&		hdr ) = 0 ;
+		const Header&		hdr,
+		u64_t			downloadFileBytes = 0 ) = 0 ;
 	
 	virtual void SetUploadSpeed( unsigned kbytes ) ;
 	virtual void SetDownloadSpeed( unsigned kbytes ) ;
@@ -84,6 +87,8 @@ public :
 	
 	virtual std::string Escape( const std::string& str ) = 0 ;
 	virtual std::string Unescape( const std::string& str ) = 0 ;
+
+	virtual void SetProgressReporter( Progress* ) = 0;
 } ;
 
 } } // end of namespace

libgrive/src/http/CurlAgent.cc

@@ -28,9 +28,6 @@
 
 #include <boost/throw_exception.hpp>
 
-// dependent libraries
-#include <curl/curl.h>
-
 #include <algorithm>
 #include <cassert>
 #include <cstring>
@@ -68,6 +65,7 @@ struct CurlAgent::Impl
 	std::string		error_headers ;
 	std::string		error_data ;
 	DataStream		*dest ;
+	u64_t			total_download, total_upload ;
 } ;
 
 static struct curl_slist* SetHeader( CURL* handle, const Header& hdr );
@@ -94,6 +92,7 @@ void CurlAgent::Init()
 	m_pimpl->error_headers = "";
 	m_pimpl->error_data = "";
 	m_pimpl->dest = NULL;
+	m_pimpl->total_download = m_pimpl->total_upload = 0;
 }
 
 CurlAgent::~CurlAgent()
@@ -111,6 +110,11 @@ void CurlAgent::SetLog(ResponseLog *log)
 	m_log.reset( log );
 }
 
+void CurlAgent::SetProgressReporter(Progress *progress)
+{
+	m_pb = progress;
+}
+
 std::size_t CurlAgent::HeaderCallback( void *ptr, size_t size, size_t nmemb, CurlAgent *pthis )
 {
 	char *str = static_cast<char*>(ptr) ;
@@ -131,7 +135,7 @@ std::size_t CurlAgent::HeaderCallback( void *ptr, size_t size, size_t nmemb, Cur
 	if ( pos != line.npos )
 	{
 		std::size_t end_pos = line.find( "\r\n", pos ) ;
-		pthis->m_pimpl->location = line.substr( loc.size(), end_pos - loc.size() ) ;
+		pthis->m_pimpl->location = line.substr( pos+loc.size(), end_pos - loc.size() ) ;
 	}
 	
 	return size*nmemb ;
@@ -142,6 +146,7 @@ std::size_t CurlAgent::Receive( void* ptr, size_t size, size_t nmemb, CurlAgent
 	assert( pthis != 0 ) ;
 	if ( pthis->m_log.get() )
 		pthis->m_log->Write( (const char*)ptr, size*nmemb );
+
 	if ( pthis->m_pimpl->error && pthis->m_pimpl->error_data.size() < 65536 )
 	{
 		// Do not feed error responses to destination stream
@@ -151,6 +156,19 @@ std::size_t CurlAgent::Receive( void* ptr, size_t size, size_t nmemb, CurlAgent
 	return pthis->m_pimpl->dest->Write( static_cast<char*>(ptr), size * nmemb ) ;
 }
 
+int CurlAgent::progress_callback( CurlAgent *pthis, curl_off_t totalDownload, curl_off_t finishedDownload, curl_off_t totalUpload, curl_off_t finishedUpload )
+{
+	// Only report download progress when set explicitly
+	totalDownload = pthis->m_pimpl->total_download;
+	if ( !totalUpload )
+		totalUpload = pthis->m_pimpl->total_upload;
+	pthis->m_pb->reportProgress(
+		totalDownload > 0 ? totalDownload : totalUpload,
+		totalDownload > 0 ? finishedDownload : finishedUpload
+	);
+	return 0;
+}
+
 long CurlAgent::ExecCurl(
 	const std::string&	url,
 	DataStream			*dest,
@@ -168,6 +186,10 @@ long CurlAgent::ExecCurl(
 
 	struct curl_slist *slist = SetHeader( m_pimpl->curl, hdr ) ;
 
+	curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L);
+	curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, progress_callback);
+	curl_easy_setopt(curl, CURLOPT_XFERINFODATA, this);
+
 	CURLcode curl_code = ::curl_easy_perform(curl);
 
 	curl_slist_free_all(slist);
@@ -202,11 +224,13 @@ long CurlAgent::Request(
 	const std::string&	url,
 	SeekStream			*in,
 	DataStream			*dest,
-	const Header&		hdr )
+	const Header&		hdr,
+	u64_t			downloadFileBytes )
 {
 	Trace("HTTP %1% \"%2%\"", method, url ) ;
 
 	Init() ;
+	m_pimpl->total_download = downloadFileBytes ;
 	CURL *curl = m_pimpl->curl ;
 
 	// set common options

libgrive/src/http/CurlAgent.hh

@@ -24,6 +24,8 @@
 #include <memory>
 #include <string>
 
+#include <curl/curl.h>
+
 namespace gr {
 
 class DataStream ;
@@ -43,13 +45,15 @@ public :
 
 	ResponseLog* GetLog() const ;
 	void SetLog( ResponseLog *log ) ;
+	void SetProgressReporter( Progress *progress ) ;
 
 	long Request(
 		const std::string&	method,
 		const std::string&	url,
 		SeekStream			*in,
 		DataStream			*dest,
-		const Header&		hdr ) ;
+		const Header&		hdr,
+		u64_t			downloadFileBytes = 0 ) ;
 
 	std::string LastError() const ;
 	std::string LastErrorHeaders() const ;
@@ -59,6 +63,8 @@ public :
 	std::string Escape( const std::string& str ) ;
 	std::string Unescape( const std::string& str ) ;
 
+	static int progress_callback( CurlAgent *pthis, curl_off_t totalDownload, curl_off_t finishedDownload, curl_off_t totalUpload, curl_off_t finishedUpload );
+
 private :
 	static std::size_t HeaderCallback( void *ptr, size_t size, size_t nmemb, CurlAgent *pthis ) ;
 	static std::size_t Receive( void* ptr, size_t size, size_t nmemb, CurlAgent *pthis ) ;
@@ -72,8 +78,9 @@ private :
 
 private :
 	struct Impl ;
-	std::unique_ptr<Impl>	m_pimpl ;
-	std::unique_ptr<ResponseLog>	m_log ;
+	std::unique_ptr<Impl> m_pimpl ;
+	std::unique_ptr<ResponseLog> m_log ;
+	Progress* m_pb ;
 } ;
 
 } } // end of namespace

libgrive/src/protocol/AuthAgent.cc

@@ -48,6 +48,11 @@ void AuthAgent::SetLog( http::ResponseLog *log )
 	return m_agent->SetLog( log );
 }
 
+void AuthAgent::SetProgressReporter( Progress *progress )
+{
+	m_agent->SetProgressReporter( progress );
+}
+
 void AuthAgent::SetUploadSpeed( unsigned kbytes )
 {
 	m_agent->SetUploadSpeed( kbytes );
@@ -71,7 +76,8 @@ long AuthAgent::Request(
 	const std::string&	url,
 	SeekStream			*in,
 	DataStream			*dest,
-	const http::Header&	hdr )
+	const http::Header&	hdr,
+	u64_t			downloadFileBytes )
 {
 	long response;
 	Header auth;
@@ -80,7 +86,7 @@ long AuthAgent::Request(
 		auth = AppendHeader( hdr );
 		if ( in )
 			in->Seek( 0, 0 );
-		response = m_agent->Request( method, url, in, dest, auth );
+		response = m_agent->Request( method, url, in, dest, auth, downloadFileBytes );
 	} while ( CheckRetry( response ) );
 	return CheckHttpResponse( response, url, auth );
 }

libgrive/src/protocol/AuthAgent.hh

@@ -44,7 +44,8 @@ public :
 		const std::string&	url,
 		SeekStream			*in,
 		DataStream			*dest,
-		const http::Header&	hdr ) ;
+		const http::Header&	hdr,
+		u64_t			downloadFileBytes = 0 ) ;
 	
 	std::string LastError() const ;
 	std::string LastErrorHeaders() const ;
@@ -57,6 +58,8 @@ public :
 	void SetUploadSpeed( unsigned kbytes ) ;
 	void SetDownloadSpeed( unsigned kbytes ) ;
 
+	void SetProgressReporter( Progress *progress ) ;
+
 private :
 	http::Header AppendHeader( const http::Header& hdr ) const ;
 	bool CheckRetry( long response ) ;

libgrive/src/util/Progress.hh

@@ -0,0 +1,14 @@
+#pragma once
+
+#include "util/Types.hh"
+
+namespace gr {
+
+class Progress
+{
+public:
+	virtual void reportProgress(u64_t total, u64_t processed) = 0;
+};
+
+}
+;

libgrive/src/util/ProgressBar.cc

@@ -0,0 +1,86 @@
+#include <sys/ioctl.h>
+#include <math.h>
+#include <unistd.h>
+#include <stdio.h>
+
+#include "ProgressBar.hh"
+
+namespace gr
+{
+
+ProgressBar::ProgressBar(): showProgressBar(false), last(1000)
+{
+}
+
+ProgressBar::~ProgressBar()
+{
+}
+
+void ProgressBar::setShowProgressBar(bool showProgressBar)
+{
+	this->showProgressBar = showProgressBar;
+}
+
+unsigned short int ProgressBar::determineTerminalSize()
+{
+	struct winsize w;
+	ioctl(STDOUT_FILENO, TIOCGWINSZ, &w);
+	return w.ws_col;
+}
+
+void ProgressBar::printBytes(u64_t bytes)
+{
+	if (bytes >= 1024*1024*1024)
+		printf("%.3f GB", (double)bytes/1024/1024/1024);
+	else if (bytes >= 1024*1024)
+		printf("%.3f MB", (double)bytes/1024/1024);
+	else
+		printf("%.3f KB", (double)bytes/1024);
+}
+
+void ProgressBar::reportProgress(u64_t total, u64_t processed)
+{
+	if (showProgressBar && total)
+	{
+		// libcurl seems to process more bytes then the actual file size :)
+		if (processed > total)
+			processed = total;
+		double fraction = (double)processed/total;
+
+		int point = round(fraction*1000);
+		if (point != this->last)
+		{
+			// only print progress after >= 0.1% change
+			this->last = point;
+
+			// 10 for prefix of percent and 22 for suffix of file size
+			int availableSize = determineTerminalSize() - 32;
+			int totalDots;
+			if (availableSize > 100)
+				totalDots = 100;
+			else if (availableSize < 0)
+				totalDots = 10;
+			else
+				totalDots = availableSize;
+
+			int dotz = round(fraction * totalDots);
+			int count = 0;
+			// delete previous output line
+			printf("\r\33[2K  [%3.0f%%] [", fraction * 100);
+			for (; count < dotz - 1; count++)
+				putchar('=');
+			putchar('>');
+			for (; count < totalDots - 1; count++)
+				putchar(' ');
+			printf("] ");
+			printBytes(processed);
+			putchar('/');
+			printBytes(total);
+			if (point == 1000)
+				putchar('\n');
+			fflush(stdout);
+		}
+	}
+}
+
+}

libgrive/src/util/ProgressBar.hh

@@ -0,0 +1,25 @@
+#pragma once
+
+#include "util/Progress.hh"
+
+namespace gr {
+
+class ProgressBar: public Progress
+{
+public:
+	ProgressBar();
+	virtual ~ProgressBar();
+
+	void reportProgress(u64_t total, u64_t processed);
+	void setShowProgressBar(bool showProgressBar);
+
+private:
+	static void printBytes(u64_t bytes);
+	static unsigned short int determineTerminalSize();
+
+	bool showProgressBar;
+	int last;
+};
+
+}
+;