vitalif
/
grive2
mirror of https://github.com/vitalif/grive2
1
0
Fork 0
Code Issues Releases Wiki Activity

Output added for total number of files to be synchronized

progressbar_old
username 2016-08-05 19:29:48 +02:00 committed by Vitaliy Filippov
parent d35c849468
commit c192d530e8
23 changed files with 302 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
grive/src/main.cc
View File

@ -45,6 +45,7 @@
#include <iostream>
#include <unistd.h>
const std::string client_id = "22314510474.apps.googleusercontent.com" ;
const std::string client_secret = "bl4ufi89h-9MkFlypcI7R785" ;
@ -126,6 +127,7 @@ int Main( int argc, char **argv )
( "ignore", po::value<std::string>(), "Perl RegExp to ignore files (matched against relative paths)." )
( "upload-speed,U", po::value<unsigned>(), "Limit upload speed in kbytes per second" )
( "download-speed,D", po::value<unsigned>(), "Limit download speed in kbytes per second" )
( "progress-bar", "Enable progress bar for upload/download of files")
;
po::variables_map vm;
@ -149,13 +151,16 @@ int Main( int argc, char **argv )
InitLog(vm) ;
Config config(vm) ;
Log( "config file name %1%", config.Filename(), log::verbose );
std::unique_ptr<http::Agent> http( new http::CurlAgent );
if ( vm.count( "log-http" ) )
http->SetLog( new http::ResponseLog( vm["log-http"].as<std::string>(), ".txt" ) );
ProgressBar *pb = new ProgressBar(vm.count( "progress-bar" ) != 0);
http->SetProgressBar(pb);
if ( vm.count( "auth" ) )
{
OAuth2 token( http.get(), client_id, client_secret ) ;
@ -178,6 +183,7 @@ int Main( int argc, char **argv )
config.Set( "refresh_token", Val( token.RefreshToken() ) ) ;
config.Save() ;
}
std::string refresh_token ;
try
@ -206,14 +212,21 @@ int Main( int argc, char **argv )
Drive drive( &syncer, config.GetAll() ) ;
drive.DetectChanges() ;
Log( "%1% total changes", drive.getTotalChanges(), log::info ) ;
if ( vm.count( "dry-run" ) == 0 )
{
//The progress bar should just be enabled when actual file transfers take place
pb->SetShowProgressBar(true);
drive.Update() ;
pb->SetShowProgressBar(false);
drive.UpdateChangeStamp();
drive.SaveState() ;
}
else
drive.DryRun() ;
config.Save() ;
Log( "Finished!", log::info ) ;
return 0 ;

7
libgrive/src/base/Drive.cc
View File

@ -92,6 +92,11 @@ void Drive::DetectChanges()
m_state.ResolveEntry() ;
}
size_t Drive::getTotalChanges() {
return m_state.getTotalChanges();
}
// pull the changes feed
// FIXME: unused until Grive will use the feed-based sync instead of reading full tree
void Drive::ReadChanges()
@ -115,8 +120,6 @@ void Drive::Update()
{
Log( "Synchronizing files", log::info ) ;
m_state.Sync( m_syncer, m_options ) ;
UpdateChangeStamp( ) ;
}
void Drive::DryRun()

6
libgrive/src/base/Drive.hh
View File

@ -44,15 +44,17 @@ public :
void Update() ;
void DryRun() ;
void SaveState() ;
void UpdateChangeStamp() ;
struct Error : virtual Exception {} ;
size_t getTotalChanges();
private :
void ReadChanges() ;
void FromRemote( const Entry& entry ) ;
void FromChange( const Entry& entry ) ;
void UpdateChangeStamp( ) ;
private :
Syncer *m_syncer ;
fs::path m_root ;

5
libgrive/src/base/Entry.cc
View File

@ -120,4 +120,9 @@ std::string Entry::Name() const
return !m_filename.empty() ? m_filename : m_title ;
}
long Entry::Size() const
{
return m_size ;
}
} // end of namespace gr

3
libgrive/src/base/Entry.hh
View File

@ -61,6 +61,8 @@ public :
const std::vector<std::string>& ParentHrefs() const ;
long Size() const ;
protected :
std::string m_title ;
std::string m_filename ;
@ -80,6 +82,7 @@ protected :
DateTime m_mtime ;
bool m_is_removed ;
long m_size ;
} ;
} // end of namespace gr

6
libgrive/src/base/Resource.cc
View File

@ -149,6 +149,7 @@ void Resource::AssignIDs( const Entry& remote )
m_content = remote.ContentSrc() ;
m_is_editable = remote.IsEditable() ;
m_etag = remote.ETag() ;
m_downloadFileBytes = remote.Size() ;
}
}
@ -747,4 +748,9 @@ bool Resource::HasID() const
return !m_href.empty() && !m_id.empty() ;
}
long Resource::DownloadFileBytes() const
{
return m_downloadFileBytes;
}
} // end of namespace

2
libgrive/src/base/Resource.hh
View File

@ -97,6 +97,7 @@ public :
std::string ETag() const ;
std::string ResourceID() const ;
State GetState() const;
long DownloadFileBytes() const;
const Resource* Parent() const ;
Resource* Parent() ;
@ -163,6 +164,7 @@ private :
State m_state ;
Val* m_json ;
bool m_local_exists ;
long m_downloadFileBytes;
} ;
} // end of namespace gr::v1

5
libgrive/src/base/State.cc
View File

@ -81,6 +81,10 @@ void State::FromLocal( const fs::path& p )
FromLocal( p, m_res.Root(), m_st.Item( "tree" ) ) ;
}
size_t State::getTotalChanges() {
return m_res.Root()->size();
}
bool State::IsIgnore( const std::string& filename )
{
return regex_search( filename.c_str(), m_ign_re );
@ -205,6 +209,7 @@ void State::FromChange( const Entry& e )
m_res.Update( res, e ) ;
}
bool State::Update( const Entry& e )
{
assert( !e.IsChange() ) ;

2
libgrive/src/base/State.hh
View File

@ -63,6 +63,8 @@ public :
long ChangeStamp() const ;
void ChangeStamp( long cstamp ) ;
size_t getTotalChanges();
private :
void FromLocal( const fs::path& p, Resource *folder, Val& tree ) ;
void FromChange( const Entry& e ) ;

2
libgrive/src/base/Syncer.cc
View File

@ -41,7 +41,7 @@ http::Agent* Syncer::Agent() const
void Syncer::Download( Resource *res, const fs::path& file )
{
http::Download dl( file.string(), http::Download::NoChecksum() ) ;
long r = m_http->Get( res->ContentSrc(), &dl, http::Header() ) ;
long r = m_http->Get( res->ContentSrc(), &dl, http::Header(), res->DownloadFileBytes() ) ;
if ( r <= 400 )
{
if ( res->ServerTime() != DateTime() )

2
libgrive/src/drive/Feed1.cc
View File

@ -45,7 +45,7 @@ bool Feed1::GetNext( http::Agent *http )
if ( m_next.empty() )
return false;
http->Get( m_next, &xrsp, http::Header() ) ;
http->Get( m_next, &xrsp, http::Header(), 0 ) ;
xml::Node m_root = xrsp.Response() ;
xml::NodeSet xe = m_root["entry"] ;

6
libgrive/src/drive/Syncer1.cc
View File

@ -72,10 +72,10 @@ void Syncer1::DeleteRemote( Resource *res )
// don't know why, but an update before deleting seems to work always
http::XmlResponse xml ;
m_http->Get( res->SelfHref(), &xml, hdr ) ;
m_http->Get( res->SelfHref(), &xml, hdr, res->DownloadFileBytes() ) ;
AssignIDs( res, Entry1( xml.Response() ) ) ;
m_http->Request( "DELETE", res->SelfHref(), NULL, &str, hdr ) ;
m_http->Request( "DELETE", res->SelfHref(), NULL, &str, hdr, 0 ) ;
}
catch ( Exception& e )
{
@ -263,7 +263,7 @@ std::unique_ptr<Feed> Syncer1::GetChanges( long min_cstamp )
long Syncer1::GetChangeStamp( long min_cstamp )
{
http::XmlResponse xrsp ;
m_http->Get( ChangesFeed( min_cstamp ), &xrsp, http::Header() ) ;
m_http->Get( ChangesFeed( min_cstamp ), &xrsp, http::Header(), 0) ;
return std::atoi( xrsp.Response()["docs:largestChangestamp"]["@value"].front().Value().c_str() );
}

9
libgrive/src/drive2/Entry2.cc
View File

@ -64,6 +64,15 @@ void Entry2::Update( const Val& item )
m_is_dir = file["mimeType"].Str() == mime_types::folder ;
m_is_editable = file["editable"].Bool() ;
m_is_removed = file["labels"]["trashed"].Bool() ;
if (file.Has("fileSize"))
{
m_size = file["fileSize"].U64() ;
} else
{
m_size = 0;
}
if ( !m_is_dir )
{
if ( !file.Has( "md5Checksum" ) || !file.Has("downloadUrl") )

2
libgrive/src/drive2/Feed2.cc
View File

@ -42,7 +42,7 @@ bool Feed2::GetNext( http::Agent *http )
return false ;
http::ValResponse out ;
http->Get( m_next, &out, http::Header() ) ;
http->Get( m_next, &out, http::Header(), 0 ) ;
Val m_content = out.Response() ;
Val::Array items = m_content["items"].AsArray() ;

4
libgrive/src/drive2/Syncer2.cc
View File

@ -140,6 +140,8 @@ std::string to_string( uint64_t n )
return s.str();
}
bool Syncer2::Upload( Resource *res, bool new_rev )
{
Val meta;
@ -235,7 +237,7 @@ std::unique_ptr<Feed> Syncer2::GetChanges( long min_cstamp )
long Syncer2::GetChangeStamp( long min_cstamp )
{
http::ValResponse res ;
m_http->Get( ChangesFeed( min_cstamp, 1 ), &res, http::Header() ) ;
m_http->Get( ChangesFeed( min_cstamp, 1 ), &res, http::Header(), 0 ) ;
return std::atoi( res.Response()["largestChangeId"].Str().c_str() );
}

12
libgrive/src/http/Agent.cc
View File

@ -20,6 +20,7 @@
#include "Agent.hh"
#include "Header.hh"
#include "util/StringStream.hh"
#include <iostream>
namespace gr {
@ -37,7 +38,7 @@ long Agent::Put(
const Header& hdr )
{
StringStream s( data );
return Request( "PUT", url, &s, dest, hdr );
return Request( "PUT", url, &s, dest, hdr);
}
long Agent::Put(
@ -46,15 +47,16 @@ long Agent::Put(
DataStream *dest,
const Header& hdr )
{
return Request( "PUT", url, (SeekStream*)file, dest, hdr );
return Request( "PUT", url, (SeekStream*)file, dest, hdr);
}
long Agent::Get(
const std::string& url,
DataStream *dest,
const Header& hdr )
const Header& hdr,
const long downloadFileBytes)
{
return Request( "GET", url, NULL, dest, hdr );
return Request( "GET", url, NULL, dest, hdr, downloadFileBytes);
}
long Agent::Post(
@ -66,7 +68,7 @@ long Agent::Post(
Header h( hdr ) ;
StringStream s( data );
h.Add( "Content-Type: application/x-www-form-urlencoded" );
return Request( "POST", url, &s, dest, h );
return Request( "POST", url, &s, dest, h);
}
void Agent::SetUploadSpeed( unsigned kbytes )

9
libgrive/src/http/Agent.hh
View File

@ -22,6 +22,7 @@
#include <string>
#include "ResponseLog.hh"
#include "util/Types.hh"
#include "util/ProgressBar.hh"
namespace gr {
@ -59,7 +60,8 @@ public :
virtual long Get(
const std::string& url,
DataStream *dest,
const Header& hdr ) ;
const Header& hdr,
const long downloadFileBytes = 0) ;
virtual long Post(
const std::string& url,
@ -72,7 +74,8 @@ public :
const std::string& url,
SeekStream *in,
DataStream *dest,
const Header& hdr ) = 0 ;
const Header& hdr,
const long downloadFileBytes = 0) = 0 ;
virtual void SetUploadSpeed( unsigned kbytes ) ;
virtual void SetDownloadSpeed( unsigned kbytes ) ;
@ -84,6 +87,8 @@ public :
virtual std::string Escape( const std::string& str ) = 0 ;
virtual std::string Unescape( const std::string& str ) = 0 ;
virtual void SetProgressBar( ProgressBar* ) = 0;
} ;
} } // end of namespace

42
libgrive/src/http/CurlAgent.cc
View File

@ -27,9 +27,8 @@
#include "util/File.hh"
#include <boost/throw_exception.hpp>
// dependent libraries
#include <curl/curl.h>
#include <boost/algorithm/string/predicate.hpp>
#include <boost/lexical_cast.hpp>
#include <algorithm>
#include <cassert>
@ -111,11 +110,16 @@ void CurlAgent::SetLog(ResponseLog *log)
m_log.reset( log );
}
void CurlAgent::SetProgressBar(ProgressBar *progressbar)
{
m_pb.reset(progressbar);
}
std::size_t CurlAgent::HeaderCallback( void *ptr, size_t size, size_t nmemb, CurlAgent *pthis )
{
char *str = static_cast<char*>(ptr) ;
std::string line( str, str + size*nmemb ) ;
// Check for error (HTTP 400 and above)
if ( line.substr( 0, 5 ) == "HTTP/" && line[9] >= '4' )
pthis->m_pimpl->error = true;
@ -125,7 +129,7 @@ std::size_t CurlAgent::HeaderCallback( void *ptr, size_t size, size_t nmemb, Cur
if ( pthis->m_log.get() )
pthis->m_log->Write( str, size*nmemb );
static const std::string loc = "Location: " ;
std::size_t pos = line.find( loc ) ;
if ( pos != line.npos )
@ -133,15 +137,23 @@ std::size_t CurlAgent::HeaderCallback( void *ptr, size_t size, size_t nmemb, Cur
std::size_t end_pos = line.find( "\r\n", pos ) ;
pthis->m_pimpl->location = line.substr( loc.size(), end_pos - loc.size() ) ;
}
return size*nmemb ;
}
std::size_t CurlAgent::Receive( void* ptr, size_t size, size_t nmemb, CurlAgent *pthis )
{
assert( pthis != 0 ) ;
if ( pthis->m_log.get() )
pthis->m_log->Write( (const char*)ptr, size*nmemb );
if ( pthis->totalDownloadSize > 0 )
{
pthis->downloadedBytes += (curl_off_t)size*nmemb;
CurlAgent::progress_callback(pthis, pthis->totalDownloadSize, pthis->downloadedBytes, 0L, 0L);
}
if ( pthis->m_pimpl->error && pthis->m_pimpl->error_data.size() < 65536 )
{
// Do not feed error responses to destination stream
@ -151,6 +163,14 @@ std::size_t CurlAgent::Receive( void* ptr, size_t size, size_t nmemb, CurlAgent
return pthis->m_pimpl->dest->Write( static_cast<char*>(ptr), size * nmemb ) ;
}
int CurlAgent::progress_callback(void *ptr, curl_off_t totalDownloadSize, curl_off_t finishedDownloadSize, curl_off_t totalToUpload, curl_off_t finishedUploaded)
{
((CurlAgent*)ptr)->m_pb->PrintProgressBar(totalDownloadSize, finishedDownloadSize, totalToUpload, finishedUploaded);
return 0;
}
long CurlAgent::ExecCurl(
const std::string& url,
DataStream *dest,
@ -168,6 +188,10 @@ long CurlAgent::ExecCurl(
struct curl_slist *slist = SetHeader( m_pimpl->curl, hdr ) ;
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L);
curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, progress_callback);
curl_easy_setopt(curl, CURLOPT_XFERINFODATA, this);
CURLcode curl_code = ::curl_easy_perform(curl);
curl_slist_free_all(slist);
@ -202,13 +226,17 @@ long CurlAgent::Request(
const std::string& url,
SeekStream *in,
DataStream *dest,
const Header& hdr )
const Header& hdr,
const long downloadFileBytes)
{
Trace("HTTP %1% \"%2%\"", method, url ) ;
Init() ;
CURL *curl = m_pimpl->curl ;
this->totalDownloadSize = downloadFileBytes;
// set common options
::curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, method.c_str() );
if ( in )

17
libgrive/src/http/CurlAgent.hh
View File

@ -24,6 +24,9 @@
#include <memory>
#include <string>
// dependent libraries
#include <curl/curl.h>
namespace gr {
class DataStream ;
@ -43,13 +46,15 @@ public :
ResponseLog* GetLog() const ;
void SetLog( ResponseLog *log ) ;
void SetProgressBar( ProgressBar *progressbar) ;
long Request(
const std::string& method,
const std::string& url,
SeekStream *in,
DataStream *dest,
const Header& hdr ) ;
const Header& hdr,
const long downloadFileBytes = 0) ;
std::string LastError() const ;
std::string LastErrorHeaders() const ;
@ -59,6 +64,8 @@ public :
std::string Escape( const std::string& str ) ;
std::string Unescape( const std::string& str ) ;
static int progress_callback(void *ptr, curl_off_t TotalDownloadSize, curl_off_t finishedDownloadSize, curl_off_t TotalToUpload, curl_off_t NowUploaded);
private :
static std::size_t HeaderCallback( void *ptr, size_t size, size_t nmemb, CurlAgent *pthis ) ;
static std::size_t Receive( void* ptr, size_t size, size_t nmemb, CurlAgent *pthis ) ;
@ -72,8 +79,12 @@ private :
private :
struct Impl ;
std::unique_ptr<Impl> m_pimpl ;
std::unique_ptr<ResponseLog> m_log ;
std::unique_ptr<Impl> m_pimpl ;
std::unique_ptr<ResponseLog> m_log ;
std::unique_ptr<ProgressBar> m_pb ;
long totalDownloadSize;
long downloadedBytes;
} ;
} } // end of namespace

10
libgrive/src/protocol/AuthAgent.cc
View File

@ -48,6 +48,11 @@ void AuthAgent::SetLog( http::ResponseLog *log )
return m_agent->SetLog( log );
}
void AuthAgent::SetProgressBar( ProgressBar *progressbar )
{
return m_agent->SetProgressBar( progressbar );
}
void AuthAgent::SetUploadSpeed( unsigned kbytes )
{
m_agent->SetUploadSpeed( kbytes );
@ -71,7 +76,8 @@ long AuthAgent::Request(
const std::string& url,
SeekStream *in,
DataStream *dest,
const http::Header& hdr )
const http::Header& hdr,
const long downloadFileBytes)
{
long response;
Header auth;
@ -80,7 +86,7 @@ long AuthAgent::Request(
auth = AppendHeader( hdr );
if ( in )
in->Seek( 0, 0 );
response = m_agent->Request( method, url, in, dest, auth );
response = m_agent->Request( method, url, in, dest, auth, downloadFileBytes );
} while ( CheckRetry( response ) );
return CheckHttpResponse( response, url, auth );
}

5
libgrive/src/protocol/AuthAgent.hh
View File

@ -44,7 +44,8 @@ public :
const std::string& url,
SeekStream *in,
DataStream *dest,
const http::Header& hdr ) ;
const http::Header& hdr,
const long downloadFileBytes = 0) ;
std::string LastError() const ;
std::string LastErrorHeaders() const ;
@ -57,6 +58,8 @@ public :
void SetUploadSpeed( unsigned kbytes ) ;
void SetDownloadSpeed( unsigned kbytes ) ;
void SetProgressBar( ProgressBar *progressbar ) ;
private :
http::Header AppendHeader( const http::Header& hdr ) const ;
bool CheckRetry( long response ) ;

133
libgrive/src/util/ProgressBar.cc Normal file
View File

@ -0,0 +1,133 @@
#include "ProgressBar.hh"
#include <sys/ioctl.h>
#include <math.h>
#include <unistd.h>
#include <sstream>
#include <iomanip>
#include <iostream>
namespace gr
{
ProgressBar::ProgressBar(bool enable_parameter)
{
this->progressBar_enabled = enable_parameter;
}
ProgressBar::~ProgressBar()
{
}
void ProgressBar::SetShowProgressBar(bool showProgressBar)
{
this->showProgressBar = showProgressBar;
}
unsigned short int ProgressBar::DetermineTerminalSize()
{
struct winsize w;
ioctl(STDOUT_FILENO, TIOCGWINSZ, &w);
return w.ws_col;
}
std::string ProgressBar::CalculateByteSize(long bytes)
{
long double KB = bytes / 1024;
long double MB = KB / 1024;
long double GB = MB / 1024;
std::string res;
std::string suffix;
std::ostringstream ss;
ss << std::fixed << std::setprecision(2);
if (GB > 1)
{
ss << GB;
suffix = "GB";
}
else if (MB > 1)
{
ss << MB;
suffix = "MB";
}
else
{
ss << KB;
suffix = "KB";
}
res = ss.str() + suffix;
return res;
}
void ProgressBar::PrintProgressBar(long TotalDownloadSize, long finishedDownloadSize, long TotalToUpload, long NowUploaded)
{
if (this->showProgressBar && this->progressBar_enabled)
{
long processed = (TotalDownloadSize > TotalToUpload) ? finishedDownloadSize : NowUploaded;
long total = (TotalDownloadSize > TotalToUpload) ? TotalDownloadSize : TotalToUpload;
if (total <= 0.0)
return;
//libcurl seems to process more bytes then the actual file size :)
if (processed > total)
processed = total;
int availableSize = this->DetermineTerminalSize() - 32; //10 for prefix of percent and 22 for suffix of file size
int totalDots;
if (availableSize > 100)
totalDots = 100;
else if (availableSize < 0)
totalDots = 10;
else
totalDots = availableSize;
double fraction = (float) processed / total;
if ((fraction * 100) < 100.0)
this->hundredpercentDone = false;
if (!this->hundredpercentDone)
{
printf("\33[2K\r"); //delete previous output line
int dotz = round(fraction * totalDots);
int count = 0;
printf(" [%3.0f%%] [", fraction * 100);
for (; count < dotz - 1; count++)
{
printf("=");
}
printf(">");
for (; count < totalDots - 1; count++)
{
printf(" ");
}
printf("] ");
printf("%s/%s", this->CalculateByteSize(processed).c_str(), this->CalculateByteSize(total).c_str());
printf("\r");
if ((fraction * 100) >= 100.0)
{
this->hundredpercentDone = true;
printf("\n");
}
fflush(stdout);
}
}
}
}

29
libgrive/src/util/ProgressBar.hh Normal file
View File

@ -0,0 +1,29 @@
#include <string>
#pragma once
namespace gr
{
class ProgressBar
{
public:
ProgressBar(bool enable_parameter);
virtual ~ProgressBar();
void PrintProgressBar(long TotalDownloadSize, long finishedDownloadSize, long TotalToUpload, long NowUploaded);
void SetShowProgressBar(bool showProgressBar);
private:
static std::string CalculateByteSize(long bytes);
static unsigned short int DetermineTerminalSize();
bool hundredpercentDone = false;
bool showProgressBar = false;
bool progressBar_enabled = false;
};
}
;