vitalif
/
grive2
mirror of https://github.com/vitalif/grive2
1
0
Fork 0
Code Issues Releases Wiki Activity

Split Resource to global (base/Resource) and API-specific ({drive,drive2}/Syncer) parts

pull/40/head
Vitaliy Filippov 2015-05-17 02:00:00 +03:00
parent f41e4f6dd3
commit 6a15dd09a5
15 changed files with 526 additions and 719 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

287
libgrive/src/drive/Resource.cc → libgrive/src/base/Resource.cc
View File

@ -18,55 +18,33 @@
*/
#include "Resource.hh"
#include "CommonUri.hh"
#include "Entry1.hh"
#include "Entry.hh"
#include "Syncer.hh"
#include "http/Agent.hh"
#include "http/Download.hh"
#include "http/Header.hh"
// #include "http/ResponseLog.hh"
#include "http/StringResponse.hh"
#include "http/XmlResponse.hh"
#include "json/Val.hh"
#include "util/CArray.hh"
#include "util/Crypt.hh"
#include "util/log/Log.hh"
#include "util/OS.hh"
#include "util/File.hh"
#include "xml/Node.hh"
#include "xml/NodeSet.hh"
#include "xml/String.hh"
#include "xml/TreeBuilder.hh"
#include <boost/bind.hpp>
#include <boost/exception/all.hpp>
#include <cassert>
// for debugging
#include <iostream>
namespace gr { namespace v1 {
// hard coded XML file
const std::string xml_meta =
"<?xml version='1.0' encoding='UTF-8'?>\n"
"<entry xmlns=\"http://www.w3.org/2005/Atom\" xmlns:docs=\"http://schemas.google.com/docs/2007\">"
"<category scheme=\"http://schemas.google.com/g/2005#kind\" "
"term=\"http://schemas.google.com/docs/2007#%1%\"/>"
"<title>%2%</title>"
"</entry>" ;
namespace gr {
/// default constructor creates the root folder
Resource::Resource(const fs::path& root_folder) :
m_name ( root_folder.string() ),
m_kind ( "folder" ),
m_id ( "folder:root" ),
m_href ( root_href ),
m_create ( root_create ),
m_parent ( 0 ),
m_state ( sync )
m_state ( sync ),
m_is_editable( true )
{
}
@ -74,7 +52,8 @@ Resource::Resource( const std::string& name, const std::string& kind ) :
m_name ( name ),
m_kind ( kind ),
m_parent ( 0 ),
m_state ( unknown )
m_state ( unknown ),
m_is_editable( true )
{
}
@ -166,9 +145,8 @@ void Resource::AssignIDs( const Entry& remote )
{
m_id = remote.ResourceID() ;
m_href = remote.SelfHref() ;
m_edit = remote.IsEditable() ? feed_base + "/" + remote.ResourceID() : "";
m_create = remote.IsEditable() && remote.IsDir() ? root_create + (remote.ResourceID() == "folder:root" ? "" : "/" + remote.ResourceID() + "/contents") : "";
m_content = remote.ContentSrc() ;
m_is_editable = remote.IsEditable() ;
m_etag = remote.ETag() ;
}
}
@ -283,16 +261,41 @@ std::string Resource::SelfHref() const
return m_href ;
}
std::string Resource::ContentSrc() const
{
return m_content ;
}
std::string Resource::ETag() const
{
return m_etag ;
}
std::string Resource::Name() const
{
return m_name ;
}
std::string Resource::Kind() const
{
return m_kind ;
}
DateTime Resource::MTime() const
{
return m_mtime ;
}
std::string Resource::ResourceID() const
{
return m_id ;
}
Resource::State Resource::GetState() const
{
return m_state ;
}
const Resource* Resource::Parent() const
{
assert( m_parent == 0 || m_parent->IsFolder() ) ;
@ -320,6 +323,11 @@ bool Resource::IsFolder() const
return m_kind == "folder" ;
}
bool Resource::IsEditable() const
{
return m_is_editable ;
}
fs::path Resource::Path() const
{
assert( m_parent != this ) ;
@ -331,7 +339,7 @@ fs::path Resource::Path() const
bool Resource::IsInRootTree() const
{
assert( m_parent == 0 || m_parent->IsFolder() ) ;
return m_parent == 0 ? (SelfHref() == root_href) : m_parent->IsInRootTree() ;
return m_parent == 0 ? SelfHref().empty() : m_parent->IsInRootTree() ;
}
Resource* Resource::FindChild( const std::string& name )
@ -346,12 +354,12 @@ Resource* Resource::FindChild( const std::string& name )
}
// try to change the state to "sync"
void Resource::Sync( http::Agent *http, DateTime& sync_time, const Val& options )
void Resource::Sync( Syncer *syncer, DateTime& sync_time, const Val& options )
{
assert( m_state != unknown ) ;
assert( !IsRoot() || m_state == sync ) ; // root folder is already synced
SyncSelf( http, options ) ;
SyncSelf( syncer, options ) ;
// we want the server sync time, so we will take the server time of the last file uploaded to store as the sync time
// m_mtime is updated to server modified time when the file is uploaded
@ -360,13 +368,13 @@ void Resource::Sync( http::Agent *http, DateTime& sync_time, const Val& options
// if myself is deleted, no need to do the childrens
if ( m_state != local_deleted && m_state != remote_deleted )
std::for_each( m_child.begin(), m_child.end(),
boost::bind( &Resource::Sync, _1, http, boost::ref(sync_time), options ) ) ;
boost::bind( &Resource::Sync, _1, syncer, boost::ref(sync_time), options ) ) ;
}
void Resource::SyncSelf( http::Agent* http, const Val& options )
void Resource::SyncSelf( Syncer* syncer, const Val& options )
{
assert( !IsRoot() || m_state == sync ) ; // root is always sync
assert( IsRoot() || http == 0 || m_parent->IsFolder() ) ;
assert( IsRoot() || !syncer || m_parent->IsFolder() ) ;
assert( IsRoot() || m_parent->m_state != remote_deleted ) ;
assert( IsRoot() || m_parent->m_state != local_deleted ) ;
@ -377,30 +385,30 @@ void Resource::SyncSelf( http::Agent* http, const Val& options )
case local_new :
Log( "sync %1% doesn't exist in server, uploading", path, log::info ) ;
if ( http != 0 && Create( http ) )
if ( syncer && syncer->Create( this ) )
m_state = sync ;
break ;
case local_deleted :
Log( "sync %1% deleted in local. deleting remote", path, log::info ) ;
if ( http != 0 )
DeleteRemote( http ) ;
if ( syncer )
syncer->DeleteRemote( this ) ;
break ;
case local_changed :
Log( "sync %1% changed in local. uploading", path, log::info ) ;
if ( http != 0 && EditContent( http, options["new-rev"].Bool() ) )
if ( syncer && syncer->EditContent( this, options["new-rev"].Bool() ) )
m_state = sync ;
break ;
case remote_new :
Log( "sync %1% created in remote. creating local", path, log::info ) ;
if ( http != 0 )
if ( syncer )
{
if ( IsFolder() )
fs::create_directories( path ) ;
else
Download( http, path ) ;
syncer->Download( this, path ) ;
m_state = sync ;
}
@ -409,16 +417,16 @@ void Resource::SyncSelf( http::Agent* http, const Val& options )
case remote_changed :
assert( !IsFolder() ) ;
Log( "sync %1% changed in remote. downloading", path, log::info ) ;
if ( http != 0 )
if ( syncer )
{
Download( http, path ) ;
syncer->Download( this, path ) ;
m_state = sync ;
}
break ;
case remote_deleted :
Log( "sync %1% deleted in remote. deleting local", path, log::info ) ;
if ( http != 0 )
if ( syncer )
DeleteLocal() ;
break ;
@ -459,193 +467,6 @@ void Resource::DeleteLocal()
}
}
void Resource::DeleteRemote( http::Agent *http )
{
assert( http != 0 ) ;
http::StringResponse str ;
try
{
http::Header hdr ;
hdr.Add( "If-Match: " + m_etag ) ;
// doesn't know why, but an update before deleting seems to work always
http::XmlResponse xml ;
http->Get( m_href, &xml, hdr ) ;
AssignIDs( Entry1( xml.Response() ) ) ;
http->Custom( "DELETE", m_href, &str, hdr ) ;
}
catch ( Exception& e )
{
// don't rethrow here. there are some cases that I don't know why
// the delete will fail.
Trace( "Exception %1% %2%",
boost::diagnostic_information(e),
str.Response() ) ;
}
}
void Resource::Download( http::Agent* http, const fs::path& file ) const
{
assert( http != 0 ) ;
http::Download dl( file.string(), http::Download::NoChecksum() ) ;
long r = http->Get( m_content, &dl, http::Header() ) ;
if ( r <= 400 )
{
if ( m_mtime != DateTime() )
os::SetFileTime( file, m_mtime ) ;
else
Log( "encountered zero date time after downloading %1%", file, log::warning ) ;
}
}
bool Resource::EditContent( http::Agent* http, bool new_rev )
{
assert( http != 0 ) ;
assert( m_parent != 0 ) ;
assert( m_parent->m_state == sync ) ;
// upload link missing means that file is read only
if ( m_edit.empty() )
{
Log( "Cannot upload %1%: file read-only. %2%", m_name, m_state, log::warning ) ;
return false ;
}
return Upload( http, m_edit + (new_rev ? "?new-revision=true" : ""), false ) ;
}
bool Resource::Create( http::Agent* http )
{
assert( http != 0 ) ;
assert( m_parent != 0 ) ;
assert( m_parent->IsFolder() ) ;
assert( m_parent->m_state == sync ) ;
if ( IsFolder() )
{
std::string uri = feed_base ;
if ( !m_parent->IsRoot() )
uri += ("/" + http->Escape(m_parent->m_id) + "/contents") ;
std::string meta = (boost::format( xml_meta )
% "folder"
% xml::Escape(m_name)
).str() ;
http::Header hdr ;
hdr.Add( "Content-Type: application/atom+xml" ) ;
http::XmlResponse xml ;
// http::ResponseLog log( "create", ".xml", &xml ) ;
http->Post( uri, meta, &xml, hdr ) ;
AssignIDs( Entry1( xml.Response() ) ) ;
return true ;
}
else if ( !m_parent->m_create.empty() )
{
return Upload( http, m_parent->m_create + "?convert=false", true ) ;
}
else
{
Log( "parent of %1% does not exist: cannot upload", Name(), log::warning ) ;
return false ;
}
}
bool Resource::Upload(
http::Agent* http,
const std::string& link,
bool post)
{
assert( http != 0 ) ;
File file( Path() ) ;
std::ostringstream xcontent_len ;
xcontent_len << "X-Upload-Content-Length: " << file.Size() ;
http::Header hdr ;
hdr.Add( "Content-Type: application/atom+xml" ) ;
hdr.Add( "X-Upload-Content-Type: application/octet-stream" ) ;
hdr.Add( xcontent_len.str() ) ;
hdr.Add( "If-Match: " + m_etag ) ;
hdr.Add( "Expect:" ) ;
std::string meta = (boost::format( xml_meta )
% m_kind
% xml::Escape(m_name)
).str() ;
bool retrying=false;
while ( true ) {
if ( retrying ) {
file.Seek( 0, SEEK_SET );
os::Sleep( 5 );
}
try {
http::StringResponse str ;
if ( post )
http->Post( link, meta, &str, hdr ) ;
else
http->Put( link, meta, &str, hdr ) ;
} catch ( Error &e ) {
std::string const *info = boost::get_error_info<xml::TreeBuilder::ExpatApiError>(e);
if ( info && (*info == "XML_Parse") ) {
Log( "Error parsing pre-upload response XML, retrying whole upload in 5s",
log::warning );
retrying = true;
continue;
} else {
throw e;
}
}
http::Header uphdr ;
uphdr.Add( "Expect:" ) ;
uphdr.Add( "Accept:" ) ;
// the content upload URL is in the "Location" HTTP header
std::string uplink = http->RedirLocation() ;
http::XmlResponse xml ;
long http_code = 0;
try {
http_code = http->Put( uplink, &file, &xml, uphdr ) ;
} catch ( Error &e ) {
std::string const *info = boost::get_error_info<xml::TreeBuilder::ExpatApiError>(e);
if ( info && (*info == "XML_Parse") ) {
Log( "Error parsing response XML, retrying whole upload in 5s",
log::warning );
retrying = true;
continue;
} else {
throw e;
}
}
if ( http_code == 410 || http_code == 412 ) {
Log( "request failed with %1%, retrying whole upload in 5s", http_code,
log::warning ) ;
retrying = true;
continue;
}
if ( retrying )
Log( "upload succeeded on retry", log::warning );
Entry1 responseEntry = Entry1( xml.Response() );
AssignIDs( responseEntry ) ;
m_mtime = responseEntry.MTime();
break;
}
return true ;
}
Resource::iterator Resource::begin() const
{
return m_child.begin() ;
@ -694,4 +515,4 @@ bool Resource::HasID() const
return !m_href.empty() && !m_id.empty() ;
}
} } // end of namespace
} // end of namespace

109
libgrive/src/drive/Resource.hh → libgrive/src/base/Resource.hh
View File

@ -29,17 +29,12 @@
namespace gr {
namespace http
{
class Agent ;
}
class Syncer ;
class Val ;
class Entry ;
namespace v1 {
/*! \brief A resource can be a file or a folder in the google drive
The google drive contains a number of resources, which is represented by this class.
@ -48,45 +43,9 @@ namespace v1 {
class Resource
{
public :
struct Error : virtual Exception {} ;
typedef std::vector<Resource*> Children ;
typedef Children::const_iterator iterator ;
public :
Resource(const fs::path& root_folder) ;
Resource( const std::string& name, const std::string& kind ) ;
bool IsFolder() const ;
std::string Name() const ;
std::string SelfHref() const ;
std::string ResourceID() const ;
const Resource* Parent() const ;
Resource* Parent() ;
void AddChild( Resource *child ) ;
Resource* FindChild( const std::string& title ) ;
fs::path Path() const ;
bool IsInRootTree() const ;
bool IsRoot() const ;
bool HasID() const ;
std::string MD5() const ;
void FromRemote( const Entry& remote, const DateTime& last_sync ) ;
void FromLocal( const DateTime& last_sync ) ;
void Sync( http::Agent* http, DateTime& sync_time, const Val& options ) ;
// children access
iterator begin() const ;
iterator end() const ;
std::size_t size() const ;
std::string StateStr() const ;
private :
/// State of the resource. indicating what to do with the resource
enum State
{
@ -119,43 +78,79 @@ private :
unknown
} ;
friend std::ostream& operator<<( std::ostream& os, State s ) ;
public :
Resource(const fs::path& root_folder) ;
Resource( const std::string& name, const std::string& kind ) ;
bool IsFolder() const ;
bool IsEditable() const ;
std::string Name() const ;
std::string Kind() const ;
DateTime MTime() const ;
std::string SelfHref() const ;
std::string ContentSrc() const ;
std::string ETag() const ;
std::string ResourceID() const ;
State GetState() const;
const Resource* Parent() const ;
Resource* Parent() ;
void AddChild( Resource *child ) ;
Resource* FindChild( const std::string& title ) ;
fs::path Path() const ;
bool IsInRootTree() const ;
bool IsRoot() const ;
bool HasID() const ;
std::string MD5() const ;
void FromRemote( const Entry& remote, const DateTime& last_sync ) ;
void FromLocal( const DateTime& last_sync ) ;
void Sync( Syncer* syncer, DateTime& sync_time, const Val& options ) ;
// children access
iterator begin() const ;
iterator end() const ;
std::size_t size() const ;
std::string StateStr() const ;
private :
void AssignIDs( const Entry& remote ) ;
friend std::ostream& operator<<( std::ostream& os, State s ) ;
friend class Syncer ;
private :
void SetState( State new_state ) ;
void Download( http::Agent* http, const fs::path& file ) const ;
bool EditContent( http::Agent* http, bool new_rev ) ;
bool Create( http::Agent* http ) ;
bool Upload( http::Agent* http, const std::string& link, bool post ) ;
void FromRemoteFolder( const Entry& remote, const DateTime& last_sync ) ;
void FromRemoteFile( const Entry& remote, const DateTime& last_sync ) ;
void DeleteLocal() ;
void DeleteRemote( http::Agent* http ) ;
void AssignIDs( const Entry& remote ) ;
void SyncSelf( http::Agent* http, const Val& options ) ;
void SyncSelf( Syncer* syncer, const Val& options ) ;
private :
std::string m_name ;
std::string m_kind ;
std::string m_md5 ;
DateTime m_mtime ;
std::string m_id ;
std::string m_href ;
std::string m_edit ;
std::string m_create ;
std::string m_content ;
std::string m_etag ;
bool m_is_editable ;
// not owned
Resource *m_parent ;
std::vector<Resource*> m_child ;
State m_state ;
} ;
} } // end of namespace gr::v1
} // end of namespace gr::v1

42
libgrive/src/base/Syncer.cc Normal file
View File

@ -0,0 +1,42 @@
/*
grive: an GPL program to sync a local directory with Google Drive
Copyright (C) 2012 Wan Wai Ho
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation version 2
of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#include "Syncer.hh"
#include "Resource.hh"
#include "Entry.hh"
#include "http/Agent.hh"
namespace gr {
Syncer::Syncer( http::Agent *http ):
m_http( http )
{
}
void Syncer::AssignIDs( Resource *res, const Entry& remote )
{
res->AssignIDs( remote );
}
void Syncer::AssignMTime( Resource *res, const DateTime& mtime )
{
res->m_mtime = mtime;
}
} // end of namespace gr

62
libgrive/src/base/Syncer.hh Normal file
View File

@ -0,0 +1,62 @@
/*
grive: an GPL program to sync a local directory with Google Drive
Copyright (C) 2012 Wan Wai Ho
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation version 2
of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#pragma once
#include "util/FileSystem.hh"
#include <string>
#include <vector>
#include <iosfwd>
namespace gr {
namespace http
{
class Agent ;
}
class DateTime ;
class Resource ;
class Entry ;
/*! \brief A Syncer incapsulates all resource-related upload/download/edit methods */
class Syncer
{
public :
Syncer( http::Agent *http );
virtual void DeleteRemote( Resource *res ) = 0;
virtual void Download( Resource *res, const fs::path& file ) = 0;
virtual bool EditContent( Resource *res, bool new_rev ) = 0;
virtual bool Create( Resource *res ) = 0;
protected:
http::Agent *m_http;
void AssignIDs( Resource *res, const Entry& remote );
void AssignMTime( Resource *res, const DateTime& mtime );
} ;
} // end of namespace gr

7
libgrive/src/drive/Drive.cc
View File

@ -22,6 +22,7 @@
#include "CommonUri.hh"
#include "base/Entry.hh"
#include "Feed.hh"
#include "Syncer1.hh"
#include "http/Agent.hh"
#include "http/ResponseLog.hh"
@ -107,7 +108,7 @@ void Drive::SyncFolders( )
// first, get all collections from the query result
for ( Feed::iterator i = feed.begin() ; i != feed.end() ; ++i )
{
Entry e( *i ) ;
const Entry &e = *i ;
if ( e.IsDir() )
{
if ( e.ParentHrefs().size() != 1 )
@ -171,8 +172,10 @@ void Drive::DetectChanges()
void Drive::Update()
{
Syncer1 syncer( m_http );
Log( "Synchronizing files", log::info ) ;
m_state.Sync( m_http, m_options ) ;
m_state.Sync( &syncer, m_options ) ;
UpdateChangeStamp( ) ;
}

2
libgrive/src/drive/ResourceTree.hh
View File

@ -19,7 +19,7 @@
#pragma once
#include "Resource.hh"
#include "base/Resource.hh"
#include "util/FileSystem.hh"

8
libgrive/src/drive/State.cc
View File

@ -20,10 +20,10 @@
#include "State.hh"
#include "base/Entry.hh"
#include "Resource.hh"
#include "base/Resource.hh"
#include "base/Syncer.hh"
#include "CommonUri.hh"
#include "http/Agent.hh"
#include "util/Crypt.hh"
#include "util/File.hh"
#include "util/log/Log.hh"
@ -273,7 +273,7 @@ void State::Write( const fs::path& filename ) const
fs << result ;
}
void State::Sync( http::Agent *http, const Val& options )
void State::Sync( Syncer *syncer, const Val& options )
{
// set the last sync time from the time returned by the server for the last file synced
// if the sync time hasn't changed (i.e. now files have been uploaded)
@ -283,7 +283,7 @@ void State::Sync( http::Agent *http, const Val& options )
// TODO - WARNING - do we use the last sync time to compare to client file times
// need to check if this introduces a new problem
DateTime last_sync_time = m_last_sync;
m_res.Root()->Sync( http, last_sync_time, options ) ;
m_res.Root()->Sync( syncer, last_sync_time, options ) ;
if ( last_sync_time == m_last_sync )
{

11
libgrive/src/drive/State.hh
View File

@ -28,19 +28,16 @@
namespace gr {
namespace http
{
class Agent ;
}
class Val ;
class Entry ;
namespace v1 {
class Syncer ;
class Resource ;
namespace v1 {
class State
{
public :
@ -60,7 +57,7 @@ public :
Resource* FindByHref( const std::string& href ) ;
Resource* FindByID( const std::string& id ) ;
void Sync( http::Agent *http, const Val& options ) ;
void Sync( Syncer *syncer, const Val& options ) ;
iterator begin() ;
iterator end() ;

255
libgrive/src/drive/Syncer1.cc Normal file
View File

@ -0,0 +1,255 @@
/*
grive: an GPL program to sync a local directory with Google Drive
Copyright (C) 2012 Wan Wai Ho
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation version 2
of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#include "base/Resource.hh"
#include "CommonUri.hh"
#include "Entry1.hh"
#include "Syncer1.hh"
#include "http/Agent.hh"
#include "http/Download.hh"
#include "http/Header.hh"
//#include "http/ResponseLog.hh"
#include "http/StringResponse.hh"
#include "http/XmlResponse.hh"
#include "xml/Node.hh"
#include "xml/NodeSet.hh"
#include "xml/String.hh"
#include "xml/TreeBuilder.hh"
#include "util/OS.hh"
#include "util/log/Log.hh"
#include <boost/exception/all.hpp>
#include <cassert>
// for debugging
#include <iostream>
namespace gr { namespace v1 {
// hard coded XML file
const std::string xml_meta =
"<?xml version='1.0' encoding='UTF-8'?>\n"
"<entry xmlns=\"http://www.w3.org/2005/Atom\" xmlns:docs=\"http://schemas.google.com/docs/2007\">"
"<category scheme=\"http://schemas.google.com/g/2005#kind\" "
"term=\"http://schemas.google.com/docs/2007#%1%\"/>"
"<title>%2%</title>"
"</entry>" ;
Syncer1::Syncer1( http::Agent *http ):
Syncer( http )
{
}
void Syncer1::DeleteRemote( Resource *res )
{
http::StringResponse str ;
try
{
http::Header hdr ;
hdr.Add( "If-Match: " + res->ETag() ) ;
// don't know why, but an update before deleting seems to work always
http::XmlResponse xml ;
m_http->Get( res->SelfHref(), &xml, hdr ) ;
AssignIDs( res, Entry1( xml.Response() ) ) ;
m_http->Custom( "DELETE", res->SelfHref(), &str, hdr ) ;
}
catch ( Exception& e )
{
// don't rethrow here. there are some cases that I don't know why
// the delete will fail.
Trace( "Exception %1% %2%",
boost::diagnostic_information(e),
str.Response() ) ;
}
}
void Syncer1::Download( Resource *res, const fs::path& file )
{
http::Download dl( file.string(), http::Download::NoChecksum() ) ;
long r = m_http->Get( res->ContentSrc(), &dl, http::Header() ) ;
if ( r <= 400 )
{
if ( res->MTime() != DateTime() )
os::SetFileTime( file, res->MTime() ) ;
else
Log( "encountered zero date time after downloading %1%", file, log::warning ) ;
}
}
bool Syncer1::EditContent( Resource *res, bool new_rev )
{
assert( res->Parent() ) ;
assert( res->Parent()->GetState() == Resource::sync ) ;
if ( !res->IsEditable() )
{
Log( "Cannot upload %1%: file read-only. %2%", res->Name(), res->StateStr(), log::warning ) ;
return false ;
}
return Upload( res, feed_base + "/" + res->ResourceID() + ( new_rev ? "?new-revision=true" : "" ), false ) ;
}
bool Syncer1::Create( Resource *res )
{
assert( res->Parent() ) ;
assert( res->Parent()->IsFolder() ) ;
assert( res->Parent()->GetState() == Resource::sync ) ;
if ( res->IsFolder() )
{
std::string uri = feed_base ;
if ( !res->Parent()->IsRoot() )
uri += ( "/" + m_http->Escape( res->Parent()->ResourceID() ) + "/contents" ) ;
std::string meta = (boost::format( xml_meta )
% "folder"
% xml::Escape( res->Name() )
).str() ;
http::Header hdr ;
hdr.Add( "Content-Type: application/atom+xml" ) ;
http::XmlResponse xml ;
// http::ResponseLog log( "create", ".xml", &xml ) ;
m_http->Post( uri, meta, &xml, hdr ) ;
AssignIDs( res, Entry1( xml.Response() ) ) ;
return true ;
}
else if ( res->Parent()->IsEditable() )
{
return Upload( res, root_create + (res->Parent()->ResourceID() == "folder:root"
? "" : "/" + res->Parent()->ResourceID() + "/contents") + "?convert=false", true ) ;
}
else
{
Log( "parent of %1% does not exist: cannot upload", res->Name(), log::warning ) ;
return false ;
}
}
bool Syncer1::Upload( Resource *res,
const std::string& link,
bool post )
{
File file( res->Path() ) ;
std::ostringstream xcontent_len ;
xcontent_len << "X-Upload-Content-Length: " << file.Size() ;
http::Header hdr ;
hdr.Add( "Content-Type: application/atom+xml" ) ;
hdr.Add( "X-Upload-Content-Type: application/octet-stream" ) ;
hdr.Add( xcontent_len.str() ) ;
hdr.Add( "If-Match: " + res->ETag() ) ;
hdr.Add( "Expect:" ) ;
std::string meta = (boost::format( xml_meta )
% res->Kind()
% xml::Escape( res->Name() )
).str() ;
bool retrying = false;
while ( true )
{
if ( retrying )
{
file.Seek( 0, SEEK_SET );
os::Sleep( 2 );
}
try
{
http::StringResponse str ;
if ( post )
m_http->Post( link, meta, &str, hdr ) ;
else
m_http->Put( link, meta, &str, hdr ) ;
}
catch ( Exception &e )
{
std::string const *info = boost::get_error_info<xml::TreeBuilder::ExpatApiError>(e);
if ( info && (*info == "XML_Parse") )
{
Log( "Error parsing pre-upload response XML, retrying whole upload in 5s",
log::warning );
retrying = true;
continue;
}
else
{
throw e;
}
}
http::Header uphdr ;
uphdr.Add( "Expect:" ) ;
uphdr.Add( "Accept:" ) ;
// the content upload URL is in the "Location" HTTP header
std::string uplink = m_http->RedirLocation() ;
http::XmlResponse xml ;
long http_code = 0;
try
{
http_code = m_http->Put( uplink, &file, &xml, uphdr ) ;
}
catch ( Exception &e )
{
std::string const *info = boost::get_error_info<xml::TreeBuilder::ExpatApiError>(e);
if ( info && (*info == "XML_Parse") )
{
Log( "Error parsing response XML, retrying whole upload in 5s",
log::warning );
retrying = true;
continue;
}
else
{
throw e;
}
}
if ( http_code == 410 || http_code == 412 )
{
Log( "request failed with %1%, retrying whole upload in 5s", http_code, log::warning ) ;
retrying = true;
continue;
}
if ( retrying )
Log( "upload succeeded on retry", log::warning );
Entry1 responseEntry = Entry1( xml.Response() );
AssignIDs( res, responseEntry ) ;
AssignMTime( res, responseEntry.MTime() );
break;
}
return true ;
}
} } // end of namespace gr::v1

46
libgrive/src/drive/Syncer1.hh Normal file
View File

@ -0,0 +1,46 @@
/*
grive: an GPL program to sync a local directory with Google Drive
Copyright (C) 2012 Wan Wai Ho
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation version 2
of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#pragma once
#include "base/Syncer.hh"
namespace gr {
namespace v1 {
class Syncer1: public Syncer
{
public :
Syncer1( http::Agent *http );
void DeleteRemote( Resource *res );
void Download( Resource *res, const fs::path& file );
bool EditContent( Resource *res, bool new_rev );
bool Create( Resource *res );
private :
bool Upload( Resource *res, const std::string& link, bool post);
} ;
} } // end of namespace gr::v1

164
libgrive/src/drive2/Drive.cc
View File

@ -1,164 +0,0 @@
/*
grive: an GPL program to sync a local directory with Google Drive
Copyright (C) 2013 Wan Wai Ho
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation version 2
of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA.
*/
#include "Drive.hh"
#include "CommonUri.hh"
#include "Feed.hh"
#include "json/Val.hh"
#include "util/Exception.hh"
#include <boost/bind.hpp>
#include <algorithm>
#include <iostream>
#include <iterator>
#include <cassert>
namespace gr { namespace v2 {
Drive::Drive( ) :
m_root( 0 )
{
}
void Drive::Refresh( http::Agent *agent )
{
// find root node ID
assert( m_root == 0 ) ;
m_root = NewResource( agent, "root" ) ;
// get all folders first
Feed folders( feeds::files ) ;
folders.Query( "mimeType", mime_types::folder ) ;
NewResource( agent, folders ) ;
// get all files
Feed files( feeds::files ) ;
NewResource( agent, files ) ;
// build parent-child linkage between folders
for ( details::DB::iterator i = m_db.begin() ; i != m_db.end() ; ++i )
{
Resource *parent = Find( (*i)->Parent() ), *child = *i ;
assert( child != 0 ) ;
if ( parent != 0 )
{
// initialize parent IDs
parent->AddChild( child->ID() ) ;
child->SetParent( parent->ID() ) ;
}
}
}
void Drive::NewResource( http::Agent *agent, Feed& items )
{
assert( agent != 0 ) ;
while ( items.Next( agent ) )
{
std::vector<Val> item_json = items.Content()["items"].AsArray() ;
for ( std::vector<Val>::iterator i = item_json.begin() ; i != item_json.end() ; ++i )
NewResource( *i ) ;
}
}
/// Create resource base on ID
Resource* Drive::NewResource( http::Agent *agent, const std::string& id )
{
Feed feed( feeds::files + "/" + id ) ;
feed.Next( agent ) ;
return NewResource( feed.Content() ) ;
}
Resource* Drive::NewResource( const Val& item )
{
// assume resource is directly under root
std::string parent_id = m_root != 0 ? m_root->ID() : "" ;
Val parents ;
if ( item.Get( "parents", parents ) )
{
std::vector<Val> pids_val = parents.Select( "id" ) ;
std::vector<std::string> pids ;
std::transform( pids_val.begin(), pids_val.end(),
std::back_inserter( pids ),
boost::bind( &Val::Str, _1 ) ) ;
// only the first parent counts
if ( !pids.empty() )
parent_id = pids.front() ;
}
Resource *r = new Resource(
item["id"].Str(),
item["mimeType"].Str(),
item["title"].Str(),
parent_id ) ;
m_db.insert(r) ;
assert( Find(r->ID()) == r ) ;
return r ;
}
Resource* Drive::Find( const std::string& id )
{
details::ID::iterator i = m_db.get<details::ByID>().find(id) ;
return i != m_db.get<details::ByID>().end() ? *i : 0 ;
}
const Resource* Drive::Find( const std::string& id ) const
{
details::ID::const_iterator i = m_db.get<details::ByID>().find(id) ;
return i != m_db.get<details::ByID>().end() ? *i : 0 ;
}
Resource* Drive::Root()
{
return m_root ;
}
const Resource* Drive::Root() const
{
return m_root ;
}
const Resource* Drive::Child( const Resource *parent, std::size_t idx ) const
{
if ( idx >= parent->ChildCount() )
BOOST_THROW_EXCEPTION(
Exception()
) ;
return Find( parent->At(idx) ) ;
}
const Resource* Drive::Parent( const Resource *child ) const
{
return Find( child->Parent() ) ;
}
} } // end of namespace gr::v2

89
libgrive/src/drive2/Drive.hh
View File

@ -1,89 +0,0 @@
/*
grive: an GPL program to sync a local directory with Google Drive
Copyright (C) 2013 Wan Wai Ho
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation version 2
of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA.
*/
#pragma once
#include "Resource.hh"
#include <boost/multi_index_container.hpp>
#include <boost/multi_index/hashed_index.hpp>
#include <boost/multi_index/identity.hpp>
#include <boost/multi_index/mem_fun.hpp>
namespace gr {
namespace http
{
class Agent ;
}
class Val ;
namespace v2 {
namespace details
{
using namespace boost::multi_index ;
struct ByID {} ;
struct ByHref {} ;
struct ByIdentity {} ;
typedef multi_index_container<
Resource*,
indexed_by<
hashed_unique<tag<ByIdentity>, identity<Resource*> >,
hashed_non_unique<tag<ByID>, const_mem_fun<Resource, std::string, &Resource::ID> >
>
> DB ;
typedef DB::index<ByID>::type ID ;
typedef DB::index<ByIdentity>::type Set ;
}
class Feed ;
class Drive
{
public :
Drive( ) ;
void Refresh( http::Agent *agent ) ;
Resource* Find( const std::string& id ) ;
const Resource* Find( const std::string& id ) const ;
Resource* Root() ;
const Resource* Root() const ;
const Resource* Child( const Resource *parent, std::size_t idx ) const ;
const Resource* Parent( const Resource *child ) const ;
private :
Resource* NewResource( const Val& item ) ;
Resource* NewResource( http::Agent *agent, const std::string& id ) ;
void NewResource( http::Agent *agent, Feed& items ) ;
private :
details::DB m_db ;
Resource *m_root ;
} ;
} } // end of namespace gr::v2

1
libgrive/src/drive2/Feed.cc
View File

@ -34,6 +34,7 @@ Feed::Feed( const std::string& base ) :
m_content.Add( "nextLink", Val(base) ) ;
}
// for example to find dirs: Query( "mimeType", mime_types::folder )
void Feed::Query( const std::string& field, const std::string& value )
{
std::string url = m_content["nextLink"].Str() ;

100
libgrive/src/drive2/Resource.cc
View File

@ -1,100 +0,0 @@
/*
grive: an GPL program to sync a local directory with Google Drive
Copyright (C) 2013 Wan Wai Ho
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation version 2
of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA.
*/
#include "Resource.hh"
#include "CommonUri.hh"
#include <algorithm>
namespace gr { namespace v2 {
/** Default constructor construct the resource of the root folder
*/
Resource::Resource() :
m_id( "root" ),
m_mime( mime_types::folder ),
m_title( "Root folder" )
{
}
Resource::Resource(
const std::string& id,
const std::string& mime,
const std::string& title,
const std::string& parent ) :
m_id ( id ),
m_mime ( mime ),
m_title ( title ),
m_parent( parent )
{
}
std::string Resource::ID() const
{
return m_id ;
}
std::string Resource::Mime() const
{
return m_mime ;
}
std::string Resource::Title() const
{
return m_title ;
}
bool Resource::IsFolder() const
{
return m_mime == "application/vnd.google-apps.folder" ;
}
void Resource::AddChild( const std::string& child )
{
m_children.push_back( child ) ;
}
void Resource::SetParent( const std::string& parent )
{
m_parent = parent ;
}
std::size_t Resource::ChildCount() const
{
return m_children.size() ;
}
std::string Resource::At( std::size_t idx ) const
{
return m_children.at(idx) ;
}
std::string Resource::Parent() const
{
return m_parent ;
}
std::size_t Resource::Index( const std::string& child ) const
{
return std::find( m_children.begin(), m_children.end(), child ) - m_children.begin() ;
}
} } // end of namespace gr::v2

62
libgrive/src/drive2/Resource.hh
View File

@ -1,62 +0,0 @@
/*
grive: an GPL program to sync a local directory with Google Drive
Copyright (C) 2013 Wan Wai Ho
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation version 2
of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA.
*/
#pragma once
#include <string>
#include <vector>
namespace gr { namespace v2 {
class Resource
{
public :
Resource() ;
Resource(
const std::string& id,
const std::string& mime,
const std::string& title,
const std::string& parent ) ;
std::string ID() const ;
std::string Mime() const ;
std::string Title() const ;
bool IsFolder() const ;
void AddChild( const std::string& child ) ;
void SetParent( const std::string& parent ) ;
std::size_t ChildCount() const ;
std::string At( std::size_t idx ) const ;
std::string Parent() const ;
std::size_t Index( const std::string& child ) const ;
private :
std::string m_id ;
std::string m_mime ;
std::string m_title ;
std::vector<std::string> m_children ;
std::string m_parent ;
} ;
} } // end of namespace gr::v2