From 6a15dd09a5d80c9b079ebb32af7082aeb014b442 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Sun, 17 May 2015 02:00:00 +0300 Subject: [PATCH] Split Resource to global (base/Resource) and API-specific ({drive,drive2}/Syncer) parts --- libgrive/src/{drive => base}/Resource.cc | 287 +++++------------------ libgrive/src/{drive => base}/Resource.hh | 109 ++++----- libgrive/src/base/Syncer.cc | 42 ++++ libgrive/src/base/Syncer.hh | 62 +++++ libgrive/src/drive/Drive.cc | 7 +- libgrive/src/drive/ResourceTree.hh | 2 +- libgrive/src/drive/State.cc | 8 +- libgrive/src/drive/State.hh | 11 +- libgrive/src/drive/Syncer1.cc | 255 ++++++++++++++++++++ libgrive/src/drive/Syncer1.hh | 46 ++++ libgrive/src/drive2/Drive.cc | 164 ------------- libgrive/src/drive2/Drive.hh | 89 ------- libgrive/src/drive2/Feed.cc | 1 + libgrive/src/drive2/Resource.cc | 100 -------- libgrive/src/drive2/Resource.hh | 62 ----- 15 files changed, 526 insertions(+), 719 deletions(-) rename libgrive/src/{drive => base}/Resource.cc (63%) rename libgrive/src/{drive => base}/Resource.hh (86%) create mode 100644 libgrive/src/base/Syncer.cc create mode 100644 libgrive/src/base/Syncer.hh create mode 100644 libgrive/src/drive/Syncer1.cc create mode 100644 libgrive/src/drive/Syncer1.hh delete mode 100644 libgrive/src/drive2/Drive.cc delete mode 100644 libgrive/src/drive2/Drive.hh delete mode 100644 libgrive/src/drive2/Resource.cc delete mode 100644 libgrive/src/drive2/Resource.hh diff --git a/libgrive/src/drive/Resource.cc b/libgrive/src/base/Resource.cc similarity index 63% rename from libgrive/src/drive/Resource.cc rename to libgrive/src/base/Resource.cc index 53ba6a4..ef995ab 100644 --- a/libgrive/src/drive/Resource.cc +++ b/libgrive/src/base/Resource.cc @@ -18,55 +18,33 @@ */ #include "Resource.hh" -#include "CommonUri.hh" -#include "Entry1.hh" +#include "Entry.hh" +#include "Syncer.hh" -#include "http/Agent.hh" -#include "http/Download.hh" -#include "http/Header.hh" -// #include "http/ResponseLog.hh" -#include "http/StringResponse.hh" -#include "http/XmlResponse.hh" #include "json/Val.hh" #include "util/CArray.hh" #include "util/Crypt.hh" #include "util/log/Log.hh" #include "util/OS.hh" #include "util/File.hh" -#include "xml/Node.hh" -#include "xml/NodeSet.hh" -#include "xml/String.hh" -#include "xml/TreeBuilder.hh" #include -#include #include // for debugging #include -namespace gr { namespace v1 { - -// hard coded XML file -const std::string xml_meta = - "\n" - "" - "" - "%2%" - "" ; - +namespace gr { /// default constructor creates the root folder Resource::Resource(const fs::path& root_folder) : m_name ( root_folder.string() ), m_kind ( "folder" ), m_id ( "folder:root" ), - m_href ( root_href ), - m_create ( root_create ), m_parent ( 0 ), - m_state ( sync ) + m_state ( sync ), + m_is_editable( true ) { } @@ -74,7 +52,8 @@ Resource::Resource( const std::string& name, const std::string& kind ) : m_name ( name ), m_kind ( kind ), m_parent ( 0 ), - m_state ( unknown ) + m_state ( unknown ), + m_is_editable( true ) { } @@ -166,9 +145,8 @@ void Resource::AssignIDs( const Entry& remote ) { m_id = remote.ResourceID() ; m_href = remote.SelfHref() ; - m_edit = remote.IsEditable() ? feed_base + "/" + remote.ResourceID() : ""; - m_create = remote.IsEditable() && remote.IsDir() ? root_create + (remote.ResourceID() == "folder:root" ? "" : "/" + remote.ResourceID() + "/contents") : ""; m_content = remote.ContentSrc() ; + m_is_editable = remote.IsEditable() ; m_etag = remote.ETag() ; } } @@ -283,16 +261,41 @@ std::string Resource::SelfHref() const return m_href ; } +std::string Resource::ContentSrc() const +{ + return m_content ; +} + +std::string Resource::ETag() const +{ + return m_etag ; +} + std::string Resource::Name() const { return m_name ; } +std::string Resource::Kind() const +{ + return m_kind ; +} + +DateTime Resource::MTime() const +{ + return m_mtime ; +} + std::string Resource::ResourceID() const { return m_id ; } +Resource::State Resource::GetState() const +{ + return m_state ; +} + const Resource* Resource::Parent() const { assert( m_parent == 0 || m_parent->IsFolder() ) ; @@ -320,6 +323,11 @@ bool Resource::IsFolder() const return m_kind == "folder" ; } +bool Resource::IsEditable() const +{ + return m_is_editable ; +} + fs::path Resource::Path() const { assert( m_parent != this ) ; @@ -331,7 +339,7 @@ fs::path Resource::Path() const bool Resource::IsInRootTree() const { assert( m_parent == 0 || m_parent->IsFolder() ) ; - return m_parent == 0 ? (SelfHref() == root_href) : m_parent->IsInRootTree() ; + return m_parent == 0 ? SelfHref().empty() : m_parent->IsInRootTree() ; } Resource* Resource::FindChild( const std::string& name ) @@ -346,12 +354,12 @@ Resource* Resource::FindChild( const std::string& name ) } // try to change the state to "sync" -void Resource::Sync( http::Agent *http, DateTime& sync_time, const Val& options ) +void Resource::Sync( Syncer *syncer, DateTime& sync_time, const Val& options ) { assert( m_state != unknown ) ; assert( !IsRoot() || m_state == sync ) ; // root folder is already synced - SyncSelf( http, options ) ; + SyncSelf( syncer, options ) ; // we want the server sync time, so we will take the server time of the last file uploaded to store as the sync time // m_mtime is updated to server modified time when the file is uploaded @@ -360,13 +368,13 @@ void Resource::Sync( http::Agent *http, DateTime& sync_time, const Val& options // if myself is deleted, no need to do the childrens if ( m_state != local_deleted && m_state != remote_deleted ) std::for_each( m_child.begin(), m_child.end(), - boost::bind( &Resource::Sync, _1, http, boost::ref(sync_time), options ) ) ; + boost::bind( &Resource::Sync, _1, syncer, boost::ref(sync_time), options ) ) ; } -void Resource::SyncSelf( http::Agent* http, const Val& options ) +void Resource::SyncSelf( Syncer* syncer, const Val& options ) { assert( !IsRoot() || m_state == sync ) ; // root is always sync - assert( IsRoot() || http == 0 || m_parent->IsFolder() ) ; + assert( IsRoot() || !syncer || m_parent->IsFolder() ) ; assert( IsRoot() || m_parent->m_state != remote_deleted ) ; assert( IsRoot() || m_parent->m_state != local_deleted ) ; @@ -377,30 +385,30 @@ void Resource::SyncSelf( http::Agent* http, const Val& options ) case local_new : Log( "sync %1% doesn't exist in server, uploading", path, log::info ) ; - if ( http != 0 && Create( http ) ) + if ( syncer && syncer->Create( this ) ) m_state = sync ; break ; case local_deleted : Log( "sync %1% deleted in local. deleting remote", path, log::info ) ; - if ( http != 0 ) - DeleteRemote( http ) ; + if ( syncer ) + syncer->DeleteRemote( this ) ; break ; case local_changed : Log( "sync %1% changed in local. uploading", path, log::info ) ; - if ( http != 0 && EditContent( http, options["new-rev"].Bool() ) ) + if ( syncer && syncer->EditContent( this, options["new-rev"].Bool() ) ) m_state = sync ; break ; case remote_new : Log( "sync %1% created in remote. creating local", path, log::info ) ; - if ( http != 0 ) + if ( syncer ) { if ( IsFolder() ) fs::create_directories( path ) ; else - Download( http, path ) ; + syncer->Download( this, path ) ; m_state = sync ; } @@ -409,16 +417,16 @@ void Resource::SyncSelf( http::Agent* http, const Val& options ) case remote_changed : assert( !IsFolder() ) ; Log( "sync %1% changed in remote. downloading", path, log::info ) ; - if ( http != 0 ) + if ( syncer ) { - Download( http, path ) ; + syncer->Download( this, path ) ; m_state = sync ; } break ; case remote_deleted : Log( "sync %1% deleted in remote. deleting local", path, log::info ) ; - if ( http != 0 ) + if ( syncer ) DeleteLocal() ; break ; @@ -459,193 +467,6 @@ void Resource::DeleteLocal() } } -void Resource::DeleteRemote( http::Agent *http ) -{ - assert( http != 0 ) ; - http::StringResponse str ; - - try - { - http::Header hdr ; - hdr.Add( "If-Match: " + m_etag ) ; - - // doesn't know why, but an update before deleting seems to work always - http::XmlResponse xml ; - http->Get( m_href, &xml, hdr ) ; - AssignIDs( Entry1( xml.Response() ) ) ; - - http->Custom( "DELETE", m_href, &str, hdr ) ; - } - catch ( Exception& e ) - { - // don't rethrow here. there are some cases that I don't know why - // the delete will fail. - Trace( "Exception %1% %2%", - boost::diagnostic_information(e), - str.Response() ) ; - } -} - - -void Resource::Download( http::Agent* http, const fs::path& file ) const -{ - assert( http != 0 ) ; - - http::Download dl( file.string(), http::Download::NoChecksum() ) ; - long r = http->Get( m_content, &dl, http::Header() ) ; - if ( r <= 400 ) - { - if ( m_mtime != DateTime() ) - os::SetFileTime( file, m_mtime ) ; - else - Log( "encountered zero date time after downloading %1%", file, log::warning ) ; - } -} - -bool Resource::EditContent( http::Agent* http, bool new_rev ) -{ - assert( http != 0 ) ; - assert( m_parent != 0 ) ; - assert( m_parent->m_state == sync ) ; - - // upload link missing means that file is read only - if ( m_edit.empty() ) - { - Log( "Cannot upload %1%: file read-only. %2%", m_name, m_state, log::warning ) ; - return false ; - } - - return Upload( http, m_edit + (new_rev ? "?new-revision=true" : ""), false ) ; -} - -bool Resource::Create( http::Agent* http ) -{ - assert( http != 0 ) ; - assert( m_parent != 0 ) ; - assert( m_parent->IsFolder() ) ; - assert( m_parent->m_state == sync ) ; - - if ( IsFolder() ) - { - std::string uri = feed_base ; - if ( !m_parent->IsRoot() ) - uri += ("/" + http->Escape(m_parent->m_id) + "/contents") ; - - std::string meta = (boost::format( xml_meta ) - % "folder" - % xml::Escape(m_name) - ).str() ; - - http::Header hdr ; - hdr.Add( "Content-Type: application/atom+xml" ) ; - - http::XmlResponse xml ; -// http::ResponseLog log( "create", ".xml", &xml ) ; - http->Post( uri, meta, &xml, hdr ) ; - AssignIDs( Entry1( xml.Response() ) ) ; - - return true ; - } - else if ( !m_parent->m_create.empty() ) - { - return Upload( http, m_parent->m_create + "?convert=false", true ) ; - } - else - { - Log( "parent of %1% does not exist: cannot upload", Name(), log::warning ) ; - return false ; - } -} - -bool Resource::Upload( - http::Agent* http, - const std::string& link, - bool post) -{ - assert( http != 0 ) ; - - File file( Path() ) ; - std::ostringstream xcontent_len ; - xcontent_len << "X-Upload-Content-Length: " << file.Size() ; - - http::Header hdr ; - hdr.Add( "Content-Type: application/atom+xml" ) ; - hdr.Add( "X-Upload-Content-Type: application/octet-stream" ) ; - hdr.Add( xcontent_len.str() ) ; - hdr.Add( "If-Match: " + m_etag ) ; - hdr.Add( "Expect:" ) ; - - std::string meta = (boost::format( xml_meta ) - % m_kind - % xml::Escape(m_name) - ).str() ; - - bool retrying=false; - while ( true ) { - if ( retrying ) { - file.Seek( 0, SEEK_SET ); - os::Sleep( 5 ); - } - - try { - http::StringResponse str ; - if ( post ) - http->Post( link, meta, &str, hdr ) ; - else - http->Put( link, meta, &str, hdr ) ; - } catch ( Error &e ) { - std::string const *info = boost::get_error_info(e); - if ( info && (*info == "XML_Parse") ) { - Log( "Error parsing pre-upload response XML, retrying whole upload in 5s", - log::warning ); - retrying = true; - continue; - } else { - throw e; - } - } - - http::Header uphdr ; - uphdr.Add( "Expect:" ) ; - uphdr.Add( "Accept:" ) ; - - // the content upload URL is in the "Location" HTTP header - std::string uplink = http->RedirLocation() ; - http::XmlResponse xml ; - - long http_code = 0; - try { - http_code = http->Put( uplink, &file, &xml, uphdr ) ; - } catch ( Error &e ) { - std::string const *info = boost::get_error_info(e); - if ( info && (*info == "XML_Parse") ) { - Log( "Error parsing response XML, retrying whole upload in 5s", - log::warning ); - retrying = true; - continue; - } else { - throw e; - } - } - - if ( http_code == 410 || http_code == 412 ) { - Log( "request failed with %1%, retrying whole upload in 5s", http_code, - log::warning ) ; - retrying = true; - continue; - } - - if ( retrying ) - Log( "upload succeeded on retry", log::warning ); - Entry1 responseEntry = Entry1( xml.Response() ); - AssignIDs( responseEntry ) ; - m_mtime = responseEntry.MTime(); - break; - } - - return true ; -} - Resource::iterator Resource::begin() const { return m_child.begin() ; @@ -694,4 +515,4 @@ bool Resource::HasID() const return !m_href.empty() && !m_id.empty() ; } -} } // end of namespace +} // end of namespace diff --git a/libgrive/src/drive/Resource.hh b/libgrive/src/base/Resource.hh similarity index 86% rename from libgrive/src/drive/Resource.hh rename to libgrive/src/base/Resource.hh index 62cea69..efb0d5f 100644 --- a/libgrive/src/drive/Resource.hh +++ b/libgrive/src/base/Resource.hh @@ -29,17 +29,12 @@ namespace gr { -namespace http -{ - class Agent ; -} +class Syncer ; class Val ; class Entry ; -namespace v1 { - /*! \brief A resource can be a file or a folder in the google drive The google drive contains a number of resources, which is represented by this class. @@ -48,45 +43,9 @@ namespace v1 { class Resource { public : - struct Error : virtual Exception {} ; - typedef std::vector Children ; typedef Children::const_iterator iterator ; -public : - Resource(const fs::path& root_folder) ; - Resource( const std::string& name, const std::string& kind ) ; - - bool IsFolder() const ; - - std::string Name() const ; - std::string SelfHref() const ; - std::string ResourceID() const ; - - const Resource* Parent() const ; - Resource* Parent() ; - void AddChild( Resource *child ) ; - Resource* FindChild( const std::string& title ) ; - - fs::path Path() const ; - bool IsInRootTree() const ; - bool IsRoot() const ; - bool HasID() const ; - std::string MD5() const ; - - void FromRemote( const Entry& remote, const DateTime& last_sync ) ; - void FromLocal( const DateTime& last_sync ) ; - - void Sync( http::Agent* http, DateTime& sync_time, const Val& options ) ; - - // children access - iterator begin() const ; - iterator end() const ; - std::size_t size() const ; - - std::string StateStr() const ; - -private : /// State of the resource. indicating what to do with the resource enum State { @@ -119,43 +78,79 @@ private : unknown } ; - friend std::ostream& operator<<( std::ostream& os, State s ) ; +public : + Resource(const fs::path& root_folder) ; + Resource( const std::string& name, const std::string& kind ) ; + bool IsFolder() const ; + bool IsEditable() const ; + + std::string Name() const ; + std::string Kind() const ; + DateTime MTime() const ; + std::string SelfHref() const ; + std::string ContentSrc() const ; + std::string ETag() const ; + std::string ResourceID() const ; + State GetState() const; + + const Resource* Parent() const ; + Resource* Parent() ; + void AddChild( Resource *child ) ; + Resource* FindChild( const std::string& title ) ; + + fs::path Path() const ; + bool IsInRootTree() const ; + bool IsRoot() const ; + bool HasID() const ; + std::string MD5() const ; + + void FromRemote( const Entry& remote, const DateTime& last_sync ) ; + void FromLocal( const DateTime& last_sync ) ; + + void Sync( Syncer* syncer, DateTime& sync_time, const Val& options ) ; + + // children access + iterator begin() const ; + iterator end() const ; + std::size_t size() const ; + + std::string StateStr() const ; + +private : + + void AssignIDs( const Entry& remote ) ; + + friend std::ostream& operator<<( std::ostream& os, State s ) ; + friend class Syncer ; + private : void SetState( State new_state ) ; - - void Download( http::Agent* http, const fs::path& file ) const ; - bool EditContent( http::Agent* http, bool new_rev ) ; - bool Create( http::Agent* http ) ; - bool Upload( http::Agent* http, const std::string& link, bool post ) ; void FromRemoteFolder( const Entry& remote, const DateTime& last_sync ) ; void FromRemoteFile( const Entry& remote, const DateTime& last_sync ) ; void DeleteLocal() ; - void DeleteRemote( http::Agent* http ) ; - - void AssignIDs( const Entry& remote ) ; - void SyncSelf( http::Agent* http, const Val& options ) ; + void SyncSelf( Syncer* syncer, const Val& options ) ; + private : std::string m_name ; std::string m_kind ; std::string m_md5 ; DateTime m_mtime ; - + std::string m_id ; std::string m_href ; - std::string m_edit ; - std::string m_create ; std::string m_content ; std::string m_etag ; + bool m_is_editable ; // not owned Resource *m_parent ; std::vector m_child ; - + State m_state ; } ; -} } // end of namespace gr::v1 +} // end of namespace gr::v1 diff --git a/libgrive/src/base/Syncer.cc b/libgrive/src/base/Syncer.cc new file mode 100644 index 0000000..ef6d7de --- /dev/null +++ b/libgrive/src/base/Syncer.cc @@ -0,0 +1,42 @@ +/* + grive: an GPL program to sync a local directory with Google Drive + Copyright (C) 2012 Wan Wai Ho + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#include "Syncer.hh" +#include "Resource.hh" +#include "Entry.hh" +#include "http/Agent.hh" + +namespace gr { + +Syncer::Syncer( http::Agent *http ): + m_http( http ) +{ +} + +void Syncer::AssignIDs( Resource *res, const Entry& remote ) +{ + res->AssignIDs( remote ); +} + +void Syncer::AssignMTime( Resource *res, const DateTime& mtime ) +{ + res->m_mtime = mtime; +} + +} // end of namespace gr diff --git a/libgrive/src/base/Syncer.hh b/libgrive/src/base/Syncer.hh new file mode 100644 index 0000000..6665e5c --- /dev/null +++ b/libgrive/src/base/Syncer.hh @@ -0,0 +1,62 @@ +/* + grive: an GPL program to sync a local directory with Google Drive + Copyright (C) 2012 Wan Wai Ho + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#pragma once + +#include "util/FileSystem.hh" + +#include +#include +#include + +namespace gr { + +namespace http +{ + class Agent ; +} + +class DateTime ; + +class Resource ; + +class Entry ; + +/*! \brief A Syncer incapsulates all resource-related upload/download/edit methods */ +class Syncer +{ +public : + + Syncer( http::Agent *http ); + + virtual void DeleteRemote( Resource *res ) = 0; + virtual void Download( Resource *res, const fs::path& file ) = 0; + virtual bool EditContent( Resource *res, bool new_rev ) = 0; + virtual bool Create( Resource *res ) = 0; + +protected: + + http::Agent *m_http; + + void AssignIDs( Resource *res, const Entry& remote ); + void AssignMTime( Resource *res, const DateTime& mtime ); + +} ; + +} // end of namespace gr diff --git a/libgrive/src/drive/Drive.cc b/libgrive/src/drive/Drive.cc index a8b99b6..e31c2c8 100644 --- a/libgrive/src/drive/Drive.cc +++ b/libgrive/src/drive/Drive.cc @@ -22,6 +22,7 @@ #include "CommonUri.hh" #include "base/Entry.hh" #include "Feed.hh" +#include "Syncer1.hh" #include "http/Agent.hh" #include "http/ResponseLog.hh" @@ -107,7 +108,7 @@ void Drive::SyncFolders( ) // first, get all collections from the query result for ( Feed::iterator i = feed.begin() ; i != feed.end() ; ++i ) { - Entry e( *i ) ; + const Entry &e = *i ; if ( e.IsDir() ) { if ( e.ParentHrefs().size() != 1 ) @@ -171,8 +172,10 @@ void Drive::DetectChanges() void Drive::Update() { + Syncer1 syncer( m_http ); + Log( "Synchronizing files", log::info ) ; - m_state.Sync( m_http, m_options ) ; + m_state.Sync( &syncer, m_options ) ; UpdateChangeStamp( ) ; } diff --git a/libgrive/src/drive/ResourceTree.hh b/libgrive/src/drive/ResourceTree.hh index d0a391c..1c34172 100644 --- a/libgrive/src/drive/ResourceTree.hh +++ b/libgrive/src/drive/ResourceTree.hh @@ -19,7 +19,7 @@ #pragma once -#include "Resource.hh" +#include "base/Resource.hh" #include "util/FileSystem.hh" diff --git a/libgrive/src/drive/State.cc b/libgrive/src/drive/State.cc index d172fcc..a74d414 100644 --- a/libgrive/src/drive/State.cc +++ b/libgrive/src/drive/State.cc @@ -20,10 +20,10 @@ #include "State.hh" #include "base/Entry.hh" -#include "Resource.hh" +#include "base/Resource.hh" +#include "base/Syncer.hh" #include "CommonUri.hh" -#include "http/Agent.hh" #include "util/Crypt.hh" #include "util/File.hh" #include "util/log/Log.hh" @@ -273,7 +273,7 @@ void State::Write( const fs::path& filename ) const fs << result ; } -void State::Sync( http::Agent *http, const Val& options ) +void State::Sync( Syncer *syncer, const Val& options ) { // set the last sync time from the time returned by the server for the last file synced // if the sync time hasn't changed (i.e. now files have been uploaded) @@ -283,7 +283,7 @@ void State::Sync( http::Agent *http, const Val& options ) // TODO - WARNING - do we use the last sync time to compare to client file times // need to check if this introduces a new problem DateTime last_sync_time = m_last_sync; - m_res.Root()->Sync( http, last_sync_time, options ) ; + m_res.Root()->Sync( syncer, last_sync_time, options ) ; if ( last_sync_time == m_last_sync ) { diff --git a/libgrive/src/drive/State.hh b/libgrive/src/drive/State.hh index f2acfb9..3fe68ce 100644 --- a/libgrive/src/drive/State.hh +++ b/libgrive/src/drive/State.hh @@ -28,19 +28,16 @@ namespace gr { -namespace http -{ - class Agent ; -} - class Val ; class Entry ; -namespace v1 { +class Syncer ; class Resource ; +namespace v1 { + class State { public : @@ -60,7 +57,7 @@ public : Resource* FindByHref( const std::string& href ) ; Resource* FindByID( const std::string& id ) ; - void Sync( http::Agent *http, const Val& options ) ; + void Sync( Syncer *syncer, const Val& options ) ; iterator begin() ; iterator end() ; diff --git a/libgrive/src/drive/Syncer1.cc b/libgrive/src/drive/Syncer1.cc new file mode 100644 index 0000000..816572f --- /dev/null +++ b/libgrive/src/drive/Syncer1.cc @@ -0,0 +1,255 @@ +/* + grive: an GPL program to sync a local directory with Google Drive + Copyright (C) 2012 Wan Wai Ho + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#include "base/Resource.hh" +#include "CommonUri.hh" +#include "Entry1.hh" +#include "Syncer1.hh" + +#include "http/Agent.hh" +#include "http/Download.hh" +#include "http/Header.hh" +//#include "http/ResponseLog.hh" +#include "http/StringResponse.hh" +#include "http/XmlResponse.hh" + +#include "xml/Node.hh" +#include "xml/NodeSet.hh" +#include "xml/String.hh" +#include "xml/TreeBuilder.hh" + +#include "util/OS.hh" +#include "util/log/Log.hh" + +#include + +#include + +// for debugging +#include + +namespace gr { namespace v1 { + +// hard coded XML file +const std::string xml_meta = + "\n" + "" + "" + "%2%" + "" ; + +Syncer1::Syncer1( http::Agent *http ): + Syncer( http ) +{ +} + +void Syncer1::DeleteRemote( Resource *res ) +{ + http::StringResponse str ; + + try + { + http::Header hdr ; + hdr.Add( "If-Match: " + res->ETag() ) ; + + // don't know why, but an update before deleting seems to work always + http::XmlResponse xml ; + m_http->Get( res->SelfHref(), &xml, hdr ) ; + AssignIDs( res, Entry1( xml.Response() ) ) ; + + m_http->Custom( "DELETE", res->SelfHref(), &str, hdr ) ; + } + catch ( Exception& e ) + { + // don't rethrow here. there are some cases that I don't know why + // the delete will fail. + Trace( "Exception %1% %2%", + boost::diagnostic_information(e), + str.Response() ) ; + } +} + +void Syncer1::Download( Resource *res, const fs::path& file ) +{ + http::Download dl( file.string(), http::Download::NoChecksum() ) ; + long r = m_http->Get( res->ContentSrc(), &dl, http::Header() ) ; + if ( r <= 400 ) + { + if ( res->MTime() != DateTime() ) + os::SetFileTime( file, res->MTime() ) ; + else + Log( "encountered zero date time after downloading %1%", file, log::warning ) ; + } +} + +bool Syncer1::EditContent( Resource *res, bool new_rev ) +{ + assert( res->Parent() ) ; + assert( res->Parent()->GetState() == Resource::sync ) ; + + if ( !res->IsEditable() ) + { + Log( "Cannot upload %1%: file read-only. %2%", res->Name(), res->StateStr(), log::warning ) ; + return false ; + } + + return Upload( res, feed_base + "/" + res->ResourceID() + ( new_rev ? "?new-revision=true" : "" ), false ) ; +} + +bool Syncer1::Create( Resource *res ) +{ + assert( res->Parent() ) ; + assert( res->Parent()->IsFolder() ) ; + assert( res->Parent()->GetState() == Resource::sync ) ; + + if ( res->IsFolder() ) + { + std::string uri = feed_base ; + if ( !res->Parent()->IsRoot() ) + uri += ( "/" + m_http->Escape( res->Parent()->ResourceID() ) + "/contents" ) ; + + std::string meta = (boost::format( xml_meta ) + % "folder" + % xml::Escape( res->Name() ) + ).str() ; + + http::Header hdr ; + hdr.Add( "Content-Type: application/atom+xml" ) ; + + http::XmlResponse xml ; +// http::ResponseLog log( "create", ".xml", &xml ) ; + m_http->Post( uri, meta, &xml, hdr ) ; + AssignIDs( res, Entry1( xml.Response() ) ) ; + + return true ; + } + else if ( res->Parent()->IsEditable() ) + { + return Upload( res, root_create + (res->Parent()->ResourceID() == "folder:root" + ? "" : "/" + res->Parent()->ResourceID() + "/contents") + "?convert=false", true ) ; + } + else + { + Log( "parent of %1% does not exist: cannot upload", res->Name(), log::warning ) ; + return false ; + } +} + +bool Syncer1::Upload( Resource *res, + const std::string& link, + bool post ) +{ + File file( res->Path() ) ; + std::ostringstream xcontent_len ; + xcontent_len << "X-Upload-Content-Length: " << file.Size() ; + + http::Header hdr ; + hdr.Add( "Content-Type: application/atom+xml" ) ; + hdr.Add( "X-Upload-Content-Type: application/octet-stream" ) ; + hdr.Add( xcontent_len.str() ) ; + hdr.Add( "If-Match: " + res->ETag() ) ; + hdr.Add( "Expect:" ) ; + + std::string meta = (boost::format( xml_meta ) + % res->Kind() + % xml::Escape( res->Name() ) + ).str() ; + + bool retrying = false; + while ( true ) + { + if ( retrying ) + { + file.Seek( 0, SEEK_SET ); + os::Sleep( 2 ); + } + + try + { + http::StringResponse str ; + if ( post ) + m_http->Post( link, meta, &str, hdr ) ; + else + m_http->Put( link, meta, &str, hdr ) ; + } + catch ( Exception &e ) + { + std::string const *info = boost::get_error_info(e); + if ( info && (*info == "XML_Parse") ) + { + Log( "Error parsing pre-upload response XML, retrying whole upload in 5s", + log::warning ); + retrying = true; + continue; + } + else + { + throw e; + } + } + + http::Header uphdr ; + uphdr.Add( "Expect:" ) ; + uphdr.Add( "Accept:" ) ; + + // the content upload URL is in the "Location" HTTP header + std::string uplink = m_http->RedirLocation() ; + http::XmlResponse xml ; + + long http_code = 0; + try + { + http_code = m_http->Put( uplink, &file, &xml, uphdr ) ; + } + catch ( Exception &e ) + { + std::string const *info = boost::get_error_info(e); + if ( info && (*info == "XML_Parse") ) + { + Log( "Error parsing response XML, retrying whole upload in 5s", + log::warning ); + retrying = true; + continue; + } + else + { + throw e; + } + } + + if ( http_code == 410 || http_code == 412 ) + { + Log( "request failed with %1%, retrying whole upload in 5s", http_code, log::warning ) ; + retrying = true; + continue; + } + + if ( retrying ) + Log( "upload succeeded on retry", log::warning ); + Entry1 responseEntry = Entry1( xml.Response() ); + AssignIDs( res, responseEntry ) ; + AssignMTime( res, responseEntry.MTime() ); + break; + } + + return true ; +} + +} } // end of namespace gr::v1 diff --git a/libgrive/src/drive/Syncer1.hh b/libgrive/src/drive/Syncer1.hh new file mode 100644 index 0000000..7f3d1cc --- /dev/null +++ b/libgrive/src/drive/Syncer1.hh @@ -0,0 +1,46 @@ +/* + grive: an GPL program to sync a local directory with Google Drive + Copyright (C) 2012 Wan Wai Ho + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation version 2 + of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#pragma once + +#include "base/Syncer.hh" + +namespace gr { + +namespace v1 { + +class Syncer1: public Syncer +{ + +public : + + Syncer1( http::Agent *http ); + + void DeleteRemote( Resource *res ); + void Download( Resource *res, const fs::path& file ); + bool EditContent( Resource *res, bool new_rev ); + bool Create( Resource *res ); + +private : + + bool Upload( Resource *res, const std::string& link, bool post); + +} ; + +} } // end of namespace gr::v1 diff --git a/libgrive/src/drive2/Drive.cc b/libgrive/src/drive2/Drive.cc deleted file mode 100644 index feaa84b..0000000 --- a/libgrive/src/drive2/Drive.cc +++ /dev/null @@ -1,164 +0,0 @@ -/* - grive: an GPL program to sync a local directory with Google Drive - Copyright (C) 2013 Wan Wai Ho - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation version 2 - of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. -*/ - -#include "Drive.hh" - -#include "CommonUri.hh" -#include "Feed.hh" -#include "json/Val.hh" -#include "util/Exception.hh" - -#include - -#include -#include -#include - -#include - -namespace gr { namespace v2 { - -Drive::Drive( ) : - m_root( 0 ) -{ - -} - -void Drive::Refresh( http::Agent *agent ) -{ - // find root node ID - assert( m_root == 0 ) ; - m_root = NewResource( agent, "root" ) ; - - // get all folders first - Feed folders( feeds::files ) ; - folders.Query( "mimeType", mime_types::folder ) ; - NewResource( agent, folders ) ; - - // get all files - Feed files( feeds::files ) ; - NewResource( agent, files ) ; - - // build parent-child linkage between folders - for ( details::DB::iterator i = m_db.begin() ; i != m_db.end() ; ++i ) - { - Resource *parent = Find( (*i)->Parent() ), *child = *i ; - assert( child != 0 ) ; - - if ( parent != 0 ) - { - // initialize parent IDs - parent->AddChild( child->ID() ) ; - child->SetParent( parent->ID() ) ; - } - } -} - -void Drive::NewResource( http::Agent *agent, Feed& items ) -{ - assert( agent != 0 ) ; - - while ( items.Next( agent ) ) - { - std::vector item_json = items.Content()["items"].AsArray() ; - for ( std::vector::iterator i = item_json.begin() ; i != item_json.end() ; ++i ) - NewResource( *i ) ; - } -} - -/// Create resource base on ID -Resource* Drive::NewResource( http::Agent *agent, const std::string& id ) -{ - Feed feed( feeds::files + "/" + id ) ; - feed.Next( agent ) ; - - return NewResource( feed.Content() ) ; -} - -Resource* Drive::NewResource( const Val& item ) -{ - // assume resource is directly under root - std::string parent_id = m_root != 0 ? m_root->ID() : "" ; - - Val parents ; - if ( item.Get( "parents", parents ) ) - { - std::vector pids_val = parents.Select( "id" ) ; - std::vector pids ; - std::transform( pids_val.begin(), pids_val.end(), - std::back_inserter( pids ), - boost::bind( &Val::Str, _1 ) ) ; - - // only the first parent counts - if ( !pids.empty() ) - parent_id = pids.front() ; - } - - Resource *r = new Resource( - item["id"].Str(), - item["mimeType"].Str(), - item["title"].Str(), - parent_id ) ; - - m_db.insert(r) ; - assert( Find(r->ID()) == r ) ; - - return r ; -} - -Resource* Drive::Find( const std::string& id ) -{ - details::ID::iterator i = m_db.get().find(id) ; - return i != m_db.get().end() ? *i : 0 ; -} - -const Resource* Drive::Find( const std::string& id ) const -{ - details::ID::const_iterator i = m_db.get().find(id) ; - return i != m_db.get().end() ? *i : 0 ; -} - -Resource* Drive::Root() -{ - return m_root ; -} - -const Resource* Drive::Root() const -{ - return m_root ; -} - -const Resource* Drive::Child( const Resource *parent, std::size_t idx ) const -{ - if ( idx >= parent->ChildCount() ) - BOOST_THROW_EXCEPTION( - Exception() - ) ; - - return Find( parent->At(idx) ) ; -} - -const Resource* Drive::Parent( const Resource *child ) const -{ - return Find( child->Parent() ) ; -} - -} } // end of namespace gr::v2 - diff --git a/libgrive/src/drive2/Drive.hh b/libgrive/src/drive2/Drive.hh deleted file mode 100644 index 33014b0..0000000 --- a/libgrive/src/drive2/Drive.hh +++ /dev/null @@ -1,89 +0,0 @@ -/* - grive: an GPL program to sync a local directory with Google Drive - Copyright (C) 2013 Wan Wai Ho - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation version 2 - of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. -*/ - -#pragma once - -#include "Resource.hh" - -#include -#include -#include -#include - -namespace gr { - -namespace http -{ - class Agent ; -} - -class Val ; - -namespace v2 { - -namespace details -{ - using namespace boost::multi_index ; - struct ByID {} ; - struct ByHref {} ; - struct ByIdentity {} ; - - typedef multi_index_container< - Resource*, - indexed_by< - hashed_unique, identity >, - hashed_non_unique, const_mem_fun > - > - > DB ; - - typedef DB::index::type ID ; - typedef DB::index::type Set ; -} - -class Feed ; - -class Drive -{ -public : - Drive( ) ; - - void Refresh( http::Agent *agent ) ; - - Resource* Find( const std::string& id ) ; - const Resource* Find( const std::string& id ) const ; - - Resource* Root() ; - const Resource* Root() const ; - - const Resource* Child( const Resource *parent, std::size_t idx ) const ; - const Resource* Parent( const Resource *child ) const ; - -private : - Resource* NewResource( const Val& item ) ; - Resource* NewResource( http::Agent *agent, const std::string& id ) ; - void NewResource( http::Agent *agent, Feed& items ) ; - -private : - details::DB m_db ; - - Resource *m_root ; -} ; - -} } // end of namespace gr::v2 diff --git a/libgrive/src/drive2/Feed.cc b/libgrive/src/drive2/Feed.cc index 78f67f0..93f5a24 100644 --- a/libgrive/src/drive2/Feed.cc +++ b/libgrive/src/drive2/Feed.cc @@ -34,6 +34,7 @@ Feed::Feed( const std::string& base ) : m_content.Add( "nextLink", Val(base) ) ; } +// for example to find dirs: Query( "mimeType", mime_types::folder ) void Feed::Query( const std::string& field, const std::string& value ) { std::string url = m_content["nextLink"].Str() ; diff --git a/libgrive/src/drive2/Resource.cc b/libgrive/src/drive2/Resource.cc deleted file mode 100644 index a583279..0000000 --- a/libgrive/src/drive2/Resource.cc +++ /dev/null @@ -1,100 +0,0 @@ -/* - grive: an GPL program to sync a local directory with Google Drive - Copyright (C) 2013 Wan Wai Ho - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation version 2 - of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. -*/ - -#include "Resource.hh" - -#include "CommonUri.hh" - -#include - -namespace gr { namespace v2 { - -/** Default constructor construct the resource of the root folder -*/ -Resource::Resource() : - m_id( "root" ), - m_mime( mime_types::folder ), - m_title( "Root folder" ) -{ -} - -Resource::Resource( - const std::string& id, - const std::string& mime, - const std::string& title, - const std::string& parent ) : - m_id ( id ), - m_mime ( mime ), - m_title ( title ), - m_parent( parent ) -{ -} - -std::string Resource::ID() const -{ - return m_id ; -} - -std::string Resource::Mime() const -{ - return m_mime ; -} - -std::string Resource::Title() const -{ - return m_title ; -} - -bool Resource::IsFolder() const -{ - return m_mime == "application/vnd.google-apps.folder" ; -} - -void Resource::AddChild( const std::string& child ) -{ - m_children.push_back( child ) ; -} - -void Resource::SetParent( const std::string& parent ) -{ - m_parent = parent ; -} - -std::size_t Resource::ChildCount() const -{ - return m_children.size() ; -} - -std::string Resource::At( std::size_t idx ) const -{ - return m_children.at(idx) ; -} - -std::string Resource::Parent() const -{ - return m_parent ; -} - -std::size_t Resource::Index( const std::string& child ) const -{ - return std::find( m_children.begin(), m_children.end(), child ) - m_children.begin() ; -} - -} } // end of namespace gr::v2 diff --git a/libgrive/src/drive2/Resource.hh b/libgrive/src/drive2/Resource.hh deleted file mode 100644 index 8513c60..0000000 --- a/libgrive/src/drive2/Resource.hh +++ /dev/null @@ -1,62 +0,0 @@ -/* - grive: an GPL program to sync a local directory with Google Drive - Copyright (C) 2013 Wan Wai Ho - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation version 2 - of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. -*/ - -#pragma once - -#include -#include - -namespace gr { namespace v2 { - -class Resource -{ -public : - Resource() ; - Resource( - const std::string& id, - const std::string& mime, - const std::string& title, - const std::string& parent ) ; - - std::string ID() const ; - std::string Mime() const ; - std::string Title() const ; - - bool IsFolder() const ; - - void AddChild( const std::string& child ) ; - void SetParent( const std::string& parent ) ; - - std::size_t ChildCount() const ; - std::string At( std::size_t idx ) const ; - std::string Parent() const ; - std::size_t Index( const std::string& child ) const ; - -private : - std::string m_id ; - std::string m_mime ; - std::string m_title ; - - std::vector m_children ; - - std::string m_parent ; -} ; - -} } // end of namespace gr::v2