From 2d34d7708b4f8c2f3b2f8384395ba58aca728775 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Sun, 17 May 2015 20:10:03 +0300 Subject: [PATCH] Implement upload methods for REST API Syncer (basic file upload to root now works) --- grive/src/main.cc | 5 +- libgrive/src/base/Syncer.cc | 17 +++++ libgrive/src/base/Syncer.hh | 2 +- libgrive/src/drive/Syncer1.cc | 17 +---- libgrive/src/drive/Syncer1.hh | 1 - libgrive/src/drive2/CommonUri.hh | 2 + libgrive/src/drive2/Entry2.cc | 10 +-- libgrive/src/drive2/Syncer2.cc | 117 ++++++++++++++++++++++++++++--- libgrive/src/drive2/Syncer2.hh | 3 +- libgrive/src/http/CurlAgent.cc | 7 +- libgrive/src/http/CurlAgent.hh | 2 +- libgrive/src/json/JsonParser.cc | 2 +- libgrive/src/json/JsonWriter.cc | 10 ++- libgrive/src/json/JsonWriter.hh | 3 + libgrive/src/json/Val.cc | 9 +++ 15 files changed, 166 insertions(+), 41 deletions(-) diff --git a/grive/src/main.cc b/grive/src/main.cc index b323c05..f151e4b 100644 --- a/grive/src/main.cc +++ b/grive/src/main.cc @@ -20,7 +20,7 @@ #include "util/Config.hh" #include "base/Drive.hh" -#include "drive/Syncer1.hh" +#include "drive2/Syncer2.hh" #include "http/CurlAgent.hh" #include "protocol/AuthAgent.hh" @@ -49,7 +49,6 @@ const std::string client_id = "22314510474.apps.googleusercontent.com" ; const std::string client_secret = "bl4ufi89h-9MkFlypcI7R785" ; using namespace gr ; -using namespace gr::v1 ; namespace po = boost::program_options; // libgcrypt insist this to be done in application, not library @@ -186,7 +185,7 @@ int Main( int argc, char **argv ) OAuth2 token( refresh_token, client_id, client_secret ) ; AuthAgent agent( token, std::auto_ptr( new http::CurlAgent ) ) ; - Syncer1 syncer( &agent ); + v2::Syncer2 syncer( &agent ); Drive drive( &syncer, config.GetAll() ) ; drive.DetectChanges() ; diff --git a/libgrive/src/base/Syncer.cc b/libgrive/src/base/Syncer.cc index 98d2aca..aa3b031 100644 --- a/libgrive/src/base/Syncer.cc +++ b/libgrive/src/base/Syncer.cc @@ -21,6 +21,10 @@ #include "Resource.hh" #include "Entry.hh" #include "http/Agent.hh" +#include "http/Header.hh" +#include "http/Download.hh" +#include "util/OS.hh" +#include "util/log/Log.hh" namespace gr { @@ -34,6 +38,19 @@ http::Agent* Syncer::Agent() const return m_http; } +void Syncer::Download( Resource *res, const fs::path& file ) +{ + http::Download dl( file.string(), http::Download::NoChecksum() ) ; + long r = m_http->Get( res->ContentSrc(), &dl, http::Header() ) ; + if ( r <= 400 ) + { + if ( res->MTime() != DateTime() ) + os::SetFileTime( file, res->MTime() ) ; + else + Log( "encountered zero date time after downloading %1%", file, log::warning ) ; + } +} + void Syncer::AssignIDs( Resource *res, const Entry& remote ) { res->AssignIDs( remote ); diff --git a/libgrive/src/base/Syncer.hh b/libgrive/src/base/Syncer.hh index efe04bc..33375ee 100644 --- a/libgrive/src/base/Syncer.hh +++ b/libgrive/src/base/Syncer.hh @@ -50,7 +50,7 @@ public : http::Agent* Agent() const; virtual void DeleteRemote( Resource *res ) = 0; - virtual void Download( Resource *res, const fs::path& file ) = 0; + virtual void Download( Resource *res, const fs::path& file ); virtual bool EditContent( Resource *res, bool new_rev ) = 0; virtual bool Create( Resource *res ) = 0; diff --git a/libgrive/src/drive/Syncer1.cc b/libgrive/src/drive/Syncer1.cc index 3cde417..d6d906d 100644 --- a/libgrive/src/drive/Syncer1.cc +++ b/libgrive/src/drive/Syncer1.cc @@ -24,7 +24,6 @@ #include "Syncer1.hh" #include "http/Agent.hh" -#include "http/Download.hh" #include "http/Header.hh" //#include "http/ResponseLog.hh" #include "http/StringResponse.hh" @@ -35,6 +34,7 @@ #include "xml/String.hh" #include "xml/TreeBuilder.hh" +#include "util/File.hh" #include "util/OS.hh" #include "util/log/Log.hh" @@ -88,19 +88,6 @@ void Syncer1::DeleteRemote( Resource *res ) } } -void Syncer1::Download( Resource *res, const fs::path& file ) -{ - http::Download dl( file.string(), http::Download::NoChecksum() ) ; - long r = m_http->Get( res->ContentSrc(), &dl, http::Header() ) ; - if ( r <= 400 ) - { - if ( res->MTime() != DateTime() ) - os::SetFileTime( file, res->MTime() ) ; - else - Log( "encountered zero date time after downloading %1%", file, log::warning ) ; - } -} - bool Syncer1::EditContent( Resource *res, bool new_rev ) { assert( res->Parent() ) ; @@ -180,7 +167,7 @@ bool Syncer1::Upload( Resource *res, if ( retrying ) { file.Seek( 0, SEEK_SET ); - os::Sleep( 2 ); + os::Sleep( 5 ); } try diff --git a/libgrive/src/drive/Syncer1.hh b/libgrive/src/drive/Syncer1.hh index b45e71c..924908b 100644 --- a/libgrive/src/drive/Syncer1.hh +++ b/libgrive/src/drive/Syncer1.hh @@ -35,7 +35,6 @@ public : Syncer1( http::Agent *http ); void DeleteRemote( Resource *res ); - void Download( Resource *res, const fs::path& file ); bool EditContent( Resource *res, bool new_rev ); bool Create( Resource *res ); diff --git a/libgrive/src/drive2/CommonUri.hh b/libgrive/src/drive2/CommonUri.hh index 34c6f04..6c2cee4 100644 --- a/libgrive/src/drive2/CommonUri.hh +++ b/libgrive/src/drive2/CommonUri.hh @@ -23,6 +23,8 @@ namespace gr { namespace v2 { +const std::string upload_base = "https://www.googleapis.com/upload/drive/v2/files" ; + namespace feeds { const std::string files = "https://www.googleapis.com/drive/v2/files" ; diff --git a/libgrive/src/drive2/Entry2.cc b/libgrive/src/drive2/Entry2.cc index 321539f..ec1c382 100644 --- a/libgrive/src/drive2/Entry2.cc +++ b/libgrive/src/drive2/Entry2.cc @@ -55,15 +55,17 @@ void Entry2::Update( const Val& item ) { m_title = file["title"] ; m_etag = file["etag"] ; - m_filename = file["originalFilename"] ; + Val fn; + m_filename = file.Get( "originalFilename", fn ) ? fn.Str() : std::string(); m_content_src = file["downloadUrl"] ; m_self_href = file["selfLink"] ; - m_mtime = DateTime( file["modificationDate"] ) ; + m_mtime = DateTime( file["modifiedDate"] ) ; - m_resource_id = file["id"]; // file#id ? + m_resource_id = file["id"]; m_md5 = file["md5Checksum"] ; - m_is_dir = file["mimeType"].Str() == v2::mime_types::folder ; + m_is_dir = file["mimeType"].Str() == mime_types::folder ; m_is_editable = file["editable"].Bool() ; + m_is_removed = file["labels"]["trashed"].Bool() ; m_parent_hrefs.clear( ) ; diff --git a/libgrive/src/drive2/Syncer2.cc b/libgrive/src/drive2/Syncer2.cc index 1855e1c..f45ff7c 100644 --- a/libgrive/src/drive2/Syncer2.cc +++ b/libgrive/src/drive2/Syncer2.cc @@ -26,8 +26,10 @@ #include "http/Agent.hh" #include "http/Download.hh" #include "http/Header.hh" +#include "http/StringResponse.hh" //#include "http/ResponseLog.hh" #include "json/ValResponse.hh" +#include "json/JsonWriter.hh" #include "util/OS.hh" #include "util/log/Log.hh" @@ -49,30 +51,129 @@ Syncer2::Syncer2( http::Agent *http ): void Syncer2::DeleteRemote( Resource *res ) { -} - -void Syncer2::Download( Resource *res, const fs::path& file ) -{ + http::StringResponse str ; + http::Header hdr ; + hdr.Add( "If-Match: " + res->ETag() ) ; + m_http->Post( res->SelfHref() + "/trash", "", &str, hdr ) ; } bool Syncer2::EditContent( Resource *res, bool new_rev ) { - return false ; + assert( res->Parent() ) ; + assert( !res->ResourceID().empty() ) ; + assert( res->Parent()->GetState() == Resource::sync ) ; + + if ( !res->IsEditable() ) + { + Log( "Cannot upload %1%: file read-only. %2%", res->Name(), res->StateStr(), log::warning ) ; + return false ; + } + + return Upload( res ) ; } bool Syncer2::Create( Resource *res ) { - return false ; + assert( res->Parent() ) ; + assert( res->Parent()->IsFolder() ) ; + assert( res->Parent()->GetState() == Resource::sync ) ; + assert( res->ResourceID().empty() ) ; + + if ( !res->Parent()->IsEditable() ) + { + Log( "Cannot upload %1%: parent directory read-only. %2%", res->Name(), res->StateStr(), log::warning ) ; + return false ; + } + + return Upload( res ); +} + +bool Syncer2::Upload( Resource *res ) +{ + File file( res->Path() ) ; + std::ostringstream xcontent_len ; + xcontent_len << "Content-Length: " << file.Size() ; + + http::Header hdr ; + hdr.Add( "Content-Type: application/octet-stream" ) ; + hdr.Add( xcontent_len.str() ) ; + if ( !res->ETag().empty() ) + hdr.Add( "If-Match: " + res->ETag() ) ; + + Val meta; + meta.Add( "title", Val( res->Name() ) ); + if ( res->IsFolder() ) + { + meta.Add( "mimeType", Val( mime_types::folder ) ); + } + if ( !res->Parent()->IsRoot() ) + { + Val parent; + parent.Add( "id", Val( res->Parent()->ResourceID() ) ); + Val parents( Val::array_type ); + parents.Add( parent ); + meta.Add( "parents", parents ); + } + std::string json_meta = WriteJson( meta ); + + Val valr ; + + // Issue metadata update request + { + http::Header hdr2 ; + hdr2.Add( "Content-Type: application/json" ); + http::ValResponse vrsp ; + long http_code = 0; + if ( res->ResourceID().empty() ) + http_code = m_http->Post( feeds::files, json_meta, &vrsp, hdr2 ) ; + else + http_code = m_http->Put( feeds::files + "/" + res->ResourceID(), json_meta, &vrsp, hdr2 ) ; + valr = vrsp.Response(); + assert( !(valr["id"].Str().empty()) ); + } + + bool retrying = false; + while ( true ) + { + if ( retrying ) + { + file.Seek( 0, SEEK_SET ); + os::Sleep( 5 ); + } + + if ( !res->IsFolder() ) + { + http::ValResponse vrsp; + long http_code = m_http->Put( upload_base + "/" + valr["id"].Str() + "?uploadType=media", &file, &vrsp, hdr ) ; + if ( http_code == 410 || http_code == 412 ) + { + Log( "request failed with %1%, retrying whole upload in 5s", http_code, log::warning ) ; + retrying = true; + continue; + } + valr = vrsp.Response(); + assert( !(valr["id"].Str().empty()) ); + } + + if ( retrying ) + Log( "upload succeeded on retry", log::warning ); + Entry2 responseEntry = Entry2( valr ); + AssignIDs( res, responseEntry ) ; + AssignMTime( res, responseEntry.MTime() ); + break; + } + + return true ; } std::auto_ptr Syncer2::GetFolders() { - return std::auto_ptr( new Feed2( feeds::files + "?maxResults=1000&q=%27me%27+in+readers+and+mimeType%3d%27" + mime_types::folder + "%27" ) ); + return std::auto_ptr( new Feed2( feeds::files + "?maxResults=1000&q=%27me%27+in+readers+and+trashed%3dfalse+and+mimeType%3d%27" + mime_types::folder + "%27" ) ); } std::auto_ptr Syncer2::GetAll() { - return std::auto_ptr( new Feed2( feeds::files + "?maxResults=1000&q=%27me%27+in+readers" ) ); + return std::auto_ptr( new Feed2( feeds::files + "?maxResults=1000&q=%27me%27+in+readers+and+trashed%3dfalse" ) ); } std::string ChangesFeed( long changestamp, int maxResults = 1000 ) diff --git a/libgrive/src/drive2/Syncer2.hh b/libgrive/src/drive2/Syncer2.hh index 35142a9..52da64c 100644 --- a/libgrive/src/drive2/Syncer2.hh +++ b/libgrive/src/drive2/Syncer2.hh @@ -35,7 +35,6 @@ public : Syncer2( http::Agent *http ); void DeleteRemote( Resource *res ); - void Download( Resource *res, const fs::path& file ); bool EditContent( Resource *res, bool new_rev ); bool Create( Resource *res ); @@ -46,6 +45,8 @@ public : private : + bool Upload( Resource *res ); + } ; } } // end of namespace gr::v2 diff --git a/libgrive/src/http/CurlAgent.cc b/libgrive/src/http/CurlAgent.cc index d725e86..53d1e27 100644 --- a/libgrive/src/http/CurlAgent.cc +++ b/libgrive/src/http/CurlAgent.cc @@ -232,18 +232,15 @@ long CurlAgent::Get( long CurlAgent::Post( const std::string& url, - const std::string& data, + const std::string& post_data, DataStream *dest, const Header& hdr ) { - Trace("HTTP POST \"%1%\" with \"%2%\"", url, data ) ; + Trace("HTTP POST \"%1%\" with \"%2%\"", url, post_data ) ; Init() ; CURL *curl = m_pimpl->curl ; - // make a copy because the parameter is const - std::string post_data = data ; - // set post specific options ::curl_easy_setopt(curl, CURLOPT_POST, 1L); ::curl_easy_setopt(curl, CURLOPT_POSTFIELDS, &post_data[0] ) ; diff --git a/libgrive/src/http/CurlAgent.hh b/libgrive/src/http/CurlAgent.hh index c142c3b..7b4f2b7 100644 --- a/libgrive/src/http/CurlAgent.hh +++ b/libgrive/src/http/CurlAgent.hh @@ -43,7 +43,7 @@ public : long Put( const std::string& url, - const std::string& data, + const std::string& post_data, DataStream *dest, const Header& hdr ) ; diff --git a/libgrive/src/json/JsonParser.cc b/libgrive/src/json/JsonParser.cc index 50092cf..9727b8e 100644 --- a/libgrive/src/json/JsonParser.cc +++ b/libgrive/src/json/JsonParser.cc @@ -39,7 +39,7 @@ namespace int OnBool( void *ctx, int value ) { ValVisitor *b = reinterpret_cast(ctx) ; - b->Visit( static_cast(value) ) ; + b->Visit( static_cast(value) ) ; return true ; } diff --git a/libgrive/src/json/JsonWriter.cc b/libgrive/src/json/JsonWriter.cc index 9fbcc2b..700c10c 100644 --- a/libgrive/src/json/JsonWriter.cc +++ b/libgrive/src/json/JsonWriter.cc @@ -19,7 +19,7 @@ */ #include "JsonWriter.hh" -#include "util/DataStream.hh" +#include "util/StringStream.hh" #include @@ -108,4 +108,12 @@ void JsonWriter::WriteCallback( void *ctx, const char *str, std::size_t size ) pthis->m_impl->out->Write( str, size ) ; } +std::string WriteJson( const Val& val ) +{ + StringStream ss ; + JsonWriter wr( &ss ) ; + val.Visit( &wr ) ; + return ss.Str() ; +} + } // end of namespace diff --git a/libgrive/src/json/JsonWriter.hh b/libgrive/src/json/JsonWriter.hh index 350357d..66f95bf 100644 --- a/libgrive/src/json/JsonWriter.hh +++ b/libgrive/src/json/JsonWriter.hh @@ -20,6 +20,7 @@ #pragma once +#include "Val.hh" #include "ValVisitor.hh" #include @@ -53,5 +54,7 @@ private : std::auto_ptr m_impl ; } ; +std::string WriteJson( const Val& val ); + } // end of namespace diff --git a/libgrive/src/json/Val.cc b/libgrive/src/json/Val.cc index 31944ff..1debddc 100644 --- a/libgrive/src/json/Val.cc +++ b/libgrive/src/json/Val.cc @@ -114,11 +114,15 @@ Val::operator std::string() const int Val::Int() const { + if ( Type() == string_type ) + return std::atoi( As().c_str() ); return static_cast(As()) ; } double Val::Double() const { + if ( Type() == string_type ) + return std::atof( As().c_str() ); return As() ; } @@ -161,6 +165,11 @@ void Val::Add( const std::string& key, const Val& value ) As().insert( std::make_pair(key, value) ) ; } +void Val::Add( const Val& json ) +{ + As().push_back( json ) ; +} + void Val::Visit( ValVisitor *visitor ) const { switch ( Type() )