Implement upload methods for REST API Syncer (basic file upload to root now works)

grive/src/main.cc

@@ -20,7 +20,7 @@
 #include "util/Config.hh"
 
 #include "base/Drive.hh"
-#include "drive/Syncer1.hh"
+#include "drive2/Syncer2.hh"
 
 #include "http/CurlAgent.hh"
 #include "protocol/AuthAgent.hh"
@@ -49,7 +49,6 @@ const std::string client_id		= "22314510474.apps.googleusercontent.com" ;
 const std::string client_secret	= "bl4ufi89h-9MkFlypcI7R785" ;
 
 using namespace gr ;
-using namespace gr::v1 ;
 namespace po = boost::program_options;
 
 // libgcrypt insist this to be done in application, not library
@@ -186,7 +185,7 @@ int Main( int argc, char **argv )
 	
 	OAuth2 token( refresh_token, client_id, client_secret ) ;
 	AuthAgent agent( token, std::auto_ptr<http::Agent>( new http::CurlAgent ) ) ;
-	Syncer1 syncer( &agent );
+	v2::Syncer2 syncer( &agent );
 
 	Drive drive( &syncer, config.GetAll() ) ;
 	drive.DetectChanges() ;

libgrive/src/base/Syncer.cc

@@ -21,6 +21,10 @@
 #include "Resource.hh"
 #include "Entry.hh"
 #include "http/Agent.hh"
+#include "http/Header.hh"
+#include "http/Download.hh"
+#include "util/OS.hh"
+#include "util/log/Log.hh"
 
 namespace gr {
 
@@ -34,6 +38,19 @@ http::Agent* Syncer::Agent() const
 	return m_http;
 }
 
+void Syncer::Download( Resource *res, const fs::path& file )
+{
+	http::Download dl( file.string(), http::Download::NoChecksum() ) ;
+	long r = m_http->Get( res->ContentSrc(), &dl, http::Header() ) ;
+	if ( r <= 400 )
+	{
+		if ( res->MTime() != DateTime() )
+			os::SetFileTime( file, res->MTime() ) ;
+		else
+			Log( "encountered zero date time after downloading %1%", file, log::warning ) ;
+	}
+}
+
 void Syncer::AssignIDs( Resource *res, const Entry& remote )
 {
 	res->AssignIDs( remote );

libgrive/src/base/Syncer.hh

@@ -50,7 +50,7 @@ public :
 	http::Agent* Agent() const;
 
 	virtual void DeleteRemote( Resource *res ) = 0;
-	virtual void Download( Resource *res, const fs::path& file ) = 0;
+	virtual void Download( Resource *res, const fs::path& file );
 	virtual bool EditContent( Resource *res, bool new_rev ) = 0;
 	virtual bool Create( Resource *res ) = 0;
 

libgrive/src/drive/Syncer1.cc

@@ -24,7 +24,6 @@
 #include "Syncer1.hh"
 
 #include "http/Agent.hh"
-#include "http/Download.hh"
 #include "http/Header.hh"
 //#include "http/ResponseLog.hh"
 #include "http/StringResponse.hh"
@@ -35,6 +34,7 @@
 #include "xml/String.hh"
 #include "xml/TreeBuilder.hh"
 
+#include "util/File.hh"
 #include "util/OS.hh"
 #include "util/log/Log.hh"
 
@@ -88,19 +88,6 @@ void Syncer1::DeleteRemote( Resource *res )
 	}
 }
 
-void Syncer1::Download( Resource *res, const fs::path& file )
-{
-	http::Download dl( file.string(), http::Download::NoChecksum() ) ;
-	long r = m_http->Get( res->ContentSrc(), &dl, http::Header() ) ;
-	if ( r <= 400 )
-	{
-		if ( res->MTime() != DateTime() )
-			os::SetFileTime( file, res->MTime() ) ;
-		else
-			Log( "encountered zero date time after downloading %1%", file, log::warning ) ;
-	}
-}
-
 bool Syncer1::EditContent( Resource *res, bool new_rev )
 {
 	assert( res->Parent() ) ;
@@ -180,7 +167,7 @@ bool Syncer1::Upload( Resource *res,
 		if ( retrying )
 		{
 			file.Seek( 0, SEEK_SET );
-			os::Sleep( 2 );
+			os::Sleep( 5 );
 		}
 
 		try

libgrive/src/drive/Syncer1.hh

@@ -35,7 +35,6 @@ public :
 	Syncer1( http::Agent *http );
 
 	void DeleteRemote( Resource *res );
-	void Download( Resource *res, const fs::path& file );
 	bool EditContent( Resource *res, bool new_rev );
 	bool Create( Resource *res );
 

libgrive/src/drive2/CommonUri.hh

@@ -23,6 +23,8 @@
 
 namespace gr { namespace v2 {
 
+const std::string upload_base = "https://www.googleapis.com/upload/drive/v2/files" ;
+
 namespace feeds
 {
 	const std::string files		= "https://www.googleapis.com/drive/v2/files" ;

libgrive/src/drive2/Entry2.cc

@@ -55,15 +55,17 @@ void Entry2::Update( const Val& item )
 	{
 		m_title			= file["title"] ;
 		m_etag			= file["etag"] ;
-		m_filename		= file["originalFilename"] ;
+		Val fn;
+		m_filename		= file.Get( "originalFilename", fn ) ? fn.Str() : std::string();
 		m_content_src	= file["downloadUrl"] ;
 		m_self_href		= file["selfLink"] ;
-		m_mtime			= DateTime( file["modificationDate"] ) ;
+		m_mtime			= DateTime( file["modifiedDate"] ) ;
 
-		m_resource_id	= file["id"]; // file#id ?
+		m_resource_id	= file["id"];
 		m_md5			= file["md5Checksum"] ;
-		m_is_dir		= file["mimeType"].Str() == v2::mime_types::folder ;
+		m_is_dir		= file["mimeType"].Str() == mime_types::folder ;
 		m_is_editable	= file["editable"].Bool() ;
+		m_is_removed	= file["labels"]["trashed"].Bool() ;
 
 		m_parent_hrefs.clear( ) ;
 

libgrive/src/drive2/Syncer2.cc

@@ -26,8 +26,10 @@
 #include "http/Agent.hh"
 #include "http/Download.hh"
 #include "http/Header.hh"
+#include "http/StringResponse.hh"
 //#include "http/ResponseLog.hh"
 #include "json/ValResponse.hh"
+#include "json/JsonWriter.hh"
 
 #include "util/OS.hh"
 #include "util/log/Log.hh"
@@ -49,30 +51,129 @@ Syncer2::Syncer2( http::Agent *http ):
 
 void Syncer2::DeleteRemote( Resource *res )
 {
-}
-
-void Syncer2::Download( Resource *res, const fs::path& file )
-{
+	http::StringResponse str ;
+	http::Header hdr ;
+	hdr.Add( "If-Match: " + res->ETag() ) ;
+	m_http->Post( res->SelfHref() + "/trash", "", &str, hdr ) ;
 }
 
 bool Syncer2::EditContent( Resource *res, bool new_rev )
 {
-	return false ;
+	assert( res->Parent() ) ;
+	assert( !res->ResourceID().empty() ) ;
+	assert( res->Parent()->GetState() == Resource::sync ) ;
+
+	if ( !res->IsEditable() )
+	{
+		Log( "Cannot upload %1%: file read-only. %2%", res->Name(), res->StateStr(), log::warning ) ;
+		return false ;
+	}
+
+	return Upload( res ) ;
 }
 
 bool Syncer2::Create( Resource *res )
 {
-	return false ;
+	assert( res->Parent() ) ;
+	assert( res->Parent()->IsFolder() ) ;
+	assert( res->Parent()->GetState() == Resource::sync ) ;
+	assert( res->ResourceID().empty() ) ;
+	
+	if ( !res->Parent()->IsEditable() )
+	{
+		Log( "Cannot upload %1%: parent directory read-only. %2%", res->Name(), res->StateStr(), log::warning ) ;
+		return false ;
+	}
+	
+	return Upload( res );
+}
+
+bool Syncer2::Upload( Resource *res )
+{
+	File file( res->Path() ) ;
+	std::ostringstream xcontent_len ;
+	xcontent_len << "Content-Length: " << file.Size() ;
+
+	http::Header hdr ;
+	hdr.Add( "Content-Type: application/octet-stream" ) ;
+	hdr.Add( xcontent_len.str() ) ;
+	if ( !res->ETag().empty() )
+		hdr.Add( "If-Match: " + res->ETag() ) ;
+
+	Val meta;
+	meta.Add( "title", Val( res->Name() ) );
+	if ( res->IsFolder() )
+	{
+		meta.Add( "mimeType", Val( mime_types::folder ) );
+	}
+	if ( !res->Parent()->IsRoot() )
+	{
+		Val parent;
+		parent.Add( "id", Val( res->Parent()->ResourceID() ) );
+		Val parents( Val::array_type );
+		parents.Add( parent );
+		meta.Add( "parents", parents );
+	}
+	std::string json_meta = WriteJson( meta );
+
+	Val valr ;
+
+	// Issue metadata update request
+	{
+		http::Header hdr2 ;
+		hdr2.Add( "Content-Type: application/json" );
+		http::ValResponse vrsp ;
+		long http_code = 0;
+		if ( res->ResourceID().empty() )
+			http_code = m_http->Post( feeds::files, json_meta, &vrsp, hdr2 ) ;
+		else
+			http_code = m_http->Put( feeds::files + "/" + res->ResourceID(), json_meta, &vrsp, hdr2 ) ;
+		valr = vrsp.Response();
+		assert( !(valr["id"].Str().empty()) );
+	}
+
+	bool retrying = false;
+	while ( true )
+	{
+		if ( retrying )
+		{
+			file.Seek( 0, SEEK_SET );
+			os::Sleep( 5 );
+		}
+
+		if ( !res->IsFolder() )
+		{
+			http::ValResponse vrsp;
+			long http_code = m_http->Put( upload_base + "/" + valr["id"].Str() + "?uploadType=media", &file, &vrsp, hdr ) ;
+			if ( http_code == 410 || http_code == 412 )
+			{
+				Log( "request failed with %1%, retrying whole upload in 5s", http_code, log::warning ) ;
+				retrying = true;
+				continue;
+			}
+			valr = vrsp.Response();
+			assert( !(valr["id"].Str().empty()) );
+		}
+
+		if ( retrying )
+			Log( "upload succeeded on retry", log::warning );
+		Entry2 responseEntry = Entry2( valr );
+		AssignIDs( res, responseEntry ) ;
+		AssignMTime( res, responseEntry.MTime() );
+		break;
+	}
+	
+	return true ;
 }
 
 std::auto_ptr<Feed> Syncer2::GetFolders()
 {
-	return std::auto_ptr<Feed>( new Feed2( feeds::files + "?maxResults=1000&q=%27me%27+in+readers+and+mimeType%3d%27" + mime_types::folder + "%27" ) );
+	return std::auto_ptr<Feed>( new Feed2( feeds::files + "?maxResults=1000&q=%27me%27+in+readers+and+trashed%3dfalse+and+mimeType%3d%27" + mime_types::folder + "%27" ) );
 }
 
 std::auto_ptr<Feed> Syncer2::GetAll()
 {
-	return std::auto_ptr<Feed>( new Feed2( feeds::files + "?maxResults=1000&q=%27me%27+in+readers" ) );
+	return std::auto_ptr<Feed>( new Feed2( feeds::files + "?maxResults=1000&q=%27me%27+in+readers+and+trashed%3dfalse" ) );
 }
 
 std::string ChangesFeed( long changestamp, int maxResults = 1000 )

libgrive/src/drive2/Syncer2.hh

@@ -35,7 +35,6 @@ public :
 	Syncer2( http::Agent *http );
 
 	void DeleteRemote( Resource *res );
-	void Download( Resource *res, const fs::path& file );
 	bool EditContent( Resource *res, bool new_rev );
 	bool Create( Resource *res );
 
@@ -46,6 +45,8 @@ public :
 
 private :
 
+	bool Upload( Resource *res );
+
 } ;
 
 } } // end of namespace gr::v2

libgrive/src/http/CurlAgent.cc

@@ -232,18 +232,15 @@ long CurlAgent::Get(
 
 long CurlAgent::Post(
 	const std::string& 		url,
-	const std::string&		data,
+	const std::string&		post_data,
 	DataStream				*dest,
 	const Header&			hdr )
 {
-	Trace("HTTP POST \"%1%\" with \"%2%\"", url, data ) ;
+	Trace("HTTP POST \"%1%\" with \"%2%\"", url, post_data ) ;
 
 	Init() ;
 	CURL *curl = m_pimpl->curl ;
 
-	// make a copy because the parameter is const
-	std::string post_data = data ;
-	
 	// set post specific options
 	::curl_easy_setopt(curl, CURLOPT_POST, 			1L);
 	::curl_easy_setopt(curl, CURLOPT_POSTFIELDS,	&post_data[0] ) ;

libgrive/src/http/CurlAgent.hh

@@ -43,7 +43,7 @@ public :
 	
 	long Put(
 		const std::string&	url,
-		const std::string&	data,
+		const std::string&	post_data,
 		DataStream			*dest,
 		const Header&		hdr ) ;
 	

libgrive/src/json/JsonParser.cc

@@ -39,7 +39,7 @@ namespace
 	int OnBool( void *ctx, int value )
 	{
 		ValVisitor *b = reinterpret_cast<ValVisitor*>(ctx) ;
-		b->Visit( static_cast<long long>(value) ) ;
+		b->Visit( static_cast<bool>(value) ) ;
 		return true ;
 	}
 	

libgrive/src/json/JsonWriter.cc

@@ -19,7 +19,7 @@
 */
 
 #include "JsonWriter.hh"
-#include "util/DataStream.hh"
+#include "util/StringStream.hh"
 
 #include <yajl/yajl_gen.h>
 
@@ -108,4 +108,12 @@ void JsonWriter::WriteCallback( void *ctx, const char *str, std::size_t size )
 	pthis->m_impl->out->Write( str, size ) ;
 }
 
+std::string WriteJson( const Val& val )
+{
+	StringStream ss ;
+	JsonWriter wr( &ss ) ;
+	val.Visit( &wr ) ;
+	return ss.Str() ;
+}
+
 } // end of namespace

libgrive/src/json/JsonWriter.hh

@@ -20,6 +20,7 @@
 
 #pragma once
 
+#include "Val.hh"
 #include "ValVisitor.hh"
 #include <memory>
 
@@ -53,5 +54,7 @@ private :
 	std::auto_ptr<Impl>	m_impl ;
 } ;
 
+std::string WriteJson( const Val& val );
+
 } // end of namespace
 

libgrive/src/json/Val.cc

@@ -114,11 +114,15 @@ Val::operator std::string() const
 
 int Val::Int() const
 {
+	if ( Type() == string_type )
+		return std::atoi( As<std::string>().c_str() );
 	return static_cast<int>(As<long long>()) ;
 }
 
 double Val::Double() const
 {
+	if ( Type() == string_type )
+		return std::atof( As<std::string>().c_str() );
 	return As<double>() ;
 }
 
@@ -161,6 +165,11 @@ void Val::Add( const std::string& key, const Val& value )
 	As<Object>().insert( std::make_pair(key, value) ) ;
 }
 
+void Val::Add( const Val& json )
+{
+	As<Array>().push_back( json ) ;
+}
+
 void Val::Visit( ValVisitor *visitor ) const
 {
 	switch ( Type() )