From 91cd90b19ae0c7529947aacc55ff169e82c6a38e Mon Sep 17 00:00:00 2001 From: Matchman Green Date: Sun, 29 Apr 2012 10:22:58 +0800 Subject: [PATCH 1/4] fixed still memcpy bug --- src/drive/Drive.cc | 39 +++++++++++++++++-- src/protocol/HTTP.cc | 89 +++++++++++++++++++++++++++++++++++++++++--- src/protocol/HTTP.hh | 2 + 3 files changed, 121 insertions(+), 9 deletions(-) diff --git a/src/drive/Drive.cc b/src/drive/Drive.cc index c3f09a9..50e4942 100644 --- a/src/drive/Drive.cc +++ b/src/drive/Drive.cc @@ -31,6 +31,7 @@ #include #include #include +#include // for debugging only #include @@ -190,7 +191,7 @@ void Drive::UpdateFile( const Json& entry ) DateTime local = ifile ? os::FileMTime( path ) : DateTime() ; // remote file is newer, download file - if ( remote > local ) + if ( !ifile || remote > local ) { std::cout << "downloading " << path << std::endl ; http::GetFile( url, path, m_http_hdr ) ; @@ -213,9 +214,41 @@ void Drive::UploadFile( const Json& entry ) "http://schemas.google.com/g/2005#resumable-edit-media" )["href"] ; std::cout << resume_link.As() << std::endl ; - std::string resp = http::Put( resume_link.Get(), "", m_http_hdr ) ; + http::Headers hdr( m_http_hdr ) ; + hdr.push_back( "Expect:" ) ; + hdr.push_back( "If-Match: *" ) ; + hdr.push_back( "X-Upload-Content-Length: 29" ) ; + hdr.push_back( "X-Upload-Content-Type: text/plain" ) ; - std::cout << "resp " << resp ; + std::istringstream resp( http::Put( resume_link.Get(), "", hdr ) ) ; + + std::string line ; + while ( std::getline( resp, line ) ) + { + // find the location header + static const std::string location = "Location: " ; + if ( line.substr( 0, location.size() ) == location ) + { + std::string upload_uri = line.substr( location.size() ) ; + upload_uri = upload_uri.substr( 0, upload_uri.size() - 1 ) ; + std::cout << upload_uri << std::endl ; + + std::string upload_content = "this is the new text file!!!!" ; + + std::ostringstream content_range ; + content_range << "Content-Range: 0-" << upload_content.size()-1 << '/' << upload_content.size() ; + + http::Headers upload_hdr/*( m_http_hdr )*/ ; + upload_hdr.push_back( "Expect:" ) ; + upload_hdr.push_back( "Accept:" ) ; + upload_hdr.push_back( "Host:" ) ; +// upload_hdr.push_back( "Content-Type: text/plain" ) ; + upload_hdr.push_back( content_range.str() ) ; + + std::string resp2 = http::Put( upload_uri, upload_content, upload_hdr ) ; + std::cout << resp2 << std::endl ; + } + } } } // end of namespace diff --git a/src/protocol/HTTP.cc b/src/protocol/HTTP.cc index 452fddc..e084f3f 100644 --- a/src/protocol/HTTP.cc +++ b/src/protocol/HTTP.cc @@ -54,9 +54,10 @@ size_t ReadCallback( void *ptr, std::size_t size, std::size_t nmemb, std::string std::size_t count = std::min( size * nmemb, data->size() ) ; if ( count > 0 ) { - std::memcpy( &(*data)[0], ptr, count ) ; + std::memcpy( ptr, &(*data)[0], count ) ; data->erase( 0, count ) ; } + return count ; } @@ -162,7 +163,7 @@ std::string PostData( const std::string& url, const std::string& data, const Hea std::string post_data = data ; - curl_easy_setopt(curl, CURLOPT_POST, 1); + curl_easy_setopt(curl, CURLOPT_POST, 1L); curl_easy_setopt(curl, CURLOPT_POSTFIELDS, &post_data[0] ) ; curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, post_data.size() ) ; @@ -183,18 +184,94 @@ std::string Put( { std::string resp ; CURL *curl = InitCurl( url, &resp, hdr ) ; - - std::string put_data = data ; - curl_easy_setopt(curl, CURLOPT_UPLOAD, 1); + std::string put_data = data ; + + // set common options + curl_easy_setopt(curl, CURLOPT_HEADER, 1L ); + curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L ) ; curl_easy_setopt(curl, CURLOPT_READFUNCTION, &ReadCallback ) ; curl_easy_setopt(curl, CURLOPT_READDATA , &put_data ) ; curl_easy_setopt(curl, CURLOPT_INFILESIZE, put_data.size() ) ; - + curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L ) ; + DoCurl( curl ) ; return resp; } +void Custom( const std::string& url, const std::string& host, const Headers& headers ) +{ + CURL *curl = curl_easy_init(); + if ( curl == 0 ) + throw std::bad_alloc() ; + + // set common options + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_CONNECT_ONLY, 1L ); + curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L ) ; + if ( curl_easy_perform( curl ) != 0 ) + throw std::runtime_error( "curl perform fail" ) ; + + long sockextr; + curl_easy_getinfo(curl, CURLINFO_LASTSOCKET, &sockextr); + curl_socket_t sockfd = sockextr ; + + struct timeval tv; + tv.tv_sec = 60 ; + tv.tv_usec = 0 ; + + fd_set infd, outfd, errfd; + FD_ZERO(&infd); + FD_ZERO(&outfd); + FD_ZERO(&errfd); + + FD_SET(sockfd, &errfd); /* always check for error */ + FD_SET(sockfd, &outfd); + if ( select(sockfd + 1, &infd, &outfd, &errfd, &tv) == -1 ) + throw std::runtime_error( "select fail" ) ; + + std::string path = url.substr( host.size() ) ; + std::cout << ("PUT " + path + "HTTP/1.1") << std::endl ; + + std::ostringstream req ; + req << ("PUT " + path + "HTTP/1.1\n") + << "Host: " << "docs.google.com" << "\n" ; + + for ( Headers::const_iterator i = headers.begin() ; i != headers.end() ; ++i ) + req << *i << '\n' ; + + std::string data = "hahaha this is the new file!!!!!" ; + req << "Content-Length: " << data.size() << '\n' + << "Content-Range: 0-" << data.size()-1 << '/' << data.size() << "\n\n\n" + << data ; + + std::string reqstr = req.str() ; + std::cout << "requesting: \n" << reqstr << std::endl ; + + std::size_t iolen ; + if ( curl_easy_send(curl, &reqstr[0], reqstr.size(), &iolen) != CURLE_OK ) + throw std::runtime_error( "cannot send" ) ; + + while ( true ) + { + char buf[1024+1] ; + FD_ZERO(&infd); + FD_ZERO(&outfd); + FD_ZERO(&errfd); + + FD_SET(sockfd, &errfd); /* always check for error */ + FD_SET(sockfd, &infd); + if ( select(sockfd + 1, &infd, &outfd, &errfd, &tv) == -1 ) + throw std::runtime_error( "select fail" ) ; + + if ( curl_easy_recv(curl, buf, 1024, &iolen) != CURLE_OK ) + throw std::runtime_error( "cannot send" ) ; + + buf[iolen] = '\0' ; + std::cout << "read: " << buf << std::endl ; + } +} + std::string Escape( const std::string& str ) { CURL *curl = curl_easy_init(); diff --git a/src/protocol/HTTP.hh b/src/protocol/HTTP.hh index 669d57c..107f2ef 100644 --- a/src/protocol/HTTP.hh +++ b/src/protocol/HTTP.hh @@ -53,6 +53,8 @@ namespace gr { namespace http const std::string& data, const Headers& hdr = Headers() ) ; + void Custom( const std::string& url, const std::string& host, const Headers& headers ) ; + std::string Escape( const std::string& str ) ; std::string Unescape( const std::string& str ) ; From 011155e6a22f3050114d0a49b1dd259404bd1168 Mon Sep 17 00:00:00 2001 From: Matchman Green Date: Sun, 29 Apr 2012 17:15:03 +0800 Subject: [PATCH 2/4] can upload a new file now! --- src/drive/Drive.cc | 54 +++++++++++++++++++++----------------------- src/protocol/HTTP.cc | 18 +++++++++++++++ src/protocol/HTTP.hh | 4 ++++ 3 files changed, 48 insertions(+), 28 deletions(-) diff --git a/src/drive/Drive.cc b/src/drive/Drive.cc index 50e4942..938ff7b 100644 --- a/src/drive/Drive.cc +++ b/src/drive/Drive.cc @@ -200,7 +200,7 @@ std::cout << "downloading " << path << std::endl ; else { std::cout << "local " << filename << " is newer" << std::endl ; -// UploadFile( entry ) ; + UploadFile( entry ) ; } } } @@ -208,45 +208,43 @@ std::cout << "local " << filename << " is newer" << std::endl ; void Drive::UploadFile( const Json& entry ) { -// std::cout << "entry:\n" << entry << std::endl ; + std::string meta = + "" + "" + "" + "Grive Document" + "" ; - Json resume_link = entry["link"].FindInArray( "rel", - "http://schemas.google.com/g/2005#resumable-edit-media" )["href"] ; - std::cout << resume_link.As() << std::endl ; - http::Headers hdr( m_http_hdr ) ; - hdr.push_back( "Expect:" ) ; - hdr.push_back( "If-Match: *" ) ; - hdr.push_back( "X-Upload-Content-Length: 29" ) ; + hdr.push_back( "Slug: Grive Document" ) ; hdr.push_back( "X-Upload-Content-Type: text/plain" ) ; + hdr.push_back( "X-Upload-Content-Length: 10" ) ; - std::istringstream resp( http::Put( resume_link.Get(), "", hdr ) ) ; + std::string resp = http::PostDataWithHeader( + "https://docs.google.com/feeds/upload/create-session/default/private/full?convert=false", + "", + m_http_hdr ) ; + + std::cout << "resp " << resp ; + std::istringstream ss( resp ) ; std::string line ; - while ( std::getline( resp, line ) ) + while ( std::getline( ss, line ) ) { - // find the location header static const std::string location = "Location: " ; if ( line.substr( 0, location.size() ) == location ) { - std::string upload_uri = line.substr( location.size() ) ; - upload_uri = upload_uri.substr( 0, upload_uri.size() - 1 ) ; - std::cout << upload_uri << std::endl ; + std::string uplink = line.substr( location.size() ) ; + uplink = uplink.substr( 0, uplink.size() -1 ) ; - std::string upload_content = "this is the new text file!!!!" ; + std::string data( 10, '!' ) ; + http::Headers uphdr ; + uphdr.push_back( "Content-Type: text/plain" ) ; + uphdr.push_back( "Content-Range: bytes 0-9/10" ) ; - std::ostringstream content_range ; - content_range << "Content-Range: 0-" << upload_content.size()-1 << '/' << upload_content.size() ; - - http::Headers upload_hdr/*( m_http_hdr )*/ ; - upload_hdr.push_back( "Expect:" ) ; - upload_hdr.push_back( "Accept:" ) ; - upload_hdr.push_back( "Host:" ) ; -// upload_hdr.push_back( "Content-Type: text/plain" ) ; - upload_hdr.push_back( content_range.str() ) ; - - std::string resp2 = http::Put( upload_uri, upload_content, upload_hdr ) ; - std::cout << resp2 << std::endl ; + std::string resp = http::Put( uplink, data, uphdr ) ; + std::cout << "put response = " << resp << std::endl ; } } } diff --git a/src/protocol/HTTP.cc b/src/protocol/HTTP.cc index e084f3f..f030a9e 100644 --- a/src/protocol/HTTP.cc +++ b/src/protocol/HTTP.cc @@ -166,6 +166,24 @@ std::string PostData( const std::string& url, const std::string& data, const Hea curl_easy_setopt(curl, CURLOPT_POST, 1L); curl_easy_setopt(curl, CURLOPT_POSTFIELDS, &post_data[0] ) ; curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, post_data.size() ) ; + curl_easy_setopt(curl, CURLOPT_VERBOSE, 1 ) ; + + DoCurl( curl ) ; + return resp; +} + +std::string PostDataWithHeader( const std::string& url, const std::string& data, const Headers& hdr ) +{ + std::string resp ; + CURL *curl = InitCurl( url, &resp, hdr ) ; + + std::string post_data = data ; + + curl_easy_setopt(curl, CURLOPT_POST, 1L); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, &post_data[0] ) ; + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, post_data.size() ) ; + curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L ) ; + curl_easy_setopt(curl, CURLOPT_HEADER, 1L ); DoCurl( curl ) ; return resp; diff --git a/src/protocol/HTTP.hh b/src/protocol/HTTP.hh index 107f2ef..df53a1e 100644 --- a/src/protocol/HTTP.hh +++ b/src/protocol/HTTP.hh @@ -43,6 +43,10 @@ namespace gr { namespace http const std::string& url, const std::string& data, const Headers& hdr = Headers() ) ; + std::string PostDataWithHeader( + const std::string& url, + const std::string& data, + const Headers& hdr = Headers() ) ; std::string PostFile( const std::string& url, const std::string& filename, From a681cb2b39170ddccbb1c7eaffd066f099ef07ce Mon Sep 17 00:00:00 2001 From: Andrea Scarpino Date: Sun, 29 Apr 2012 18:40:13 +0300 Subject: [PATCH 3/4] Fix build with GCC 4.7 --- src/main.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main.cc b/src/main.cc index c542f23..ef14874 100644 --- a/src/main.cc +++ b/src/main.cc @@ -25,6 +25,7 @@ #include #include #include +#include #include #include From db092f30513b27934816559cb4d8c28a5832c6fb Mon Sep 17 00:00:00 2001 From: Matchman Green Date: Mon, 30 Apr 2012 00:03:04 +0800 Subject: [PATCH 4/4] working on upload problem. trying to exclude unit test if cppunit is not found --- CMakeLists.txt | 42 +++++++++++++++++++++++++++- src/drive/Drive.cc | 65 ++++++++++++++++++++++++++++++++------------ src/drive/Drive.hh | 2 ++ src/protocol/HTTP.cc | 26 ++++++++++++++++-- 4 files changed, 114 insertions(+), 21 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e2f363a..37ee53d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,8 +4,42 @@ cmake_minimum_required(VERSION 2.8) include(FindOpenSSL) +############################################################################### +# finding cppunit +############################################################################### +find_path( CPPUNIT_INCLUDE_DIR cppunit/TestFixture.h /usr/include + /usr/local/include + ${CPPUNIT_PREFIX}/include ) +find_library( CPPUNIT_LIBRARY_DEBUG NAMES cppunit cppunit_dll + PATHS /usr/lib + /usr/lib64 + /usr/local/lib + /usr/local/lib64 + ${CPPUNIT_PREFIX}/lib + PATH_SUFFIXES debug ) + +find_library( CPPUNIT_LIBRARY_RELEASE NAMES cppunit cppunit_dll + PATHS /usr/lib + /usr/lib64 + /usr/local/lib + /usr/local/lib64 + ${CPPUNIT_PREFIX}/lib + PATH_SUFFIXES release ) + +set( CPPUNIT_LIBRARY debug ${CPPUNIT_LIBRARY_DEBUG} + optimized ${CPPUNIT_LIBRARY_RELEASE} ) + +if ( CPPUNIT_INCLUDE_DIR AND CPPUNIT_LIBRARY ) + message( STATUS "found cppunit" ) + set( CPPUNIT_FOUND TRUE ) + set( OPT_INCS ${CPPUNIT_INCLUDE_DIR} ) +endif ( CPPUNIT_INCLUDE_DIR AND CPPUNIT_LIBRARY ) + +############################################################################### + include_directories( ${grive_SOURCE_DIR}/src + ${OPT_INCS} ) add_executable( grive @@ -27,6 +61,8 @@ target_link_libraries( grive ${OPENSSL_LIBRARIES} ) +if ( CPPUNIT_FOUND ) + add_executable( unittest test/UnitTest.cc src/util/DateTime.cc @@ -35,5 +71,9 @@ add_executable( unittest ) target_link_libraries( unittest - cppunit + ${CPPUNIT_LIBRARY} ) + +else ( CPPUNIT_FOUND ) + message( STATUS "skip building unittest" ) +endif ( CPPUNIT_FOUND ) diff --git a/src/drive/Drive.cc b/src/drive/Drive.cc index 938ff7b..6d7db36 100644 --- a/src/drive/Drive.cc +++ b/src/drive/Drive.cc @@ -47,8 +47,17 @@ Drive::Drive( OAuth2& auth ) : m_http_hdr.push_back( "Authorization: Bearer " + m_auth.AccessToken() ) ; m_http_hdr.push_back( "GData-Version: 3.0" ) ; - Json resp = Json::Parse( http::Get( root_url + "?alt=json&showfolders=true", m_http_hdr )) ; + + std::cout << http::Get( "https://docs.google.com/feeds/metadata/default", m_http_hdr ) ; + + Json resume_link ; + if ( resp["feed"]["link"].FindInArray( "rel", "http://schemas.google.com/g/2005#resumable-create-media", resume_link ) ) + { + m_resume_link = resume_link["href"].As() ; + std::cout << "resume_link = " << resume_link << std::endl ; + } + Json::Array entries = resp["feed"]["entry"].As() ; ConstructDirTree( entries ) ; @@ -158,7 +167,13 @@ void Drive::UpdateFile( const Json& entry ) std::string filename = entry["docs$filename"]["$t"].Get() ; std::string url = entry["content"]["src"].Get() ; std::string parent_href = Parent( entry ) ; - +/* + Json kind_json ; + if ( entry["category"].FindInArray( "scheme", "http://schemas.google.com/g/2005#kind", kind_json ) ) + { + std::cout << filename << " kind = " << kind_json << std::endl ; + } +*/ bool changed = true ; std::string path = "./" + filename ; @@ -208,25 +223,37 @@ std::cout << "local " << filename << " is newer" << std::endl ; void Drive::UploadFile( const Json& entry ) { - std::string meta = +/* std::string meta = "" - "" - "" - "Grive Document" + "" + "" + "Test document" "" ; - +*/ http::Headers hdr( m_http_hdr ) ; - hdr.push_back( "Slug: Grive Document" ) ; +// hdr.push_back( "Slug: Grive Document" ) ; +// hdr.push_back( "Content-Type: application/atom+xml" ) ; hdr.push_back( "X-Upload-Content-Type: text/plain" ) ; - hdr.push_back( "X-Upload-Content-Length: 10" ) ; - + hdr.push_back( "X-Upload-Content-Length: 1000" ) ; + hdr.push_back( "Expect:" ) ; +/* std::string resp = http::PostDataWithHeader( - "https://docs.google.com/feeds/upload/create-session/default/private/full?convert=false", - "", - m_http_hdr ) ; - - std::cout << "resp " << resp ; + m_resume_link + "?convert=false", + meta, + hdr ) ; +*/ + Json resume_link = entry["link"].FindInArray( "rel", + "http://schemas.google.com/g/2005#resumable-edit-media" )["href"] ; + std::cout << resume_link.As() << std::endl ; + + std::string etag = entry["gd$etag"].As() ; + std::cout << "etag = " << etag << std::endl ; + + hdr.push_back( "If-Match: " + etag ) ; + std::string resp = http::Put( resume_link.Get(), "", hdr ) ; + + std::cout << "resp " << resp << std::endl ; std::istringstream ss( resp ) ; std::string line ; @@ -238,10 +265,12 @@ void Drive::UploadFile( const Json& entry ) std::string uplink = line.substr( location.size() ) ; uplink = uplink.substr( 0, uplink.size() -1 ) ; - std::string data( 10, '!' ) ; + std::string data( 1000, 'x' ) ; http::Headers uphdr ; uphdr.push_back( "Content-Type: text/plain" ) ; - uphdr.push_back( "Content-Range: bytes 0-9/10" ) ; + uphdr.push_back( "Content-Range: bytes 0-999/1000" ) ; + uphdr.push_back( "Expect:" ) ; + uphdr.push_back( "Accept:" ) ; std::string resp = http::Put( uplink, data, uphdr ) ; std::cout << "put response = " << resp << std::endl ; diff --git a/src/drive/Drive.hh b/src/drive/Drive.hh index 768869c..c88113e 100644 --- a/src/drive/Drive.hh +++ b/src/drive/Drive.hh @@ -53,6 +53,8 @@ private : private : OAuth2& m_auth ; std::vector m_http_hdr ; + + std::string m_resume_link ; FolderList m_coll ; Collection m_root ; diff --git a/src/protocol/HTTP.cc b/src/protocol/HTTP.cc index f030a9e..00897fb 100644 --- a/src/protocol/HTTP.cc +++ b/src/protocol/HTTP.cc @@ -51,6 +51,8 @@ size_t ReadCallback( void *ptr, std::size_t size, std::size_t nmemb, std::string assert( ptr != 0 ) ; assert( data != 0 ) ; +std::cout << "reading " << (size*nmemb) << " bytes " << data->size() << std::endl ; + std::size_t count = std::min( size * nmemb, data->size() ) ; if ( count > 0 ) { @@ -58,6 +60,8 @@ size_t ReadCallback( void *ptr, std::size_t size, std::size_t nmemb, std::string data->erase( 0, count ) ; } +std::cout << "readed " << count << " bytes " << data->size() << std::endl ; + return count ; } @@ -185,7 +189,16 @@ std::string PostDataWithHeader( const std::string& url, const std::string& data, curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L ) ; curl_easy_setopt(curl, CURLOPT_HEADER, 1L ); - DoCurl( curl ) ; + try + { + DoCurl( curl ) ; + } + catch ( ... ) + { + std::cout << "response = " << resp << std::endl ; + throw ; + } + return resp; } @@ -213,7 +226,16 @@ std::string Put( curl_easy_setopt(curl, CURLOPT_INFILESIZE, put_data.size() ) ; curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L ) ; - DoCurl( curl ) ; + try + { + DoCurl( curl ) ; + } + catch ( ... ) + { + std::cout << "response = " << resp << std::endl ; + throw ; + } + return resp; }