mirror of https://github.com/vitalif/grive2
checked MD5 checksum before downloading
parent
b717bfb0d2
commit
947664937a
|
@ -1,12 +1,18 @@
|
||||||
project(grive)
|
project(grive)
|
||||||
|
|
||||||
|
cmake_minimum_required(VERSION 2.8)
|
||||||
|
|
||||||
|
include(FindOpenSSL)
|
||||||
|
|
||||||
add_executable( grive
|
add_executable( grive
|
||||||
src/main.cc
|
src/main.cc
|
||||||
src/OAuth2.cc
|
src/OAuth2.cc
|
||||||
src/HTTP.cc
|
src/HTTP.cc
|
||||||
src/Json.cc
|
src/Json.cc
|
||||||
src/Drive.cc )
|
src/Drive.cc
|
||||||
|
src/Download.cc )
|
||||||
|
|
||||||
target_link_libraries( grive
|
target_link_libraries( grive
|
||||||
curl
|
curl
|
||||||
json )
|
json
|
||||||
|
${OPENSSL_LIBRARIES} )
|
||||||
|
|
|
@ -0,0 +1,90 @@
|
||||||
|
/*
|
||||||
|
grive: an GPL program to sync a local directory with Google Drive
|
||||||
|
Copyright (C) 2012 Wan Wai Ho
|
||||||
|
|
||||||
|
This program is free software; you can redistribute it and/or
|
||||||
|
modify it under the terms of the GNU General Public License
|
||||||
|
as published by the Free Software Foundation; either version 2
|
||||||
|
of the License, or (at your option) any later version.
|
||||||
|
|
||||||
|
This program is distributed in the hope that it will be useful,
|
||||||
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
GNU General Public License for more details.
|
||||||
|
|
||||||
|
You should have received a copy of the GNU General Public License
|
||||||
|
along with this program; if not, write to the Free Software
|
||||||
|
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include "Download.hh"
|
||||||
|
|
||||||
|
#include <openssl/evp.h>
|
||||||
|
|
||||||
|
#include <cassert>
|
||||||
|
#include <new>
|
||||||
|
#include <stdexcept>
|
||||||
|
|
||||||
|
namespace gr {
|
||||||
|
|
||||||
|
Download::Download( const std::string& filename ) :
|
||||||
|
m_file( filename.c_str(), std::ios::out | std::ios::binary ),
|
||||||
|
m_mdctx( ::EVP_MD_CTX_create() )
|
||||||
|
{
|
||||||
|
if ( m_mdctx == 0 )
|
||||||
|
throw std::bad_alloc() ;
|
||||||
|
|
||||||
|
if ( ::EVP_DigestInit_ex( m_mdctx, ::EVP_md5(), 0 ) != 1 )
|
||||||
|
throw std::runtime_error( "cannot create MD5 digest context" ) ;
|
||||||
|
|
||||||
|
if ( !m_file )
|
||||||
|
throw std::runtime_error( "cannot open file " + filename + " for writing" ) ;
|
||||||
|
}
|
||||||
|
|
||||||
|
Download::Download( const std::string& filename, NoChecksum ) :
|
||||||
|
m_file( filename.c_str(), std::ios::out | std::ios::binary ),
|
||||||
|
m_mdctx( 0 )
|
||||||
|
{
|
||||||
|
if ( !m_file )
|
||||||
|
throw std::runtime_error( "cannot open file " + filename + " for writing" ) ;
|
||||||
|
}
|
||||||
|
|
||||||
|
Download::~Download( )
|
||||||
|
{
|
||||||
|
if ( m_mdctx != 0 )
|
||||||
|
::EVP_MD_CTX_destroy( m_mdctx ) ;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string Download::Finish() const
|
||||||
|
{
|
||||||
|
std::string result ;
|
||||||
|
|
||||||
|
// get the checksum and return it ;
|
||||||
|
if ( m_mdctx != 0 )
|
||||||
|
{
|
||||||
|
unsigned int size = EVP_MAX_MD_SIZE ;
|
||||||
|
result.resize( size ) ;
|
||||||
|
|
||||||
|
if ( ::EVP_DigestFinal_ex( m_mdctx, reinterpret_cast<unsigned char*>(&result[0]), &size ) != 1 )
|
||||||
|
throw std::runtime_error( "cannot calculate checksum" ) ;
|
||||||
|
|
||||||
|
result.resize( size ) ;
|
||||||
|
}
|
||||||
|
|
||||||
|
return result ;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::size_t Download::Callback( char *data, std::size_t size, std::size_t nmemb, Download *pthis )
|
||||||
|
{
|
||||||
|
assert( pthis != 0 ) ;
|
||||||
|
assert( data != 0 ) ;
|
||||||
|
|
||||||
|
std::size_t count = size * nmemb ;
|
||||||
|
|
||||||
|
if ( pthis->m_mdctx != 0 )
|
||||||
|
::EVP_DigestUpdate( pthis->m_mdctx, data, count ) ;
|
||||||
|
|
||||||
|
return pthis->m_file.rdbuf()->sputn( data, count ) ;
|
||||||
|
}
|
||||||
|
|
||||||
|
} // end of namespace
|
|
@ -0,0 +1,46 @@
|
||||||
|
/*
|
||||||
|
grive: an GPL program to sync a local directory with Google Drive
|
||||||
|
Copyright (C) 2012 Wan Wai Ho
|
||||||
|
|
||||||
|
This program is free software; you can redistribute it and/or
|
||||||
|
modify it under the terms of the GNU General Public License
|
||||||
|
as published by the Free Software Foundation; either version 2
|
||||||
|
of the License, or (at your option) any later version.
|
||||||
|
|
||||||
|
This program is distributed in the hope that it will be useful,
|
||||||
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
GNU General Public License for more details.
|
||||||
|
|
||||||
|
You should have received a copy of the GNU General Public License
|
||||||
|
along with this program; if not, write to the Free Software
|
||||||
|
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <string>
|
||||||
|
#include <fstream>
|
||||||
|
|
||||||
|
#include <openssl/evp.h>
|
||||||
|
|
||||||
|
namespace gr {
|
||||||
|
|
||||||
|
class Download
|
||||||
|
{
|
||||||
|
public :
|
||||||
|
struct NoChecksum {} ;
|
||||||
|
Download( const std::string& filename ) ;
|
||||||
|
Download( const std::string& filename, NoChecksum ) ;
|
||||||
|
~Download( ) ;
|
||||||
|
|
||||||
|
std::string Finish() const ;
|
||||||
|
|
||||||
|
static std::size_t Callback( char *data, std::size_t size, std::size_t nmemb, Download *pthis ) ;
|
||||||
|
|
||||||
|
private :
|
||||||
|
std::ofstream m_file ;
|
||||||
|
EVP_MD_CTX *m_mdctx ;
|
||||||
|
} ;
|
||||||
|
|
||||||
|
} // end of namespace
|
61
src/Drive.cc
61
src/Drive.cc
|
@ -23,11 +23,42 @@
|
||||||
#include "Json.hh"
|
#include "Json.hh"
|
||||||
#include "OAuth2.hh"
|
#include "OAuth2.hh"
|
||||||
|
|
||||||
|
#include <openssl/evp.h>
|
||||||
|
|
||||||
|
#include <algorithm>
|
||||||
|
#include <fstream>
|
||||||
|
#include <iomanip>
|
||||||
|
#include <sstream>
|
||||||
|
|
||||||
// for debugging only
|
// for debugging only
|
||||||
#include <iostream>
|
#include <iostream>
|
||||||
|
|
||||||
namespace gr {
|
namespace gr {
|
||||||
|
|
||||||
|
std::string MD5( std::streambuf *file )
|
||||||
|
{
|
||||||
|
char buf[64 * 1024] ;
|
||||||
|
EVP_MD_CTX md ;
|
||||||
|
EVP_MD_CTX_init( &md );
|
||||||
|
EVP_DigestInit_ex( &md, EVP_md5(), 0 ) ;
|
||||||
|
|
||||||
|
std::size_t count = 0 ;
|
||||||
|
while ( (count = file->sgetn( buf, sizeof(buf) )) > 0 )
|
||||||
|
{
|
||||||
|
EVP_DigestUpdate( &md, buf, count ) ;
|
||||||
|
}
|
||||||
|
|
||||||
|
unsigned int md5_size = EVP_MAX_MD_SIZE ;
|
||||||
|
unsigned char md5[EVP_MAX_MD_SIZE] ;
|
||||||
|
EVP_DigestFinal_ex( &md, md5, &md5_size ) ;
|
||||||
|
|
||||||
|
std::ostringstream ss ;
|
||||||
|
for ( unsigned int i = 0 ; i < md5_size ; i++ )
|
||||||
|
ss << std::hex << std::setw(2) << std::setfill('0') << static_cast<int>(md5[i]) ;
|
||||||
|
|
||||||
|
return ss.str() ;
|
||||||
|
}
|
||||||
|
|
||||||
Drive::Drive( OAuth2& auth ) :
|
Drive::Drive( OAuth2& auth ) :
|
||||||
m_auth( auth )
|
m_auth( auth )
|
||||||
{
|
{
|
||||||
|
@ -51,11 +82,11 @@ Drive::Drive( OAuth2& auth ) :
|
||||||
|
|
||||||
void Drive::DownloadEntry( const Json& entry )
|
void Drive::DownloadEntry( const Json& entry )
|
||||||
{
|
{
|
||||||
Json::Object map = entry.As<Json::Object>() ;
|
// Json::Object map = entry.As<Json::Object>() ;
|
||||||
for ( Json::Object::iterator i = map.begin() ; i != map.end() ; ++i )
|
// for ( Json::Object::iterator i = map.begin() ; i != map.end() ; ++i )
|
||||||
{
|
// {
|
||||||
// std::cout << i->first << "\t" << i->second.DataType() << std::endl ;
|
// std::cout << i->first << "\t" << i->second.DataType() << std::endl ;
|
||||||
}
|
// }
|
||||||
|
|
||||||
// only handle uploaded files
|
// only handle uploaded files
|
||||||
if ( entry.Has( "docs$filename" ) )
|
if ( entry.Has( "docs$filename" ) )
|
||||||
|
@ -64,9 +95,25 @@ void Drive::DownloadEntry( const Json& entry )
|
||||||
std::string filename = entry["docs$filename"]["$t"].As<std::string>() ;
|
std::string filename = entry["docs$filename"]["$t"].As<std::string>() ;
|
||||||
std::string url = entry["content"]["src"].As<std::string>() ;
|
std::string url = entry["content"]["src"].As<std::string>() ;
|
||||||
|
|
||||||
std::cout << "downloading " << filename << " from " << url << std::endl ;
|
bool changed = true ;
|
||||||
|
|
||||||
HttpGetFile( url, filename, m_http_hdr ) ;
|
// compare checksum first if file exists
|
||||||
|
std::ifstream ifile( filename.c_str(), std::ios::binary | std::ios::out ) ;
|
||||||
|
if ( ifile && entry.Has("docs$md5Checksum") )
|
||||||
|
{
|
||||||
|
std::string remote_md5 = entry["docs$md5Checksum"]["$t"].As<std::string>() ;
|
||||||
|
std::string local_md5 = MD5( ifile.rdbuf() ) ;
|
||||||
|
|
||||||
|
std::transform( remote_md5.begin(), remote_md5.end(), remote_md5.begin(), tolower ) ;
|
||||||
|
if ( local_md5 == remote_md5 )
|
||||||
|
changed = false ;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ( changed )
|
||||||
|
{
|
||||||
|
std::cout << "downloading " << filename << std::endl ;
|
||||||
|
HttpGetFile( url, filename, m_http_hdr ) ;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
32
src/HTTP.cc
32
src/HTTP.cc
|
@ -19,9 +19,12 @@
|
||||||
|
|
||||||
#include "HTTP.hh"
|
#include "HTTP.hh"
|
||||||
|
|
||||||
|
#include "Download.hh"
|
||||||
|
|
||||||
|
// dependent libraries
|
||||||
#include <curl/curl.h>
|
#include <curl/curl.h>
|
||||||
|
|
||||||
#include <cassert>
|
#include <cassert>
|
||||||
#include <fstream>
|
|
||||||
#include <iostream>
|
#include <iostream>
|
||||||
#include <sstream>
|
#include <sstream>
|
||||||
#include <streambuf>
|
#include <streambuf>
|
||||||
|
@ -51,15 +54,6 @@ std::size_t WriteCallback( char *data, size_t size, size_t nmemb, std::string *r
|
||||||
return count ;
|
return count ;
|
||||||
}
|
}
|
||||||
|
|
||||||
// libcurl callback to write to a file
|
|
||||||
std::size_t DownloadCallback( char *data, size_t size, size_t nmemb, std::streambuf *file )
|
|
||||||
{
|
|
||||||
assert( file != 0 ) ;
|
|
||||||
assert( data != 0 ) ;
|
|
||||||
|
|
||||||
return file->sputn( data, size * nmemb ) ;
|
|
||||||
}
|
|
||||||
|
|
||||||
CURL* InitCurl( const std::string& url, std::string *resp, const Headers& hdr )
|
CURL* InitCurl( const std::string& url, std::string *resp, const Headers& hdr )
|
||||||
{
|
{
|
||||||
CURL *curl = curl_easy_init();
|
CURL *curl = curl_easy_init();
|
||||||
|
@ -122,17 +116,23 @@ void HttpGetFile(
|
||||||
const std::string& filename,
|
const std::string& filename,
|
||||||
const Headers& hdr )
|
const Headers& hdr )
|
||||||
{
|
{
|
||||||
std::ofstream file( filename.c_str(), std::ios::binary | std::ios::out ) ;
|
Download dl( filename, Download::NoChecksum() ) ;
|
||||||
if ( !file )
|
|
||||||
throw std::runtime_error( "cannot open file " + filename + " for writing" ) ;
|
|
||||||
|
|
||||||
CURL *curl = InitCurl( url, 0, hdr ) ;
|
CURL *curl = InitCurl( url, 0, hdr ) ;
|
||||||
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, DownloadCallback);
|
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &Download::Callback);
|
||||||
curl_easy_setopt(curl, CURLOPT_WRITEDATA, file.rdbuf() ) ;
|
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &dl ) ;
|
||||||
curl_easy_setopt(curl, CURLOPT_HTTPGET, 1L);
|
curl_easy_setopt(curl, CURLOPT_HTTPGET, 1L);
|
||||||
DoCurl( curl ) ;
|
DoCurl( curl ) ;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void HttpGetFile(
|
||||||
|
const std::string& url,
|
||||||
|
const std::string& filename,
|
||||||
|
std::string& md5sum,
|
||||||
|
const Headers& hdr )
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
std::string HttpPostData( const std::string& url, const std::string& data, const Headers& hdr )
|
std::string HttpPostData( const std::string& url, const std::string& data, const Headers& hdr )
|
||||||
{
|
{
|
||||||
std::string resp ;
|
std::string resp ;
|
||||||
|
|
|
@ -33,6 +33,12 @@ namespace gr
|
||||||
const std::string& filename,
|
const std::string& filename,
|
||||||
const Headers& hdr = Headers() ) ;
|
const Headers& hdr = Headers() ) ;
|
||||||
|
|
||||||
|
void HttpGetFile(
|
||||||
|
const std::string& url,
|
||||||
|
const std::string& filename,
|
||||||
|
std::string& md5sum,
|
||||||
|
const Headers& hdr = Headers() ) ;
|
||||||
|
|
||||||
std::string HttpPostData(
|
std::string HttpPostData(
|
||||||
const std::string& url,
|
const std::string& url,
|
||||||
const std::string& data,
|
const std::string& data,
|
||||||
|
|
|
@ -40,7 +40,6 @@ namespace gr
|
||||||
|
|
||||||
Json ReadConfig()
|
Json ReadConfig()
|
||||||
{
|
{
|
||||||
std::cout << ConfigFilename() << std::endl ;
|
|
||||||
std::ifstream ifile( ConfigFilename().c_str() ) ;
|
std::ifstream ifile( ConfigFilename().c_str() ) ;
|
||||||
|
|
||||||
if ( ifile )
|
if ( ifile )
|
||||||
|
|
Loading…
Reference in New Issue