vitalif
/
grive2
mirror of https://github.com/vitalif/grive2
1
0
Fork 0
Code Issues Releases Wiki Activity

Cache file checksums and modification times in local index (in JSON in .grive_state)

syncer-move-dates
Vitaliy Filippov 2016-01-01 02:27:21 +03:00
parent 5381919e5b
commit 4edff0a816
14 changed files with 214 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
libgrive/src/base/Drive.cc
View File

@ -62,11 +62,11 @@ void Drive::FromRemote( const Entry& entry )
// so these checkings are done in normal entries only
Resource *parent = m_state.FindByHref( entry.ParentHref() ) ;
if ( parent != 0 && !parent->IsFolder() )
if ( parent && !parent->IsFolder() )
Log( "warning: entry %1% has parent %2% which is not a folder, ignored",
entry.Title(), parent->Name(), log::verbose ) ;
else if ( parent == 0 || !parent->IsInRootTree() )
else if ( !parent || !parent->IsInRootTree() )
Log( "file \"%1%\" parent doesn't exist, ignored", entry.Title(), log::verbose ) ;
else

107
libgrive/src/base/Resource.cc
View File

@ -48,8 +48,9 @@ Resource::Resource( const fs::path& root_folder ) :
m_id ( "folder:root" ),
m_href ( "root" ),
m_is_editable( true ),
m_parent ( NULL ),
m_state ( sync )
m_parent ( 0 ),
m_state ( sync ),
m_json ( NULL )
{
}
@ -57,8 +58,9 @@ Resource::Resource( const std::string& name, const std::string& kind ) :
m_name ( name ),
m_kind ( kind ),
m_is_editable( true ),
m_parent ( NULL ),
m_state ( unknown )
m_parent ( 0 ),
m_state ( unknown ),
m_json ( NULL )
{
}
@ -137,10 +139,8 @@ void Resource::FromRemote( const Entry& remote, const DateTime& last_change )
assert( m_state != unknown ) ;
if ( m_state == remote_new || m_state == remote_changed )
{
m_md5 = remote.MD5() ;
m_mtime = remote.MTime() ;
}
m_md5 = remote.MD5() ;
m_mtime = remote.MTime() ;
}
void Resource::AssignIDs( const Entry& remote )
@ -233,15 +233,40 @@ void Resource::FromRemoteFile( const Entry& remote, const DateTime& last_change
/// Update the resource with the attributes of local file or directory. This
/// function will propulate the fields in m_entry.
void Resource::FromLocal( const DateTime& last_sync )
void Resource::FromLocal( const DateTime& last_sync, Val& state )
{
fs::path path = Path() ;
//assert( fs::exists( path ) ) ;
assert( !m_json );
m_json = &state;
// root folder is always in sync
if ( !IsRoot() )
{
m_mtime = os::FileCTime( path ) ;
fs::path path = Path() ;
bool is_dir;
os::Stat( path, &m_ctime, &m_size, &is_dir ) ;
m_name = path.filename().string() ;
if ( !is_dir )
{
m_kind = "file";
if ( state.Has( "ctime" ) && state.Has( "md5" ) && (u64_t) m_ctime.Sec() == state["ctime"].U64() )
m_md5 = state["md5"];
else
{
m_md5 = crypt::MD5::Get( path );
state.Set( "md5", Val( m_md5 ) );
state.Set( "ctime", Val( m_ctime.Sec() ) );
}
if ( state.Has( "srv_time" ) && m_mtime != DateTime() )
m_mtime.Assign( state[ "srv_time" ].U64(), 0 ) ;
}
else
{
m_kind = "folder";
state.Del( "md5" );
state.Del( "ctime" );
state.Del( "srv_time" );
}
// follow parent recursively
if ( m_parent->m_state == local_new || m_parent->m_state == local_deleted )
@ -251,11 +276,7 @@ void Resource::FromLocal( const DateTime& last_sync )
// remote_deleted first, it will be updated to sync/remote_changed
// in FromRemote()
else
m_state = ( m_mtime > last_sync ? local_new : remote_deleted ) ;
m_name = path.filename().string() ;
m_kind = IsFolder() ? "folder" : "file" ;
m_md5 = IsFolder() ? "" : crypt::MD5::Get( path ) ;
m_state = ( m_ctime > last_sync ? local_new : remote_deleted ) ;
}
assert( m_state != unknown ) ;
@ -286,7 +307,7 @@ std::string Resource::Kind() const
return m_kind ;
}
DateTime Resource::MTime() const
DateTime Resource::ServerTime() const
{
return m_mtime ;
}
@ -395,8 +416,26 @@ void Resource::Sync( Syncer *syncer, DateTime& sync_time, const Val& options )
// if myself is deleted, no need to do the childrens
if ( m_state != local_deleted && m_state != remote_deleted )
{
std::for_each( m_child.begin(), m_child.end(),
boost::bind( &Resource::Sync, _1, syncer, boost::ref(sync_time), options ) ) ;
if ( IsFolder() )
{
// delete state of removed files
Val& tree = (*m_json)["tree"];
Val::Object leftover = tree.AsObject();
for( iterator i = m_child.begin(); i != m_child.end(); i++ )
{
Resource *r = *i;
if ( r->m_state != local_deleted && r->m_state != remote_deleted )
leftover.erase( r->Name() );
else
r->m_json = NULL;
}
for( Val::Object::iterator i = leftover.begin(); i != leftover.end(); i++ )
tree.Del( i->first );
}
}
}
void Resource::SyncSelf( Syncer* syncer, const Val& options )
@ -438,7 +477,7 @@ void Resource::SyncSelf( Syncer* syncer, const Val& options )
fs::create_directories( path ) ;
else
syncer->Download( this, path ) ;
SetIndex() ;
m_state = sync ;
}
break ;
@ -449,6 +488,7 @@ void Resource::SyncSelf( Syncer* syncer, const Val& options )
if ( syncer )
{
syncer->Download( this, path ) ;
SetIndex() ;
m_state = sync ;
}
break ;
@ -462,7 +502,7 @@ void Resource::SyncSelf( Syncer* syncer, const Val& options )
case sync :
Log( "sync %1% already in sync", path, log::verbose ) ;
break ;
// shouldn't go here
case unknown :
assert( false ) ;
@ -471,6 +511,17 @@ void Resource::SyncSelf( Syncer* syncer, const Val& options )
default :
break ;
}
if ( m_state != local_deleted && m_state != remote_deleted && !IsFolder() )
{
// Update server time of this file
m_json->Set( "srv_time", Val( m_mtime.Sec() ) );
}
}
void Resource::SetServerTime( const DateTime& time )
{
m_mtime = time ;
}
/// this function doesn't really remove the local file. it renames it.
@ -478,7 +529,7 @@ void Resource::DeleteLocal()
{
static const boost::format trash_file( "%1%-%2%" ) ;
assert( m_parent != 0 ) ;
assert( m_parent != NULL ) ;
Resource* p = m_parent;
fs::path destdir;
while ( !p->IsRoot() )
@ -503,6 +554,20 @@ void Resource::DeleteLocal()
}
}
void Resource::SetIndex()
{
assert( m_parent->m_json != NULL );
if ( !m_json )
m_json = &((*m_parent->m_json)["tree"]).Item( Name() );
bool is_dir;
os::Stat( Path(), &m_ctime, &m_size, &is_dir );
if ( !is_dir )
{
m_json->Set( "ctime", Val( m_ctime.Sec() ) );
m_json->Set( "md5", Val( m_md5 ) );
}
}
Resource::iterator Resource::begin() const
{
return m_child.begin() ;

8
libgrive/src/base/Resource.hh
View File

@ -87,7 +87,7 @@ public :
std::string Name() const ;
std::string Kind() const ;
DateTime MTime() const ;
DateTime ServerTime() const ;
std::string SelfHref() const ;
std::string ContentSrc() const ;
std::string ETag() const ;
@ -107,9 +107,10 @@ public :
std::string MD5() const ;
void FromRemote( const Entry& remote, const DateTime& last_change ) ;
void FromLocal( const DateTime& last_sync ) ;
void FromLocal( const DateTime& last_sync, Val& state ) ;
void Sync( Syncer* syncer, DateTime& sync_time, const Val& options ) ;
void SetServerTime( const DateTime& time ) ;
// children access
iterator begin() const ;
@ -132,6 +133,7 @@ private :
void FromRemoteFile( const Entry& remote, const DateTime& last_change ) ;
void DeleteLocal() ;
void SetIndex() ;
void SyncSelf( Syncer* syncer, const Val& options ) ;
@ -140,6 +142,8 @@ private :
std::string m_kind ;
std::string m_md5 ;
DateTime m_mtime ;
DateTime m_ctime ;
off_t m_size ;
std::string m_id ;
std::string m_href ;

66
libgrive/src/base/State.cc
View File

@ -26,7 +26,6 @@
#include "util/Crypt.hh"
#include "util/File.hh"
#include "util/log/Log.hh"
#include "json/Val.hh"
#include "json/JsonParser.hh"
#include <fstream>
@ -84,7 +83,10 @@ State::~State()
/// of local directory.
void State::FromLocal( const fs::path& p )
{
FromLocal( p, m_res.Root() ) ;
if ( !m_st.Has( "tree" ) )
m_st.Add( "tree", Val() );
m_res.Root()->FromLocal( m_last_sync, m_st ) ;
FromLocal( p, m_res.Root(), m_st["tree"] ) ;
}
bool State::IsIgnore( const std::string& filename )
@ -92,44 +94,39 @@ bool State::IsIgnore( const std::string& filename )
return regex_search( filename.c_str(), m_ign_re );
}
void State::FromLocal( const fs::path& p, Resource* folder )
void State::FromLocal( const fs::path& p, Resource* folder, Val& tree )
{
assert( folder != 0 ) ;
assert( folder->IsFolder() ) ;
// sync the folder itself
folder->FromLocal( m_last_sync ) ;
for ( fs::directory_iterator i( p ) ; i != fs::directory_iterator() ; ++i )
{
std::string fname = i->path().filename().string() ;
fs::file_status st = fs::status(i->path());
std::string path = folder->IsRoot() ? fname : ( folder->RelPath() / fname ).string();
if ( IsIgnore( path ) )
Log( "file %1% is ignored by grive", path, log::verbose ) ;
// check for broken symblic links
else if ( st.type() == fs::file_not_found )
Log( "file %1% doesn't exist (broken link?), ignored", i->path(), log::verbose ) ;
else
{
bool is_dir = st.type() == fs::directory_file;
// if the Resource object of the child already exists, it should
// have been so no need to do anything here
Resource *c = folder->FindChild( fname ) ;
if ( c == 0 )
if ( !c )
{
c = new Resource( fname, is_dir ? "folder" : "file" ) ;
c = new Resource( fname, "" ) ;
folder->AddChild( c ) ;
m_res.Insert( c ) ;
}
c->FromLocal( m_last_sync ) ;
if ( is_dir )
FromLocal( *i, c ) ;
if ( !tree.Has( fname ) )
tree.Add( fname, Val() );
Val& rec = tree[fname];
c->FromLocal( m_last_sync, rec ) ;
if ( c->IsFolder() )
{
if ( !rec.Has("tree") )
rec.Add( "tree", Val() );
FromLocal( *i, c, rec["tree"] ) ;
}
}
}
}
@ -140,7 +137,7 @@ void State::FromRemote( const Entry& e )
std::string k = e.IsDir() ? "folder" : "file";
// common checkings
if ( !e.IsDir() && (fn.empty() || e.ContentSrc().empty()) )
if ( !e.IsDir() && ( fn.empty() || e.ContentSrc().empty() ) )
Log( "%1% \"%2%\" is a google document, ignored", k, e.Name(), log::verbose ) ;
else if ( fn.find('/') != fn.npos )
@ -225,7 +222,7 @@ bool State::Update( const Entry& e )
// see if the entry already exist in local
std::string name = e.Name() ;
Resource *child = parent->FindChild( name ) ;
if ( child != 0 )
if ( child )
{
// since we are updating the ID and Href, we need to remove it and re-add it.
m_res.Update( child, e, m_last_change ) ;
@ -273,21 +270,21 @@ void State::Read( const fs::path& filename )
{
File file( filename ) ;
Val json = ParseJson( file );
Val last_sync = json["last_sync"] ;
Val last_change = json.Has( "last_change" ) ? json["last_change"] : json["last_sync"] ;
m_st = ParseJson( file );
Val last_sync = m_st["last_sync"] ;
Val last_change = m_st.Has( "last_change" ) ? m_st["last_change"] : m_st["last_sync"] ;
m_last_sync.Assign( last_sync["sec"].Int(), last_sync["nsec"].Int() ) ;
m_last_change.Assign( last_change["sec"].Int(), last_change["nsec"].Int() ) ;
m_ign = json.Has( "ignore_regexp" ) ? json["ignore_regexp"].Str() : std::string();
m_ign = m_st.Has( "ignore_regexp" ) ? m_st["ignore_regexp"].Str() : std::string();
m_cstamp = json["change_stamp"].Int() ;
m_cstamp = m_st["change_stamp"].Int() ;
}
catch ( Exception& )
{
}
}
void State::Write( const fs::path& filename ) const
void State::Write( const fs::path& filename )
{
Val last_sync ;
last_sync.Add( "sec", Val( (int)m_last_sync.Sec() ) );
@ -297,14 +294,13 @@ void State::Write( const fs::path& filename ) const
last_change.Add( "sec", Val( (int)m_last_change.Sec() ) );
last_change.Add( "nsec", Val( (unsigned)m_last_change.NanoSec() ) );
Val result ;
result.Add( "last_sync", last_sync ) ;
result.Add( "last_change", last_change ) ;
result.Add( "change_stamp", Val(m_cstamp) ) ;
result.Add( "ignore_regexp", Val(m_ign) ) ;
m_st.Set( "last_sync", last_sync ) ;
m_st.Set( "last_change", last_change ) ;
m_st.Set( "change_stamp", Val( m_cstamp ) ) ;
m_st.Set( "ignore_regexp", Val( m_ign ) ) ;
std::ofstream fs( filename.string().c_str() ) ;
fs << result ;
fs << m_st ;
}
void State::Sync( Syncer *syncer, const Val& options )

8
libgrive/src/base/State.hh
View File

@ -23,14 +23,13 @@
#include "util/DateTime.hh"
#include "util/FileSystem.hh"
#include "json/Val.hh"
#include <memory>
#include <boost/regex.hpp>
namespace gr {
class Val ;
class Entry ;
class Syncer ;
@ -51,7 +50,7 @@ public :
void ResolveEntry() ;
void Read( const fs::path& filename ) ;
void Write( const fs::path& filename ) const ;
void Write( const fs::path& filename ) ;
Resource* FindByHref( const std::string& href ) ;
Resource* FindByID( const std::string& id ) ;
@ -66,7 +65,7 @@ public :
bool Move( Syncer* syncer, fs::path old_p, fs::path new_p, fs::path grive_root );
private :
void FromLocal( const fs::path& p, Resource *folder ) ;
void FromLocal( const fs::path& p, Resource *folder, Val& tree ) ;
void FromChange( const Entry& e ) ;
bool Update( const Entry& e ) ;
std::size_t TryResolveEntry() ;
@ -80,6 +79,7 @@ private :
int m_cstamp ;
std::string m_ign ;
boost::regex m_ign_re ;
Val m_st ;
std::vector<Entry> m_unresolved ;
} ;

9
libgrive/src/base/Syncer.cc
View File

@ -44,8 +44,8 @@ void Syncer::Download( Resource *res, const fs::path& file )
long r = m_http->Get( res->ContentSrc(), &dl, http::Header() ) ;
if ( r <= 400 )
{
if ( res->MTime() != DateTime() )
os::SetFileTime( file, res->MTime() ) ;
if ( res->ServerTime() != DateTime() )
os::SetFileTime( file, res->ServerTime() ) ;
else
Log( "encountered zero date time after downloading %1%", file, log::warning ) ;
}
@ -56,9 +56,4 @@ void Syncer::AssignIDs( Resource *res, const Entry& remote )
res->AssignIDs( remote );
}
void Syncer::AssignMTime( Resource *res, const DateTime& mtime )
{
res->m_mtime = mtime;
}
} // end of namespace gr

1
libgrive/src/base/Syncer.hh
View File

@ -65,7 +65,6 @@ protected:
http::Agent *m_http;
void AssignIDs( Resource *res, const Entry& remote );
void AssignMTime( Resource *res, const DateTime& mtime );
} ;

2
libgrive/src/drive/Syncer1.cc
View File

@ -232,7 +232,7 @@ bool Syncer1::Upload( Resource *res,
Log( "upload succeeded on retry", log::warning );
Entry1 responseEntry = Entry1( xml.Response() );
AssignIDs( res, responseEntry ) ;
AssignMTime( res, responseEntry.MTime() );
res->SetServerTime( responseEntry.MTime() );
break;
}

2
libgrive/src/drive2/Syncer2.cc
View File

@ -201,7 +201,7 @@ bool Syncer2::Upload( Resource *res )
Entry2 responseEntry = Entry2( valr ) ;
AssignIDs( res, responseEntry ) ;
AssignMTime( res, responseEntry.MTime() ) ;
res->SetServerTime( responseEntry.MTime() );
return true ;
}

50
libgrive/src/json/Val.cc
View File

@ -91,6 +91,18 @@ const Val& Val::operator[]( const std::string& key ) const
throw ;
}
Val& Val::operator[]( const std::string& key )
{
Object& obj = As<Object>() ;
Object::iterator i = obj.find(key) ;
if ( i != obj.end() )
return i->second ;
// shut off compiler warning
BOOST_THROW_EXCEPTION(Error() << NoKey_(key)) ;
throw ;
}
const Val& Val::operator[]( std::size_t index ) const
{
const Array& ar = As<Array>() ;
@ -119,6 +131,13 @@ int Val::Int() const
return static_cast<int>(As<long long>()) ;
}
unsigned long long Val::U64() const
{
if ( Type() == string_type )
return strtoull( As<std::string>().c_str(), NULL, 10 );
return static_cast<unsigned long long>(As<long long>()) ;
}
double Val::Double() const
{
if ( Type() == string_type )
@ -136,17 +155,38 @@ const Val::Array& Val::AsArray() const
return As<Array>() ;
}
Val::Array& Val::AsArray()
{
return As<Array>() ;
}
const Val::Object& Val::AsObject() const
{
return As<Object>() ;
}
Val::Object& Val::AsObject()
{
return As<Object>() ;
}
bool Val::Has( const std::string& key ) const
{
const Object& obj = As<Object>() ;
return obj.find(key) != obj.end() ;
}
bool Val::Del( const std::string& key )
{
Object& obj = As<Object>() ;
return obj.erase(key) > 0 ;
}
Val& Val::Item( const std::string& key )
{
return As<Object>()[key];
}
bool Val::Get( const std::string& key, Val& val ) const
{
const Object& obj = As<Object>() ;
@ -165,6 +205,16 @@ void Val::Add( const std::string& key, const Val& value )
As<Object>().insert( std::make_pair(key, value) ) ;
}
void Val::Set( const std::string& key, const Val& value )
{
Object& obj = As<Object>();
Object::iterator i = obj.find(key);
if (i == obj.end())
obj.insert(std::make_pair(key, value));
else
i->second = value;
}
void Val::Add( const Val& json )
{
As<Array>().push_back( json ) ;

23
libgrive/src/json/Val.hh
View File

@ -94,24 +94,29 @@ public :
TypeEnum Type() const ;
const Val& operator[]( const std::string& key ) const ;
const Val& operator[]( std::size_t index ) const ;
// shortcuts for As<>()
std::string Str() const ;
int Int() const ;
long Long() const ;
unsigned long long U64() const ;
double Double() const ;
bool Bool() const ;
const Array& AsArray() const ;
Array& AsArray() ;
const Object& AsObject() const ;
Object& AsObject() ;
// shortcuts for objects
bool Has( const std::string& key ) const ;
bool Get( const std::string& key, Val& val ) const ;
void Add( const std::string& key, const Val& val ) ;
Val& operator[]( const std::string& key ) ; // get updatable ref or throw
const Val& operator[]( const std::string& key ) const ; // get const ref or throw
Val& Item( const std::string& key ) ; // insert if not exists and get
bool Has( const std::string& key ) const ; // check if exists
bool Get( const std::string& key, Val& val ) const ; // get or return false
void Add( const std::string& key, const Val& val ) ; // insert or do nothing
void Set( const std::string& key, const Val& val ) ; // insert or update
bool Del( const std::string& key ); // delete or do nothing
// shortcuts for array (and array of objects)
const Val& operator[]( std::size_t index ) const ;
void Add( const Val& json ) ;
std::vector<Val> Select( const std::string& key ) const ;
@ -191,7 +196,7 @@ const T& Val::As() const
{
try
{
const Impl<T> *impl = &dynamic_cast<const Impl<T>&>( *m_base ) ;
const Impl<T> *impl = dynamic_cast<const Impl<T> *>( m_base.get() ) ;
return impl->val ;
}
catch ( std::exception& e )
@ -208,7 +213,7 @@ T& Val::As()
{
try
{
Impl<T> *impl = &dynamic_cast<Impl<T>&>( *m_base ) ;
Impl<T> *impl = dynamic_cast<Impl<T> *>( m_base.get() ) ;
return impl->val ;
}
catch ( std::exception& e )

17
libgrive/src/util/OS.cc
View File

@ -39,12 +39,12 @@
namespace gr { namespace os {
DateTime FileCTime( const fs::path& filename )
void Stat( const fs::path& filename, DateTime *t, off_t *size, bool *is_dir )
{
return FileCTime( filename.string() ) ;
Stat( filename.string(), t, size, is_dir ) ;
}
DateTime FileCTime( const std::string& filename )
void Stat( const std::string& filename, DateTime *t, off_t *size, bool *is_dir )
{
struct stat s = {} ;
if ( ::stat( filename.c_str(), &s ) != 0 )
@ -57,11 +57,18 @@ DateTime FileCTime( const std::string& filename )
) ;
}
if (t)
{
#if defined __APPLE__ && defined __DARWIN_64_BIT_INO_T
return DateTime( s.st_ctimespec.tv_sec, s.st_ctimespec.tv_nsec ) ;
*t = DateTime( s.st_ctimespec.tv_sec, s.st_ctimespec.tv_nsec ) ;
#else
return DateTime( s.st_ctim.tv_sec, s.st_ctim.tv_nsec);
*t = DateTime( s.st_ctim.tv_sec, s.st_ctim.tv_nsec);
#endif
}
if (size)
*size = s.st_size;
if (is_dir)
*is_dir = S_ISDIR(s.st_mode) ? true : false;
}
void SetFileTime( const fs::path& filename, const DateTime& t )

4
libgrive/src/util/OS.hh
View File

@ -33,8 +33,8 @@ namespace os
{
struct Error : virtual Exception {} ;
DateTime FileCTime( const std::string& filename ) ;
DateTime FileCTime( const fs::path& filename ) ;
void Stat( const std::string& filename, DateTime *t, off_t *size, bool *is_dir ) ;
void Stat( const fs::path& filename, DateTime *t, off_t *size, bool *is_dir ) ;
void SetFileTime( const std::string& filename, const DateTime& t ) ;
void SetFileTime( const fs::path& filename, const DateTime& t ) ;

4
libgrive/test/base/ResourceTest.cc
View File

@ -25,6 +25,7 @@
#include "drive/Entry1.hh"
#include "xml/Node.hh"
#include "json/Val.hh"
#include <iostream>
@ -53,7 +54,8 @@ void ResourceTest::TestNormal( )
GRUT_ASSERT_EQUAL( subject.Path(), fs::path( TEST_DATA ) / "entry.xml" ) ;
subject.FromLocal( DateTime() ) ;
Val st;
subject.FromLocal( DateTime(), st ) ;
GRUT_ASSERT_EQUAL( subject.MD5(), "c0742c0a32b2c909b6f176d17a6992d0" ) ;
GRUT_ASSERT_EQUAL( subject.StateStr(), "local_new" ) ;