vitalif
/
grive2
mirror of https://github.com/vitalif/grive2
1
0
Fork 0
Code Issues Releases Wiki Activity

Use stored server times to check for server-side modifications 

(broken because deleted/new detection logic relies on last_sync/last_change)
syncer-move-dates
Vitaliy Filippov 2016-01-01 22:04:39 +03:00
parent 4edff0a816
commit af05c7c626
7 changed files with 78 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

118
libgrive/src/base/Resource.cc
View File

@ -77,13 +77,13 @@ void Resource::SetState( State new_state )
boost::bind( &Resource::SetState, _1, new_state ) ) ;
}
void Resource::FromRemoteFolder( const Entry& remote, const DateTime& last_change )
void Resource::FromRemoteFolder( const Entry& remote )
{
fs::path path = Path() ;
if ( !remote.IsEditable() )
Log( "folder %1% is read-only", path, log::verbose ) ;
// already sync
if ( fs::is_directory( path ) )
{
@ -91,48 +91,39 @@ void Resource::FromRemoteFolder( const Entry& remote, const DateTime& last_chang
m_state = sync ;
}
// remote file created after last sync, so remote is newer
else if ( remote.MTime() > last_change )
else if ( fs::exists( path ) )
{
if ( fs::exists( path ) )
{
// TODO: handle type change
Log( "%1% changed from folder to file", path, log::verbose ) ;
m_state = sync ;
}
else
{
// make all children as remote_new, if any
Log( "folder %1% is created in remote", path, log::verbose ) ;
SetState( remote_new ) ;
}
// TODO: handle type change
Log( "%1% changed from folder to file", path, log::verbose ) ;
m_state = sync ;
}
// remote file created after last sync, so remote is newer
// TODO: Check local index instead of last_sync time
else if ( remote.MTime() > last_sync )
{
// make all children as remote_new, if any
Log( "folder %1% is created in remote", path, log::verbose ) ;
SetState( remote_new ) ;
}
else
{
if ( fs::exists( path ) )
{
// TODO: handle type chage
Log( "%1% changed from file to folder", path, log::verbose ) ;
m_state = sync ;
}
else
{
Log( "folder %1% is deleted in local", path, log::verbose ) ;
SetState( local_deleted ) ;
}
Log( "folder %1% is deleted in local", path, log::verbose ) ;
SetState( local_deleted ) ;
}
}
/// Update the state according to information (i.e. Entry) from remote. This function
/// compares the modification time and checksum of both copies and determine which
/// one is newer.
void Resource::FromRemote( const Entry& remote, const DateTime& last_change )
void Resource::FromRemote( const Entry& remote )
{
// sync folder
if ( remote.IsDir() && IsFolder() )
FromRemoteFolder( remote, last_change ) ;
FromRemoteFolder( remote ) ;
else
FromRemoteFile( remote, last_change ) ;
FromRemoteFile( remote ) ;
AssignIDs( remote ) ;
@ -156,7 +147,7 @@ void Resource::AssignIDs( const Entry& remote )
}
}
void Resource::FromRemoteFile( const Entry& remote, const DateTime& last_change )
void Resource::FromRemoteFile( const Entry& remote )
{
assert( m_parent != 0 ) ;
@ -179,7 +170,8 @@ void Resource::FromRemoteFile( const Entry& remote, const DateTime& last_change
{
Trace( "file %1% change stamp = %2%", Path(), remote.ChangeStamp() ) ;
if ( remote.MTime() > last_change || remote.ChangeStamp() > 0 )
// TODO: Check local index instead of last_sync time
if ( remote.MTime() > last_sync || remote.ChangeStamp() > 0 )
{
Log( "file %1% is created in remote (change %2%)", path,
remote.ChangeStamp(), log::verbose ) ;
@ -214,7 +206,7 @@ void Resource::FromRemoteFile( const Entry& remote, const DateTime& last_change
assert( m_state != unknown ) ;
// if remote is modified
if ( remote.MTime() > m_mtime )
if ( remote.MTime().Sec() > m_mtime.Sec() )
{
Log( "file %1% is changed in remote", path, log::verbose ) ;
m_state = remote_changed ;
@ -233,7 +225,7 @@ void Resource::FromRemoteFile( const Entry& remote, const DateTime& last_change
/// Update the resource with the attributes of local file or directory. This
/// function will propulate the fields in m_entry.
void Resource::FromLocal( const DateTime& last_sync, Val& state )
void Resource::FromLocal( Val& state )
{
assert( !m_json );
m_json = &state;
@ -246,37 +238,43 @@ void Resource::FromLocal( const DateTime& last_sync, Val& state )
os::Stat( path, &m_ctime, &m_size, &is_dir ) ;
m_name = path.filename().string() ;
if ( !is_dir )
m_kind = is_dir ? "folder" : "file";
bool is_changed;
if ( state.Has( "ctime" ) && (u64_t) m_ctime.Sec() <= state["ctime"].U64() &&
( is_dir || state.Has( "md5" ) ) )
{
m_kind = "file";
if ( state.Has( "ctime" ) && state.Has( "md5" ) && (u64_t) m_ctime.Sec() == state["ctime"].U64() )
if ( !is_dir )
m_md5 = state["md5"];
else
{
m_md5 = crypt::MD5::Get( path );
state.Set( "md5", Val( m_md5 ) );
state.Set( "ctime", Val( m_ctime.Sec() ) );
}
if ( state.Has( "srv_time" ) && m_mtime != DateTime() )
m_mtime.Assign( state[ "srv_time" ].U64(), 0 ) ;
is_changed = false;
}
else
{
m_kind = "folder";
state.Del( "md5" );
state.Del( "ctime" );
state.Del( "srv_time" );
if ( !is_dir )
{
m_md5 = crypt::MD5::Get( path );
// File is changed locally. TODO: Detect conflicts
is_changed = state.Has( "md5" ) && m_md5 != state["md5"].Str();
state.Set( "md5", Val( m_md5 ) );
}
else
is_changed = true;
state.Set( "ctime", Val( m_ctime.Sec() ) );
}
if ( state.Has( "srv_time" ) && m_mtime != DateTime() )
m_mtime.Assign( state[ "srv_time" ].U64(), 0 ) ;
if ( is_dir )
state.Del( "md5" );
// follow parent recursively
if ( m_parent->m_state == local_new || m_parent->m_state == local_deleted )
m_state = local_new ;
// if the file is not created after last sync, assume file is
// remote_deleted first, it will be updated to sync/remote_changed
// in FromRemote()
m_state = m_parent->m_state ;
else
m_state = ( m_ctime > last_sync ? local_new : remote_deleted ) ;
{
// Upload file if it is changed and remove if not.
// State will be updated to sync/remote_changed in FromRemote()
m_state = is_changed ? local_new : remote_deleted;
}
}
assert( m_state != unknown ) ;
@ -389,7 +387,7 @@ Resource* Resource::FindChild( const std::string& name )
}
// try to change the state to "sync"
void Resource::Sync( Syncer *syncer, DateTime& sync_time, const Val& options )
void Resource::Sync( Syncer *syncer, const Val& options )
{
assert( m_state != unknown ) ;
assert( !IsRoot() || m_state == sync ) ; // root folder is already synced
@ -410,15 +408,11 @@ void Resource::Sync( Syncer *syncer, DateTime& sync_time, const Val& options )
return;
}
// we want the server sync time, so we will take the server time of the last file uploaded to store as the sync time
// m_mtime is updated to server modified time when the file is uploaded
sync_time = std::max(sync_time, m_mtime);
// if myself is deleted, no need to do the childrens
if ( m_state != local_deleted && m_state != remote_deleted )
{
std::for_each( m_child.begin(), m_child.end(),
boost::bind( &Resource::Sync, _1, syncer, boost::ref(sync_time), options ) ) ;
boost::bind( &Resource::Sync, _1, syncer, options ) ) ;
if ( IsFolder() )
{
// delete state of removed files
@ -512,7 +506,7 @@ void Resource::SyncSelf( Syncer* syncer, const Val& options )
break ;
}
if ( m_state != local_deleted && m_state != remote_deleted && !IsFolder() )
if ( m_state != local_deleted && m_state != remote_deleted )
{
// Update server time of this file
m_json->Set( "srv_time", Val( m_mtime.Sec() ) );

10
libgrive/src/base/Resource.hh
View File

@ -106,10 +106,10 @@ public :
bool HasID() const ;
std::string MD5() const ;
void FromRemote( const Entry& remote, const DateTime& last_change ) ;
void FromLocal( const DateTime& last_sync, Val& state ) ;
void FromRemote( const Entry& remote ) ;
void FromLocal( Val& state ) ;
void Sync( Syncer* syncer, DateTime& sync_time, const Val& options ) ;
void Sync( Syncer* syncer, const Val& options ) ;
void SetServerTime( const DateTime& time ) ;
// children access
@ -129,8 +129,8 @@ private :
private :
void SetState( State new_state ) ;
void FromRemoteFolder( const Entry& remote, const DateTime& last_change ) ;
void FromRemoteFile( const Entry& remote, const DateTime& last_change ) ;
void FromRemoteFolder( const Entry& remote ) ;
void FromRemoteFile( const Entry& remote ) ;
void DeleteLocal() ;
void SetIndex() ;

4
libgrive/src/base/ResourceTree.cc
View File

@ -123,11 +123,11 @@ void ResourceTree::Erase( Resource *coll )
s.erase( s.find( coll ) ) ;
}
void ResourceTree::Update( Resource *coll, const Entry& e, const DateTime& last_change )
void ResourceTree::Update( Resource *coll, const Entry& e )
{
assert( coll != 0 ) ;
coll->FromRemote( e, last_change ) ;
coll->FromRemote( e ) ;
ReInsert( coll ) ;
}

2
libgrive/src/base/ResourceTree.hh
View File

@ -75,7 +75,7 @@ public :
void Insert( Resource *coll ) ;
void Erase( Resource *coll ) ;
void Update( Resource *coll, const Entry& e, const DateTime& last_change ) ;
void Update( Resource *coll, const Entry& e ) ;
Resource* Root() ;
const Resource* Root() const ;

48
libgrive/src/base/State.cc
View File

@ -65,14 +65,11 @@ State::State( const fs::path& filename, const Val& options ) :
// the "-f" option will make grive always think remote is newer
if ( force )
{
m_last_change = DateTime() ;
m_last_sync = DateTime::Now() ;
}
m_last_sync = new DateTime() ;
m_ign_re = boost::regex( m_ign.empty() ? "^\\.(grive|grive_state|trash)" : ( m_ign+"|^\\.(grive|grive_state|trash)" ) );
Log( "last server change time: %1%; last sync time: %2%", m_last_change, m_last_sync, log::verbose ) ;
Log( "last sync time: %2%", m_last_sync, log::verbose ) ;
}
State::~State()
@ -85,7 +82,9 @@ void State::FromLocal( const fs::path& p )
{
if ( !m_st.Has( "tree" ) )
m_st.Add( "tree", Val() );
m_res.Root()->FromLocal( m_last_sync, m_st ) ;
// Remember m_update_sync just before reading local file tree
m_update_sync = DateTime::Now() ;
m_res.Root()->FromLocal( m_st ) ;
FromLocal( p, m_res.Root(), m_st["tree"] ) ;
}
@ -120,7 +119,7 @@ void State::FromLocal( const fs::path& p, Resource* folder, Val& tree )
if ( !tree.Has( fname ) )
tree.Add( fname, Val() );
Val& rec = tree[fname];
c->FromLocal( m_last_sync, rec ) ;
c->FromLocal( rec ) ;
if ( c->IsFolder() )
{
if ( !rec.Has("tree") )
@ -189,7 +188,7 @@ void State::FromChange( const Entry& e )
// entries in the change feed is always treated as newer in remote,
// so we override the last sync time to 0
if ( Resource *res = m_res.FindByHref( e.SelfHref() ) )
m_res.Update( res, e, DateTime() ) ;
m_res.Update( res, e ) ;
}
bool State::Update( const Entry& e )
@ -205,7 +204,7 @@ bool State::Update( const Entry& e )
Log( "%1% is ignored by grive", path, log::verbose ) ;
return true;
}
m_res.Update( res, e, m_last_change ) ;
m_res.Update( res, e ) ;
return true;
}
else if ( Resource *parent = m_res.FindByHref( e.ParentHref() ) )
@ -225,7 +224,7 @@ bool State::Update( const Entry& e )
if ( child )
{
// since we are updating the ID and Href, we need to remove it and re-add it.
m_res.Update( child, e, m_last_change ) ;
m_res.Update( child, e ) ;
}
// folder entry exist in google drive, but not local. we should create
@ -238,7 +237,7 @@ bool State::Update( const Entry& e )
m_res.Insert( child ) ;
// update the state of the resource
m_res.Update( child, e, m_last_change ) ;
m_res.Update( child, e ) ;
}
return true ;
@ -265,16 +264,13 @@ State::iterator State::end()
void State::Read( const fs::path& filename )
{
m_last_sync.Assign( 0 ) ;
m_last_change.Assign( 0 ) ;
try
{
File file( filename ) ;
m_st = ParseJson( file );
Val last_sync = m_st["last_sync"] ;
Val last_change = m_st.Has( "last_change" ) ? m_st["last_change"] : m_st["last_sync"] ;
m_last_sync.Assign( last_sync["sec"].Int(), last_sync["nsec"].Int() ) ;
m_last_change.Assign( last_change["sec"].Int(), last_change["nsec"].Int() ) ;
m_ign = m_st.Has( "ignore_regexp" ) ? m_st["ignore_regexp"].Str() : std::string();
m_cstamp = m_st["change_stamp"].Int() ;
@ -290,12 +286,7 @@ void State::Write( const fs::path& filename )
last_sync.Add( "sec", Val( (int)m_last_sync.Sec() ) );
last_sync.Add( "nsec", Val( (unsigned)m_last_sync.NanoSec() ) );
Val last_change ;
last_change.Add( "sec", Val( (int)m_last_change.Sec() ) );
last_change.Add( "nsec", Val( (unsigned)m_last_change.NanoSec() ) );
m_st.Set( "last_sync", last_sync ) ;
m_st.Set( "last_change", last_change ) ;
m_st.Set( "change_stamp", Val( m_cstamp ) ) ;
m_st.Set( "ignore_regexp", Val( m_ign ) ) ;
@ -305,24 +296,9 @@ void State::Write( const fs::path& filename )
void State::Sync( Syncer *syncer, const Val& options )
{
// set the last sync time from the time returned by the server for the last file synced
// if the sync time hasn't changed (i.e. now files have been uploaded)
// set the last sync time to the time on the client
// ideally because we compare server file times to the last sync time
// the last sync time would always be a server time rather than a client time
// TODO - WARNING - do we use the last sync time to compare to client file times
// need to check if this introduces a new problem
DateTime last_change_time = m_last_change;
m_res.Root()->Sync( syncer, last_change_time, options ) ;
if ( last_change_time == m_last_change )
Trace( "nothing changed at the server side since %1%", m_last_change ) ;
else
{
Trace( "updating last server-side change time to %1%", last_change_time ) ;
m_last_change = last_change_time;
}
m_last_sync = DateTime::Now();
m_res.Root()->Sync( syncer, options ) ;
m_last_sync = m_update_sync;
}
long State::ChangeStamp() const

2
libgrive/src/base/State.hh
View File

@ -75,7 +75,7 @@ private :
private :
ResourceTree m_res ;
DateTime m_last_sync ;
DateTime m_last_change ;
DateTime m_update_sync ;
int m_cstamp ;
std::string m_ign ;
boost::regex m_ign_re ;

2
libgrive/test/base/ResourceTest.cc
View File

@ -63,7 +63,7 @@ void ResourceTest::TestNormal( )
entry.AddElement( "updated" ).AddText( "2012-05-09T16:13:22.401Z" ) ;
Entry1 remote( entry ) ;
subject.FromRemote( remote, DateTime() ) ;
subject.FromRemote( remote ) ;
GRUT_ASSERT_EQUAL( "local_changed", subject.StateStr() ) ;
}