vitalif
/
grive2
mirror of https://github.com/vitalif/grive2
1
0
Fork 0
Code Issues Releases Wiki Activity

Check local index instead of last_sync time, totally disable separate folder and changes sync passes

syncer-move-dates
Vitaliy Filippov 2016-01-02 15:04:28 +03:00
parent af05c7c626
commit 8f640ebdad
6 changed files with 83 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
libgrive/src/base/Drive.cc
View File

@ -58,19 +58,7 @@ Drive::Drive( Syncer *syncer, const Val& options ) :
void Drive::FromRemote( const Entry& entry )
{
// entries from change feed does not have the parent HREF,
// so these checkings are done in normal entries only
Resource *parent = m_state.FindByHref( entry.ParentHref() ) ;
if ( parent && !parent->IsFolder() )
Log( "warning: entry %1% has parent %2% which is not a folder, ignored",
entry.Title(), parent->Name(), log::verbose ) ;
else if ( !parent || !parent->IsInRootTree() )
Log( "file \"%1%\" parent doesn't exist, ignored", entry.Title(), log::verbose ) ;
else
m_state.FromRemote( entry ) ;
m_state.FromRemote( entry ) ;
}
void Drive::FromChange( const Entry& entry )
@ -88,43 +76,10 @@ void Drive::SaveState()
m_state.Write( m_root / state_file ) ;
}
void Drive::SyncFolders( )
{
Log( "Synchronizing folders", log::info ) ;
std::auto_ptr<Feed> feed = m_syncer->GetFolders() ;
while ( feed->GetNext( m_syncer->Agent() ) )
{
// first, get all collections from the query result
for ( Feed::iterator i = feed->begin() ; i != feed->end() ; ++i )
{
const Entry &e = *i ;
if ( e.IsDir() )
{
if ( e.ParentHrefs().size() != 1 )
Log( "folder \"%1%\" has multiple parents, ignored", e.Title(), log::verbose ) ;
else if ( e.Title().find('/') != std::string::npos )
Log( "folder \"%1%\" contains a slash in its name, ignored", e.Title(), log::verbose ) ;
else
m_state.FromRemote( e ) ;
}
}
}
m_state.ResolveEntry() ;
}
void Drive::DetectChanges()
{
Log( "Reading local directories", log::info ) ;
m_state.FromLocal( m_root ) ;
long prev_stamp = m_state.ChangeStamp() ;
Trace( "previous change stamp is %1%", prev_stamp ) ;
SyncFolders( ) ;
Log( "Reading remote server file list", log::info ) ;
std::auto_ptr<Feed> feed = m_syncer->GetAll() ;
@ -135,12 +90,19 @@ void Drive::DetectChanges()
feed->begin(), feed->end(),
boost::bind( &Drive::FromRemote, this, _1 ) ) ;
}
// pull the changes feed
m_state.ResolveEntry() ;
}
// pull the changes feed
// FIXME: unused until Grive will use the feed-based sync instead of reading full tree
void Drive::ReadChanges()
{
long prev_stamp = m_state.ChangeStamp() ;
if ( prev_stamp != -1 )
{
Trace( "previous change stamp is %1%", prev_stamp ) ;
Log( "Detecting changes from last sync", log::info ) ;
feed = m_syncer->GetChanges( prev_stamp+1 ) ;
std::auto_ptr<Feed> feed = m_syncer->GetChanges( prev_stamp+1 ) ;
while ( feed->GetNext( m_syncer->Agent() ) )
{
std::for_each(

2
libgrive/src/base/Drive.hh
View File

@ -49,7 +49,7 @@ public :
struct Error : virtual Exception {} ;
private :
void SyncFolders( ) ;
void ReadChanges() ;
void FromRemote( const Entry& entry ) ;
void FromChange( const Entry& entry ) ;
void UpdateChangeStamp( ) ;

76
libgrive/src/base/Resource.cc
View File

@ -90,23 +90,18 @@ void Resource::FromRemoteFolder( const Entry& remote )
Log( "folder %1% is in sync", path, log::verbose ) ;
m_state = sync ;
}
else if ( fs::exists( path ) )
{
// TODO: handle type change
Log( "%1% changed from folder to file", path, log::verbose ) ;
m_state = sync ;
}
// remote file created after last sync, so remote is newer
// TODO: Check local index instead of last_sync time
else if ( remote.MTime() > last_sync )
else if ( remote.MTime().Sec() > m_mtime.Sec() ) // FIXME only seconds are stored in local index
{
// make all children as remote_new, if any
// remote folder created after last sync, so remote is newer
Log( "folder %1% is created in remote", path, log::verbose ) ;
SetState( remote_new ) ;
}
else
{
Log( "folder %1% is deleted in local", path, log::verbose ) ;
@ -131,6 +126,7 @@ void Resource::FromRemote( const Entry& remote )
if ( m_state == remote_new || m_state == remote_changed )
m_md5 = remote.MD5() ;
m_mtime = remote.MTime() ;
}
@ -170,8 +166,7 @@ void Resource::FromRemoteFile( const Entry& remote )
{
Trace( "file %1% change stamp = %2%", Path(), remote.ChangeStamp() ) ;
// TODO: Check local index instead of last_sync time
if ( remote.MTime() > last_sync || remote.ChangeStamp() > 0 )
if ( remote.MTime().Sec() > m_mtime.Sec() || remote.MD5() != m_md5 || remote.ChangeStamp() > 0 )
{
Log( "file %1% is created in remote (change %2%)", path,
remote.ChangeStamp(), log::verbose ) ;
@ -223,6 +218,19 @@ void Resource::FromRemoteFile( const Entry& remote )
}
}
void Resource::FromDeleted( Val& state )
{
assert( !m_json );
m_json = &state;
if ( state.Has( "ctime" ) )
m_ctime.Assign( state["ctime"].U64(), 0 );
if ( state.Has( "md5" ) )
m_md5 = state["md5"];
if ( state.Has( "srv_time" ) )
m_mtime.Assign( state[ "srv_time" ].U64(), 0 ) ;
m_state = both_deleted;
}
/// Update the resource with the attributes of local file or directory. This
/// function will propulate the fields in m_entry.
void Resource::FromLocal( Val& state )
@ -235,7 +243,7 @@ void Resource::FromLocal( Val& state )
{
fs::path path = Path() ;
bool is_dir;
os::Stat( path, &m_ctime, &m_size, &is_dir ) ;
os::Stat( path, &m_ctime, NULL, &is_dir ) ;
m_name = path.filename().string() ;
m_kind = is_dir ? "folder" : "file";
@ -254,7 +262,7 @@ void Resource::FromLocal( Val& state )
{
m_md5 = crypt::MD5::Get( path );
// File is changed locally. TODO: Detect conflicts
is_changed = state.Has( "md5" ) && m_md5 != state["md5"].Str();
is_changed = !state.Has( "md5" ) || m_md5 != state["md5"].Str();
state.Set( "md5", Val( m_md5 ) );
}
else
@ -265,9 +273,11 @@ void Resource::FromLocal( Val& state )
m_mtime.Assign( state[ "srv_time" ].U64(), 0 ) ;
if ( is_dir )
state.Del( "md5" );
else
state.Del( "tree" );
// follow parent recursively
if ( m_parent->m_state == local_new || m_parent->m_state == local_deleted )
if ( m_parent->m_state == local_new || m_parent->m_state == remote_deleted )
m_state = m_parent->m_state ;
else
{
@ -413,22 +423,6 @@ void Resource::Sync( Syncer *syncer, const Val& options )
{
std::for_each( m_child.begin(), m_child.end(),
boost::bind( &Resource::Sync, _1, syncer, options ) ) ;
if ( IsFolder() )
{
// delete state of removed files
Val& tree = (*m_json)["tree"];
Val::Object leftover = tree.AsObject();
for( iterator i = m_child.begin(); i != m_child.end(); i++ )
{
Resource *r = *i;
if ( r->m_state != local_deleted && r->m_state != remote_deleted )
leftover.erase( r->Name() );
else
r->m_json = NULL;
}
for( Val::Object::iterator i = leftover.begin(); i != leftover.end(); i++ )
tree.Del( i->first );
}
}
}
@ -454,7 +448,10 @@ void Resource::SyncSelf( Syncer* syncer, const Val& options )
case local_deleted :
Log( "sync %1% deleted in local. deleting remote", path, log::info ) ;
if ( syncer )
{
syncer->DeleteRemote( this ) ;
DeleteIndex() ;
}
break ;
case local_changed :
@ -490,7 +487,15 @@ void Resource::SyncSelf( Syncer* syncer, const Val& options )
case remote_deleted :
Log( "sync %1% deleted in remote. deleting local", path, log::info ) ;
if ( syncer )
{
DeleteLocal() ;
DeleteIndex() ;
}
break ;
case both_deleted :
if ( syncer )
DeleteIndex() ;
break ;
case sync :
@ -506,7 +511,7 @@ void Resource::SyncSelf( Syncer* syncer, const Val& options )
break ;
}
if ( m_state != local_deleted && m_state != remote_deleted )
if ( syncer && m_json )
{
// Update server time of this file
m_json->Set( "srv_time", Val( m_mtime.Sec() ) );
@ -548,18 +553,27 @@ void Resource::DeleteLocal()
}
}
void Resource::DeleteIndex()
{
(*m_parent->m_json)["tree"].Del( Name() );
m_json = NULL;
}
void Resource::SetIndex()
{
assert( m_parent->m_json != NULL );
if ( !m_json )
m_json = &((*m_parent->m_json)["tree"]).Item( Name() );
bool is_dir;
os::Stat( Path(), &m_ctime, &m_size, &is_dir );
os::Stat( Path(), &m_ctime, NULL, &is_dir );
if ( !is_dir )
{
m_json->Set( "ctime", Val( m_ctime.Sec() ) );
m_json->Set( "md5", Val( m_md5 ) );
m_json->Del( "tree" );
}
else // check if tree item exists
m_json->Item( "tree" );
}
Resource::iterator Resource::begin() const
@ -582,7 +596,7 @@ std::ostream& operator<<( std::ostream& os, Resource::State s )
static const char *state[] =
{
"sync", "local_new", "local_changed", "local_deleted", "remote_new",
"remote_changed", "remote_deleted"
"remote_changed", "remote_deleted", "both_deleted"
} ;
assert( s >= 0 && s < Count(state) ) ;
return os << state[s] ;

7
libgrive/src/base/Resource.hh
View File

@ -67,12 +67,14 @@ public :
/// We should download the file.
remote_new,
/// Resource exists in both local & remote, but remote is newer.
/// Resource exists in both local & remote, but remote is newer.
remote_changed,
/// Resource delete in remote, need to delete in local
remote_deleted,
/// Both deleted. State is used to remove leftover files from the index after sync.
both_deleted,
/// invalid value
unknown
@ -107,6 +109,7 @@ public :
std::string MD5() const ;
void FromRemote( const Entry& remote ) ;
void FromDeleted( Val& state ) ;
void FromLocal( Val& state ) ;
void Sync( Syncer* syncer, const Val& options ) ;
@ -133,6 +136,7 @@ private :
void FromRemoteFile( const Entry& remote ) ;
void DeleteLocal() ;
void DeleteIndex() ;
void SetIndex() ;
void SyncSelf( Syncer* syncer, const Val& options ) ;
@ -143,7 +147,6 @@ private :
std::string m_md5 ;
DateTime m_mtime ;
DateTime m_ctime ;
off_t m_size ;
std::string m_id ;
std::string m_href ;

32
libgrive/src/base/State.cc
View File

@ -65,7 +65,7 @@ State::State( const fs::path& filename, const Val& options ) :
// the "-f" option will make grive always think remote is newer
if ( force )
m_last_sync = new DateTime() ;
m_last_sync = DateTime() ;
m_ign_re = boost::regex( m_ign.empty() ? "^\\.(grive|grive_state|trash)" : ( m_ign+"|^\\.(grive|grive_state|trash)" ) );
@ -80,12 +80,10 @@ State::~State()
/// of local directory.
void State::FromLocal( const fs::path& p )
{
if ( !m_st.Has( "tree" ) )
m_st.Add( "tree", Val() );
// Remember m_update_sync just before reading local file tree
m_update_sync = DateTime::Now() ;
m_res.Root()->FromLocal( m_st ) ;
FromLocal( p, m_res.Root(), m_st["tree"] ) ;
FromLocal( p, m_res.Root(), m_st.Item( "tree" ) ) ;
}
bool State::IsIgnore( const std::string& filename )
@ -98,6 +96,8 @@ void State::FromLocal( const fs::path& p, Resource* folder, Val& tree )
assert( folder != 0 ) ;
assert( folder->IsFolder() ) ;
Val::Object leftover = tree.AsObject();
for ( fs::directory_iterator i( p ) ; i != fs::directory_iterator() ; ++i )
{
std::string fname = i->path().filename().string() ;
@ -116,18 +116,26 @@ void State::FromLocal( const fs::path& p, Resource* folder, Val& tree )
folder->AddChild( c ) ;
m_res.Insert( c ) ;
}
if ( !tree.Has( fname ) )
tree.Add( fname, Val() );
Val& rec = tree[fname];
leftover.erase( fname );
Val& rec = tree.Item( fname );
c->FromLocal( rec ) ;
if ( c->IsFolder() )
{
if ( !rec.Has("tree") )
rec.Add( "tree", Val() );
FromLocal( *i, c, rec["tree"] ) ;
}
FromLocal( *i, c, rec.Item( "tree" ) ) ;
}
}
for( Val::Object::iterator i = leftover.begin(); i != leftover.end(); i++ )
{
// Restore state of locally deleted files
Resource *c = folder->FindChild( i->first ) ;
if ( !c )
{
c = new Resource( i->first, i->second.Has( "tree" ) ? "folder" : "file" ) ;
folder->AddChild( c ) ;
m_res.Insert( c ) ;
}
c->FromDeleted( tree.Item( i->first ) );
}
}
void State::FromRemote( const Entry& e )

2
libgrive/test/base/ResourceTest.cc
View File

@ -55,7 +55,7 @@ void ResourceTest::TestNormal( )
GRUT_ASSERT_EQUAL( subject.Path(), fs::path( TEST_DATA ) / "entry.xml" ) ;
Val st;
subject.FromLocal( DateTime(), st ) ;
subject.FromLocal( st ) ;
GRUT_ASSERT_EQUAL( subject.MD5(), "c0742c0a32b2c909b6f176d17a6992d0" ) ;
GRUT_ASSERT_EQUAL( subject.StateStr(), "local_new" ) ;