vitalif
/
grive2
mirror of https://github.com/vitalif/grive2
1
0
Fork 0
Code Issues Releases Wiki Activity

Implement automatic rename detection

syncer-move-dates
Vitaliy Filippov 2016-01-03 01:57:54 +03:00
parent 23fa985bdb
commit 00311e8365
11 changed files with 76 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
README.md
View File

@ -1,6 +1,6 @@
# Grive2 0.5-dev
02 Jan 2016, Vitaliy Filippov
3 Jan 2016, Vitaliy Filippov
http://yourcmc.ru/wiki/Grive2
@ -73,6 +73,7 @@ Enjoy!
### Grive2 v0.5 (unreleased)
- Much faster and more correct synchronisation using local modification time and checksum cache (similar to git index)
- Automatic move/rename detection, -m option removed
- force option works again
### Grive2 v0.4.2

16
grive/src/main.cc
View File

@ -121,7 +121,6 @@ int Main( int argc, char **argv )
( "dry-run", "Only detect which files need to be uploaded/downloaded, "
"without actually performing them." )
( "ignore", po::value<std::string>(), "Perl RegExp to ignore files (matched against relative paths, remembered for next runs)." )
( "move,m", po::value<std::vector<std::string> >()->multitoken(), "Syncs, then moves a file (first argument) to new location (second argument) without reuploading or redownloading." )
;
po::variables_map vm;
@ -205,21 +204,6 @@ int Main( int argc, char **argv )
else
drive.DryRun() ;
if ( vm.count ( "move" ) > 0 && vm.count( "dry-run" ) == 0 )
{
if (vm["move"].as<std::vector<std::string> >().size() < 2 )
Log( "Not enough arguments for move. Move failed.", log::error );
else
{
bool success = drive.Move( vm["move"].as<std::vector<std::string> >()[0],
vm["move"].as<std::vector<std::string> >()[1] );
if (success)
Log( "Move successful!", log::info );
else
Log( "Move failed.", log::error);
}
}
config.Save() ;
Log( "Finished!", log::info ) ;
return 0 ;

5
libgrive/src/base/Drive.cc
View File

@ -112,11 +112,6 @@ void Drive::ReadChanges()
}
}
bool Drive::Move( fs::path old_p, fs::path new_p )
{
return m_state.Move( m_syncer, old_p, new_p, m_options["path"].Str() );
}
void Drive::Update()
{
Log( "Synchronizing files", log::info ) ;

1
libgrive/src/base/Drive.hh
View File

@ -41,7 +41,6 @@ public :
Drive( Syncer *syncer, const Val& options ) ;
void DetectChanges() ;
bool Move( fs::path old_p, fs::path new_p );
void Update() ;
void DryRun() ;
void SaveState() ;

55
libgrive/src/base/Resource.cc
View File

@ -18,6 +18,7 @@
*/
#include "Resource.hh"
#include "ResourceTree.hh"
#include "Entry.hh"
#include "Syncer.hh"
@ -269,7 +270,7 @@ void Resource::FromLocal( Val& state )
is_changed = true;
state.Set( "ctime", Val( m_ctime.Sec() ) );
}
if ( state.Has( "srv_time" ) && m_mtime != DateTime() )
if ( state.Has( "srv_time" ) )
m_mtime.Assign( state[ "srv_time" ].U64(), 0 ) ;
if ( is_dir )
state.Del( "md5" );
@ -397,14 +398,14 @@ Resource* Resource::FindChild( const std::string& name )
}
// try to change the state to "sync"
void Resource::Sync( Syncer *syncer, const Val& options )
void Resource::Sync( Syncer *syncer, ResourceTree *res_tree, const Val& options )
{
assert( m_state != unknown ) ;
assert( !IsRoot() || m_state == sync ) ; // root folder is already synced
try
{
SyncSelf( syncer, options ) ;
SyncSelf( syncer, res_tree, options ) ;
}
catch ( File::Error &e )
{
@ -422,11 +423,11 @@ void Resource::Sync( Syncer *syncer, const Val& options )
if ( m_state != local_deleted && m_state != remote_deleted )
{
std::for_each( m_child.begin(), m_child.end(),
boost::bind( &Resource::Sync, _1, syncer, options ) ) ;
boost::bind( &Resource::Sync, _1, syncer, res_tree, options ) ) ;
}
}
void Resource::SyncSelf( Syncer* syncer, const Val& options )
void Resource::SyncSelf( Syncer* syncer, ResourceTree *res_tree, const Val& options )
{
assert( !IsRoot() || m_state == sync ) ; // root is always sync
assert( IsRoot() || !syncer || m_parent->IsFolder() ) ;
@ -435,6 +436,50 @@ void Resource::SyncSelf( Syncer* syncer, const Val& options )
const fs::path path = Path() ;
// Detect renames
if ( !IsFolder() && ( m_state == local_new || m_state == local_deleted ||
m_state == remote_new || m_state == remote_deleted ) )
{
details::MD5Range moved = res_tree->FindByMD5( m_md5 );
bool is_local = m_state == local_new || m_state == local_deleted;
State other;
if ( m_state == local_new )
other = local_deleted;
else if ( m_state == local_deleted )
other = local_new;
else if ( m_state == remote_new )
other = remote_deleted;
else
other = remote_new;
for ( details::MD5Map::iterator i = moved.first ; i != moved.second; i++ )
{
Resource *m = *i;
if ( m->m_state == other )
{
Resource* from = m_state == local_new || m_state == remote_new ? m : this;
Resource* to = m_state == local_new || m_state == remote_new ? this : m;
Log( "sync %1% moved to %2%. moving %3%", from->Path(), to->Path(),
is_local ? "remote" : "local", log::info );
if ( syncer )
{
if ( is_local )
syncer->Move( from, to->Parent(), to->Name() );
else
{
fs::rename( from->Path(), to->Path() );
to->SetIndex();
}
to->m_mtime = from->m_mtime;
to->m_json->Set( "srv_time", Val( from->m_mtime.Sec() ) );
from->DeleteIndex();
}
from->m_state = both_deleted;
to->m_state = sync;
return;
}
}
}
switch ( m_state )
{
case local_new :

6
libgrive/src/base/Resource.hh
View File

@ -29,6 +29,8 @@
namespace gr {
class ResourceTree ;
class Syncer ;
class Val ;
@ -112,7 +114,7 @@ public :
void FromDeleted( Val& state ) ;
void FromLocal( Val& state ) ;
void Sync( Syncer* syncer, const Val& options ) ;
void Sync( Syncer* syncer, ResourceTree *res_tree, const Val& options ) ;
void SetServerTime( const DateTime& time ) ;
// children access
@ -139,7 +141,7 @@ private :
void DeleteIndex() ;
void SetIndex() ;
void SyncSelf( Syncer* syncer, const Val& options ) ;
void SyncSelf( Syncer* syncer, ResourceTree *res_tree, const Val& options ) ;
private :
std::string m_name ;

8
libgrive/src/base/ResourceTree.cc
View File

@ -97,6 +97,14 @@ const Resource* ResourceTree::FindByHref( const std::string& href ) const
return i != map.end() ? *i : 0 ;
}
MD5Range ResourceTree::FindByMD5( const std::string& md5 )
{
MD5Map& map = m_set.get<ByMD5>() ;
if ( !md5.empty() )
return map.equal_range( md5 );
return MD5Range( map.end(), map.end() ) ;
}
/// Reinsert should be called when the ID/HREF were updated
bool ResourceTree::ReInsert( Resource *coll )
{

10
libgrive/src/base/ResourceTree.hh
View File

@ -33,7 +33,7 @@ namespace gr {
namespace details
{
using namespace boost::multi_index ;
struct ByID {} ;
struct ByMD5 {} ;
struct ByHref {} ;
struct ByIdentity {} ;
@ -41,14 +41,15 @@ namespace details
Resource*,
indexed_by<
hashed_non_unique<tag<ByHref>, const_mem_fun<Resource, std::string, &Resource::SelfHref> >,
hashed_non_unique<tag<ByID>, const_mem_fun<Resource, std::string, &Resource::ResourceID> >,
hashed_non_unique<tag<ByMD5>, const_mem_fun<Resource, std::string, &Resource::MD5> >,
hashed_unique<tag<ByIdentity>, identity<Resource*> >
>
> Folders ;
typedef Folders::index<ByID>::type IDMap ;
typedef Folders::index<ByMD5>::type MD5Map ;
typedef Folders::index<ByHref>::type HrefMap ;
typedef Folders::index<ByIdentity>::type Set ;
typedef std::pair<MD5Map::iterator, MD5Map::iterator> MD5Range ;
}
/*! \brief A simple container for storing folders
@ -68,9 +69,8 @@ public :
Resource* FindByHref( const std::string& href ) ;
const Resource* FindByHref( const std::string& href ) const ;
details::MD5Range FindByMD5( const std::string& md5 ) ;
Resource* FindByID( const std::string& id ) ;
bool ReInsert( Resource *coll ) ;
void Insert( Resource *coll ) ;

63
libgrive/src/base/State.cc
View File

@ -295,7 +295,7 @@ void State::Write( const fs::path& filename )
void State::Sync( Syncer *syncer, const Val& options )
{
// set the last sync time to the time on the client
m_res.Root()->Sync( syncer, options ) ;
m_res.Root()->Sync( syncer, &m_res, options ) ;
}
long State::ChangeStamp() const
@ -309,65 +309,4 @@ void State::ChangeStamp( long cstamp )
m_cstamp = cstamp ;
}
bool State::Move( Syncer* syncer, fs::path old_p, fs::path new_p, fs::path grive_root )
{
// Convert paths to canonical representations
// Also seems to remove trailing / at the end of directory paths
old_p = fs::canonical( old_p );
grive_root = fs::canonical( grive_root );
// new_p is a little special because fs::canonical() requires that the path exists
if ( new_p.string()[ new_p.string().size() - 1 ] == '/') // If new_p ends with a /, remove it
new_p = new_p.parent_path();
new_p = fs::canonical( new_p.parent_path() ) / new_p.filename();
// Fails if source file doesn't exist, or if destination file already
// exists and is not a directory, or if the source and destination are exactly the same
if ( (fs::exists(new_p) && !fs::is_directory(new_p)) || !fs::exists(old_p) || fs::equivalent( old_p, new_p ) )
return false;
// If new path is an existing directory, move the file into the directory
// instead of trying to rename it
if ( fs::is_directory( new_p ) )
new_p = new_p / old_p.filename();
// Get the paths relative to grive root.
// Just finds the substring from the end of the grive_root to the end of the path
// +1s are to exclude slash at beginning of relative path
std::string root( grive_root.string() + "/" );
if ( new_p.string().substr( 0, root.length() ).compare( root ) != 0 ||
old_p.string().substr( 0, root.length() ).compare( root ) != 0 )
return false;
fs::path new_p_rootrel( new_p.string().substr( root.length() ) );
fs::path old_p_rootrel( old_p.string().substr( root.length() ) );
//Get resources
Resource* res = m_res.Root();
Resource* newParentRes = m_res.Root();
for ( fs::path::iterator it = old_p_rootrel.begin(); it != old_p_rootrel.end(); ++it )
{
if ( *it != "." && *it != ".." && res )
res = res->FindChild(it->string());
if ( *it == ".." )
res = res->Parent();
}
for ( fs::path::iterator it = new_p_rootrel.begin(); it != new_p_rootrel.end(); ++it )
{
if ( *it != "." && *it != ".." && *it != new_p.filename() && newParentRes )
newParentRes = newParentRes->FindChild(it->string());
if ( *it == "..")
res = res->Parent();
}
//These conditions should only occur if everything is not up-to-date
if ( res == 0 || newParentRes == 0 || res->GetState() != Resource::sync ||
newParentRes->GetState() != Resource::sync ||
newParentRes->FindChild( new_p.filename().string() ) != 0 )
return false;
fs::rename(old_p, new_p); //Moves local file
syncer->Move(res, newParentRes, new_p.filename().string()); //Moves server file
return true;
}
} // end of namespace gr

1
libgrive/src/base/State.hh
View File

@ -62,7 +62,6 @@ public :
long ChangeStamp() const ;
void ChangeStamp( long cstamp ) ;
bool Move( Syncer* syncer, fs::path old_p, fs::path new_p, fs::path grive_root );
private :
void FromLocal( const fs::path& p, Resource *folder, Val& tree ) ;

8
libgrive/src/drive2/Syncer2.cc
View File

@ -121,11 +121,15 @@ bool Syncer2::Move( Resource* res, Resource* newParentRes, std::string newFilena
http::Header hdr2 ;
hdr2.Add( "Content-Type: application/json" );
http::ValResponse vrsp ;
//Don't change modified date because we're only moving
long http_code = m_http->Put( feeds::files + "/" + res->ResourceID() + "?modifiedDateBehavior=noChange" + addRemoveParents, json_meta, &vrsp, hdr2 ) ;
// Don't change modified date because we're only moving
long http_code = m_http->Put(
feeds::files + "/" + res->ResourceID() + "?modifiedDateBehavior=noChange" + addRemoveParents,
json_meta, &vrsp, hdr2
) ;
valr = vrsp.Response();
assert( http_code == 200 && !( valr["id"].Str().empty() ) );
}
return true;
}