diff --git a/README.md b/README.md index 8d4bea4..756873b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Grive2 0.5-dev -02 Jan 2016, Vitaliy Filippov +3 Jan 2016, Vitaliy Filippov http://yourcmc.ru/wiki/Grive2 @@ -73,6 +73,7 @@ Enjoy! ### Grive2 v0.5 (unreleased) - Much faster and more correct synchronisation using local modification time and checksum cache (similar to git index) +- Automatic move/rename detection, -m option removed - force option works again ### Grive2 v0.4.2 diff --git a/grive/src/main.cc b/grive/src/main.cc index 347f323..b0fb4ae 100644 --- a/grive/src/main.cc +++ b/grive/src/main.cc @@ -121,7 +121,6 @@ int Main( int argc, char **argv ) ( "dry-run", "Only detect which files need to be uploaded/downloaded, " "without actually performing them." ) ( "ignore", po::value(), "Perl RegExp to ignore files (matched against relative paths, remembered for next runs)." ) - ( "move,m", po::value >()->multitoken(), "Syncs, then moves a file (first argument) to new location (second argument) without reuploading or redownloading." ) ; po::variables_map vm; @@ -205,21 +204,6 @@ int Main( int argc, char **argv ) else drive.DryRun() ; - if ( vm.count ( "move" ) > 0 && vm.count( "dry-run" ) == 0 ) - { - if (vm["move"].as >().size() < 2 ) - Log( "Not enough arguments for move. Move failed.", log::error ); - else - { - bool success = drive.Move( vm["move"].as >()[0], - vm["move"].as >()[1] ); - if (success) - Log( "Move successful!", log::info ); - else - Log( "Move failed.", log::error); - } - } - config.Save() ; Log( "Finished!", log::info ) ; return 0 ; diff --git a/libgrive/src/base/Drive.cc b/libgrive/src/base/Drive.cc index bd306a1..9dd8089 100644 --- a/libgrive/src/base/Drive.cc +++ b/libgrive/src/base/Drive.cc @@ -112,11 +112,6 @@ void Drive::ReadChanges() } } -bool Drive::Move( fs::path old_p, fs::path new_p ) -{ - return m_state.Move( m_syncer, old_p, new_p, m_options["path"].Str() ); -} - void Drive::Update() { Log( "Synchronizing files", log::info ) ; diff --git a/libgrive/src/base/Drive.hh b/libgrive/src/base/Drive.hh index a930192..5f99be2 100644 --- a/libgrive/src/base/Drive.hh +++ b/libgrive/src/base/Drive.hh @@ -41,7 +41,6 @@ public : Drive( Syncer *syncer, const Val& options ) ; void DetectChanges() ; - bool Move( fs::path old_p, fs::path new_p ); void Update() ; void DryRun() ; void SaveState() ; diff --git a/libgrive/src/base/Resource.cc b/libgrive/src/base/Resource.cc index d87c42a..66a4ac3 100644 --- a/libgrive/src/base/Resource.cc +++ b/libgrive/src/base/Resource.cc @@ -18,6 +18,7 @@ */ #include "Resource.hh" +#include "ResourceTree.hh" #include "Entry.hh" #include "Syncer.hh" @@ -269,7 +270,7 @@ void Resource::FromLocal( Val& state ) is_changed = true; state.Set( "ctime", Val( m_ctime.Sec() ) ); } - if ( state.Has( "srv_time" ) && m_mtime != DateTime() ) + if ( state.Has( "srv_time" ) ) m_mtime.Assign( state[ "srv_time" ].U64(), 0 ) ; if ( is_dir ) state.Del( "md5" ); @@ -397,14 +398,14 @@ Resource* Resource::FindChild( const std::string& name ) } // try to change the state to "sync" -void Resource::Sync( Syncer *syncer, const Val& options ) +void Resource::Sync( Syncer *syncer, ResourceTree *res_tree, const Val& options ) { assert( m_state != unknown ) ; assert( !IsRoot() || m_state == sync ) ; // root folder is already synced try { - SyncSelf( syncer, options ) ; + SyncSelf( syncer, res_tree, options ) ; } catch ( File::Error &e ) { @@ -422,11 +423,11 @@ void Resource::Sync( Syncer *syncer, const Val& options ) if ( m_state != local_deleted && m_state != remote_deleted ) { std::for_each( m_child.begin(), m_child.end(), - boost::bind( &Resource::Sync, _1, syncer, options ) ) ; + boost::bind( &Resource::Sync, _1, syncer, res_tree, options ) ) ; } } -void Resource::SyncSelf( Syncer* syncer, const Val& options ) +void Resource::SyncSelf( Syncer* syncer, ResourceTree *res_tree, const Val& options ) { assert( !IsRoot() || m_state == sync ) ; // root is always sync assert( IsRoot() || !syncer || m_parent->IsFolder() ) ; @@ -435,6 +436,50 @@ void Resource::SyncSelf( Syncer* syncer, const Val& options ) const fs::path path = Path() ; + // Detect renames + if ( !IsFolder() && ( m_state == local_new || m_state == local_deleted || + m_state == remote_new || m_state == remote_deleted ) ) + { + details::MD5Range moved = res_tree->FindByMD5( m_md5 ); + bool is_local = m_state == local_new || m_state == local_deleted; + State other; + if ( m_state == local_new ) + other = local_deleted; + else if ( m_state == local_deleted ) + other = local_new; + else if ( m_state == remote_new ) + other = remote_deleted; + else + other = remote_new; + for ( details::MD5Map::iterator i = moved.first ; i != moved.second; i++ ) + { + Resource *m = *i; + if ( m->m_state == other ) + { + Resource* from = m_state == local_new || m_state == remote_new ? m : this; + Resource* to = m_state == local_new || m_state == remote_new ? this : m; + Log( "sync %1% moved to %2%. moving %3%", from->Path(), to->Path(), + is_local ? "remote" : "local", log::info ); + if ( syncer ) + { + if ( is_local ) + syncer->Move( from, to->Parent(), to->Name() ); + else + { + fs::rename( from->Path(), to->Path() ); + to->SetIndex(); + } + to->m_mtime = from->m_mtime; + to->m_json->Set( "srv_time", Val( from->m_mtime.Sec() ) ); + from->DeleteIndex(); + } + from->m_state = both_deleted; + to->m_state = sync; + return; + } + } + } + switch ( m_state ) { case local_new : diff --git a/libgrive/src/base/Resource.hh b/libgrive/src/base/Resource.hh index 97c55d4..354369f 100644 --- a/libgrive/src/base/Resource.hh +++ b/libgrive/src/base/Resource.hh @@ -29,6 +29,8 @@ namespace gr { +class ResourceTree ; + class Syncer ; class Val ; @@ -112,7 +114,7 @@ public : void FromDeleted( Val& state ) ; void FromLocal( Val& state ) ; - void Sync( Syncer* syncer, const Val& options ) ; + void Sync( Syncer* syncer, ResourceTree *res_tree, const Val& options ) ; void SetServerTime( const DateTime& time ) ; // children access @@ -139,7 +141,7 @@ private : void DeleteIndex() ; void SetIndex() ; - void SyncSelf( Syncer* syncer, const Val& options ) ; + void SyncSelf( Syncer* syncer, ResourceTree *res_tree, const Val& options ) ; private : std::string m_name ; diff --git a/libgrive/src/base/ResourceTree.cc b/libgrive/src/base/ResourceTree.cc index 363c5c6..16ac534 100644 --- a/libgrive/src/base/ResourceTree.cc +++ b/libgrive/src/base/ResourceTree.cc @@ -97,6 +97,14 @@ const Resource* ResourceTree::FindByHref( const std::string& href ) const return i != map.end() ? *i : 0 ; } +MD5Range ResourceTree::FindByMD5( const std::string& md5 ) +{ + MD5Map& map = m_set.get() ; + if ( !md5.empty() ) + return map.equal_range( md5 ); + return MD5Range( map.end(), map.end() ) ; +} + /// Reinsert should be called when the ID/HREF were updated bool ResourceTree::ReInsert( Resource *coll ) { diff --git a/libgrive/src/base/ResourceTree.hh b/libgrive/src/base/ResourceTree.hh index f550004..b1fc728 100644 --- a/libgrive/src/base/ResourceTree.hh +++ b/libgrive/src/base/ResourceTree.hh @@ -33,7 +33,7 @@ namespace gr { namespace details { using namespace boost::multi_index ; - struct ByID {} ; + struct ByMD5 {} ; struct ByHref {} ; struct ByIdentity {} ; @@ -41,14 +41,15 @@ namespace details Resource*, indexed_by< hashed_non_unique, const_mem_fun >, - hashed_non_unique, const_mem_fun >, + hashed_non_unique, const_mem_fun >, hashed_unique, identity > > > Folders ; - typedef Folders::index::type IDMap ; + typedef Folders::index::type MD5Map ; typedef Folders::index::type HrefMap ; typedef Folders::index::type Set ; + typedef std::pair MD5Range ; } /*! \brief A simple container for storing folders @@ -68,9 +69,8 @@ public : Resource* FindByHref( const std::string& href ) ; const Resource* FindByHref( const std::string& href ) const ; + details::MD5Range FindByMD5( const std::string& md5 ) ; - Resource* FindByID( const std::string& id ) ; - bool ReInsert( Resource *coll ) ; void Insert( Resource *coll ) ; diff --git a/libgrive/src/base/State.cc b/libgrive/src/base/State.cc index 8f61b59..f743033 100644 --- a/libgrive/src/base/State.cc +++ b/libgrive/src/base/State.cc @@ -295,7 +295,7 @@ void State::Write( const fs::path& filename ) void State::Sync( Syncer *syncer, const Val& options ) { // set the last sync time to the time on the client - m_res.Root()->Sync( syncer, options ) ; + m_res.Root()->Sync( syncer, &m_res, options ) ; } long State::ChangeStamp() const @@ -309,65 +309,4 @@ void State::ChangeStamp( long cstamp ) m_cstamp = cstamp ; } -bool State::Move( Syncer* syncer, fs::path old_p, fs::path new_p, fs::path grive_root ) -{ - // Convert paths to canonical representations - // Also seems to remove trailing / at the end of directory paths - old_p = fs::canonical( old_p ); - grive_root = fs::canonical( grive_root ); - - // new_p is a little special because fs::canonical() requires that the path exists - if ( new_p.string()[ new_p.string().size() - 1 ] == '/') // If new_p ends with a /, remove it - new_p = new_p.parent_path(); - new_p = fs::canonical( new_p.parent_path() ) / new_p.filename(); - - // Fails if source file doesn't exist, or if destination file already - // exists and is not a directory, or if the source and destination are exactly the same - if ( (fs::exists(new_p) && !fs::is_directory(new_p)) || !fs::exists(old_p) || fs::equivalent( old_p, new_p ) ) - return false; - - // If new path is an existing directory, move the file into the directory - // instead of trying to rename it - if ( fs::is_directory( new_p ) ) - new_p = new_p / old_p.filename(); - - // Get the paths relative to grive root. - // Just finds the substring from the end of the grive_root to the end of the path - // +1s are to exclude slash at beginning of relative path - std::string root( grive_root.string() + "/" ); - if ( new_p.string().substr( 0, root.length() ).compare( root ) != 0 || - old_p.string().substr( 0, root.length() ).compare( root ) != 0 ) - return false; - fs::path new_p_rootrel( new_p.string().substr( root.length() ) ); - fs::path old_p_rootrel( old_p.string().substr( root.length() ) ); - - //Get resources - Resource* res = m_res.Root(); - Resource* newParentRes = m_res.Root(); - for ( fs::path::iterator it = old_p_rootrel.begin(); it != old_p_rootrel.end(); ++it ) - { - if ( *it != "." && *it != ".." && res ) - res = res->FindChild(it->string()); - if ( *it == ".." ) - res = res->Parent(); - } - for ( fs::path::iterator it = new_p_rootrel.begin(); it != new_p_rootrel.end(); ++it ) - { - if ( *it != "." && *it != ".." && *it != new_p.filename() && newParentRes ) - newParentRes = newParentRes->FindChild(it->string()); - if ( *it == "..") - res = res->Parent(); - } - - //These conditions should only occur if everything is not up-to-date - if ( res == 0 || newParentRes == 0 || res->GetState() != Resource::sync || - newParentRes->GetState() != Resource::sync || - newParentRes->FindChild( new_p.filename().string() ) != 0 ) - return false; - - fs::rename(old_p, new_p); //Moves local file - syncer->Move(res, newParentRes, new_p.filename().string()); //Moves server file - return true; -} - } // end of namespace gr diff --git a/libgrive/src/base/State.hh b/libgrive/src/base/State.hh index 939a371..1d5dcc1 100644 --- a/libgrive/src/base/State.hh +++ b/libgrive/src/base/State.hh @@ -62,7 +62,6 @@ public : long ChangeStamp() const ; void ChangeStamp( long cstamp ) ; - bool Move( Syncer* syncer, fs::path old_p, fs::path new_p, fs::path grive_root ); private : void FromLocal( const fs::path& p, Resource *folder, Val& tree ) ; diff --git a/libgrive/src/drive2/Syncer2.cc b/libgrive/src/drive2/Syncer2.cc index 9d77fa5..66ad64f 100644 --- a/libgrive/src/drive2/Syncer2.cc +++ b/libgrive/src/drive2/Syncer2.cc @@ -121,11 +121,15 @@ bool Syncer2::Move( Resource* res, Resource* newParentRes, std::string newFilena http::Header hdr2 ; hdr2.Add( "Content-Type: application/json" ); http::ValResponse vrsp ; - //Don't change modified date because we're only moving - long http_code = m_http->Put( feeds::files + "/" + res->ResourceID() + "?modifiedDateBehavior=noChange" + addRemoveParents, json_meta, &vrsp, hdr2 ) ; + // Don't change modified date because we're only moving + long http_code = m_http->Put( + feeds::files + "/" + res->ResourceID() + "?modifiedDateBehavior=noChange" + addRemoveParents, + json_meta, &vrsp, hdr2 + ) ; valr = vrsp.Response(); assert( http_code == 200 && !( valr["id"].Str().empty() ) ); } + return true; }