mirror of https://github.com/vitalif/grive2
Implement automatic rename detection
parent
23fa985bdb
commit
00311e8365
|
@ -1,6 +1,6 @@
|
||||||
# Grive2 0.5-dev
|
# Grive2 0.5-dev
|
||||||
|
|
||||||
02 Jan 2016, Vitaliy Filippov
|
3 Jan 2016, Vitaliy Filippov
|
||||||
|
|
||||||
http://yourcmc.ru/wiki/Grive2
|
http://yourcmc.ru/wiki/Grive2
|
||||||
|
|
||||||
|
@ -73,6 +73,7 @@ Enjoy!
|
||||||
### Grive2 v0.5 (unreleased)
|
### Grive2 v0.5 (unreleased)
|
||||||
|
|
||||||
- Much faster and more correct synchronisation using local modification time and checksum cache (similar to git index)
|
- Much faster and more correct synchronisation using local modification time and checksum cache (similar to git index)
|
||||||
|
- Automatic move/rename detection, -m option removed
|
||||||
- force option works again
|
- force option works again
|
||||||
|
|
||||||
### Grive2 v0.4.2
|
### Grive2 v0.4.2
|
||||||
|
|
|
@ -121,7 +121,6 @@ int Main( int argc, char **argv )
|
||||||
( "dry-run", "Only detect which files need to be uploaded/downloaded, "
|
( "dry-run", "Only detect which files need to be uploaded/downloaded, "
|
||||||
"without actually performing them." )
|
"without actually performing them." )
|
||||||
( "ignore", po::value<std::string>(), "Perl RegExp to ignore files (matched against relative paths, remembered for next runs)." )
|
( "ignore", po::value<std::string>(), "Perl RegExp to ignore files (matched against relative paths, remembered for next runs)." )
|
||||||
( "move,m", po::value<std::vector<std::string> >()->multitoken(), "Syncs, then moves a file (first argument) to new location (second argument) without reuploading or redownloading." )
|
|
||||||
;
|
;
|
||||||
|
|
||||||
po::variables_map vm;
|
po::variables_map vm;
|
||||||
|
@ -205,21 +204,6 @@ int Main( int argc, char **argv )
|
||||||
else
|
else
|
||||||
drive.DryRun() ;
|
drive.DryRun() ;
|
||||||
|
|
||||||
if ( vm.count ( "move" ) > 0 && vm.count( "dry-run" ) == 0 )
|
|
||||||
{
|
|
||||||
if (vm["move"].as<std::vector<std::string> >().size() < 2 )
|
|
||||||
Log( "Not enough arguments for move. Move failed.", log::error );
|
|
||||||
else
|
|
||||||
{
|
|
||||||
bool success = drive.Move( vm["move"].as<std::vector<std::string> >()[0],
|
|
||||||
vm["move"].as<std::vector<std::string> >()[1] );
|
|
||||||
if (success)
|
|
||||||
Log( "Move successful!", log::info );
|
|
||||||
else
|
|
||||||
Log( "Move failed.", log::error);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
config.Save() ;
|
config.Save() ;
|
||||||
Log( "Finished!", log::info ) ;
|
Log( "Finished!", log::info ) ;
|
||||||
return 0 ;
|
return 0 ;
|
||||||
|
|
|
@ -112,11 +112,6 @@ void Drive::ReadChanges()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
bool Drive::Move( fs::path old_p, fs::path new_p )
|
|
||||||
{
|
|
||||||
return m_state.Move( m_syncer, old_p, new_p, m_options["path"].Str() );
|
|
||||||
}
|
|
||||||
|
|
||||||
void Drive::Update()
|
void Drive::Update()
|
||||||
{
|
{
|
||||||
Log( "Synchronizing files", log::info ) ;
|
Log( "Synchronizing files", log::info ) ;
|
||||||
|
|
|
@ -41,7 +41,6 @@ public :
|
||||||
Drive( Syncer *syncer, const Val& options ) ;
|
Drive( Syncer *syncer, const Val& options ) ;
|
||||||
|
|
||||||
void DetectChanges() ;
|
void DetectChanges() ;
|
||||||
bool Move( fs::path old_p, fs::path new_p );
|
|
||||||
void Update() ;
|
void Update() ;
|
||||||
void DryRun() ;
|
void DryRun() ;
|
||||||
void SaveState() ;
|
void SaveState() ;
|
||||||
|
|
|
@ -18,6 +18,7 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#include "Resource.hh"
|
#include "Resource.hh"
|
||||||
|
#include "ResourceTree.hh"
|
||||||
#include "Entry.hh"
|
#include "Entry.hh"
|
||||||
#include "Syncer.hh"
|
#include "Syncer.hh"
|
||||||
|
|
||||||
|
@ -269,7 +270,7 @@ void Resource::FromLocal( Val& state )
|
||||||
is_changed = true;
|
is_changed = true;
|
||||||
state.Set( "ctime", Val( m_ctime.Sec() ) );
|
state.Set( "ctime", Val( m_ctime.Sec() ) );
|
||||||
}
|
}
|
||||||
if ( state.Has( "srv_time" ) && m_mtime != DateTime() )
|
if ( state.Has( "srv_time" ) )
|
||||||
m_mtime.Assign( state[ "srv_time" ].U64(), 0 ) ;
|
m_mtime.Assign( state[ "srv_time" ].U64(), 0 ) ;
|
||||||
if ( is_dir )
|
if ( is_dir )
|
||||||
state.Del( "md5" );
|
state.Del( "md5" );
|
||||||
|
@ -397,14 +398,14 @@ Resource* Resource::FindChild( const std::string& name )
|
||||||
}
|
}
|
||||||
|
|
||||||
// try to change the state to "sync"
|
// try to change the state to "sync"
|
||||||
void Resource::Sync( Syncer *syncer, const Val& options )
|
void Resource::Sync( Syncer *syncer, ResourceTree *res_tree, const Val& options )
|
||||||
{
|
{
|
||||||
assert( m_state != unknown ) ;
|
assert( m_state != unknown ) ;
|
||||||
assert( !IsRoot() || m_state == sync ) ; // root folder is already synced
|
assert( !IsRoot() || m_state == sync ) ; // root folder is already synced
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
SyncSelf( syncer, options ) ;
|
SyncSelf( syncer, res_tree, options ) ;
|
||||||
}
|
}
|
||||||
catch ( File::Error &e )
|
catch ( File::Error &e )
|
||||||
{
|
{
|
||||||
|
@ -422,11 +423,11 @@ void Resource::Sync( Syncer *syncer, const Val& options )
|
||||||
if ( m_state != local_deleted && m_state != remote_deleted )
|
if ( m_state != local_deleted && m_state != remote_deleted )
|
||||||
{
|
{
|
||||||
std::for_each( m_child.begin(), m_child.end(),
|
std::for_each( m_child.begin(), m_child.end(),
|
||||||
boost::bind( &Resource::Sync, _1, syncer, options ) ) ;
|
boost::bind( &Resource::Sync, _1, syncer, res_tree, options ) ) ;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void Resource::SyncSelf( Syncer* syncer, const Val& options )
|
void Resource::SyncSelf( Syncer* syncer, ResourceTree *res_tree, const Val& options )
|
||||||
{
|
{
|
||||||
assert( !IsRoot() || m_state == sync ) ; // root is always sync
|
assert( !IsRoot() || m_state == sync ) ; // root is always sync
|
||||||
assert( IsRoot() || !syncer || m_parent->IsFolder() ) ;
|
assert( IsRoot() || !syncer || m_parent->IsFolder() ) ;
|
||||||
|
@ -435,6 +436,50 @@ void Resource::SyncSelf( Syncer* syncer, const Val& options )
|
||||||
|
|
||||||
const fs::path path = Path() ;
|
const fs::path path = Path() ;
|
||||||
|
|
||||||
|
// Detect renames
|
||||||
|
if ( !IsFolder() && ( m_state == local_new || m_state == local_deleted ||
|
||||||
|
m_state == remote_new || m_state == remote_deleted ) )
|
||||||
|
{
|
||||||
|
details::MD5Range moved = res_tree->FindByMD5( m_md5 );
|
||||||
|
bool is_local = m_state == local_new || m_state == local_deleted;
|
||||||
|
State other;
|
||||||
|
if ( m_state == local_new )
|
||||||
|
other = local_deleted;
|
||||||
|
else if ( m_state == local_deleted )
|
||||||
|
other = local_new;
|
||||||
|
else if ( m_state == remote_new )
|
||||||
|
other = remote_deleted;
|
||||||
|
else
|
||||||
|
other = remote_new;
|
||||||
|
for ( details::MD5Map::iterator i = moved.first ; i != moved.second; i++ )
|
||||||
|
{
|
||||||
|
Resource *m = *i;
|
||||||
|
if ( m->m_state == other )
|
||||||
|
{
|
||||||
|
Resource* from = m_state == local_new || m_state == remote_new ? m : this;
|
||||||
|
Resource* to = m_state == local_new || m_state == remote_new ? this : m;
|
||||||
|
Log( "sync %1% moved to %2%. moving %3%", from->Path(), to->Path(),
|
||||||
|
is_local ? "remote" : "local", log::info );
|
||||||
|
if ( syncer )
|
||||||
|
{
|
||||||
|
if ( is_local )
|
||||||
|
syncer->Move( from, to->Parent(), to->Name() );
|
||||||
|
else
|
||||||
|
{
|
||||||
|
fs::rename( from->Path(), to->Path() );
|
||||||
|
to->SetIndex();
|
||||||
|
}
|
||||||
|
to->m_mtime = from->m_mtime;
|
||||||
|
to->m_json->Set( "srv_time", Val( from->m_mtime.Sec() ) );
|
||||||
|
from->DeleteIndex();
|
||||||
|
}
|
||||||
|
from->m_state = both_deleted;
|
||||||
|
to->m_state = sync;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
switch ( m_state )
|
switch ( m_state )
|
||||||
{
|
{
|
||||||
case local_new :
|
case local_new :
|
||||||
|
|
|
@ -29,6 +29,8 @@
|
||||||
|
|
||||||
namespace gr {
|
namespace gr {
|
||||||
|
|
||||||
|
class ResourceTree ;
|
||||||
|
|
||||||
class Syncer ;
|
class Syncer ;
|
||||||
|
|
||||||
class Val ;
|
class Val ;
|
||||||
|
@ -112,7 +114,7 @@ public :
|
||||||
void FromDeleted( Val& state ) ;
|
void FromDeleted( Val& state ) ;
|
||||||
void FromLocal( Val& state ) ;
|
void FromLocal( Val& state ) ;
|
||||||
|
|
||||||
void Sync( Syncer* syncer, const Val& options ) ;
|
void Sync( Syncer* syncer, ResourceTree *res_tree, const Val& options ) ;
|
||||||
void SetServerTime( const DateTime& time ) ;
|
void SetServerTime( const DateTime& time ) ;
|
||||||
|
|
||||||
// children access
|
// children access
|
||||||
|
@ -139,7 +141,7 @@ private :
|
||||||
void DeleteIndex() ;
|
void DeleteIndex() ;
|
||||||
void SetIndex() ;
|
void SetIndex() ;
|
||||||
|
|
||||||
void SyncSelf( Syncer* syncer, const Val& options ) ;
|
void SyncSelf( Syncer* syncer, ResourceTree *res_tree, const Val& options ) ;
|
||||||
|
|
||||||
private :
|
private :
|
||||||
std::string m_name ;
|
std::string m_name ;
|
||||||
|
|
|
@ -97,6 +97,14 @@ const Resource* ResourceTree::FindByHref( const std::string& href ) const
|
||||||
return i != map.end() ? *i : 0 ;
|
return i != map.end() ? *i : 0 ;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
MD5Range ResourceTree::FindByMD5( const std::string& md5 )
|
||||||
|
{
|
||||||
|
MD5Map& map = m_set.get<ByMD5>() ;
|
||||||
|
if ( !md5.empty() )
|
||||||
|
return map.equal_range( md5 );
|
||||||
|
return MD5Range( map.end(), map.end() ) ;
|
||||||
|
}
|
||||||
|
|
||||||
/// Reinsert should be called when the ID/HREF were updated
|
/// Reinsert should be called when the ID/HREF were updated
|
||||||
bool ResourceTree::ReInsert( Resource *coll )
|
bool ResourceTree::ReInsert( Resource *coll )
|
||||||
{
|
{
|
||||||
|
|
|
@ -33,7 +33,7 @@ namespace gr {
|
||||||
namespace details
|
namespace details
|
||||||
{
|
{
|
||||||
using namespace boost::multi_index ;
|
using namespace boost::multi_index ;
|
||||||
struct ByID {} ;
|
struct ByMD5 {} ;
|
||||||
struct ByHref {} ;
|
struct ByHref {} ;
|
||||||
struct ByIdentity {} ;
|
struct ByIdentity {} ;
|
||||||
|
|
||||||
|
@ -41,14 +41,15 @@ namespace details
|
||||||
Resource*,
|
Resource*,
|
||||||
indexed_by<
|
indexed_by<
|
||||||
hashed_non_unique<tag<ByHref>, const_mem_fun<Resource, std::string, &Resource::SelfHref> >,
|
hashed_non_unique<tag<ByHref>, const_mem_fun<Resource, std::string, &Resource::SelfHref> >,
|
||||||
hashed_non_unique<tag<ByID>, const_mem_fun<Resource, std::string, &Resource::ResourceID> >,
|
hashed_non_unique<tag<ByMD5>, const_mem_fun<Resource, std::string, &Resource::MD5> >,
|
||||||
hashed_unique<tag<ByIdentity>, identity<Resource*> >
|
hashed_unique<tag<ByIdentity>, identity<Resource*> >
|
||||||
>
|
>
|
||||||
> Folders ;
|
> Folders ;
|
||||||
|
|
||||||
typedef Folders::index<ByID>::type IDMap ;
|
typedef Folders::index<ByMD5>::type MD5Map ;
|
||||||
typedef Folders::index<ByHref>::type HrefMap ;
|
typedef Folders::index<ByHref>::type HrefMap ;
|
||||||
typedef Folders::index<ByIdentity>::type Set ;
|
typedef Folders::index<ByIdentity>::type Set ;
|
||||||
|
typedef std::pair<MD5Map::iterator, MD5Map::iterator> MD5Range ;
|
||||||
}
|
}
|
||||||
|
|
||||||
/*! \brief A simple container for storing folders
|
/*! \brief A simple container for storing folders
|
||||||
|
@ -68,9 +69,8 @@ public :
|
||||||
|
|
||||||
Resource* FindByHref( const std::string& href ) ;
|
Resource* FindByHref( const std::string& href ) ;
|
||||||
const Resource* FindByHref( const std::string& href ) const ;
|
const Resource* FindByHref( const std::string& href ) const ;
|
||||||
|
details::MD5Range FindByMD5( const std::string& md5 ) ;
|
||||||
|
|
||||||
Resource* FindByID( const std::string& id ) ;
|
|
||||||
|
|
||||||
bool ReInsert( Resource *coll ) ;
|
bool ReInsert( Resource *coll ) ;
|
||||||
|
|
||||||
void Insert( Resource *coll ) ;
|
void Insert( Resource *coll ) ;
|
||||||
|
|
|
@ -295,7 +295,7 @@ void State::Write( const fs::path& filename )
|
||||||
void State::Sync( Syncer *syncer, const Val& options )
|
void State::Sync( Syncer *syncer, const Val& options )
|
||||||
{
|
{
|
||||||
// set the last sync time to the time on the client
|
// set the last sync time to the time on the client
|
||||||
m_res.Root()->Sync( syncer, options ) ;
|
m_res.Root()->Sync( syncer, &m_res, options ) ;
|
||||||
}
|
}
|
||||||
|
|
||||||
long State::ChangeStamp() const
|
long State::ChangeStamp() const
|
||||||
|
@ -309,65 +309,4 @@ void State::ChangeStamp( long cstamp )
|
||||||
m_cstamp = cstamp ;
|
m_cstamp = cstamp ;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool State::Move( Syncer* syncer, fs::path old_p, fs::path new_p, fs::path grive_root )
|
|
||||||
{
|
|
||||||
// Convert paths to canonical representations
|
|
||||||
// Also seems to remove trailing / at the end of directory paths
|
|
||||||
old_p = fs::canonical( old_p );
|
|
||||||
grive_root = fs::canonical( grive_root );
|
|
||||||
|
|
||||||
// new_p is a little special because fs::canonical() requires that the path exists
|
|
||||||
if ( new_p.string()[ new_p.string().size() - 1 ] == '/') // If new_p ends with a /, remove it
|
|
||||||
new_p = new_p.parent_path();
|
|
||||||
new_p = fs::canonical( new_p.parent_path() ) / new_p.filename();
|
|
||||||
|
|
||||||
// Fails if source file doesn't exist, or if destination file already
|
|
||||||
// exists and is not a directory, or if the source and destination are exactly the same
|
|
||||||
if ( (fs::exists(new_p) && !fs::is_directory(new_p)) || !fs::exists(old_p) || fs::equivalent( old_p, new_p ) )
|
|
||||||
return false;
|
|
||||||
|
|
||||||
// If new path is an existing directory, move the file into the directory
|
|
||||||
// instead of trying to rename it
|
|
||||||
if ( fs::is_directory( new_p ) )
|
|
||||||
new_p = new_p / old_p.filename();
|
|
||||||
|
|
||||||
// Get the paths relative to grive root.
|
|
||||||
// Just finds the substring from the end of the grive_root to the end of the path
|
|
||||||
// +1s are to exclude slash at beginning of relative path
|
|
||||||
std::string root( grive_root.string() + "/" );
|
|
||||||
if ( new_p.string().substr( 0, root.length() ).compare( root ) != 0 ||
|
|
||||||
old_p.string().substr( 0, root.length() ).compare( root ) != 0 )
|
|
||||||
return false;
|
|
||||||
fs::path new_p_rootrel( new_p.string().substr( root.length() ) );
|
|
||||||
fs::path old_p_rootrel( old_p.string().substr( root.length() ) );
|
|
||||||
|
|
||||||
//Get resources
|
|
||||||
Resource* res = m_res.Root();
|
|
||||||
Resource* newParentRes = m_res.Root();
|
|
||||||
for ( fs::path::iterator it = old_p_rootrel.begin(); it != old_p_rootrel.end(); ++it )
|
|
||||||
{
|
|
||||||
if ( *it != "." && *it != ".." && res )
|
|
||||||
res = res->FindChild(it->string());
|
|
||||||
if ( *it == ".." )
|
|
||||||
res = res->Parent();
|
|
||||||
}
|
|
||||||
for ( fs::path::iterator it = new_p_rootrel.begin(); it != new_p_rootrel.end(); ++it )
|
|
||||||
{
|
|
||||||
if ( *it != "." && *it != ".." && *it != new_p.filename() && newParentRes )
|
|
||||||
newParentRes = newParentRes->FindChild(it->string());
|
|
||||||
if ( *it == "..")
|
|
||||||
res = res->Parent();
|
|
||||||
}
|
|
||||||
|
|
||||||
//These conditions should only occur if everything is not up-to-date
|
|
||||||
if ( res == 0 || newParentRes == 0 || res->GetState() != Resource::sync ||
|
|
||||||
newParentRes->GetState() != Resource::sync ||
|
|
||||||
newParentRes->FindChild( new_p.filename().string() ) != 0 )
|
|
||||||
return false;
|
|
||||||
|
|
||||||
fs::rename(old_p, new_p); //Moves local file
|
|
||||||
syncer->Move(res, newParentRes, new_p.filename().string()); //Moves server file
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
} // end of namespace gr
|
} // end of namespace gr
|
||||||
|
|
|
@ -62,7 +62,6 @@ public :
|
||||||
|
|
||||||
long ChangeStamp() const ;
|
long ChangeStamp() const ;
|
||||||
void ChangeStamp( long cstamp ) ;
|
void ChangeStamp( long cstamp ) ;
|
||||||
bool Move( Syncer* syncer, fs::path old_p, fs::path new_p, fs::path grive_root );
|
|
||||||
|
|
||||||
private :
|
private :
|
||||||
void FromLocal( const fs::path& p, Resource *folder, Val& tree ) ;
|
void FromLocal( const fs::path& p, Resource *folder, Val& tree ) ;
|
||||||
|
|
|
@ -121,11 +121,15 @@ bool Syncer2::Move( Resource* res, Resource* newParentRes, std::string newFilena
|
||||||
http::Header hdr2 ;
|
http::Header hdr2 ;
|
||||||
hdr2.Add( "Content-Type: application/json" );
|
hdr2.Add( "Content-Type: application/json" );
|
||||||
http::ValResponse vrsp ;
|
http::ValResponse vrsp ;
|
||||||
//Don't change modified date because we're only moving
|
// Don't change modified date because we're only moving
|
||||||
long http_code = m_http->Put( feeds::files + "/" + res->ResourceID() + "?modifiedDateBehavior=noChange" + addRemoveParents, json_meta, &vrsp, hdr2 ) ;
|
long http_code = m_http->Put(
|
||||||
|
feeds::files + "/" + res->ResourceID() + "?modifiedDateBehavior=noChange" + addRemoveParents,
|
||||||
|
json_meta, &vrsp, hdr2
|
||||||
|
) ;
|
||||||
valr = vrsp.Response();
|
valr = vrsp.Response();
|
||||||
assert( http_code == 200 && !( valr["id"].Str().empty() ) );
|
assert( http_code == 200 && !( valr["id"].Str().empty() ) );
|
||||||
}
|
}
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue