From bddf8cf6b0d9255c7d12a2d8453440bd64b552f3 Mon Sep 17 00:00:00 2001 From: Doron Behar Date: Sun, 19 Feb 2017 20:03:37 +0200 Subject: [PATCH 1/9] Add completion file for zsh. --- completion.zsh | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 completion.zsh diff --git a/completion.zsh b/completion.zsh new file mode 100644 index 0000000..4d3f807 --- /dev/null +++ b/completion.zsh @@ -0,0 +1,63 @@ +#compdef grive +# ------------------------------------------------------------------------------ +# Copyright (c) 2015 Github zsh-users - http://github.com/zsh-users +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the zsh-users nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL ZSH-USERS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# ------------------------------------------------------------------------------ +# Description +# ----------- +# +# Completion script for Grive (https://github.com/vitalif/grive2) +# +# ------------------------------------------------------------------------------ +# Authors +# ------- +# +# * Doron Behar +# +# ------------------------------------------------------------------------------ +# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*- +# vim: ft=zsh sw=2 ts=2 et +# ------------------------------------------------------------------------------ + +local curcontext="$curcontext" state line ret=1 +typeset -A opt_args + +_arguments -C \ + '(-h --help)'{-h,--help}'[Produce help message]' \ + '(-v --version)'{-v,--version}'[Display Grive version]' \ + '(-a --auth)'{-a,--auth}'[Request authorization token]' \ + '(-p --path)'{-p,--path}'[Root directory to sync]' \ + '(-s --dir)'{-s,--dir}'[Single subdirectory to sync (remembered for next runs)]' \ + '(-V --verbose)'{-V,--verbose}'[Verbose mode. Enable more messages than normal.]' \ + '(--log-http)--log-http[Log all HTTP responses in this file for debugging.]' \ + '(--new-rev)--new-rev[Create,new revisions in server for updated files.]' \ + '(-d --debug)'{-d,--debug}'[Enable debug level messages. Implies -v.]' \ + '(-l --log)'{-l,--log}'[Set log output filename.]' \ + '(-f --force)'{-f,--force}'[Force grive to always download a file from Google Drive instead of uploading it.]' \ + '(--dry-run)--dry-run[Only,detect which files need to be uploaded/downloaded,without actually performing them.]' \ + '(--ignore)--ignore[Perl,RegExp to ignore files (matched against relative paths, remembered for next runs) ]' \ + '*: :_files' && ret=0 + +return ret From 974733ff462caa17690e74b214bdc78e512f13db Mon Sep 17 00:00:00 2001 From: Jan Date: Fri, 10 Mar 2017 13:33:40 +0100 Subject: [PATCH 2/9] also compile with cURL Version less than 7.32.0 --- libgrive/src/http/CurlAgent.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/libgrive/src/http/CurlAgent.cc b/libgrive/src/http/CurlAgent.cc index eee9a8d..8a9d43f 100644 --- a/libgrive/src/http/CurlAgent.cc +++ b/libgrive/src/http/CurlAgent.cc @@ -190,8 +190,10 @@ long CurlAgent::ExecCurl( struct curl_slist *slist = SetHeader( m_pimpl->curl, hdr ) ; curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L); - curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, progress_callback); - curl_easy_setopt(curl, CURLOPT_XFERINFODATA, this); + #if LIBCURL_VERSION_NUM >= 0x072000 + curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, progress_callback); + curl_easy_setopt(curl, CURLOPT_XFERINFODATA, this); + #endif CURLcode curl_code = ::curl_easy_perform(curl); From a0aff5b1462edacd364036c465fbbf32fc00f2c1 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Sun, 19 Mar 2017 12:48:01 +0300 Subject: [PATCH 3/9] Fix #136 - Skip links, sockets, fifos and etc --- libgrive/src/base/Resource.cc | 28 ++++++++++++++++++---------- libgrive/src/util/OS.cc | 12 ++++++------ libgrive/src/util/OS.hh | 6 ++++-- 3 files changed, 28 insertions(+), 18 deletions(-) diff --git a/libgrive/src/base/Resource.cc b/libgrive/src/base/Resource.cc index 23ce335..72c8052 100644 --- a/libgrive/src/base/Resource.cc +++ b/libgrive/src/base/Resource.cc @@ -262,10 +262,10 @@ void Resource::FromLocal( Val& state ) if ( !IsRoot() ) { fs::path path = Path() ; - bool is_dir; + FileType ft ; try { - os::Stat( path, &m_ctime, (off64_t*)&m_size, &is_dir ) ; + os::Stat( path, &m_ctime, (off64_t*)&m_size, &ft ) ; } catch ( os::Error &e ) { @@ -276,22 +276,30 @@ void Resource::FromLocal( Val& state ) m_kind = "bad"; return; } + if ( ft == FT_UNKNOWN ) + { + // Skip sockets/FIFOs/etc + Log( "File %1% is not a regular file or directory; skipping file", path.string(), log::warning ); + m_state = sync; + m_kind = "bad"; + return; + } m_name = path.filename().string() ; - m_kind = is_dir ? "folder" : "file"; + m_kind = ft == FT_DIR ? "folder" : "file"; m_local_exists = true; bool is_changed; if ( state.Has( "ctime" ) && (u64_t) m_ctime.Sec() <= state["ctime"].U64() && - ( is_dir || state.Has( "md5" ) ) ) + ( ft == FT_DIR || state.Has( "md5" ) ) ) { - if ( !is_dir ) + if ( ft != FT_DIR ) m_md5 = state["md5"]; is_changed = false; } else { - if ( !is_dir ) + if ( ft != FT_DIR ) { // File is changed locally. TODO: Detect conflicts is_changed = ( state.Has( "size" ) && m_size != state["size"].U64() ) || @@ -703,13 +711,13 @@ void Resource::SetIndex( bool re_stat ) assert( m_parent && m_parent->m_json != NULL ); if ( !m_json ) m_json = &((*m_parent->m_json)["tree"]).Item( Name() ); - bool is_dir; + FileType ft; if ( re_stat ) - os::Stat( Path(), &m_ctime, NULL, &is_dir ); + os::Stat( Path(), &m_ctime, NULL, &ft ); else - is_dir = IsFolder(); + ft = IsFolder() ? FT_DIR : FT_FILE; m_json->Set( "ctime", Val( m_ctime.Sec() ) ); - if ( !is_dir ) + if ( ft != FT_DIR ) { m_json->Set( "md5", Val( m_md5 ) ); m_json->Set( "size", Val( m_size ) ); diff --git a/libgrive/src/util/OS.cc b/libgrive/src/util/OS.cc index 9095e73..e8b468f 100644 --- a/libgrive/src/util/OS.cc +++ b/libgrive/src/util/OS.cc @@ -39,12 +39,12 @@ namespace gr { namespace os { -void Stat( const fs::path& filename, DateTime *t, off_t *size, bool *is_dir ) +void Stat( const fs::path& filename, DateTime *t, off_t *size, FileType *ft ) { - Stat( filename.string(), t, size, is_dir ) ; + Stat( filename.string(), t, size, ft ) ; } -void Stat( const std::string& filename, DateTime *t, off64_t *size, bool *is_dir ) +void Stat( const std::string& filename, DateTime *t, off64_t *size, FileType *ft ) { struct stat s = {} ; if ( ::stat( filename.c_str(), &s ) != 0 ) @@ -57,7 +57,7 @@ void Stat( const std::string& filename, DateTime *t, off64_t *size, bool *is_dir ) ; } - if (t) + if ( t ) { #if defined __APPLE__ && defined __DARWIN_64_BIT_INO_T *t = DateTime( s.st_ctimespec.tv_sec, s.st_ctimespec.tv_nsec ) ; @@ -67,8 +67,8 @@ void Stat( const std::string& filename, DateTime *t, off64_t *size, bool *is_dir } if ( size ) *size = s.st_size; - if ( is_dir ) - *is_dir = S_ISDIR( s.st_mode ) ? true : false; + if ( ft ) + *ft = S_ISDIR( s.st_mode ) ? FT_DIR : ( S_ISREG( s.st_mode ) ? FT_FILE : FT_UNKNOWN ) ; } void SetFileTime( const fs::path& filename, const DateTime& t ) diff --git a/libgrive/src/util/OS.hh b/libgrive/src/util/OS.hh index 4349783..bd28e30 100644 --- a/libgrive/src/util/OS.hh +++ b/libgrive/src/util/OS.hh @@ -29,12 +29,14 @@ namespace gr { class DateTime ; class Path ; +enum FileType { FT_FILE = 1, FT_DIR = 2, FT_UNKNOWN = 3 } ; + namespace os { struct Error : virtual Exception {} ; - void Stat( const std::string& filename, DateTime *t, off64_t *size, bool *is_dir ) ; - void Stat( const fs::path& filename, DateTime *t, off64_t *size, bool *is_dir ) ; + void Stat( const std::string& filename, DateTime *t, off64_t *size, FileType *ft ) ; + void Stat( const fs::path& filename, DateTime *t, off64_t *size, FileType *ft ) ; void SetFileTime( const std::string& filename, const DateTime& t ) ; void SetFileTime( const fs::path& filename, const DateTime& t ) ; From 63bb138b2d8a23947479d3ac06eab4336abe40bc Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Sun, 19 Mar 2017 19:54:03 +0300 Subject: [PATCH 4/9] Fix #120, fix #125, fix #134 - Add .griveignore --- README.md | 19 +++++++ grive/doc/grive.1 | 35 +++++++++++-- grive/src/main.cc | 17 ++++-- libgrive/src/base/Drive.cc | 9 +--- libgrive/src/base/State.cc | 105 +++++++++++++++++++++++++++++++------ libgrive/src/base/State.hh | 8 +-- 6 files changed, 159 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 2e2ee46..51941c5 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,7 @@ Grive uses cmake to build. Basic install sequence is ### Grive2 v0.5.1-dev +- support for .griveignore - no-remote-new and upload-only modes - ignore regexp does not persist anymore (note that Grive will still track it to not accidentally delete remote files when changing ignore regexp) @@ -137,3 +138,21 @@ New features: - #87: support for revisions - #86: ~~partial sync (contributed by justin at tierramedia.com)~~ that's not partial sync, that's only support for specifying local path on command line + +## .griveignore + +Rules are similar to Git's .gitignore, but may differ slightly due to the different +implementation. + +- lines that start with # are comments +- leading and trailing spaces ignored unless escaped with \ +- non-empty lines without ! in front are treated as "exclude" patterns +- non-empty lines with ! in front are treated as "include" patterns + and have a priority over all "exclude" ones +- patterns are matched against the filenames relative to the grive root +- a/**/b matches any number of subpaths between a and b, including 0 +- **/a matches `a` inside any directory +- b/** matches everything inside `b`, but not b itself +- \* matches any number of any characters except / +- ? matches any character except / +- .griveignore itself isn't ignored by default, but you can include it in itself to ignore diff --git a/grive/doc/grive.1 b/grive/doc/grive.1 index cf0bc26..a00aef6 100644 --- a/grive/doc/grive.1 +++ b/grive/doc/grive.1 @@ -46,10 +46,6 @@ to download only files that are changed in Google Drive and already exist locall \fB\-h\fR, \fB\-\-help\fR Produces help message .TP -\fB\-\-ignore\fR -Ignore files with relative paths matching this Perl Regular Expression. -Value is remembered for next runs. -.TP \fB\-l\fR , \fB\-\-log\fR Write log output to .I @@ -81,6 +77,37 @@ Print ASCII progress bar for each downloaded/uploaded file. \fB\-V\fR, \fB\-\-verbose\fR Verbose mode. Enables more messages than usual. +.SH .griveignore +.PP +You may create .griveignore in your Grive root and use it to setup +exclusion/inclusion rules. +.PP +Rules are similar to Git's .gitignore, but may differ slightly due to the different +implementation. +.IP \[bu] +lines that start with # are comments +.IP \[bu] +leading and trailing spaces ignored unless escaped with \\ +.IP \[bu] +non-empty lines without ! in front are treated as "exclude" patterns +.IP \[bu] +non-empty lines with ! in front are treated as "include" patterns +and have a priority over all "exclude" ones +.IP \[bu] +patterns are matched against the filenames relative to the grive root +.IP \[bu] +a/**/b matches any number of subpaths between a and b, including 0 +.IP \[bu] +**/a matches `a` inside any directory +.IP \[bu] +b/** matches everything inside `b`, but not b itself +.IP \[bu] +* matches any number of any characters except / +.IP \[bu] +? matches any character except / +.IP \[bu] +\[char46]griveignore itself isn't ignored by default, but you can include it in itself to ignore + .SH AUTHORS .PP Current maintainer is Vitaliy Filippov. diff --git a/grive/src/main.cc b/grive/src/main.cc index c87a854..b5a0a00 100644 --- a/grive/src/main.cc +++ b/grive/src/main.cc @@ -124,15 +124,22 @@ int Main( int argc, char **argv ) ( "no-remote-new,n", "Download only files that are changed in Google Drive and already exist locally" ) ( "dry-run", "Only detect which files need to be uploaded/downloaded, " "without actually performing them." ) - ( "ignore", po::value(), "Perl RegExp to ignore files (matched against relative paths)." ) ( "upload-speed,U", po::value(), "Limit upload speed in kbytes per second" ) ( "download-speed,D", po::value(), "Limit download speed in kbytes per second" ) ( "progress-bar,P", "Enable progress bar for upload/download of files") ; po::variables_map vm; - po::store(po::parse_command_line( argc, argv, desc), vm ); - po::notify(vm); + try + { + po::store( po::parse_command_line( argc, argv, desc ), vm ); + } + catch( po::error &e ) + { + std::cerr << "Options are incorrect. Use -h for help\n"; + return -1; + } + po::notify( vm ); // simple commands that doesn't require log or config if ( vm.count("help") ) @@ -148,9 +155,9 @@ int Main( int argc, char **argv ) } // initialize logging - InitLog(vm) ; + InitLog( vm ) ; - Config config(vm) ; + Config config( vm ) ; Log( "config file name %1%", config.Filename(), log::verbose ); diff --git a/libgrive/src/base/Drive.cc b/libgrive/src/base/Drive.cc index fd26225..28fd29a 100644 --- a/libgrive/src/base/Drive.cc +++ b/libgrive/src/base/Drive.cc @@ -41,15 +41,10 @@ namespace gr { -namespace -{ - const std::string state_file = ".grive_state" ; -} - Drive::Drive( Syncer *syncer, const Val& options ) : m_syncer ( syncer ), m_root ( options["path"].Str() ), - m_state ( m_root / state_file, options ), + m_state ( m_root, options ), m_options ( options ) { assert( m_syncer ) ; @@ -72,7 +67,7 @@ void Drive::FromChange( const Entry& entry ) void Drive::SaveState() { - m_state.Write( m_root / state_file ) ; + m_state.Write() ; } void Drive::DetectChanges() diff --git a/libgrive/src/base/State.cc b/libgrive/src/base/State.cc index 9fddf4d..80e2a2d 100644 --- a/libgrive/src/base/State.cc +++ b/libgrive/src/base/State.cc @@ -32,17 +32,28 @@ namespace gr { -State::State( const fs::path& filename, const Val& options ) : +const std::string state_file = ".grive_state" ; +const std::string ignore_file = ".griveignore" ; +const int MAX_IGN = 65536 ; +const char* regex_escape_chars = ".^$|()[]{}*+?\\"; +const boost::regex regex_escape_re( "[.^$|()\\[\\]{}*+?\\\\]" ); + +inline std::string regex_escape( std::string s ) +{ + return regex_replace( s, regex_escape_re, "\\\\&", boost::format_sed ); +} + +State::State( const fs::path& root, const Val& options ) : + m_root ( root ), m_res ( options["path"].Str() ), m_cstamp ( -1 ) { - Read( filename ) ; - + Read() ; + // the "-f" option will make grive always think remote is newer m_force = options.Has( "force" ) ? options["force"].Bool() : false ; - + std::string m_orig_ign = m_ign; - m_ign = ""; if ( options.Has( "ignore" ) && options["ignore"].Str() != m_ign ) m_ign = options["ignore"].Str(); else if ( options.Has( "dir" ) ) @@ -52,8 +63,7 @@ State::State( const fs::path& filename, const Val& options ) : if ( !m_dir.empty() ) { // "-s" is internally converted to an ignore regexp - const boost::regex esc( "[.^$|()\\[\\]{}*+?\\\\]" ); - m_dir = regex_replace( m_dir, esc, "\\\\&", boost::format_sed ); + m_dir = regex_escape( m_dir ); size_t pos = 0; while ( ( pos = m_dir.find( '/', pos ) ) != std::string::npos ) { @@ -66,7 +76,7 @@ State::State( const fs::path& filename, const Val& options ) : } m_ign_changed = m_orig_ign != "" && m_orig_ign != m_ign; - m_ign_re = boost::regex( m_ign.empty() ? "^\\.(grive|grive_state|trash)" : ( m_ign+"|^\\.(grive|grive_state|trash)" ) ); + m_ign_re = boost::regex( m_ign.empty() ? "^\\.(grive$|grive_state$|trash)" : ( m_ign+"|^\\.(grive|grive_state|trash)" ) ); } State::~State() @@ -277,27 +287,92 @@ State::iterator State::end() return m_res.end() ; } -void State::Read( const fs::path& filename ) +void State::Read() { try { - File file( filename ) ; - - m_st = ParseJson( file ); - m_ign = m_st.Has( "ignore_regexp" ) ? m_st["ignore_regexp"].Str() : std::string(); - + File st_file( m_root / state_file ) ; + m_st = ParseJson( st_file ); m_cstamp = m_st["change_stamp"].Int() ; } catch ( Exception& ) { } + + try + { + File ign_file( m_root / ignore_file ) ; + char ign[MAX_IGN] = { 0 }; + int s = ign_file.Read( ign, MAX_IGN-1 ) ; + ParseIgnoreFile( ign, s ); + } + catch ( Exception& e ) + { + throw e; + } } -void State::Write( const fs::path& filename ) +bool State::ParseIgnoreFile( const char* buffer, int size ) +{ + const boost::regex re1( "/\\\\\\*\\\\\\*$" ); + const boost::regex re2( "^\\\\\\*\\\\\\*/" ); + const boost::regex re3( "([^\\\\](\\\\\\\\)*)/\\\\\\*\\\\\\*/" ); + const boost::regex re4( "([^\\\\](\\\\\\\\)*|^)\\\\\\*" ); + const boost::regex re5( "([^\\\\](\\\\\\\\)*|^)\\\\\\?" ); + std::string exclude_re, include_re; + int prev = 0; + for ( int i = 0; i <= size; i++ ) + { + if ( buffer[i] == '\n' || ( i == size && i > prev ) ) + { + while ( prev < i && ( buffer[prev] == ' ' || buffer[prev] == '\t' || buffer[prev] == '\r' ) ) + prev++; + if ( buffer[prev] != '#' ) + { + int j; + for ( j = i-1; j > prev; j-- ) + if ( buffer[j-1] == '\\' || ( buffer[j] != ' ' && buffer[j] != '\t' && buffer[j] != '\r' ) ) + break; + std::string str( buffer+prev, j+1-prev ); + bool inc = str[0] == '!'; + if ( inc ) + str = str.substr( 1 ); + str = regex_escape( str ); + str = regex_replace( str, re1, "/.*", boost::format_perl ); + str = regex_replace( str, re2, ".*/", boost::format_perl ); + str = regex_replace( str, re3, "$1/(.*/)*", boost::format_perl ); + str = regex_replace( str, re4, "$1[^/]*", boost::format_perl ); + std::string str1; + while (1) + { + str1 = regex_replace( str, re5, "$1[^/]", boost::format_perl ); + if ( str1.size() == str.size() ) + break; + str = str1; + } + if ( !inc ) + exclude_re = exclude_re + ( exclude_re.size() > 0 ? "|" : "" ) + str; + else + include_re = include_re + ( include_re.size() > 0 ? "|" : "" ) + str; + } + prev = i+1; + } + } + + if ( exclude_re.size() > 0 ) + { + m_ign = "^" + ( include_re.size() > 0 ? "(?!" + include_re + ")" : std::string() ) + "(" + exclude_re + ")$"; + return true; + } + return false; +} + +void State::Write() { m_st.Set( "change_stamp", Val( m_cstamp ) ) ; m_st.Set( "ignore_regexp", Val( m_ign ) ) ; + fs::path filename = m_root / state_file ; std::ofstream fs( filename.string().c_str() ) ; fs << m_st ; } diff --git a/libgrive/src/base/State.hh b/libgrive/src/base/State.hh index 68b6df8..b42a32e 100644 --- a/libgrive/src/base/State.hh +++ b/libgrive/src/base/State.hh @@ -42,15 +42,15 @@ public : typedef ResourceTree::iterator iterator ; public : - explicit State( const fs::path& filename, const Val& options ) ; + explicit State( const fs::path& root, const Val& options ) ; ~State() ; void FromLocal( const fs::path& p ) ; void FromRemote( const Entry& e ) ; void ResolveEntry() ; - void Read( const fs::path& filename ) ; - void Write( const fs::path& filename ) ; + void Read() ; + void Write() ; Resource* FindByHref( const std::string& href ) ; Resource* FindByID( const std::string& id ) ; @@ -64,6 +64,7 @@ public : void ChangeStamp( long cstamp ) ; private : + bool ParseIgnoreFile( const char* buffer, int size ) ; void FromLocal( const fs::path& p, Resource *folder, Val& tree ) ; void FromChange( const Entry& e ) ; bool Update( const Entry& e ) ; @@ -72,6 +73,7 @@ private : bool IsIgnore( const std::string& filename ) ; private : + fs::path m_root ; ResourceTree m_res ; int m_cstamp ; std::string m_ign ; From 4b6cf69cbb5bc9a47beb34f5204c92d76c558158 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Wed, 22 Mar 2017 18:28:48 +0300 Subject: [PATCH 5/9] Fix #139 --- libgrive/src/base/State.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/libgrive/src/base/State.cc b/libgrive/src/base/State.cc index 80e2a2d..6d86a92 100644 --- a/libgrive/src/base/State.cc +++ b/libgrive/src/base/State.cc @@ -308,7 +308,6 @@ void State::Read() } catch ( Exception& e ) { - throw e; } } From d996989c29a223af5a9d6e1dd42788c19815b9f7 Mon Sep 17 00:00:00 2001 From: Jan Schulz Date: Mon, 24 Apr 2017 23:58:37 +0200 Subject: [PATCH 6/9] Add user systemd units The units can be used to automatically sync a folder in the users home dir * every 3 minutes * on local file changes (requires inotify-tools). To enable the units for `~/google-drive/`: ``` systemctl --user enable grive-timer@google-drive.timer systemctl --user start grive-timer@google-drive.timer systemctl --user enable grive-changes@google-drive.service systemctl --user start grive-changes@google-drive.service ``` The units can be enabled multiple times --- CMakeLists.txt | 4 + systemd/CMakeLists.txt | 26 +++++++ systemd/grive-changes@.service.in | 11 +++ systemd/grive-sync.sh | 118 ++++++++++++++++++++++++++++++ systemd/grive-timer@.service.in | 6 ++ systemd/grive-timer@.timer | 11 +++ 6 files changed, 176 insertions(+) create mode 100644 systemd/CMakeLists.txt create mode 100644 systemd/grive-changes@.service.in create mode 100755 systemd/grive-sync.sh create mode 100644 systemd/grive-timer@.service.in create mode 100644 systemd/grive-timer@.timer diff --git a/CMakeLists.txt b/CMakeLists.txt index 95443aa..951d874 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,7 @@ cmake_minimum_required(VERSION 2.8) +include(GNUInstallDirs) + # Grive version. remember to update it for every new release! set( GRIVE_VERSION "0.5.1-dev" ) @@ -7,5 +9,7 @@ set( GRIVE_VERSION "0.5.1-dev" ) add_definitions( -DVERSION="${GRIVE_VERSION}" ) add_definitions( -D_FILE_OFFSET_BITS=64 -std=c++0x ) +add_subdirectory( systemd ) add_subdirectory( libgrive ) add_subdirectory( grive ) + \ No newline at end of file diff --git a/systemd/CMakeLists.txt b/systemd/CMakeLists.txt new file mode 100644 index 0000000..e3e57ba --- /dev/null +++ b/systemd/CMakeLists.txt @@ -0,0 +1,26 @@ +SET(GRIVE_SYNC_SH_BINARY "${CMAKE_INSTALL_PREFIX}/lib/grive/grive-sync.sh") + +CONFIGURE_FILE(grive-changes@.service.in grive-changes@.service @ONLY) +CONFIGURE_FILE(grive-timer@.service.in grive-timer@.service @ONLY) + +install( + FILES + ${CMAKE_BINARY_DIR}/systemd/grive-changes@.service + ${CMAKE_BINARY_DIR}/systemd/grive-timer@.service + DESTINATION + lib/systemd/user +) + +install( + FILES + grive-timer@.timer + DESTINATION + lib/systemd/user +) + +install( + PROGRAMS + grive-sync.sh + DESTINATION + lib/grive +) \ No newline at end of file diff --git a/systemd/grive-changes@.service.in b/systemd/grive-changes@.service.in new file mode 100644 index 0000000..172ab19 --- /dev/null +++ b/systemd/grive-changes@.service.in @@ -0,0 +1,11 @@ +[Unit] +Description=Google drive sync (changed files) + +[Service] +ExecStart=@GRIVE_SYNC_SH_BINARY@ listen "%i" +Type=simple +Restart=always +RestartSec=30 + +[Install] +WantedBy=default.target diff --git a/systemd/grive-sync.sh b/systemd/grive-sync.sh new file mode 100755 index 0000000..f070ac0 --- /dev/null +++ b/systemd/grive-sync.sh @@ -0,0 +1,118 @@ +#!/bin/bash + +# Copyright (C) 2009 Przemyslaw Pawelczyk +# (C) 2017 Jan Schulz +## +## This script is licensed under the terms of the MIT license. +## https://opensource.org/licenses/MIT + +# Fail on all errors +set -o pipefail + +# We always start in the current users home directory so that names always start there +cd ~ + + +### ARGUMENT PARSING ### +SCRIPT=${0} + +if [[ -z ${2} ]] || [[ ! -d ${2} ]] ; then + echo "Need a directory name in the current users home directory as second argument. Aborting." + exit 1 +fi + +DIRECTORY=$2 + +if [[ -z ${1} ]] ; then + echo "Need a command as first argument. Aborting." + exit 1 +else + if [[ "sync" == "${1}" ]] ; then + COMMAND=sync + elif [[ "listen" == "${1}" ]] ; then + COMMAND=listen + else + echo "Unknown command. Aborting." + exit 1 + fi +fi + + +### LOCKFILE BOILERPLATE ### +LOCKFILE="/run/user/$(id -u)/$(basename $0)_${DIRECTORY//\//_}" +LOCKFD=99 + +# PRIVATE +_lock() { flock -$1 $LOCKFD; } +_no_more_locking() { _lock u; _lock xn && rm -f $LOCKFILE; } +_prepare_locking() { eval "exec $LOCKFD>\"$LOCKFILE\""; trap _no_more_locking EXIT; } + +# ON START +_prepare_locking + +# PUBLIC +exlock_now() { _lock xn; } # obtain an exclusive lock immediately or fail +exlock() { _lock x; } # obtain an exclusive lock +shlock() { _lock s; } # obtain a shared lock +unlock() { _lock u; } # drop a lock + +### SYNC SCRIPT ### +# Idea: only let one script run, but if the sync script is called a second time +# make sure we sync a second time, too + +sync_directory() { + _directory=${1} + if ping -c1 -W1 -q accounts.google.com >/dev/null 2>&1; then + true + # pass + else + echo "Google drive server not reachable..." + exit 0 + fi + + reset_timer_and_exit() { echo "Retriggered google drive sync" && touch -m $LOCKFILE && exit; } + + exlock_now || reset_timer_and_exit + + TIME_AT_START=0 + TIME_AT_END=1 + while [[ ${TIME_AT_START} -lt ${TIME_AT_END} ]]; do + echo "Syncing ${_directory}..." + TIME_AT_START=$(stat -c %Y $LOCKFILE) + grive -p ${_directory} 2>&1 | grep -v -E "^Reading local directories$|^Reading remote server file list$|^Synchronizing files$|^Finished!$" + TIME_AT_END=$(stat -c %Y $LOCKFILE) + echo "Sync of ${_directory} done." + done + + # always exit ok, so that we never go into a wrong systemd state + exit 0 +} + +### LISTEN TO DIRECTORY CHANGES ### + + +listen_directory() { + _directory=${1} + + type inotifywait >/dev/null 2>&1 || { echo >&2 "I require inotifywait but it's not installed. Aborting."; exit 1; } + + echo "Listening for changes in ~/${_directory}" + + while true #run indefinitely + do + # Use a different call to not need to change exit into return + inotifywait -q -r -e modify,attrib,close_write,move,create,delete --exclude ".grive_state|.grive" ${_directory} > /dev/null 2>&1 && ${SCRIPT} sync ${_directory} + done + + # always exit ok, so that we never go into a wrong systemd state + exit 0 +} + +if [[ ${COMMAND} == listen ]] ; then + listen_directory ${DIRECTORY} +else + sync_directory ${DIRECTORY} +fi + +# always exit ok, so that we never go into a wrong systemd state +exit 0 \ No newline at end of file diff --git a/systemd/grive-timer@.service.in b/systemd/grive-timer@.service.in new file mode 100644 index 0000000..a6aae6a --- /dev/null +++ b/systemd/grive-timer@.service.in @@ -0,0 +1,6 @@ +[Unit] +Description=Google drive sync +After=network-online.target + +[Service] +ExecStart=@GRIVE_SYNC_SH_BINARY@ sync "%i" diff --git a/systemd/grive-timer@.timer b/systemd/grive-timer@.timer new file mode 100644 index 0000000..32f14b9 --- /dev/null +++ b/systemd/grive-timer@.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Google drive sync (fixed intervals) + +[Timer] +OnCalendar=*:0/5 +OnBootSec=3min +OnUnitActiveSec=5min +Unit=grive-timer@%i.service + +[Install] +WantedBy=timers.target From 4fe1e71d5b1bc121c67dd5a99faa4600a6a4a64a Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Mon, 5 Jun 2017 14:18:13 +0300 Subject: [PATCH 7/9] Issue #148 - add debug message for the case --- libgrive/src/base/State.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libgrive/src/base/State.cc b/libgrive/src/base/State.cc index 6d86a92..6cbf984 100644 --- a/libgrive/src/base/State.cc +++ b/libgrive/src/base/State.cc @@ -235,6 +235,12 @@ bool State::Update( const Entry& e ) } else if ( Resource *parent = m_res.FindByHref( e.ParentHref() ) ) { + if ( !parent->IsFolder() ) + { + // https://github.com/vitalif/grive2/issues/148 + Log( "%1% is owned by something that's not a directory: href=%2% name=%3%", e.Name(), e.ParentHref(), parent->RelPath(), log::error ); + return true; + } assert( parent->IsFolder() ) ; std::string path = parent->IsRoot() ? e.Name() : ( parent->RelPath() / e.Name() ).string(); From 90c603dc4c15657fb1fc2d7f3ed9d6e9170d8320 Mon Sep 17 00:00:00 2001 From: Mitos Kalandiel Date: Fri, 11 Aug 2017 12:05:53 +0100 Subject: [PATCH 8/9] added 3 more dependencies on linux mint 18.3 I also had to install "debhelper", "zlib1g-dev" & "dpkg-dev" to be able to run the dpkg-buildpackage command --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 51941c5..ace7f57 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,8 @@ On a Debian/Ubuntu/Linux Mint machine just run the following command to install these packages: sudo apt-get install git cmake build-essential libgcrypt11-dev libyajl-dev \ - libboost-all-dev libcurl4-openssl-dev libexpat1-dev libcppunit-dev binutils-dev + libboost-all-dev libcurl4-openssl-dev libexpat1-dev libcppunit-dev binutils-dev \ + debhelper zlib1g-dev dpkg-dev FreeBSD: From 4e5c61b668781043668d688519af3bb5c98bf9e7 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Tue, 21 Nov 2017 22:34:46 +0300 Subject: [PATCH 9/9] Add pkg-config (fixes #126) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ace7f57..2b26772 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ these packages: sudo apt-get install git cmake build-essential libgcrypt11-dev libyajl-dev \ libboost-all-dev libcurl4-openssl-dev libexpat1-dev libcppunit-dev binutils-dev \ - debhelper zlib1g-dev dpkg-dev + debhelper zlib1g-dev dpkg-dev pkg-config FreeBSD: