Merge branch 'master' into mac

CMakeLists.txt

@@ -3,7 +3,8 @@ cmake_minimum_required(VERSION 2.8)
 include(GNUInstallDirs)
 
 # Grive version. remember to update it for every new release!
-set( GRIVE_VERSION "0.5.1-dev" )
+set( GRIVE_VERSION "0.5.2-dev" CACHE STRING "Grive version" )
+message(WARNING "Version to build: ${GRIVE_VERSION}")
 
 # common compile options
 add_definitions( -DVERSION="${GRIVE_VERSION}" )
@@ -23,4 +24,3 @@ endif( HAVE_SYSTEMD )
 
 add_subdirectory( libgrive )
 add_subdirectory( grive )
-

README.md

@@ -1,14 +1,14 @@
-# Grive2 0.5.1-dev
+# Grive2 0.5.2-dev
 
-28 Sep 2016, Vitaliy Filippov
+13 Nov 2019, Vitaliy Filippov
 
 http://yourcmc.ru/wiki/Grive2
 
 This is the fork of original "Grive" (https://github.com/Grive/grive) Google Drive client
 with the support for the new Drive REST API and partial sync.
 
-Grive can be considered still beta or pre-beta quality. It simply downloads all the files in your
+Grive simply downloads all the files in your Google Drive into the current directory.
-Google Drive into the current directory. After you make some changes to the local files, run
+After you make some changes to the local files, run
 grive again and it will upload your changes back to your Google Drive. New files created locally
 or in Google Drive will be uploaded or downloaded respectively. Deleted files will also be "removed".
 Currently Grive will NOT destroy any of your files: it will only move the files to a
@@ -16,12 +16,12 @@ directory named .trash or put them in the Google Drive trash. You can always rec
 
 There are a few things that Grive does not do at the moment:
 - continously wait for changes in file system or in Google Drive to occur and upload.
-  A sync is only performed when you run Grive (there are workarounds for almost 
+  A sync is only performed when you run Grive (there are workarounds for almost
   continuous sync. See below).
 - symbolic links support.
 - support for Google documents.
 
-These may be added in the future, possibly the next release.
+These may be added in the future.
 
 Enjoy!
 
@@ -90,10 +90,10 @@ Prepare a Google Drive folder in your $HOME directory with `grive -a`.
 
 ```bash
 # 'google-drive' is the name of your Google Drive folder in your $HOME directory
-systemctl --user enable grive-timer@google-drive.timer
+systemctl --user enable grive-timer@$(systemd-escape google-drive).timer
-systemctl --user start grive-timer@google-drive.timer
+systemctl --user start grive-timer@$(systemd-escape google-drive).timer
-systemctl --user enable grive-changes@google-drive.service
+systemctl --user enable grive-changes@$(systemd-escape google-drive).service
-systemctl --user start grive-changes@google-drive.service
+systemctl --user start grive-changes@$(systemd-escape google-drive).service
 ```
 
 You can enable and start these two units for multiple folders in your `$HOME`
@@ -106,6 +106,37 @@ your folder. They need to be added explicitly to your Google Drive: go to the
 Google Drive website, right click on the file or folder and chose 'Add to My
 Drive'.
 
+### Different OAuth2 client to workaround over quota and google approval issues
+
+Google recently started to restrict access for unapproved applications:
+https://developers.google.com/drive/api/v3/about-auth?hl=ru
+
+Grive2 is currently awaiting approval but it seems it will take forever.
+Also even if they approve it the default Client ID supplied with grive may
+exceed quota and grive will then fail to sync.
+
+You can supply your own OAuth2 client credentials to work around these problems
+by following these steps:
+
+1. Go to https://console.developers.google.com/apis/api/drive.googleapis.com
+2. Choose a project (you might need to create one first)
+3. Go to https://console.developers.google.com/apis/library/drive.googleapis.com and
+   "Enable" the Google Drive APIs
+4. Go to https://console.cloud.google.com/apis/credentials and click "Create credentials > Help me choose"
+5. In the "Find out what credentials you need" dialog, choose:
+   - Which API are you using: "Google Drive API"
+   - Where will you be calling the API from: "Other UI (...CLI...)"
+   - What data will you be accessing: "User Data"
+6. In the next steps create a client id (name doesn't matter) and
+   setup the consent screen (defaults are ok, no need for any URLs)
+7. The needed "Client ID" and "Client Secret" are either in the shown download
+   or can later found by clicking on the created credential on
+   https://console.developers.google.com/apis/credentials/
+8. When you change client ID/secret in an existing Grive folder you must first delete
+   the old `.grive` configuration file.
+9. Call `grive -a --id <client_id> --secret <client_secret>` and follow the steps
+   to authenticate the OAuth2 client to allow it to access your drive folder.
+
 ## Installation
 
 For the detailed instructions, see http://yourcmc.ru/wiki/Grive2#Installation
@@ -129,10 +160,15 @@ There are also some optional dependencies:
 On a Debian/Ubuntu/Linux Mint machine just run the following command to install all
 these packages:
 
-    sudo apt-get install git cmake build-essential libgcrypt11-dev libyajl-dev \
+    sudo apt-get install git cmake build-essential libgcrypt20-dev libyajl-dev \
         libboost-all-dev libcurl4-openssl-dev libexpat1-dev libcppunit-dev binutils-dev \
         debhelper zlib1g-dev dpkg-dev pkg-config
 
+Fedora:
+
+    sudo dnf install git cmake libgcrypt-devel gcc-c++ libstdc++ yajl-devel boost-devel libcurl-devel expat-devel binutils zlib
+
+
 FreeBSD:
 
     pkg install git cmake boost-libs yajl libgcrypt pkgconf cppunit libbfd
@@ -154,19 +190,32 @@ Grive uses cmake to build. Basic install sequence is
     make -j4
     sudo make install
 
+Alternativly you can define your own client_id and client_secret during build
+
+    mkdir build
+    cd build
+    cmake .. "-DAPP_ID:STRING=<client_id>" "-DAPP_SECRET:STRING=<client_secret>"
+    make -j4
+    sudo make install
+
 ## Version History
 
-### Grive2 v0.5.1-dev
+### Grive2 v0.5.2-dev
 
-- support for .griveignore
+### Grive2 v0.5.1
-- automatic sync solution based on inotify-tools and systemd
+
+- Support for .griveignore
+- Automatic sync solution based on inotify-tools and systemd
 - no-remote-new and upload-only modes
-- ignore regexp does not persist anymore (note that Grive will still track it to not
+- Ignore regexp does not persist anymore (note that Grive will still track it to not
   accidentally delete remote files when changing ignore regexp)
-- added options to limit upload and download speed
+- Added options to limit upload and download speed
-- faster upload of new and changed files. now Grive uploads files without first calculating
+- Faster upload of new and changed files. Now Grive uploads files without first calculating
   md5 checksum when file is created locally or when its size changes.
-- added -P/--progress-bar option to print ASCII progress bar for each processed file (pull request by @svartkanin)
+- Added -P/--progress-bar option to print ASCII progress bar for each processed file (pull request by @svartkanin)
+- Added command-line options to specify your own client_id and client_secret
+- Now grive2 skips links, sockets, fifos and other unusual files
+- Various small build fixes
 
 ### Grive2 v0.5
 

grive/CMakeLists.txt

@@ -20,6 +20,17 @@ target_link_libraries( grive_executable
 	grive
 )
 
+set(DEFAULT_APP_ID "615557989097-i93d4d1ojpen0m0dso18ldr6orjkidgf.apps.googleusercontent.com")
+set(DEFAULT_APP_SECRET "xiM8Apu_WuRRdheNelJcNtOD")
+set(APP_ID ${DEFAULT_APP_ID} CACHE STRING "Application Id")
+set(APP_SECRET ${DEFAULT_APP_SECRET} CACHE STRING "Application Secret")
+
+target_compile_definitions ( grive_executable
+	PRIVATE
+		-DAPP_ID="${APP_ID}"
+		-DAPP_SECRET="${APP_SECRET}"
+)
+
 set_target_properties( grive_executable
 	PROPERTIES OUTPUT_NAME grive
 )

grive/src/main.cc

@@ -46,8 +46,8 @@
 #include <iostream>
 #include <unistd.h>
 
-const std::string default_id            = "22314510474.apps.googleusercontent.com" ;
+const std::string default_id            = APP_ID ;
-const std::string default_secret        = "bl4ufi89h-9MkFlypcI7R785" ;
+const std::string default_secret        = APP_SECRET ;
 
 using namespace gr ;
 namespace po = boost::program_options;
@@ -113,6 +113,7 @@ int Main( int argc, char **argv )
 		( "auth,a",		"Request authorization token" )
                 ( "id,i",               po::value<std::string>(), "Authentication ID")
                 ( "secret,e",           po::value<std::string>(), "Authentication secret")
+                ( "print-url",          "Only print url for request")
 		( "path,p",		po::value<std::string>(), "Path to working copy root")
 		( "dir,s",		po::value<std::string>(), "Single subdirectory to sync")
 		( "verbose,V",	"Verbose mode. Enable more messages than normal.")
@@ -174,23 +175,23 @@ int Main( int argc, char **argv )
 		http->SetProgressReporter( pb.get() );
 	}
 
-	std::string id = default_id;
-	std::string secret = default_secret;
-
-	if( vm.count( "id" ) )
-	{
-		id = vm["id"].as<std::string>();
-	}
-
-	if( vm.count( "secret" ) )
-	{
-		secret = vm["secret"].as<std::string>();
-	}
-
 	if ( vm.count( "auth" ) )
 	{
+		std::string id = vm.count( "id" ) > 0
+                        ? vm["id"].as<std::string>()
+                        : default_id ;
+		std::string secret = vm.count( "secret" ) > 0
+                        ? vm["secret"].as<std::string>()
+                        : default_secret ;
+
 		OAuth2 token( http.get(), id, secret ) ;
 		
+		if ( vm.count("print-url") )
+		{
+			std::cout <<  token.MakeAuthURL() << std::endl ;
+			return 0 ;
+		}
+		
 		std::cout
 			<< "-----------------------\n"
 			<< "Please go to this URL and get an authentication code:\n\n"
@@ -206,14 +207,20 @@ int Main( int argc, char **argv )
 		token.Auth( code ) ;
 		
 		// save to config
+		config.Set( "id", Val( id ) ) ;
+		config.Set( "secret", Val( secret ) ) ;
 		config.Set( "refresh_token", Val( token.RefreshToken() ) ) ;
 		config.Save() ;
 	}
 	
 	std::string refresh_token ;
+	std::string id ;
+	std::string secret ;
 	try
 	{
 		refresh_token = config.Get("refresh_token").Str() ;
+		id = config.Get("id").Str() ;
+		secret = config.Get("secret").Str() ;
 	}
 	catch ( Exception& e )
 	{

libgrive/CMakeLists.txt

@@ -4,12 +4,10 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/")
 
 find_package(LibGcrypt REQUIRED)
 find_package(CURL REQUIRED)
-find_package(EXPAT REQUIRED)
 find_package(Boost 1.40.0 COMPONENTS program_options filesystem unit_test_framework regex system REQUIRED)
 find_package(BFD)
 find_package(CppUnit)
 find_package(Iberty)
-find_package(ZLIB)
 
 find_package(PkgConfig)
 pkg_check_modules(YAJL REQUIRED yajl)
@@ -37,10 +35,6 @@ else ( IBERTY_FOUND )
   set( IBERTY_LIBRARY "" )
 endif ( IBERTY_FOUND )
 
-if ( ZLIB_FOUND )
-	set( OPT_LIBS	${OPT_LIBS}	${ZLIB_LIBRARIES} )
-endif ( ZLIB_FOUND )
-
 include_directories(
 	${libgrive_SOURCE_DIR}/src
 	${libgrive_SOURCE_DIR}/test
@@ -88,7 +82,6 @@ target_link_libraries( grive
 	${Boost_REGEX_LIBRARY}
 	${Boost_SYSTEM_LIBRARY}
 	${IBERTY_LIBRARY}
-	${EXPAT_LIBRARY}
 	${OPT_LIBS}
 )
 

libgrive/src/bfd/SymbolInfo.cc

@@ -49,9 +49,9 @@ SymbolInfo::SymbolInfo( )
 	m_impl->m_bfd		= 0 ;
 	m_impl->m_symbols	= 0 ;
 	m_impl->m_symbol_count	= 0 ;
-	
+
 	bfd_init( ) ;
-	
+
 	// opening itself
 	bfd *b = bfd_openr( "/proc/self/exe", 0 ) ;
 	if ( b == NULL )
@@ -60,13 +60,13 @@ SymbolInfo::SymbolInfo( )
 		          << bfd_errmsg( bfd_get_error() ) << std::endl ;
 		return ;
 	}
-	
+
 	if ( bfd_check_format( b, bfd_archive ) )
 	{
 		bfd_close( b ) ;
 		return ;
 	}
-	
+
 	char **matching ;
 	if ( !bfd_check_format_matches( b, bfd_object, &matching ) )
 	{
@@ -78,7 +78,7 @@ SymbolInfo::SymbolInfo( )
 			std::cerr << bfd_get_filename( b ) << ": Matching formats: " ;
 			for ( char **p = matching ; *p != 0 ; p++ )
 				std::cerr << " " << *p ;
-			
+
 			std::cerr << std::endl ;
 			std::free( matching ) ;
 		}
@@ -107,7 +107,7 @@ struct SymbolInfo::BacktraceInfo
 	const char			*m_func_name ;
 	unsigned int		m_lineno ;
 	unsigned int		m_is_found ;
-	
+
 	static void Callback( bfd *abfd, asection *section, void* addr ) ;
 } ;
 
@@ -117,17 +117,24 @@ void SymbolInfo::BacktraceInfo::Callback( bfd *abfd, asection *section,
 	BacktraceInfo *info = (BacktraceInfo *)data ;
 	if ((section->flags & SEC_ALLOC) == 0)
 		return ;
-	
+
-	bfd_vma vma = bfd_get_section_vma(abfd, section);
+	// bfd_get_section_vma works up to 7b1cfbcf1a27951fb1b3a212995075dd6fdf985b,
-	
+	// removed in 7c13bc8c91abf291f0206b6608b31955c5ea70d8 (binutils 2.33.1 or so)
+	// so it's substituted by its implementation to avoid checking for binutils
+	// version (which at least on Debian SID it's not that easy because the
+	// version.h is not included with the official package)
+	bfd_vma vma = section->vma;
+
 	unsigned long address = (unsigned long)(info->m_addr);
 	if ( address < vma )
 		return;
-	
+
-	bfd_size_type size = bfd_section_size(abfd, section);
+	// bfd_section_size changed between the two objects described above,
+	// same rationale applies
+	bfd_size_type size = section->size;
 	if ( address > (vma + size))
 		return ;
-	
+
 	const SymbolInfo *pthis	= info->m_pthis ;
 	info->m_is_found	=  bfd_find_nearest_line( abfd, section,
 	                                              pthis->m_impl->m_symbols,
@@ -149,7 +156,7 @@ void SymbolInfo::PrintTrace( void *addr, std::ostream& os, std::size_t idx )
 	{
 		this, addr, 0, 0, 0, false
 	} ;
-	
+
 	Dl_info sym ;
 	bfd_map_over_sections( m_impl->m_bfd,
 							&SymbolInfo::BacktraceInfo::Callback,
@@ -165,7 +172,7 @@ if ( btinfo.m_is_found )
 			filename.erase( pos, std::strlen( SRC_DIR ) ) ;
 #endif
 		os << "#"  << idx << " " << addr << " "
-			<< filename << ":" << btinfo.m_lineno 
+			<< filename << ":" << btinfo.m_lineno
 			<< " "
 			<< (btinfo.m_func_name != 0 ? Demangle(btinfo.m_func_name) : "" )
 			<< std::endl ;

libgrive/src/protocol/OAuth2.cc

@@ -84,12 +84,7 @@ void OAuth2::Auth( const std::string&	auth_code )
 std::string OAuth2::MakeAuthURL()
 {
 	return "https://accounts.google.com/o/oauth2/auth"
-		"?scope=" +
+		"?scope=" + m_agent->Escape( "https://www.googleapis.com/auth/drive" ) +
-			m_agent->Escape( "https://www.googleapis.com/auth/userinfo.email" )	+ "+" + 
-			m_agent->Escape( "https://www.googleapis.com/auth/userinfo.profile" )	+ "+" +
-			m_agent->Escape( "https://docs.google.com/feeds/" )					+ "+" + 
-			m_agent->Escape( "https://docs.googleusercontent.com/" )				+ "+" + 
-			m_agent->Escape( "https://spreadsheets.google.com/feeds/" )			+
 		"&redirect_uri=urn:ietf:wg:oauth:2.0:oob"
 		"&response_type=code"
 		"&client_id=" + m_client_id ;

libgrive/src/util/Config.cc

@@ -38,6 +38,10 @@ const std::string	default_root_folder = ".";
 
 Config::Config( const po::variables_map& vm )
 {
+	if ( vm.count( "id" ) > 0 )
+		m_cmd.Add( "id",	Val( vm["id"].as<std::string>() ) ) ;
+	if ( vm.count( "secret" ) > 0 )
+		m_cmd.Add( "secret",	Val( vm["secret"].as<std::string>() ) ) ;
 	m_cmd.Add( "new-rev",	Val(vm.count("new-rev") > 0) ) ;
 	m_cmd.Add( "force",		Val(vm.count("force") > 0 ) ) ;
 	m_cmd.Add( "path",		Val(vm.count("path") > 0

libgrive/src/xml/TreeBuilder.cc

@@ -23,7 +23,6 @@
 #include "Node.hh"
 #include "util/log/Log.hh"
 
-#include <expat.h>
 
 #include <cassert>
 #include <iostream>

systemd/grive-sync.sh

@@ -107,7 +107,7 @@ listen_directory() {
 	while true #run indefinitely
 	do 
 		# Use a different call to not need to change exit into return
-		inotifywait -q -r -e modify,attrib,close_write,move,create,delete --exclude ".grive_state|.grive" "${_directory}" > /dev/null 2>&1 && ${SCRIPT} sync "${_directory}"
+		inotifywait -q -r -e modify,attrib,close_write,move,create,delete --exclude ".grive_state|.grive" "${_directory}" > /dev/null 2>&1 && ${SCRIPT} sync $(systemd-escape "${_directory}")
 		#echo ${SCRIPT} "${_directory}"
 	done