Bump version to 0.5.3

fixed: CMAKE_INSTALL_FULL_LIBDIR -> CMAKE_INSTALL_FULL_LIBEXECDIR

.dockerignore

@@ -0,0 +1,4 @@
+.git
+Dockerfile
+.dockerignore
+.gitignore

.gitignore

@@ -4,3 +4,24 @@ grive.kdev4
 .project
 .cproject
 build/
+/CMakeCache.txt
+CMakeFiles
+moc_*.cxx*
+bgrive/ui_MainWindow.h
+Makefile
+*.a
+bgrive/bgrive
+grive/grive
+libgrive/btest
+*.cmake
+
+debian/debhelper-build-stamp
+debian/files
+debian/grive.debhelper.log
+debian/grive.substvars
+debian/grive/
+debian/.debhelper
+
+obj-x86_64-linux-gnu/
+
+.idea

CMakeLists.txt

@@ -1,6 +1,27 @@
 cmake_minimum_required(VERSION 2.8)
+project(grive2)
 
-set( GRIVE_VERSION "0.2.0" )
+include(GNUInstallDirs)
+
+# Grive version. remember to update it for every new release!
+set( GRIVE_VERSION "0.5.3" CACHE STRING "Grive version" )
+message(WARNING "Version to build: ${GRIVE_VERSION}")
+
+# common compile options
+add_definitions( -DVERSION="${GRIVE_VERSION}" )
+add_definitions( -D_FILE_OFFSET_BITS=64 -std=c++0x )
+if ( APPLE )
+	add_definitions( -Doff64_t=off_t )
+endif ( APPLE )
+
+find_program(
+	HAVE_SYSTEMD systemd
+	PATHS /lib/systemd /usr/lib/systemd
+	NO_DEFAULT_PATH
+)
+if ( HAVE_SYSTEMD )
+	add_subdirectory( systemd )
+endif( HAVE_SYSTEMD )
 
 add_subdirectory( libgrive )
 add_subdirectory( grive )

Dockerfile

@@ -0,0 +1,27 @@
+FROM alpine:3.7 as build
+
+RUN apk add git make cmake g++ libgcrypt-dev yajl-dev yajl \
+    boost-dev curl-dev expat-dev cppunit-dev binutils-dev \
+	pkgconfig \
+	&& git clone https://github.com/vitalif/grive2.git \
+	&& mkdir grive2/build \
+	&& cd grive2/build  \
+	&& cmake ..  \
+	&& make -j4  \
+	&& make install \
+    && cd ../.. \
+	&& rm -rf grive2 \
+	&& mkdir /drive
+
+FROM alpine:3.7
+COPY --from=build /usr/local/bin/grive /bin/grive
+ADD https://github.com/Yelp/dumb-init/releases/download/v1.2.1/dumb-init_1.2.1_amd64 /bin/dumb-init
+RUN chmod 777 /bin/dumb-init /bin/grive \
+	&& mkdir /data \
+	&& apk add yajl-dev curl-dev libgcrypt \
+	boost-program_options boost-regex libstdc++ boost-system boost-dev binutils-dev \
+	&& apk add boost-filesystem --repository=http://dl-cdn.alpinelinux.org/alpine/edge/main
+
+VOLUME /data
+WORKDIR /data
+ENTRYPOINT ["dumb-init", "grive"]

README

@@ -1,50 +0,0 @@
-Grive 0.2.0
-9 June 2012
-
-http://www.lbreda.com/grive/
-
-Grive is still considered experimental. It just downloads all the files in your google drive
-into the current directory. After you make some changes to the local files, run grive and
-it will upload your changes back to your google drive. New files created in local or google
-drive will be uploaded or downloaded respectively. Deleted files will also be "removed".
-Currently Grive will NOT destroy any of your files: it will only move the files to a
-directory named .trash, or put it in the google drive trash. You can always recover them.
-
-There are a few things that grive does not do at the moment:
-- wait for changes in file system to occur and upload. Grive only sync when you run it.
-- symbolic links support
-- support for Google documents
-- support for files >2GB
-  
-Of course these will be done in future, possibly the next release.
-
-You need the following libraries:
-
-- json-c
-- libcurl
-- libstdc++
-- libgcrypt
-- Boost (Boost filesystem and program_option are required)
-
-There are also some optional dependencies:
-- CppUnit (for unit tests)
-- libbfd (for backtrace)
-- binutils (for libiberty, required for compilation in OpenSUSE & ubuntu)
-
-Grive uses cmake to build, see the instructions in:
-
-http://www.lbreda.com/grive/installation
-
-for detailed procedures to compile Grive.
-
-When grive is ran for the first time, you should use the "-a" argument to grant
-permission to grive to access to your Google Drive. An URL should be printed.
-Go to the link. You will need to login to your google account if you haven't
-done so. After granting the permission to grive, the browser will show you
-an authenication code. Copy-and-paste that to the standard input of grive.
-
-If everything works fine, grive will create a .grive and a .grive_state file in your
-current directory. It will also start downloading files from your Google Drive to
-your current directory.
-
-Enjoy!

README.md

@@ -0,0 +1,270 @@
+# Grive2 0.5.3
+
+09 Nov 2022, Vitaliy Filippov
+
+http://yourcmc.ru/wiki/Grive2
+
+This is the fork of original "Grive" (https://github.com/Grive/grive) Google Drive client
+with the support for the new Drive REST API and partial sync.
+
+Grive simply downloads all the files in your Google Drive into the current directory.
+After you make some changes to the local files, run
+grive again and it will upload your changes back to your Google Drive. New files created locally
+or in Google Drive will be uploaded or downloaded respectively. Deleted files will also be "removed".
+Currently Grive will NOT destroy any of your files: it will only move the files to a
+directory named .trash or put them in the Google Drive trash. You can always recover them.
+
+There are a few things that Grive does not do at the moment:
+- continously wait for changes in file system or in Google Drive to occur and upload.
+  A sync is only performed when you run Grive (there are workarounds for almost
+  continuous sync. See below).
+- symbolic links support.
+- support for Google documents.
+
+These may be added in the future.
+
+Enjoy!
+
+## Usage
+
+When Grive is run for the first time, you should use the "-a" argument to grant
+permission to Grive to access to your Google Drive:
+
+```bash
+cd $HOME
+mkdir google-drive
+cd google-drive
+grive -a
+```
+
+A URL should be printed. Go to the link. You will need to login to your Google
+account if you haven't done so. After granting the permission to Grive, the
+authorization code will be forwarded to the Grive application and you will be
+redirected to a localhost web page confirming the authorization.
+
+If everything works fine, Grive will create .grive and .grive\_state files in your
+current directory. It will also start downloading files from your Google Drive to
+your current directory.
+
+To resync the direcory, run `grive` in the folder.
+
+```bash
+cd $HOME/google-drive
+grive
+```
+
+### Exclude specific files and folders from sync: .griveignore
+
+Rules are similar to Git's .gitignore, but may differ slightly due to the different
+implementation.
+
+- lines that start with # are comments
+- leading and trailing spaces ignored unless escaped with \
+- non-empty lines without ! in front are treated as "exclude" patterns
+- non-empty lines with ! in front are treated as "include" patterns
+  and have a priority over all "exclude" ones
+- patterns are matched against the filenames relative to the grive root
+- a/**/b matches any number of subpaths between a and b, including 0
+- **/a matches `a` inside any directory
+- b/** matches everything inside `b`, but not b itself
+- \* matches any number of any characters except /
+- ? matches any character except /
+- .griveignore itself isn't ignored by default, but you can include it in itself to ignore
+
+
+### Scheduled syncs and syncs on file change events
+
+There are tools which you can use to enable both scheduled syncs and syncs
+when a file changes. Together these gives you an experience almost like the
+Google Drive clients on other platforms (it misses the almost instantious
+download of changed files in the google drive).
+
+Grive installs such a basic solution which uses inotify-tools together with
+systemd timer and services. You can enable it for a folder in your `$HOME`
+directory (in this case the `$HOME/google-drive`):
+
+First install the `inotify-tools` (seems to be named like that in all major distros): 
+test that it works by calling `inotifywait -h`.
+
+Prepare a Google Drive folder in your $HOME directory with `grive -a`.
+
+```bash
+# 'google-drive' is the name of your Google Drive folder in your $HOME directory
+systemctl --user enable grive@$(systemd-escape google-drive).service
+systemctl --user start grive@$(systemd-escape google-drive).service
+```
+
+You can enable and start this unit for multiple folders in your `$HOME`
+directory if you need to sync with multiple google accounts.
+
+You can also only enable the time based syncing or the changes based syncing
+by only directly enabling and starting the corresponding unit:
+`grive-changes@$(systemd-escape google-drive).service` or 
+`grive-timer@$(systemd-escape google-drive).timer`.
+
+### Shared files
+
+Files and folders which are shared with you don't automatically show up in
+your folder. They need to be added explicitly to your Google Drive: go to the
+Google Drive website, right click on the file or folder and chose 'Add to My
+Drive'.
+
+### Different OAuth2 client to workaround over quota and google approval issues
+
+Google recently started to restrict access for unapproved applications:
+https://developers.google.com/drive/api/v3/about-auth?hl=ru
+
+Grive2 is currently awaiting approval but it seems it will take forever.
+Also even if they approve it the default Client ID supplied with grive may
+exceed quota and grive will then fail to sync.
+
+You can supply your own OAuth2 client credentials to work around these problems
+by following these steps:
+
+1. Go to https://console.developers.google.com/apis/api/drive.googleapis.com
+2. Choose a project (you might need to create one first)
+3. Go to https://console.developers.google.com/apis/library/drive.googleapis.com and
+   "Enable" the Google Drive APIs
+4. Go to https://console.cloud.google.com/apis/credentials and click "Create credentials > Help me choose"
+5. In the "Find out what credentials you need" dialog, choose:
+   - Which API are you using: "Google Drive API"
+   - Where will you be calling the API from: "Other UI (...CLI...)"
+   - What data will you be accessing: "User Data"
+6. In the next steps create a client id (name doesn't matter) and
+   setup the consent screen (defaults are ok, no need for any URLs)
+7. The needed "Client ID" and "Client Secret" are either in the shown download
+   or can later found by clicking on the created credential on
+   https://console.developers.google.com/apis/credentials/
+8. When you change client ID/secret in an existing Grive folder you must first delete
+   the old `.grive` configuration file.
+9. Call `grive -a --id <client_id> --secret <client_secret>` and follow the steps
+   to authenticate the OAuth2 client to allow it to access your drive folder.
+
+## Installation
+
+For the detailed instructions, see http://yourcmc.ru/wiki/Grive2#Installation
+
+### Install dependencies
+
+You need the following libraries:
+
+- yajl 2.x
+- libcurl
+- libstdc++
+- libgcrypt
+- Boost (Boost filesystem, program_options, regex, unit_test_framework and system are required)
+- expat
+
+There are also some optional dependencies:
+- CppUnit (for unit tests)
+- libbfd (for backtrace)
+- binutils (for libiberty, required for compilation in OpenSUSE, Ubuntu, Arch and etc)
+
+On a Debian/Ubuntu/Linux Mint machine just run the following command to install all
+these packages:
+
+    sudo apt-get install git cmake build-essential libgcrypt20-dev libyajl-dev \
+        libboost-all-dev libcurl4-openssl-dev libexpat1-dev libcppunit-dev binutils-dev \
+        debhelper zlib1g-dev dpkg-dev pkg-config
+
+Fedora:
+
+    sudo dnf install git cmake libgcrypt-devel gcc-c++ libstdc++ yajl-devel boost-devel libcurl-devel expat-devel binutils zlib
+
+
+FreeBSD:
+
+    pkg install git cmake boost-libs yajl libgcrypt pkgconf cppunit libbfd
+
+### Build Debian packages
+
+On a Debian/Ubuntu/Linux Mint you can use `dpkg-buildpackage` utility from `dpkg-dev` package
+to build grive. Just clone the repository, `cd` into it and run
+
+    dpkg-buildpackage -j4 --no-sign
+
+### Manual build
+
+Grive uses cmake to build. Basic install sequence is
+
+    mkdir build
+    cd build
+    cmake ..
+    make -j4
+    sudo make install
+
+Alternativly you can define your own client_id and client_secret during build
+
+    mkdir build
+    cd build
+    cmake .. "-DAPP_ID:STRING=<client_id>" "-DAPP_SECRET:STRING=<client_secret>"
+    make -j4
+    sudo make install
+
+## Version History
+
+### Grive2 v0.5.3
+
+- Implement Google OAuth loopback IP redirect flow
+- Various small fixes
+
+### Grive2 v0.5.1
+
+- Support for .griveignore
+- Automatic sync solution based on inotify-tools and systemd
+- no-remote-new and upload-only modes
+- Ignore regexp does not persist anymore (note that Grive will still track it to not
+  accidentally delete remote files when changing ignore regexp)
+- Added options to limit upload and download speed
+- Faster upload of new and changed files. Now Grive uploads files without first calculating
+  md5 checksum when file is created locally or when its size changes.
+- Added -P/--progress-bar option to print ASCII progress bar for each processed file (pull request by @svartkanin)
+- Added command-line options to specify your own client_id and client_secret
+- Now grive2 skips links, sockets, fifos and other unusual files
+- Various small build fixes
+
+### Grive2 v0.5
+
+- Much faster and more correct synchronisation using local modification time and checksum cache (similar to git index)
+- Automatic move/rename detection, -m option removed
+- force option works again
+- Instead of crashing on sync exceptions Grive will give a warning and attempt to sync failed files again during the next run.
+- Revision support works again. Grive 0.4.x always created new revisions for all files during sync, regardless of the absence of the --new-rev option.
+- Shared files now sync correctly
+
+### Grive2 v0.4.2
+
+- Option to exclude files by perl regexp
+- Reimplemented HTTP response logging for debug purposes
+- Use multipart uploads (update metadata and contents at the same time) for improved perfomance & stability
+- Bug fixes
+- Simple option to move/rename files and directories, via `grive -m oldpath newpath` (by Dylan Wulf, wulfd1@tcnj.edu)
+
+Known issues:
+- force option does not work as documented #51
+
+### Grive2 v0.4.1
+
+- Bug fixes
+
+### Grive2 v0.4.0
+
+First fork release, by Vitaliy Filippov / vitalif at mail*ru
+- Support for the new Google Drive REST API (old "Document List" API is shut down by Google 20 April 2015)
+- REAL support for partial sync: syncs only one subdirectory with `grive -s subdir`
+- Major refactoring - a lot of dead code removed, JSON-C is not used anymore, API-specific code is split from non-API-specific
+- Some stability fixes from Visa Putkinen https://github.com/visap/grive/commits/visa
+- Slightly reduce number of syscalls when reading local files.
+
+### Grive v0.3
+
+Bug fix & minor feature release. Fixed bugs:
+- #93: missing reference count increment in one of the Json constructors
+- #82: retry for HTTP error 500 & 503
+- #77: Fixed a bug where grive crashed on the first run.
+
+New features:
+- #87: support for revisions
+- #86: ~~partial sync (contributed by justin at tierramedia.com)~~ that's not partial sync,
+  that's only support for specifying local path on command line
+

cmake/Modules/FindBFD.cmake

@@ -1,5 +1,5 @@
-find_library( DL_LIBRARY 	NAMES dl	PATH /usr/lib /usr/lib64 )
-find_library( BFD_LIBRARY	NAMES bfd	PATH /usr/lib /usr/lib64 )
+find_library( DL_LIBRARY 	NAMES dl	PATH ${CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES} )
+find_library( BFD_LIBRARY	NAMES bfd	PATH ${CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES} )
 
 if ( DL_LIBRARY AND BFD_LIBRARY )
 	set( BFD_FOUND TRUE )
@@ -9,5 +9,4 @@ if ( BFD_FOUND )
 
 	message( STATUS "Found libbfd: ${BFD_LIBRARY}")
 
-
 endif ( BFD_FOUND )

cmake/Modules/FindGDBM.cmake

@@ -1,48 +0,0 @@
-# - Find gdbm
-# Find the native GDBM includes and library
-#
-#  GDBM_INCLUDE_DIR - where to find gdbm.h, etc.
-#  GDBM_LIBRARIES   - List of libraries when using gdbm.
-#  GDBM_FOUND       - True if gdbm found.
-
-
-IF (GDBM_INCLUDE_DIR)
-  # Already in cache, be silent
-  SET(GDBM_FIND_QUIETLY TRUE)
-ENDIF (GDBM_INCLUDE_DIR)
-
-FIND_PATH(GDBM_INCLUDE_DIR gdbm.h
-  /usr/local/include
-  /usr/include
-  /opt/local/include
-)
-
-SET(GDBM_NAMES gdbm)
-FIND_LIBRARY(GDBM_LIBRARY
-  NAMES ${GDBM_NAMES}
-  PATHS /usr/lib /usr/local/lib /opt/local/lib
-)
-
-IF (GDBM_INCLUDE_DIR AND GDBM_LIBRARY)
-   SET(GDBM_FOUND TRUE)
-    SET( GDBM_LIBRARIES ${GDBM_LIBRARY} )
-ELSE (GDBM_INCLUDE_DIR AND GDBM_LIBRARY)
-   SET(GDBM_FOUND FALSE)
-   SET( GDBM_LIBRARIES )
-ENDIF (GDBM_INCLUDE_DIR AND GDBM_LIBRARY)
-
-IF (GDBM_FOUND)
-   IF (NOT GDBM_FIND_QUIETLY)
-      MESSAGE(STATUS "Found GDBM: ${GDBM_LIBRARY}")
-   ENDIF (NOT GDBM_FIND_QUIETLY)
-ELSE (GDBM_FOUND)
-   IF (GDBM_FIND_REQUIRED)
-      MESSAGE(STATUS "Looked for gdbm libraries named ${GDBMS_NAMES}.")
-      MESSAGE(FATAL_ERROR "Could NOT find gdbm library")
-   ENDIF (GDBM_FIND_REQUIRED)
-ENDIF (GDBM_FOUND)
-
-MARK_AS_ADVANCED(
-  GDBM_LIBRARY
-  GDBM_INCLUDE_DIR
-)

cmake/Modules/FindIberty.cmake

@@ -2,13 +2,13 @@
 # This module finds libiberty.
 #
 # It sets the following variables:
-#  IBERTY_LIBRARY     - The JSON-C library to link against.
+#  IBERTY_LIBRARY     - The library to link against.
 
 FIND_LIBRARY( IBERTY_LIBRARY NAMES iberty )
 
 IF (IBERTY_LIBRARY)
 
-	# show which JSON-C was found only if not quiet
+	# show which library was found only if not quiet
 	MESSAGE( STATUS "Found libiberty: ${IBERTY_LIBRARY}")
 	
 	SET(IBERTY_FOUND TRUE)

cmake/Modules/FindJSONC.cmake

@@ -1,30 +0,0 @@
-# - Find JSON-C
-# This module finds an installed JSON-C package.
-#
-# It sets the following variables:
-#  JSONC_FOUND       - Set to false, or undefined, if JSON-C isn't found.
-#  JSONC_INCLUDE_DIR - The JSON-C include directory.
-#  JSONC_LIBRARY     - The JSON-C library to link against.
-
-FIND_PATH(JSONC_INCLUDE_DIR json/json.h)
-FIND_LIBRARY(JSONC_LIBRARY NAMES json)
-
-IF (JSONC_INCLUDE_DIR AND JSONC_LIBRARY)
-   SET(JSONC_FOUND TRUE)
-ENDIF (JSONC_INCLUDE_DIR AND JSONC_LIBRARY)
-
-IF (JSONC_FOUND)
-
-   # show which JSON-C was found only if not quiet
-   IF (NOT JSONC_FIND_QUIETLY)
-      MESSAGE(STATUS "Found JSON-C: ${JSONC_LIBRARY}")
-   ENDIF (NOT JSONC_FIND_QUIETLY)
-
-ELSE (JSONC_FOUND)
-
-   # fatal error if JSON-C is required but not found
-   IF (JSONC_FIND_REQUIRED)
-      MESSAGE(FATAL_ERROR "Could not find JSON-C")
-   ENDIF (JSONC_FIND_REQUIRED)
-
-ENDIF (JSONC_FOUND)

completion.zsh

@@ -0,0 +1,63 @@
+#compdef grive
+# ------------------------------------------------------------------------------
+# Copyright (c) 2015 Github zsh-users - http://github.com/zsh-users
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of the zsh-users nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL ZSH-USERS BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# ------------------------------------------------------------------------------
+# Description
+# -----------
+#
+#  Completion script for Grive (https://github.com/vitalif/grive2)
+#
+# ------------------------------------------------------------------------------
+# Authors
+# -------
+#
+#  * Doron Behar <https://github.com/doronbehar>
+#
+# ------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# ------------------------------------------------------------------------------
+
+local curcontext="$curcontext" state line ret=1
+typeset -A opt_args
+
+_arguments -C \
+	'(-h --help)'{-h,--help}'[Produce help message]' \
+	'(-v --version)'{-v,--version}'[Display Grive version]' \
+	'(-a --auth)'{-a,--auth}'[Request authorization token]' \
+	'(-p --path)'{-p,--path}'[Root directory to sync]' \
+	'(-s --dir)'{-s,--dir}'[Single subdirectory to sync (remembered for next runs)]' \
+	'(-V --verbose)'{-V,--verbose}'[Verbose mode. Enable more messages than normal.]' \
+	'(--log-http)--log-http[Log all HTTP responses in this file for debugging.]' \
+	'(--new-rev)--new-rev[Create,new revisions in server for updated files.]' \
+	'(-d --debug)'{-d,--debug}'[Enable debug level messages. Implies -v.]' \
+	'(-l --log)'{-l,--log}'[Set log output filename.]' \
+	'(-f --force)'{-f,--force}'[Force grive to always download a file from Google Drive instead of uploading it.]' \
+	'(--dry-run)--dry-run[Only,detect which files need to be uploaded/downloaded,without actually performing them.]' \
+	'(--ignore)--ignore[Perl,RegExp to ignore files (matched against relative paths, remembered for next runs) ]' \
+	'*: :_files' && ret=0
+
+return ret

debian/changelog

@@ -0,0 +1,99 @@
+grive2 (0.5.3) unstable; urgency=medium
+
+  * Implement Google OAuth loopback IP redirect flow
+  * Various small fixes
+
+ -- Vitaliy Filippov <vitalif@yourcmc.ru>  Wed, 09 Nov 2022 12:42:28 +0300
+
+grive2 (0.5.2+git20210315) unstable; urgency=medium
+
+  * Newer dev version
+  * Add systemd unit files and helper script for automatic syncs
+  * Add possibility to change client id and secret and save it between runs
+
+ -- Vitaliy Filippov <vitalif@yourcmc.ru>  Wed, 31 Jul 2016 22:04:53 +0300
+
+grive2 (0.5+git20160114) unstable; urgency=medium
+
+  * Newer release, with support for faster sync and rename detection
+
+ -- Vitaliy Filippov <vitalif@yourcmc.ru>  Sun, 03 Jan 2016 12:51:55 +0300
+
+grive2 (0.4.1+git20151011) unstable; urgency=medium
+
+  * Add Debian packaging scripts to the official repository
+
+ -- Vitaliy Filippov <vitalif@yourcmc.ru>  Sun, 11 Oct 2015 15:03:55 +0300
+
+grive2 (0.4.1-1+git20151008~webupd8~wily0) wily; urgency=medium
+ 
+  * new git pull
+  * For Precise and Trusty only: use libgcrypt11 instead of libgcrypt20 
+    because libcurl4-openssl-dev depends on it (and for Precise, 
+    there's no libgcrypt20)
+  
+ -- Alin Andrei <webupd8@gmail.com>  Thu, 08 Oct 2015 11:50:55 +0200
+
+grive2 (0.4.1-1+git20151007~webupd8~wily0) wily; urgency=medium
+ 
+  * new git pull
+  * For Precise and Trusty only: use libgcrypt11 instead of libgcrypt20 
+    because libcurl4-openssl-dev depends on it (and for Precise, 
+    there's no libgcrypt20)
+  
+ -- Alin Andrei <webupd8@gmail.com>  Wed, 07 Oct 2015 13:46:29 +0200
+
+grive2 (0.4.1-1+git20151001~webupd8~precise0) precise; urgency=medium
+ 
+  * new git pull
+  * For Precise and Trusty only: use libgcrypt11 instead of libgcrypt20 
+    because libcurl4-openssl-dev depends on it (and for Precise, 
+    there's no libgcrypt20)
+  
+ -- Alin Andrei <webupd8@gmail.com>  Thu, 01 Oct 2015 12:54:44 +0200
+
+grive2 (0.4.0-1+git20150928~webupd8~wily1) wily; urgency=medium
+ 
+  * new git pull
+  * build-depend on libboost-regex-dev
+  * For Precise and Trusty only: use libgcrypt11 instead of libgcrypt20 
+    because libcurl4-openssl-dev depends on it (and for Precise, 
+    there's no libgcrypt20)
+  
+ -- Alin Andrei <webupd8@gmail.com>  Mon, 28 Sep 2015 11:36:19 +0200
+
+grive2 (0.4.0-1+git20150810~webupd8~wily0) wily; urgency=medium
+ 
+  * upload for Wily
+  * For Precise and Trusty only: use libgcrypt11 instead of libgcrypt20 
+    because libcurl4-openssl-dev depends on it (and for Precise, 
+    there's no libgcrypt20)
+  
+ -- Alin Andrei <webupd8@gmail.com>  Sat, 04 Jul 2015 12:26:14 +0200
+
+grive2 (0.4.0-1+git20150810~webupd8~vivid0) vivid; urgency=medium
+ 
+  * new git pull
+  * For Precise and Trusty only: use libgcrypt11 instead of libgcrypt20 
+    because libcurl4-openssl-dev depends on it (and for Precise, 
+    there's no libgcrypt20)
+  
+ -- Alin Andrei <webupd8@gmail.com>  Sat, 04 Jul 2015 12:26:14 +0200
+
+grive2 (0.4.0-1+git20150629~webupd8~vivid0) vivid; urgency=medium
+ 
+  * new git pull
+  * For Precise and Trusty only: use libgcrypt11 instead of libgcrypt20 
+    because libcurl4-openssl-dev depends on it (and for Precise, 
+    there's no libgcrypt20)
+  
+ -- Alin Andrei <webupd8@gmail.com>  Mon, 29 Jun 2015 12:27:21 +0200
+
+grive2 (0.4.0-1~webupd8~precise3) precise; urgency=medium
+ 
+  * Initial packaging (based on grive)
+  * For Precise and Trusty only: use libgcrypt11 instead of libgcrypt20 
+    because libcurl4-openssl-dev depends on it (and for Precise, 
+    there's no libgcrypt20)
+  
+ -- Alin Andrei <webupd8@gmail.com>  Mon, 25 May 2015 15:31:24 +0200

debian/compat

@@ -0,0 +1 @@
+11

debian/control

@@ -0,0 +1,25 @@
+Source: grive2
+Section: net
+Priority: optional
+Maintainer: Vitaliy Filippov <vitalif@mail.ru>
+Build-Depends: debhelper, cmake, pkg-config, zlib1g-dev, libcurl4-openssl-dev | libcurl4-gnutls-dev, libboost-filesystem-dev, libboost-program-options-dev, libboost-test-dev, libboost-regex-dev, libexpat1-dev, libgcrypt-dev, libyajl-dev
+Standards-Version: 3.9.6
+Homepage: https://yourcmc.ru/wiki/Grive2
+
+Package: grive
+Architecture: any
+Depends: ${shlibs:Depends}, ${misc:Depends}
+Description: Grive2: an open source Linux client for Google Drive
+ .
+ This is the up-to-date fork of the original "Grive" (https://github.com/Grive/grive) 
+ Google Drive client with the support for the new Drive REST API and partial 
+ sync.
+ .
+ For the first time running grive, you should use the "-a" argument to grant
+ permission to grive to access to your Google Drive. An URL should be printed.
+ Go to the page. You will need to login to your google account if you haven't
+ done so. After granting the permission to grive, the browser will show you
+ an authenication code. Copy-and-paste that to the standard input of grive.
+ If everything works fine, grive will create .grive and .grive_state inside the
+ synchronized directory. It will also start downloading files from your Google
+ Drive to that directory.

debian/copyright

@@ -0,0 +1,20 @@
+Grive2
+
+    https://github.com/vitalif/grive2
+
+Current developers:
+
+    Vitaliy Filippov <vitalif@mail.ru>
+
+Previous developers:
+
+    Nestal Wan (me@nestal.net) 16.05.2012 — 03.05.2013
+    Matchman Green (match065@gmail.com) 26.04.2012 — 20.06.2012
+
+Contributors:
+
+    See full list here https://yourcmc.ru/wiki/Grive2#Full_list_of_contributors
+
+License:
+
+    GPL 2.0+

debian/rules

@@ -0,0 +1,7 @@
+#!/usr/bin/make -f
+
+override_dh_auto_configure:
+	dh_auto_configure -- -DHAVE_SYSTEMD=1
+
+%:
+	dh $@ --buildsystem=cmake --parallel --builddirectory=build

debian/source/format

@@ -0,0 +1 @@
+3.0 (native)

grive/CMakeLists.txt

@@ -5,12 +5,11 @@ find_package(Boost COMPONENTS program_options REQUIRED)
 include_directories(
 	${grive_SOURCE_DIR}/../libgrive/src
 	${OPT_INCS}
+	${Boost_INCLUDE_DIRS}
 )
 
-add_definitions( -DVERSION="${GRIVE_VERSION}" )
-
 file (GLOB GRIVE_EXE_SRC
-    ${grive_SOURCE_DIR}/src/*.cc
+	${grive_SOURCE_DIR}/src/*.cc
 )
 
 add_executable( grive_executable
@@ -18,13 +17,28 @@ add_executable( grive_executable
 )
 
 target_link_libraries( grive_executable
-	${Boost_LIBRARIES}
-    grive
+	grive
+)
+
+set(DEFAULT_APP_ID "615557989097-i93d4d1ojpen0m0dso18ldr6orjkidgf.apps.googleusercontent.com")
+set(DEFAULT_APP_SECRET "xiM8Apu_WuRRdheNelJcNtOD")
+set(APP_ID ${DEFAULT_APP_ID} CACHE STRING "Application Id")
+set(APP_SECRET ${DEFAULT_APP_SECRET} CACHE STRING "Application Secret")
+
+target_compile_definitions ( grive_executable
+	PRIVATE
+		-DAPP_ID="${APP_ID}"
+		-DAPP_SECRET="${APP_SECRET}"
 )
 
 set_target_properties( grive_executable
-    PROPERTIES OUTPUT_NAME grive
+	PROPERTIES OUTPUT_NAME grive
 )
 
 install(TARGETS grive_executable RUNTIME DESTINATION bin)
-install(FILES doc/grive.1 DESTINATION share/man/man1 )
+
+if ( ${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD" OR ${CMAKE_SYSTEM_NAME} MATCHES "OpenBSD" )
+    install(FILES doc/grive.1 DESTINATION man/man1 )
+else ( ${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD" OR ${CMAKE_SYSTEM_NAME} MATCHES "OpenBSD" )
+    install(FILES doc/grive.1 DESTINATION share/man/man1 )
+endif( ${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD" OR ${CMAKE_SYSTEM_NAME} MATCHES "OpenBSD" )

grive/doc/grive.1

@@ -2,7 +2,7 @@
 .\" First parameter, NAME, should be all caps
 .\" Second parameter, SECTION, should be 1-8, maybe w/ subsection
 .\" other parameters are allowed: see man(7), man(1)
-.TH "GRIVE" 1 "June 19, 2012"
+.TH "GRIVE" 1 "January 3, 2016"
 .SH NAME
 grive \- Google Drive client for GNU/Linux
 
@@ -26,33 +26,102 @@ Requests authorization token from Google
 Enable debug level messages. Implies \-V
 .TP
 \fB\-\-dry-run\fR
-Only detects which files are needed for download or upload without doing it
+Only detect which files need to be uploaded/downloaded, without actually performing changes
 .TP
 \fB\-f, \-\-force\fR
 Forces
 .I grive
 to always download a file from Google Drive instead uploading it
 .TP
+\fB\-u, \-\-upload\-only\fR
+Forces
+.I grive
+to not download anything from Google Drive and only upload local changes to server instead
+.TP
+\fB\-n, \-\-no\-remote\-new\fR
+Forces
+.I grive
+to download only files that are changed in Google Drive and already exist locally
+.TP
 \fB\-h\fR, \fB\-\-help\fR
 Produces help message
 .TP
-\fB\-l\fR filename, \fB\-\-log\fR filename
-Set log output to
-.I filename
+\fB\-\-ignore\fR <perl_regexp>
+Ignore files with relative paths matching this Perl Regular Expression.
+.TP
+\fB\-l\fR <filename>, \fB\-\-log\fR <filename>
+Write log output to
+.I <filename>
+.TP
+\fB\-\-log\-http\fR <filename_prefix>
+Log all HTTP responses in files named
+.I <filename_prefix>YYYY-MM-DD.HHMMSS.txt
+for debugging
+.TP
+\fB\-\-new\-rev\fR
+Create new revisions in server for updated files
+.TP
+\fB\-p\fR <wc_path>, \fB\-\-path\fR <wc_path>
+Use
+.I <wc_path>
+as the working copy root directory
+.TP
+\fB\-s\fR <subdir>, \fB\-\-dir\fR <subdir>
+Sync a single
+.I <subdir>
+subdirectory. Internally converted to an ignore regexp.
 .TP
 \fB\-v\fR, \fB\-\-version\fR
 Displays program version
 .TP
+\fB\-P\fR, \fB\-\-progress-bar\fR
+Print ASCII progress bar for each downloaded/uploaded file.
+.TP
 \fB\-V\fR, \fB\-\-verbose\fR
 Verbose mode. Enables more messages than usual.
 
-.SH AUTHOR
+.SH .griveignore
 .PP
-The software was developed by Nestal Wan.
+You may create .griveignore in your Grive root and use it to setup
+exclusion/inclusion rules.
 .PP
+Rules are similar to Git's .gitignore, but may differ slightly due to the different
+implementation.
+.IP \[bu]
+lines that start with # are comments
+.IP \[bu]
+leading and trailing spaces ignored unless escaped with \\
+.IP \[bu]
+non-empty lines without ! in front are treated as "exclude" patterns
+.IP \[bu]
+non-empty lines with ! in front are treated as "include" patterns
+and have a priority over all "exclude" ones
+.IP \[bu]
+patterns are matched against the filenames relative to the grive root
+.IP \[bu]
+a/**/b matches any number of subpaths between a and b, including 0
+.IP \[bu]
+**/a matches `a` inside any directory
+.IP \[bu]
+b/** matches everything inside `b`, but not b itself
+.IP \[bu]
+* matches any number of any characters except /
+.IP \[bu]
+? matches any character except /
+.IP \[bu]
+\[char46]griveignore itself isn't ignored by default, but you can include it in itself to ignore
+
+.SH AUTHORS
+.PP
+Current maintainer is Vitaliy Filippov.
+.PP
+Original author was Nestal Wan.
 This manpage was written by José Luis Segura Lucas (josel.segura@gmx.es)
+.PP
+The full list of contributors may be found here
+.I http://yourcmc.ru/wiki/Grive2#Full_list_of_contributors
 
 .SH REPORT BUGS
 .PP
-.I https://github.com/Grive/grive
+.I https://github.com/vitalif/grive2/issues
 .I https://groups.google.com/forum/?fromgroups#!forum/grive-devel

grive/src/main.cc

@@ -17,12 +17,16 @@
 	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */
 
-#include "Config.hh"
+#include "util/Config.hh"
+#include "util/ProgressBar.hh"
 
-#include "drive/Drive.hh"
+#include "base/Drive.hh"
+#include "drive2/Syncer2.hh"
 
+#include "http/CurlAgent.hh"
+#include "protocol/AuthAgent.hh"
 #include "protocol/OAuth2.hh"
-#include "protocol/Json.hh"
+#include "json/Val.hh"
 
 #include "bfd/Backtrace.hh"
 #include "util/Exception.hh"
@@ -42,16 +46,17 @@
 #include <iostream>
 #include <unistd.h>
 
-const std::string client_id		= "22314510474.apps.googleusercontent.com" ;
-const std::string client_secret	= "bl4ufi89h-9MkFlypcI7R785" ;
+const std::string default_id            = APP_ID ;
+const std::string default_secret        = APP_SECRET ;
 
 using namespace gr ;
+namespace po = boost::program_options;
 
 // libgcrypt insist this to be done in application, not library
 void InitGCrypt()
 {
 	if ( !gcry_check_version(GCRYPT_VERSION) )
-		throw Exception() << expt::ErrMsg( "libgcrypt version mismatch" ) ;
+		throw std::runtime_error( "libgcrypt version mismatch" ) ;
 
 	// disable secure memory
 	gcry_control(GCRYCTL_DISABLE_SECMEM, 0);
@@ -60,67 +65,15 @@ void InitGCrypt()
 	gcry_control(GCRYCTL_INITIALIZATION_FINISHED, 0);
 }
 
-int Main( int argc, char **argv )
+void InitLog( const po::variables_map& vm )
 {
-	InitGCrypt() ;
-	
-	Config config ;
-	
-	std::auto_ptr<log::CompositeLog> comp_log(new log::CompositeLog) ;
-	LogBase* console_log = comp_log->Add( std::auto_ptr<LogBase>( new log::DefaultLog ) ) ;
-	
-	Json options ;
-	
-	namespace po = boost::program_options;
-	
-	// construct the program options
-	po::options_description desc( "Grive options" );
-	desc.add_options()
-		( "help,h",		"Produce help message" )
-		( "version,v",	"Display Grive version" )
-		( "auth,a",		"Request authorization token" )
-		( "verbose,V",	"Verbose mode. Enable more messages than normal.")
-		( "debug,d",	"Enable debug level messages. Implies -v.")
-		( "log,l",		po::value<std::string>(), "Set log output filename." )
-		( "force,f",	"Force grive to always download a file from Google Drive "
-						"instead of uploading it." )
-		( "dry-run",	"Only detect which files need to be uploaded/downloaded, "
-						"without actually performing them." )
-	;
-	
-	po::variables_map vm;
-	po::store(po::parse_command_line( argc, argv, desc), vm );
-	po::notify(vm);
-	
-	if ( vm.count("help") )
-	{
-		std::cout << desc << std::endl ;
-		return 0 ;
-	}
-	if ( vm.count( "auth" ) )
-	{
-		std::cout
-			<< "-----------------------\n"
-			<< "Please go to this URL and get an authentication code:\n\n"
-			<< OAuth2::MakeAuthURL( client_id )
-			<< std::endl ;
-		
-		std::cout
-			<< "\n-----------------------\n"
-			<< "Please input the authentication code here: " << std::endl ;
-		std::string code ;
-		std::cin >> code ;
-		
-		OAuth2 token( client_id, client_secret ) ;
-		token.Auth( code ) ;
-		
-		// save to config
-		config.Get().Add( "refresh_token", Json( token.RefreshToken() ) ) ;
-		config.Save() ;
-	}
+	std::unique_ptr<log::CompositeLog> comp_log( new log::CompositeLog ) ;
+	std::unique_ptr<LogBase> def_log( new log::DefaultLog );
+	LogBase* console_log = comp_log->Add( def_log ) ;
+
 	if ( vm.count( "log" ) )
 	{
-		std::auto_ptr<LogBase> file_log(new log::DefaultLog( vm["log"].as<std::string>() )) ;
+		std::unique_ptr<LogBase> file_log( new log::DefaultLog( vm["log"].as<std::string>() ) ) ;
 		file_log->Enable( log::debug ) ;
 		file_log->Enable( log::verbose ) ;
 		file_log->Enable( log::info ) ;
@@ -134,32 +87,138 @@ int Main( int argc, char **argv )
 		
 		comp_log->Add( file_log ) ;
 	}
-	if ( vm.count( "version" ) )
-	{
-		std::cout
-			<< "grive version " << VERSION << ' ' << __DATE__ << ' ' << __TIME__ << std::endl ;
-		return 0 ;
-	}
+	
 	if ( vm.count( "verbose" ) )
 	{
 		console_log->Enable( log::verbose ) ;
 	}
+	
 	if ( vm.count( "debug" ) )
 	{
 		console_log->Enable( log::verbose ) ;
 		console_log->Enable( log::debug ) ;
 	}
-	if ( vm.count( "force" ) )
-	{
-		options.Add( "force", Json(true) ) ;
-	}
+	LogBase::Inst( comp_log.release() ) ;
+}
+
+int Main( int argc, char **argv )
+{
+	InitGCrypt() ;
 	
-	LogBase::Inst( std::auto_ptr<LogBase>(comp_log.release()) ) ;
+	// construct the program options
+	po::options_description desc( "Grive options" );
+	desc.add_options()
+		( "help,h",		"Produce help message" )
+		( "version,v",	"Display Grive version" )
+		( "auth,a",		"Request authorization token" )
+		( "id,i",		po::value<std::string>(), "Authentication ID")
+		( "secret,e",	po::value<std::string>(), "Authentication secret")
+		( "print-url",	"Only print url for request")
+		( "path,p",		po::value<std::string>(), "Path to working copy root")
+		( "dir,s",		po::value<std::string>(), "Single subdirectory to sync")
+		( "verbose,V",	"Verbose mode. Enable more messages than normal.")
+		( "log-http",	po::value<std::string>(), "Log all HTTP responses in this file for debugging.")
+		( "new-rev",	"Create new revisions in server for updated files.")
+		( "debug,d",	"Enable debug level messages. Implies -v.")
+		( "log,l",		po::value<std::string>(), "Set log output filename." )
+		( "force,f",	"Force grive to always download a file from Google Drive "
+						"instead of uploading it." )
+		( "upload-only,u", "Do not download anything from Google Drive, only upload local changes" )
+		( "no-remote-new,n", "Download only files that are changed in Google Drive and already exist locally" )
+		( "dry-run",	"Only detect which files need to be uploaded/downloaded, "
+						"without actually performing them." )
+		( "upload-speed,U", po::value<unsigned>(), "Limit upload speed in kbytes per second" )
+		( "download-speed,D", po::value<unsigned>(), "Limit download speed in kbytes per second" )
+		( "progress-bar,P", "Enable progress bar for upload/download of files")
+	;
 	
-	std::string refresh_token ;
+	po::variables_map vm;
 	try
 	{
-		refresh_token = config.Get()["refresh_token"].Str() ;
+		po::store( po::parse_command_line( argc, argv, desc ), vm );
+	}
+	catch( po::error &e )
+	{
+		std::cerr << "Options are incorrect. Use -h for help\n";
+		return -1;
+	}
+	po::notify( vm );
+	
+	// simple commands that doesn't require log or config
+	if ( vm.count("help") )
+	{
+		std::cout << desc << std::endl ;
+		return 0 ;
+	}
+	else if ( vm.count( "version" ) )
+	{
+		std::cout
+			<< "grive version " << VERSION << ' ' << __DATE__ << ' ' << __TIME__ << std::endl ;
+		return 0 ;
+	}
+
+	// initialize logging
+	InitLog( vm ) ;
+	
+	Config config( vm ) ;
+	
+	Log( "config file name %1%", config.Filename(), log::verbose );
+
+	std::unique_ptr<http::Agent> http( new http::CurlAgent );
+	if ( vm.count( "log-http" ) )
+		http->SetLog( new http::ResponseLog( vm["log-http"].as<std::string>(), ".txt" ) );
+
+	std::unique_ptr<ProgressBar> pb;
+	if ( vm.count( "progress-bar" ) )
+	{
+		pb.reset( new ProgressBar() );
+		http->SetProgressReporter( pb.get() );
+	}
+
+	if ( vm.count( "auth" ) )
+	{
+		std::string id = vm.count( "id" ) > 0
+                        ? vm["id"].as<std::string>()
+                        : default_id ;
+		std::string secret = vm.count( "secret" ) > 0
+                        ? vm["secret"].as<std::string>()
+                        : default_secret ;
+
+		OAuth2 token( http.get(), id, secret ) ;
+
+		if ( vm.count("print-url") )
+		{
+			std::cout << token.MakeAuthURL() << std::endl ;
+			return 0 ;
+		}
+
+		std::cout
+			<< "-----------------------\n"
+			<< "Please open this URL in your browser to authenticate Grive2:\n\n"
+			<< token.MakeAuthURL()
+			<< std::endl ;
+
+		if ( !token.GetCode() )
+		{
+			std::cout << "Authentication failed\n";
+			return -1;
+		}
+
+		// save to config
+		config.Set( "id", Val( id ) ) ;
+		config.Set( "secret", Val( secret ) ) ;
+		config.Set( "refresh_token", Val( token.RefreshToken() ) ) ;
+		config.Save() ;
+	}
+
+	std::string refresh_token ;
+	std::string id ;
+	std::string secret ;
+	try
+	{
+		refresh_token = config.Get("refresh_token").Str() ;
+		id = config.Get("id").Str() ;
+		secret = config.Get("secret").Str() ;
 	}
 	catch ( Exception& e )
 	{
@@ -170,19 +229,33 @@ int Main( int argc, char **argv )
 		
 		return -1;
 	}
-	
-	OAuth2 token( refresh_token, client_id, client_secret ) ;
-	Drive drive( token, options ) ;
+
+	OAuth2 token( http.get(), refresh_token, id, secret ) ;
+	AuthAgent agent( token, http.get() ) ;
+	v2::Syncer2 syncer( &agent );
+
+	if ( vm.count( "upload-speed" ) > 0 )
+		agent.SetUploadSpeed( vm["upload-speed"].as<unsigned>() * 1000 );
+	if ( vm.count( "download-speed" ) > 0 )
+		agent.SetDownloadSpeed( vm["download-speed"].as<unsigned>() * 1000 );
+
+	Drive drive( &syncer, config.GetAll() ) ;
 	drive.DetectChanges() ;
 
 	if ( vm.count( "dry-run" ) == 0 )
 	{
+		// The progress bar should just be enabled when actual file transfers take place
+		if ( pb )
+			pb->setShowProgressBar( true ) ;
 		drive.Update() ;
+		if ( pb )
+			pb->setShowProgressBar( false ) ;
+
 		drive.SaveState() ;
 	}
 	else
 		drive.DryRun() ;
-	
+		
 	config.Save() ;
 	Log( "Finished!", log::info ) ;
 	return 0 ;

libgrive/CMakeLists.txt

@@ -3,47 +3,45 @@ project(libgrive)
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/")
 
 find_package(LibGcrypt REQUIRED)
-find_package(JSONC REQUIRED)
 find_package(CURL REQUIRED)
-find_package(EXPAT REQUIRED)
-find_package(Boost 1.40.0 COMPONENTS filesystem system REQUIRED)
+find_package(Backtrace)
+find_package(Boost 1.40.0 COMPONENTS program_options filesystem unit_test_framework regex system REQUIRED)
 find_package(BFD)
 find_package(CppUnit)
 find_package(Iberty)
-find_package(ZLIB)
+
+find_package(PkgConfig)
+pkg_check_modules(YAJL REQUIRED yajl)
+
+add_definitions(-Wall)
 
 # additional headers if build unit tests
 IF ( CPPUNIT_FOUND )
 	set( OPT_INCS		${CPPUNIT_INCLUDE_DIR} )
 ENDIF ( CPPUNIT_FOUND )
 
-# build bfd classes if libbfd is found
-if ( BFD_FOUND )
-	set( OPT_LIBS	${DL_LIBRARY} ${BFD_LIBRARY} )
+# build bfd classes if libbfd and the backtrace library is found
+if ( BFD_FOUND AND Backtrace_FOUND )
+	set( OPT_LIBS	${DL_LIBRARY} ${BFD_LIBRARY} ${Backtrace_LIBRARY} )
 	file( GLOB OPT_SRC
 		src/bfd/*.cc
 	)
 	add_definitions( -DHAVE_BFD )
 	
-endif ( BFD_FOUND )
+endif ( BFD_FOUND AND Backtrace_FOUND )
 
 if ( IBERTY_FOUND )
 	set( OPT_LIBS	${OPT_LIBS}	${IBERTY_LIBRARY} )
+else ( IBERTY_FOUND )
+  set( IBERTY_LIBRARY "" )
 endif ( IBERTY_FOUND )
 
-if ( ZLIB_FOUND )
-	set( OPT_LIBS	${OPT_LIBS}	${ZLIB_LIBRARIES} )
-endif ( ZLIB_FOUND )
-
 include_directories(
 	${libgrive_SOURCE_DIR}/src
 	${libgrive_SOURCE_DIR}/test
-	${GDBM_INCLUDE_DIR}
+	${Boost_INCLUDE_DIRS}
 	${OPT_INCS}
-)
-
-file(GLOB DRIVE_HEADERS
-    ${libgrive_SOURCE_DIR}/src/drive/*.hh
+	${YAJL_INCLUDE_DIRS}
 )
 
 file (GLOB PROTOCOL_HEADERS
@@ -59,16 +57,16 @@ file (GLOB XML_HEADERS
 )
 
 file (GLOB LIBGRIVE_SRC
-	src/drive/*.cc
+	src/base/*.cc
+	src/drive2/*.cc
 	src/http/*.cc
 	src/protocol/*.cc
+	src/json/*.cc
 	src/util/*.cc
 	src/util/log/*.cc
-	src/xml/*.cc
 )
 
 add_definitions(
-	-DVERSION="${GRIVE_VERSION}"
 	-DTEST_DATA="${libgrive_SOURCE_DIR}/test/data/"
 	-DSRC_DIR="${libgrive_SOURCE_DIR}/src"
 )
@@ -77,13 +75,14 @@ add_definitions(
 add_library( grive STATIC ${LIBGRIVE_SRC} ${OPT_SRC} )
 
 target_link_libraries( grive
+	${YAJL_LIBRARIES}
 	${CURL_LIBRARIES}
-	${JSONC_LIBRARY}
 	${LIBGCRYPT_LIBRARIES}
-	${GDBM_LIBRARIES}
-	${Boost_LIBRARIES}
+	${Boost_FILESYSTEM_LIBRARY}
+	${Boost_PROGRAM_OPTIONS_LIBRARY}
+	${Boost_REGEX_LIBRARY}
+	${Boost_SYSTEM_LIBRARY}
 	${IBERTY_LIBRARY}
-	${EXPAT_LIBRARY}
 	${OPT_LIBS}
 )
 
@@ -112,9 +111,8 @@ IF ( CPPUNIT_FOUND )
     
     # list of test source files here
 	file(GLOB TEST_SRC
-		test/drive/*.cc
+		test/base/*.cc
 		test/util/*.cc
-		test/xml/*.cc
 	)
 
 	add_executable( unittest
@@ -128,3 +126,24 @@ IF ( CPPUNIT_FOUND )
 	)
         
 ENDIF ( CPPUNIT_FOUND )
+
+file(GLOB BTEST_SRC
+	test/btest/*.cc
+)
+
+add_executable( btest ${BTEST_SRC} )
+
+target_link_libraries( btest
+	grive
+	${Boost_UNIT_TEST_FRAMEWORK_LIBRARY}
+)
+
+if ( ${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD" )
+  set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-c++11-narrowing" )
+endif ( ${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD" )
+
+if ( WIN32 )
+else ( WIN32 )
+	set_target_properties( btest
+		PROPERTIES COMPILE_FLAGS	 -DBOOST_TEST_DYN_LINK )
+endif (WIN32)

libgrive/src/base/Drive.cc

@@ -0,0 +1,130 @@
+/*
+	grive: an GPL program to sync a local directory with Google Drive
+	Copyright (C) 2012  Wan Wai Ho
+
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License
+	as published by the Free Software Foundation version 2
+	of the License.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with this program; if not, write to the Free Software
+	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+
+#include "Drive.hh"
+
+#include "Entry.hh"
+#include "Feed.hh"
+#include "Syncer.hh"
+
+#include "http/Agent.hh"
+#include "util/Destroy.hh"
+#include "util/log/Log.hh"
+
+#include <boost/bind.hpp>
+
+// standard C++ library
+#include <algorithm>
+#include <cassert>
+#include <cstdlib>
+#include <fstream>
+#include <map>
+
+// for debugging only
+#include <iostream>
+
+namespace gr {
+
+Drive::Drive( Syncer *syncer, const Val& options ) :
+	m_syncer	( syncer ),
+	m_root		( options["path"].Str() ),
+	m_state		( m_root, options ),
+	m_options	( options )
+{
+	assert( m_syncer ) ;
+}
+
+void Drive::FromRemote( const Entry& entry )
+{
+	m_state.FromRemote( entry ) ;
+}
+
+void Drive::FromChange( const Entry& entry )
+{
+	if ( entry.IsRemoved() )
+		Log( "file \"%1%\" represents a deletion, ignored", entry.Title(), log::verbose ) ;
+	
+	// folders go directly
+	else
+		m_state.FromRemote( entry ) ;
+}
+
+void Drive::SaveState()
+{
+	m_state.Write() ;
+}
+
+void Drive::DetectChanges()
+{
+	Log( "Reading local directories", log::info ) ;
+	m_state.FromLocal( m_root ) ;
+
+	Log( "Reading remote server file list", log::info ) ;
+	std::unique_ptr<Feed> feed = m_syncer->GetAll() ;
+
+	while ( feed->GetNext( m_syncer->Agent() ) )
+	{
+		std::for_each(
+			feed->begin(), feed->end(),
+			boost::bind( &Drive::FromRemote, this, _1 ) ) ;
+	}
+	m_state.ResolveEntry() ;
+}
+
+// pull the changes feed
+// FIXME: unused until Grive will use the feed-based sync instead of reading full tree
+void Drive::ReadChanges()
+{
+	long prev_stamp = m_state.ChangeStamp() ;
+	if ( prev_stamp != -1 )
+	{
+		Trace( "previous change stamp is %1%", prev_stamp ) ;
+		Log( "Detecting changes from last sync", log::info ) ;
+		std::unique_ptr<Feed> feed = m_syncer->GetChanges( prev_stamp+1 ) ;
+		while ( feed->GetNext( m_syncer->Agent() ) )
+		{
+			std::for_each(
+				feed->begin(), feed->end(),
+				boost::bind( &Drive::FromChange, this, _1 ) ) ;
+		}
+	}
+}
+
+void Drive::Update()
+{
+	Log( "Synchronizing files", log::info ) ;
+	m_state.Sync( m_syncer, m_options ) ;
+	
+	UpdateChangeStamp( ) ;
+}
+
+void Drive::DryRun()
+{
+	Log( "Synchronizing files (dry-run)", log::info ) ;
+	m_state.Sync( NULL, m_options ) ;
+}
+
+void Drive::UpdateChangeStamp( )
+{
+	// FIXME: we should go through the changes to see if it was really Grive to made that change
+	// maybe by recording the updated timestamp and compare it?
+	m_state.ChangeStamp( m_syncer->GetChangeStamp( m_state.ChangeStamp()+1 ) );
+}
+
+} // end of namespace gr

libgrive/src/base/Drive.hh

@@ -19,9 +19,9 @@
 
 #pragma once
 
-#include "State.hh"
+#include "base/State.hh"
 
-#include "http/Header.hh"
+#include "json/Val.hh"
 #include "util/Exception.hh"
 
 #include <string>
@@ -29,19 +29,16 @@
 
 namespace gr {
 
-namespace http
-{
-	class Agent ;
-}
+class Syncer ;
 
 class Entry ;
-class OAuth2 ;
-class Json ;
+
+class State ;
 
 class Drive
 {
 public :
-	Drive( OAuth2& auth, const Json& options ) ;
+	Drive( Syncer *syncer, const Val& options ) ;
 
 	void DetectChanges() ;
 	void Update() ;
@@ -51,18 +48,16 @@ public :
 	struct Error : virtual Exception {} ;
 	
 private :
-	void SyncFolders( http::Agent *http ) ;
-    void file();
+	void ReadChanges() ;
 	void FromRemote( const Entry& entry ) ;
 	void FromChange( const Entry& entry ) ;
-	void UpdateChangeStamp( http::Agent *http ) ;
+	void UpdateChangeStamp( ) ;
 	
 private :
-	OAuth2&			m_auth ;
-	http::Header	m_http_hdr ;
-
-	std::string		m_resume_link ;
+	Syncer			*m_syncer ;
+	fs::path		m_root ;
 	State			m_state ;
+	Val				m_options ;
 } ;
 
-} // end of namespace
+} // end of namespace gr

libgrive/src/base/Entry.cc

@@ -0,0 +1,129 @@
+/*
+	grive: an GPL program to sync a local directory with Google Drive
+	Copyright (C) 2012  Wan Wai Ho
+
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License
+	as published by the Free Software Foundation version 2
+	of the License.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with this program; if not, write to the Free Software
+	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+
+#include "Entry.hh"
+
+#include "util/Crypt.hh"
+#include "util/log/Log.hh"
+#include "util/OS.hh"
+#include "xml/Node.hh"
+#include "xml/NodeSet.hh"
+
+#include <algorithm>
+#include <iterator>
+
+namespace gr {
+
+/// construct an entry for the root folder
+Entry::Entry( ) :
+	m_title			( "." ),
+	m_is_dir		( true ),
+	m_resource_id	( "folder:root" ),
+	m_change_stamp	( -1 ),
+	m_is_removed	( false ),
+	m_size			( 0 )
+{
+}
+
+const std::vector<std::string>& Entry::ParentHrefs() const
+{
+	return m_parent_hrefs ;
+}
+
+std::string Entry::Title() const
+{
+	return m_title ;
+}
+
+std::string Entry::Filename() const
+{
+	return m_filename ;
+}
+
+bool Entry::IsDir() const
+{
+	return m_is_dir ;
+}
+
+std::string Entry::MD5() const
+{
+	return m_md5 ;
+}
+
+u64_t Entry::Size() const
+{
+	return m_size ;
+}
+
+DateTime Entry::MTime() const
+{
+	return m_mtime ;
+}
+
+std::string Entry::SelfHref() const
+{
+	return m_self_href ;
+}
+
+std::string Entry::ParentHref() const
+{
+	return m_parent_hrefs.empty() ? "" : m_parent_hrefs.front() ;
+}
+
+std::string Entry::ResourceID() const
+{
+	return m_resource_id ;
+}
+
+std::string Entry::ETag() const
+{
+	return m_etag ;
+}
+
+std::string Entry::ContentSrc() const
+{
+	return m_content_src ;
+}
+
+bool Entry::IsEditable() const
+{
+	return m_is_editable ;
+}
+
+long Entry::ChangeStamp() const
+{
+	return m_change_stamp ;
+}
+
+bool Entry::IsChange() const
+{
+	return m_change_stamp != -1 ;
+}
+
+bool Entry::IsRemoved() const
+{
+	return m_is_removed ;
+}
+
+std::string Entry::Name() const
+{
+	return !m_filename.empty() ? m_filename : m_title ;
+}
+
+} // end of namespace gr

libgrive/src/base/Entry.hh

@@ -19,6 +19,7 @@
 
 #pragma once
 
+#include "util/Types.hh"
 #include "util/DateTime.hh"
 #include "util/FileSystem.hh"
 
@@ -28,11 +29,6 @@
 
 namespace gr {
 
-namespace xml
-{
-	class Node ;
-}
-
 /*!	\brief	corresponds to an "entry" in the resource feed
 
 	This class is decodes an entry in the resource feed. It will stored the properties like
@@ -43,13 +39,13 @@ class Entry
 {
 public :
 	Entry( ) ;
-	explicit Entry( const xml::Node& n ) ;
 	
 	std::string Title() const ;
 	std::string Filename() const ;
-	std::string Kind() const ;
+	bool IsDir() const ;
 	std::string MD5() const ;
 	DateTime MTime() const ;
+	u64_t Size() const ;
 	
 	std::string Name() const ;
 	
@@ -57,11 +53,9 @@ public :
 	std::string ETag() const ;
 	
 	std::string SelfHref() const ;
-	std::string AltSelf() const ;
 	std::string ParentHref() const ;
 	std::string ContentSrc() const ;
-	std::string EditLink() const ;
-	std::string CreateLink() const ;
+	bool IsEditable() const ;
 	long ChangeStamp() const ;
 	
 	bool IsChange() const ;
@@ -69,30 +63,26 @@ public :
 	
 	const std::vector<std::string>& ParentHrefs() const ;
 	
-	void Swap( Entry& e ) ;
-	
-	void Update( const xml::Node& entry ) ;
-	
-private :
+protected :
 	std::string		m_title ;
 	std::string		m_filename ;
-	std::string		m_kind ;
+	bool			m_is_dir ;
 	std::string		m_md5 ;
 	std::string		m_etag ;
 	std::string		m_resource_id ;
 
 	std::vector<std::string>	m_parent_hrefs ;
-	
+
 	std::string		m_self_href ;
-	std::string		m_alt_self ;
 	std::string		m_content_src ;
-	std::string		m_edit_link ;
-	std::string		m_create_link ;
+
+	bool			m_is_editable ;
 
 	long			m_change_stamp ;
-	
+
 	DateTime		m_mtime ;
 	bool			m_is_removed ;
+	u64_t			m_size ;
 } ;
 
-} // end of namespace
+} // end of namespace gr

libgrive/src/base/Feed.cc

@@ -17,15 +17,31 @@
 	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */
 
-#include "CommonUri.hh"
-#include <boost/format.hpp>
+#include "Feed.hh"
+
+#include "Entry.hh"
+
+#include "http/Agent.hh"
 
 namespace gr {
 
-std::string ChangesFeed( int changestamp )
+Feed::Feed( const std::string &url ):
+	m_next( url )
 {
-	boost::format feed( feed_changes + "?start-index=%1%" ) ;
-	return changestamp > 0 ? (feed%changestamp).str() : feed_changes ;
 }
 
-}
+Feed::~Feed()
+{
+}
+
+Feed::iterator Feed::begin() const
+{
+	return m_entries.begin() ;
+}
+
+Feed::iterator Feed::end() const
+{
+	return m_entries.end() ;
+}
+
+} // end of namespace gr::v1

libgrive/src/base/Feed.hh

@@ -19,28 +19,35 @@
 
 #pragma once
 
-#include "http/Receivable.hh"
-#include "http/StringResponse.hh"
+#include "base/Entry.hh"
 
-namespace gr
+#include <vector>
+
+#include <string>
+
+namespace gr {
+
+namespace http
 {
-	class Json ;
+	class Agent ;
 }
 
-namespace gr { namespace http {
-
-class JsonResponse : public Receivable
+class Feed
 {
 public :
-	JsonResponse() ;
+	typedef std::vector<Entry> Entries;
+	typedef std::vector<Entry>::const_iterator iterator;
 
-	std::size_t OnData( void *data, std::size_t count ) ;
-	void Clear() ;
+public :
+	Feed( const std::string& url );
+	virtual bool GetNext( http::Agent *http ) = 0 ;
+	virtual ~Feed() = 0 ;
+	iterator begin() const ;
+	iterator end() const ;
 
-	Json Response() const ;
-	
-private :
-	StringResponse	m_resp ;
+protected :
+	Entries m_entries ;
+	std::string m_next ;
 } ;
 
-} } // end of namespace
+} // end of namespace gr

libgrive/src/base/Resource.cc

@@ -0,0 +1,801 @@
+/*
+	grive: an GPL program to sync a local directory with Google Drive
+	Copyright (C) 2012  Wan Wai Ho
+
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License
+	as published by the Free Software Foundation version 2
+	of the License.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with this program; if not, write to the Free Software
+	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+
+#include "Resource.hh"
+#include "ResourceTree.hh"
+#include "Entry.hh"
+#include "Syncer.hh"
+
+#include "json/Val.hh"
+#include "util/CArray.hh"
+#include "util/Crypt.hh"
+#include "util/log/Log.hh"
+#include "util/OS.hh"
+#include "util/File.hh"
+#include "http/Error.hh"
+
+#include <boost/exception/all.hpp>
+#include <boost/filesystem.hpp>
+#include <boost/bind.hpp>
+
+#include <errno.h>
+
+#include <cassert>
+
+// for debugging
+#include <iostream>
+
+namespace gr {
+
+/// default constructor creates the root folder
+Resource::Resource( const fs::path& root_folder ) :
+	m_name		( root_folder.string() ),
+	m_kind		( "folder" ),
+	m_size		( 0 ),
+	m_id		( "folder:root" ),
+	m_href		( "root" ),
+	m_is_editable( true ),
+	m_parent	( 0 ),
+	m_state		( sync ),
+	m_json		( NULL ),
+	m_local_exists( true )
+{
+}
+
+Resource::Resource( const std::string& name, const std::string& kind ) :
+	m_name		( name ),
+	m_kind		( kind ),
+	m_size		( 0 ),
+	m_is_editable( true ),
+	m_parent	( 0 ),
+	m_state		( unknown ),
+	m_json		( NULL ),
+	m_local_exists( false )
+{
+}
+
+void Resource::SetState( State new_state )
+{
+	// only the new and delete states need to be set recursively
+	assert(
+		new_state == remote_new || new_state == remote_deleted ||
+		new_state == local_new  || new_state == local_deleted
+	) ;
+	
+	m_state = new_state ;
+	std::for_each( m_child.begin(), m_child.end(),
+		boost::bind( &Resource::SetState, _1, new_state ) ) ;
+}
+
+void Resource::FromRemoteFolder( const Entry& remote )
+{
+	fs::path path = Path() ;
+	
+	if ( !remote.IsEditable() )
+		Log( "folder %1% is read-only", path, log::verbose ) ;
+	
+	// already sync
+	if ( m_local_exists && m_kind == "folder" )
+	{
+		Log( "folder %1% is in sync", path, log::verbose ) ;
+		m_state = sync ;
+	}
+	else if ( m_local_exists && m_kind == "file" )
+	{
+		// TODO: handle type change
+		Log( "%1% changed from folder to file", path, log::verbose ) ;
+		m_state = sync ;
+	}
+	else if ( m_local_exists && m_kind == "bad" )
+	{
+		Log( "%1% inaccessible", path, log::verbose ) ;
+		m_state = sync ;
+	}
+	else if ( remote.MTime().Sec() > m_mtime.Sec() ) // FIXME only seconds are stored in local index
+	{
+		// remote folder created after last sync, so remote is newer
+		Log( "folder %1% is created in remote", path, log::verbose ) ;
+		SetState( remote_new ) ;
+	}
+	else
+	{
+		Log( "folder %1% is deleted in local", path, log::verbose ) ;
+		SetState( local_deleted ) ;
+	}
+}
+
+/// Update the state according to information (i.e. Entry) from remote. This function
+/// compares the modification time and checksum of both copies and determine which
+/// one is newer.
+void Resource::FromRemote( const Entry& remote )
+{
+	// sync folder
+	if ( remote.IsDir() && IsFolder() )
+		FromRemoteFolder( remote ) ;
+	else
+		FromRemoteFile( remote ) ;
+	
+	AssignIDs( remote ) ;
+	
+	assert( m_state != unknown ) ;
+	
+	if ( m_state == remote_new || m_state == remote_changed )
+		m_md5 = remote.MD5() ;
+	
+	m_mtime = remote.MTime() ;
+}
+
+void Resource::AssignIDs( const Entry& remote )
+{
+	// the IDs from change feed entries are different
+	if ( !remote.IsChange() )
+	{
+		m_id		= remote.ResourceID() ;
+		m_href		= remote.SelfHref() ;
+		m_content	= remote.ContentSrc() ;
+		m_is_editable = remote.IsEditable() ;
+		m_etag		= remote.ETag() ;
+		m_md5		= remote.MD5() ;
+	}
+}
+
+void Resource::FromRemoteFile( const Entry& remote )
+{
+	assert( m_parent != 0 ) ;
+	
+	fs::path path = Path() ;
+
+	// recursively create/delete folder
+	if ( m_parent->m_state == remote_new || m_parent->m_state == remote_deleted ||
+		 m_parent->m_state == local_new  || m_parent->m_state == local_deleted )
+	{
+		Log( "file %1% parent %2% recursively in %3% (%4%)", path,
+			( m_parent->m_state == remote_new || m_parent->m_state == local_new )      ? "created" : "deleted",
+			( m_parent->m_state == remote_new || m_parent->m_state == remote_deleted ) ? "remote"  : "local",
+			m_parent->m_state, log::verbose ) ;
+		
+		m_state = m_parent->m_state ;
+	}
+
+	else if ( m_kind == "bad" )
+	{
+		m_state = sync;
+	}
+
+	// local not exists
+	else if ( !m_local_exists )
+	{
+		Trace( "file %1% change stamp = %2%", Path(), remote.ChangeStamp() ) ;
+		
+		if ( remote.MTime().Sec() > m_mtime.Sec() || remote.MD5() != m_md5 || remote.ChangeStamp() > 0 )
+		{
+			Log( "file %1% is created in remote (change %2%)", path,
+				remote.ChangeStamp(), log::verbose ) ;
+			m_size = remote.Size();
+			m_state = remote_new ;
+		}
+		else
+		{
+			Log( "file %1% is deleted in local", path, log::verbose ) ;
+			m_state = local_deleted ;
+		}
+	}
+
+	// remote checksum unknown, assume the file is not changed in remote
+	else if ( remote.MD5().empty() )
+	{
+		Log( "file %1% has unknown checksum in remote. assumed in sync",
+			Path(), log::verbose ) ;
+		m_state = sync ;
+	}
+
+	// use mtime to check which one is more recent
+	else if ( remote.Size() != m_size || remote.MD5() != GetMD5() )
+	{
+		assert( m_state != unknown ) ;
+
+		// if remote is modified
+		if ( remote.MTime().Sec() > m_mtime.Sec() )
+		{
+			Log( "file %1% is changed in remote", path, log::verbose ) ;
+			m_size = remote.Size();
+			m_state = remote_changed ;
+		}
+		
+		// remote also has the file, so it's not new in local
+		else if ( m_state == local_new || m_state == remote_deleted )
+		{
+			Log( "file %1% is changed in local", path, log::verbose ) ;
+			m_state = local_changed ;
+		}
+		else
+			Trace( "file %1% state is %2%", m_name, m_state ) ;
+	}
+
+	// if checksum is equal, no need to compare the mtime
+	else
+	{
+		Log( "file %1% is already in sync", Path(), log::verbose ) ;
+		m_state = sync ;
+	}
+}
+
+void Resource::FromDeleted( Val& state )
+{
+	assert( !m_json );
+	m_json = &state;
+	if ( state.Has( "ctime" ) )
+		m_ctime.Assign( state["ctime"].U64(), 0 );
+	if ( state.Has( "md5" ) )
+		m_md5 = state["md5"];
+	if ( state.Has( "srv_time" ) )
+		m_mtime.Assign( state[ "srv_time" ].U64(), 0 ) ;
+	if ( state.Has( "size" ) )
+		m_size = state[ "size" ].U64();
+	m_state = both_deleted;
+}
+
+/// Update the resource with the attributes of local file or directory. This
+/// function will propulate the fields in m_entry.
+void Resource::FromLocal( Val& state )
+{
+	assert( !m_json );
+	m_json = &state;
+
+	// root folder is always in sync
+	if ( !IsRoot() )
+	{
+		fs::path path = Path() ;
+		FileType ft ;
+		try
+		{
+			os::Stat( path, &m_ctime, (off64_t*)&m_size, &ft ) ;
+		}
+		catch ( os::Error &e )
+		{
+			// invalid symlink, unreadable file or something else
+			int const* eno = boost::get_error_info< boost::errinfo_errno >(e);
+			Log( "Error accessing %1%: %2%; skipping file", path.string(), strerror( *eno ), log::warning );
+			m_state = sync;
+			m_kind = "bad";
+			return;
+		}
+		if ( ft == FT_UNKNOWN )
+		{
+			// Skip sockets/FIFOs/etc
+			Log( "File %1% is not a regular file or directory; skipping file", path.string(), log::warning );
+			m_state = sync;
+			m_kind = "bad";
+			return;
+		}
+
+		m_name = path.filename().string() ;
+		m_kind = ft == FT_DIR ? "folder" : "file";
+		m_local_exists = true;
+
+		bool is_changed;
+		if ( state.Has( "ctime" ) && (u64_t) m_ctime.Sec() <= state["ctime"].U64() &&
+			( ft == FT_DIR || state.Has( "md5" ) ) )
+		{
+			if ( ft != FT_DIR )
+				m_md5 = state["md5"];
+			is_changed = false;
+		}
+		else
+		{
+			if ( ft != FT_DIR )
+			{
+				// File is changed locally. TODO: Detect conflicts
+				is_changed = ( state.Has( "size" ) && m_size != state["size"].U64() ) ||
+					!state.Has( "md5" ) || GetMD5() != state["md5"].Str();
+			}
+			else
+				is_changed = true;
+		}
+		if ( state.Has( "srv_time" ) )
+			m_mtime.Assign( state[ "srv_time" ].U64(), 0 ) ;
+
+		// Upload file if it is changed and remove if not.
+		// State will be updated to sync/remote_changed in FromRemote()
+		m_state = is_changed ? local_new : remote_deleted;
+		if ( m_state == local_new )
+		{
+			// local_new means this file is changed in local.
+			// this means we can't delete any of its parents.
+			// make sure their state is also set to local_new.
+			Resource *p = m_parent;
+			while ( p && p->m_state == remote_deleted )
+			{
+				p->m_state = local_new;
+				p = p->m_parent;
+			}
+		}
+	}
+	
+	assert( m_state != unknown ) ;
+}
+
+std::string Resource::SelfHref() const
+{
+	return m_href ;
+}
+
+std::string Resource::ContentSrc() const
+{
+	return m_content ;
+}
+
+std::string Resource::ETag() const
+{
+	return m_etag ;
+}
+
+std::string Resource::Name() const
+{
+	return m_name ;
+}
+
+std::string Resource::Kind() const
+{
+	return m_kind ;
+}
+
+DateTime Resource::ServerTime() const
+{
+	return m_mtime ;
+}
+
+std::string Resource::ResourceID() const
+{
+	return m_id ;
+}
+
+Resource::State Resource::GetState() const
+{
+	return m_state ;
+}
+
+const Resource* Resource::Parent() const
+{
+	assert( m_parent == 0 || m_parent->IsFolder() ) ;
+	return m_parent ;
+}
+
+Resource* Resource::Parent()
+{
+	assert( m_parent == 0 || m_parent->IsFolder() ) ;
+	return m_parent ;
+}
+
+void Resource::AddChild( Resource *child )
+{
+	assert( child != 0 ) ;
+	assert( child->m_parent == 0 || child->m_parent == this ) ;
+	assert( child != this ) ;
+
+	child->m_parent = this ;
+	m_child.push_back( child ) ;
+}
+
+bool Resource::IsFolder() const
+{
+	return m_kind == "folder" ;
+}
+
+bool Resource::IsEditable() const
+{
+	return m_is_editable ;
+}
+
+fs::path Resource::Path() const
+{
+	assert( m_parent != this ) ;
+	assert( m_parent == 0 || m_parent->IsFolder() ) ;
+
+	return m_parent != 0 ? (m_parent->Path() / m_name) : m_name ;
+}
+
+// Path relative to the root directory
+fs::path Resource::RelPath() const
+{
+	assert( m_parent != this ) ;
+	assert( m_parent == 0 || m_parent->IsFolder() ) ;
+
+	return m_parent != 0 && !m_parent->IsRoot() ? (m_parent->RelPath() / m_name) : m_name ;
+}
+
+bool Resource::IsInRootTree() const
+{
+	assert( m_parent == 0 || m_parent->IsFolder() ) ;
+	return m_parent == 0 ? IsRoot() : m_parent->IsInRootTree() ;
+}
+
+Resource* Resource::FindChild( const std::string& name )
+{
+	for ( std::vector<Resource*>::iterator i = m_child.begin() ; i != m_child.end() ; ++i )
+	{
+		assert( (*i)->m_parent == this ) ;
+		if ( (*i)->m_name == name )
+			return *i ;
+	}
+	return 0 ;
+}
+
+// try to change the state to "sync"
+void Resource::Sync( Syncer *syncer, ResourceTree *res_tree, const Val& options )
+{
+	assert( m_state != unknown ) ;
+	assert( !IsRoot() || m_state == sync ) ;	// root folder is already synced
+	
+	try
+	{
+		SyncSelf( syncer, res_tree, options ) ;
+	}
+	catch ( File::Error &e )
+	{
+		int *en = boost::get_error_info< boost::errinfo_errno > ( e ) ;
+		Log( "Error syncing %1%: %2%", Path(), en ? strerror( *en ) : "", log::error );
+		return;
+	}
+	catch ( boost::filesystem::filesystem_error &e )
+	{
+		Log( "Error syncing %1%: %2%", Path(), e.what(), log::error );
+		return;
+	}
+	catch ( http::Error &e )
+	{
+		int *curlcode = boost::get_error_info< http::CurlCode > ( e ) ;
+		int *httpcode = boost::get_error_info< http::HttpResponseCode > ( e ) ;
+		std::string msg;
+		if ( curlcode )
+			msg = *( boost::get_error_info< http::CurlErrMsg > ( e ) );
+		else if ( httpcode )
+			msg = "HTTP " + boost::to_string( *httpcode );
+		else
+			msg = e.what();
+		Log( "Error syncing %1%: %2%", Path(), msg, log::error );
+		std::string *url = boost::get_error_info< http::Url > ( e );
+		std::string *resp_hdr = boost::get_error_info< http::HttpResponseHeaders > ( e );
+		std::string *resp_txt = boost::get_error_info< http::HttpResponseText > ( e );
+		http::Header *req_hdr = boost::get_error_info< http::HttpRequestHeaders > ( e );
+		if ( url )
+			Log( "Request URL: %1%", *url, log::verbose );
+		if ( req_hdr )
+			Log( "Request headers: %1%", req_hdr->Str(), log::verbose );
+		if ( resp_hdr )
+			Log( "Response headers: %1%", *resp_hdr, log::verbose );
+		if ( resp_txt )
+			Log( "Response text: %1%", *resp_txt, log::verbose );
+		return;
+	}
+	
+	// if myself is deleted, no need to do the childrens
+	if ( m_state != local_deleted && m_state != remote_deleted )
+	{
+		std::for_each( m_child.begin(), m_child.end(),
+			boost::bind( &Resource::Sync, _1, syncer, res_tree, options ) ) ;
+	}
+}
+
+bool Resource::CheckRename( Syncer* syncer, ResourceTree *res_tree )
+{
+	if ( !IsFolder() && ( m_state == local_new || m_state == remote_new ) )
+	{
+		bool is_local = m_state == local_new;
+		State other = is_local ? local_deleted : remote_deleted;
+		if ( is_local )
+		{
+			// First check size index for locally added files
+			details::SizeRange moved = res_tree->FindBySize( m_size );
+			bool found = false;
+			for ( details::SizeMap::iterator i = moved.first ; i != moved.second; i++ )
+			{
+				Resource *m = *i;
+				if ( m->m_state == other )
+				{
+					found = true;
+					break;
+				}
+			}
+			if ( !found )
+			{
+				// Don't check md5 sums if there are no deleted files with same size
+				return false;
+			}
+		}
+		details::MD5Range moved = res_tree->FindByMD5( GetMD5() );
+		for ( details::MD5Map::iterator i = moved.first ; i != moved.second; i++ )
+		{
+			Resource *m = *i;
+			if ( m->m_state == other )
+			{
+				Resource* from = m_state == local_new || m_state == remote_new ? m : this;
+				Resource* to = m_state == local_new || m_state == remote_new ? this : m;
+				Log( "sync %1% moved to %2%. moving %3%", from->Path(), to->Path(),
+					is_local ? "remote" : "local", log::info );
+				if ( syncer )
+				{
+					if ( is_local )
+					{
+						syncer->Move( from, to->Parent(), to->Name() );
+						to->SetIndex( false );
+					}
+					else
+					{
+						fs::rename( from->Path(), to->Path() );
+						to->SetIndex( true );
+					}
+					to->m_mtime = from->m_mtime;
+					to->m_json->Set( "srv_time", Val( from->m_mtime.Sec() ) );
+					from->DeleteIndex();
+				}
+				from->m_state = both_deleted;
+				to->m_state = sync;
+				return true;
+			}
+		}
+	}
+	return false;
+}
+
+void Resource::SyncSelf( Syncer* syncer, ResourceTree *res_tree, const Val& options )
+{
+	assert( !IsRoot() || m_state == sync ) ;	// root is always sync
+	assert( IsRoot() || !syncer || m_parent->IsFolder() ) ;
+	assert( IsRoot() || m_parent->m_state != remote_deleted ) ;
+	assert( IsRoot() || m_parent->m_state != local_deleted ) ;
+
+	const fs::path path = Path() ;
+
+	// Detect renames
+	if ( CheckRename( syncer, res_tree ) )
+		return;
+
+	switch ( m_state )
+	{
+	case local_new :
+		Log( "sync %1% doesn't exist in server, uploading", path, log::info ) ;
+		
+		if ( syncer && syncer->Create( this ) )
+		{
+			m_state = sync ;
+			SetIndex( false );
+		}
+		break ;
+	
+	case local_deleted :
+		Log( "sync %1% deleted in local. deleting remote", path, log::info ) ;
+		if ( syncer && !options["no-delete-remote"].Bool() )
+		{
+			syncer->DeleteRemote( this ) ;
+			DeleteIndex() ;
+		}
+		break ;
+	
+	case local_changed :
+		Log( "sync %1% changed in local. uploading", path, log::info ) ;
+		if ( syncer && syncer->EditContent( this, options["new-rev"].Bool() ) )
+		{
+			m_state = sync ;
+			SetIndex( false );
+		}
+		break ;
+	
+	case remote_new :
+		if ( options["no-remote-new"].Bool() )
+			Log( "sync %1% created in remote. skipping", path, log::info ) ;
+		else
+		{
+			Log( "sync %1% created in remote. creating local", path, log::info ) ;
+			if ( syncer )
+			{
+				if ( IsFolder() )
+					fs::create_directories( path ) ;
+				else
+					syncer->Download( this, path ) ;
+				SetIndex( true ) ;
+				m_state = sync ;
+			}
+		}
+		break ;
+	
+	case remote_changed :
+		assert( !IsFolder() ) ;
+		if ( options["upload-only"].Bool() )
+			Log( "sync %1% changed in remote. skipping", path, log::info ) ;
+		else
+		{
+			Log( "sync %1% changed in remote. downloading", path, log::info ) ;
+			if ( syncer )
+			{
+				syncer->Download( this, path ) ;
+				SetIndex( true ) ;
+				m_state = sync ;
+			}
+		}
+		break ;
+	
+	case remote_deleted :
+		Log( "sync %1% deleted in remote. deleting local", path, log::info ) ;
+		if ( syncer )
+		{
+			DeleteLocal() ;
+			DeleteIndex() ;
+		}
+		break ;
+	
+	case both_deleted :
+		if ( syncer )
+			DeleteIndex() ;
+		break ;
+	
+	case sync :
+		Log( "sync %1% already in sync", path, log::verbose ) ;
+		if ( !IsRoot() )
+			SetIndex( false ) ;
+		break ;
+	
+	// shouldn't go here
+	case unknown :
+	default :
+		assert( false ) ;
+		break ;
+	}
+	
+	if ( syncer && m_json )
+	{
+		// Update server time of this file
+		m_json->Set( "srv_time", Val( m_mtime.Sec() ) );
+	}
+}
+
+void Resource::SetServerTime( const DateTime& time )
+{
+	m_mtime = time ;
+}
+
+/// this function doesn't really remove the local file. it renames it.
+void Resource::DeleteLocal()
+{
+	static const boost::format trash_file( "%1%-%2%" ) ;
+
+	assert( m_parent != NULL ) ;
+	Resource* p = m_parent;
+	fs::path destdir;
+	while ( !p->IsRoot() )
+	{
+		destdir = p->Name() / destdir;
+		p = p->Parent();
+	}
+	destdir = p->Path() / ".trash" / destdir;
+
+	fs::path dest = destdir / Name();
+	std::size_t idx = 1 ;
+	while ( fs::exists( dest ) && idx != 0 )
+		dest = destdir / (boost::format(trash_file) % Name() % idx++).str() ;
+
+	// wrap around! just remove the file
+	if ( idx == 0 )
+		fs::remove_all( Path() ) ;
+	else
+	{
+		fs::create_directories( dest.parent_path() ) ;
+		fs::rename( Path(), dest ) ;
+	}
+}
+
+void Resource::DeleteIndex()
+{
+	(*m_parent->m_json)["tree"].Del( Name() );
+	m_json = NULL;
+}
+
+void Resource::SetIndex( bool re_stat )
+{
+	assert( m_parent && m_parent->m_json != NULL );
+	if ( !m_json )
+		m_json = &((*m_parent->m_json)["tree"]).Item( Name() );
+	FileType ft;
+	if ( re_stat )
+		os::Stat( Path(), &m_ctime, NULL, &ft );
+	else
+		ft = IsFolder() ? FT_DIR : FT_FILE;
+	m_json->Set( "ctime", Val( m_ctime.Sec() ) );
+	if ( ft != FT_DIR )
+	{
+		m_json->Set( "md5", Val( m_md5 ) );
+		m_json->Set( "size", Val( m_size ) );
+		m_json->Del( "tree" );
+	}
+	else
+	{
+		// add tree item if it does not exist
+		m_json->Item( "tree" );
+		m_json->Del( "md5" );
+		m_json->Del( "size" );
+	}
+}
+
+Resource::iterator Resource::begin() const
+{
+	return m_child.begin() ;
+}
+
+Resource::iterator Resource::end() const
+{
+	return m_child.end() ;
+}
+
+std::size_t Resource::size() const
+{
+	return m_child.size() ;
+}
+
+std::ostream& operator<<( std::ostream& os, Resource::State s )
+{
+	static const char *state[] =
+	{
+		"sync",	"local_new", "local_changed", "local_deleted", "remote_new",
+		"remote_changed", "remote_deleted", "both_deleted"
+	} ;
+	assert( s >= 0 && s < Count(state) ) ;
+	return os << state[s] ;
+}
+
+std::string Resource::StateStr() const
+{
+	std::ostringstream ss ;
+	ss << m_state ;
+	return ss.str() ;
+}
+
+u64_t Resource::Size() const
+{
+	return m_size ;
+}
+
+std::string Resource::MD5() const
+{
+	return m_md5 ;
+}
+
+std::string Resource::GetMD5()
+{
+	if ( m_md5.empty() && !IsFolder() && m_local_exists )
+	{
+		// MD5 checksum is calculated lazily and only when really needed:
+		// 1) when a local rename is supposed (when there are a new file and a deleted file of the same size)
+		// 2) when local ctime is changed, but file size isn't
+		m_md5 = crypt::MD5::Get( Path() );
+	}
+	return m_md5 ;
+}
+
+bool Resource::IsRoot() const
+{
+	// Root entry does not show up in file feeds, so we check for empty parent (and self-href)
+	return !m_parent ;
+}
+
+bool Resource::HasID() const
+{
+	return !m_href.empty() && !m_id.empty() ;
+}
+
+} // end of namespace

libgrive/src/base/Resource.hh

@@ -19,6 +19,7 @@
 
 #pragma once
 
+#include "util/Types.hh"
 #include "util/DateTime.hh"
 #include "util/Exception.hh"
 #include "util/FileSystem.hh"
@@ -29,11 +30,11 @@
 
 namespace gr {
 
-namespace http
-{
-	class Agent ;
-	class Header ;
-}
+class ResourceTree ;
+
+class Syncer ;
+
+class Val ;
 
 class Entry ;
 
@@ -45,48 +46,9 @@ class Entry ;
 class Resource
 {
 public :
-	struct Error : virtual Exception {} ;
-
 	typedef std::vector<Resource*> Children ;
 	typedef Children::const_iterator iterator ;
 	
-public :
-	Resource() ;
-	Resource( const std::string& name, const std::string& kind ) ;
-	
-	// default copy ctor & op= are fine
-	void Swap( Resource& coll ) ;
-
-	bool IsFolder() const ;
-
-	std::string Name() const ;
-	std::string SelfHref() const ;
-	std::string ResourceID() const ;
-	
-	const Resource* Parent() const ;
-	Resource* Parent() ;
-	void AddChild( Resource *child ) ;
-	Resource* FindChild( const std::string& title ) ;
-	
-	fs::path Path() const ;
-	bool IsInRootTree() const ;
-	bool IsRoot() const ;
-	bool HasID() const ;
-	std::string MD5() const ;
-
-	void FromRemote( const Entry& remote, const DateTime& last_sync ) ;
-	void FromLocal( const DateTime& last_sync ) ;
-	
-	void Sync( http::Agent* http, const http::Header& auth ) ;
-
-	// children access
-	iterator begin() const ;
-	iterator end() const ;
-	std::size_t size() const ;
-	
-	std::string StateStr() const ;
-	
-private :
 	/// State of the resource. indicating what to do with the resource
 	enum State
 	{
@@ -108,59 +70,104 @@ private :
 		/// We should download the file.
 		remote_new,
 		
-		/// Resource exists in both local & remote, but remote is newer.		
+		/// Resource exists in both local & remote, but remote is newer.
 		remote_changed,
 		
 		/// Resource delete in remote, need to delete in local
 		remote_deleted,
 		
+		/// Both deleted. State is used to remove leftover files from the index after sync.
+		both_deleted,
 		
 		/// invalid value
 		unknown
 	} ;
 
-	friend std::ostream& operator<<( std::ostream& os, State s ) ;
+public :
+	Resource(const fs::path& root_folder) ;
+	Resource( const std::string& name, const std::string& kind ) ;
 	
+	bool IsFolder() const ;
+	bool IsEditable() const ;
+
+	std::string Name() const ;
+	std::string Kind() const ;
+	DateTime ServerTime() const ;
+	std::string SelfHref() const ;
+	std::string ContentSrc() const ;
+	std::string ETag() const ;
+	std::string ResourceID() const ;
+	State GetState() const;
+	
+	const Resource* Parent() const ;
+	Resource* Parent() ;
+	void AddChild( Resource *child ) ;
+	Resource* FindChild( const std::string& title ) ;
+	
+	fs::path Path() const ;
+	fs::path RelPath() const ;
+	bool IsInRootTree() const ;
+	bool IsRoot() const ;
+	bool HasID() const ;
+	u64_t Size() const;
+	std::string MD5() const ;
+	std::string GetMD5() ;
+
+	void FromRemote( const Entry& remote ) ;
+	void FromDeleted( Val& state ) ;
+	void FromLocal( Val& state ) ;
+	
+	void Sync( Syncer* syncer, ResourceTree *res_tree, const Val& options ) ;
+	void SetServerTime( const DateTime& time ) ;
+
+	// children access
+	iterator begin() const ;
+	iterator end() const ;
+	std::size_t size() const ;
+	
+	std::string StateStr() const ;
+
+private :
+
+	void AssignIDs( const Entry& remote ) ;
+
+	friend std::ostream& operator<<( std::ostream& os, State s ) ;
+	friend class Syncer ;
+
 private :
 	void SetState( State new_state ) ;
-
-	void Download( http::Agent* http, const fs::path& file, const http::Header& auth ) const ;
-	bool EditContent( http::Agent* http, const http::Header& auth ) ;
-	bool Create( http::Agent* http, const http::Header& auth ) ;
-	bool Upload( http::Agent* http, const std::string& link, const http::Header& auth, bool post ) ;
 	
-	void FromRemoteFolder( const Entry& remote, const DateTime& last_sync ) ;
-	void FromRemoteFile( const Entry& remote, const DateTime& last_sync ) ;
+	void FromRemoteFolder( const Entry& remote ) ;
+	void FromRemoteFile( const Entry& remote ) ;
 	
 	void DeleteLocal() ;
-	void DeleteRemote( http::Agent* http, const http::Header& auth ) ;
-	
-	void AssignIDs( const Entry& remote ) ;
-	void SyncSelf( http::Agent* http, const http::Header& auth ) ;
+	void DeleteIndex() ;
+	void SetIndex( bool ) ;
 	
+	bool CheckRename( Syncer* syncer, ResourceTree *res_tree ) ;
+	void SyncSelf( Syncer* syncer, ResourceTree *res_tree, const Val& options ) ;
+
 private :
 	std::string				m_name ;
 	std::string				m_kind ;
 	std::string				m_md5 ;
 	DateTime				m_mtime ;
-	
+	DateTime				m_ctime ;
+	u64_t					m_size ;
+
 	std::string				m_id ;
 	std::string				m_href ;
-	std::string				m_edit ;
-	std::string				m_create ;
 	std::string				m_content ;
 	std::string				m_etag ;
+	bool					m_is_editable ;
 
 	// not owned
 	Resource				*m_parent ;
 	std::vector<Resource*>	m_child ;
-	
+
 	State					m_state ;
+	Val*					m_json ;
+	bool					m_local_exists ;
 } ;
 
-} // end of namespace
-
-namespace std
-{
-	void swap( gr::Resource& c1, gr::Resource& c2 ) ;
-}
+} // end of namespace gr::v1

libgrive/src/base/ResourceTree.cc

@@ -18,9 +18,7 @@
 */
 
 #include "ResourceTree.hh"
-#include "CommonUri.hh"
 
-#include "protocol/Json.hh"
 #include "util/Destroy.hh"
 #include "util/log/Log.hh"
 
@@ -31,8 +29,8 @@ namespace gr {
 
 using namespace details ;
 
-ResourceTree::ResourceTree( ) :
-	m_root( new Resource )
+ResourceTree::ResourceTree( const fs::path& rootFolder ) :
+	m_root( new Resource( rootFolder ) )
 {
 	m_set.insert( m_root ) ;
 }
@@ -44,7 +42,7 @@ ResourceTree::ResourceTree( const ResourceTree& fs ) :
 	for ( Set::const_iterator i = s.begin() ; i != s.end() ; ++i )
 	{
 		Resource *c = new Resource( **i ) ;
-		if ( c->SelfHref() == root_href )
+		if ( c->IsRoot() )
 			m_root = c ;
 		
 		m_set.insert( c ) ;
@@ -80,18 +78,6 @@ const Resource* ResourceTree::Root() const
 	return m_root ;
 }
 
-void ResourceTree::Swap( ResourceTree& fs )
-{
-	m_set.swap( fs.m_set ) ;
-}
-
-ResourceTree& ResourceTree::operator=( const ResourceTree& fs )
-{
-	ResourceTree tmp( fs ) ;
-	Swap( tmp ) ;
-	return *this ;
-}
-
 Resource* ResourceTree::FindByHref( const std::string& href )
 {
 	// for the resource that not yet have href (e.g. not yet fetched from server)
@@ -111,30 +97,21 @@ const Resource* ResourceTree::FindByHref( const std::string& href ) const
 	return i != map.end() ? *i : 0 ;
 }
 
-/// Unlike other search functions, this one does not depend on the multi-index
-/// container. It traverses the tree instead.
-Resource* ResourceTree::FindByPath( const fs::path& path )
+MD5Range ResourceTree::FindByMD5( const std::string& md5 )
 {
-	Resource *current = m_root ;
-	for ( fs::path::iterator i = path.begin() ; i != path.end() && current != 0 ; ++i )
-	{
-		Trace( "path it = %1%", *i ) ;
-		
-		// current directory
-		if ( *i == "." )
-			continue ;
-		
-		else if ( *i == ".." )
-			current = current->Parent() ;
-	
-		else
-			current = current->FindChild( Path2Str(*i) ) ;
-	}
-			
-	return current ;
+	MD5Map& map = m_set.get<ByMD5>() ;
+	if ( !md5.empty() )
+		return map.equal_range( md5 );
+	return MD5Range( map.end(), map.end() ) ;
 }
 
-///	Reinsert should be called when the ID/HREF were updated
+SizeRange ResourceTree::FindBySize( u64_t size )
+{
+	SizeMap& map = m_set.get<BySize>() ;
+	return map.equal_range( size );
+}
+
+///	Reinsert should be called when the ID/HREF/MD5 were updated
 bool ResourceTree::ReInsert( Resource *coll )
 {
 	Set& s = m_set.get<ByIdentity>() ;
@@ -160,11 +137,11 @@ void ResourceTree::Erase( Resource *coll )
 	s.erase( s.find( coll ) ) ;
 }
 
-void ResourceTree::Update( Resource *coll, const Entry& e, const DateTime& last_sync )
+void ResourceTree::Update( Resource *coll, const Entry& e )
 {
 	assert( coll != 0 ) ;
 
-	coll->FromRemote( e, last_sync ) ;
+	coll->FromRemote( e ) ;
 	ReInsert( coll ) ;
 }
 
@@ -178,4 +155,4 @@ ResourceTree::iterator ResourceTree::end()
 	return m_set.get<ByIdentity>().end() ;
 }
 
-} // end of namespace
+} // end of namespace gr

libgrive/src/base/ResourceTree.hh

@@ -30,27 +30,30 @@
 
 namespace gr {
 
-class Json ;
-
 namespace details
 {
 	using namespace boost::multi_index ;
-	struct ByID {} ;
+	struct ByMD5 {} ;
 	struct ByHref {} ;
 	struct ByIdentity {} ;
+	struct BySize {} ;
 
 	typedef multi_index_container<
 		Resource*,
 		indexed_by<
 			hashed_non_unique<tag<ByHref>,	const_mem_fun<Resource, std::string,	&Resource::SelfHref> >,
-			hashed_non_unique<tag<ByID>,	const_mem_fun<Resource, std::string,	&Resource::ResourceID> >,
+			hashed_non_unique<tag<ByMD5>,	const_mem_fun<Resource, std::string,	&Resource::MD5> >,
+			hashed_non_unique<tag<BySize>,	const_mem_fun<Resource, u64_t,			&Resource::Size> >,
 			hashed_unique<tag<ByIdentity>,	identity<Resource*> >
 		>
 	> Folders ;
 	
-	typedef Folders::index<ByID>::type			IDMap ;
+	typedef Folders::index<ByMD5>::type			MD5Map ;
 	typedef Folders::index<ByHref>::type		HrefMap ;
+	typedef Folders::index<BySize>::type		SizeMap ;
 	typedef Folders::index<ByIdentity>::type	Set ;
+	typedef std::pair<SizeMap::iterator, SizeMap::iterator> SizeRange ;
+	typedef std::pair<MD5Map::iterator, MD5Map::iterator> MD5Range ;
 }
 
 /*!	\brief	A simple container for storing folders
@@ -64,24 +67,20 @@ public :
 	typedef details::Set::iterator iterator ;
 
 public :
-	ResourceTree( ) ;
+	ResourceTree( const fs::path& rootFolder ) ;
 	ResourceTree( const ResourceTree& fs ) ;
 	~ResourceTree( ) ;
 	
-	void Swap( ResourceTree& fs ) ;
-	ResourceTree& operator=( const ResourceTree& fs ) ;
-	
 	Resource* FindByHref( const std::string& href ) ;
 	const Resource* FindByHref( const std::string& href ) const ;
+	details::MD5Range FindByMD5( const std::string& md5 ) ;
+	details::SizeRange FindBySize( u64_t size ) ;
 
-	Resource* FindByPath( const fs::path& path ) ;
-	Resource* FindByID( const std::string& id ) ;
-	
 	bool ReInsert( Resource *coll ) ;
 	
 	void Insert( Resource *coll ) ;
 	void Erase( Resource *coll ) ;
-	void Update( Resource *coll, const Entry& e, const DateTime& last_sync ) ;
+	void Update( Resource *coll, const Entry& e ) ;
 	
 	Resource* Root() ;
 	const Resource* Root() const ;
@@ -97,4 +96,4 @@ private :
 	Resource*			m_root ;
 } ;
 
-} // end of namespace
+} // end of namespace gr

libgrive/src/base/State.cc

@@ -0,0 +1,431 @@
+/*
+	grive: an GPL program to sync a local directory with Google Drive
+	Copyright (C) 2012  Wan Wai Ho
+
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License
+	as published by the Free Software Foundation version 2
+	of the License.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with this program; if not, write to the Free Software
+	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+
+#include "State.hh"
+
+#include "Entry.hh"
+#include "Resource.hh"
+#include "Syncer.hh"
+
+#include "util/Crypt.hh"
+#include "util/File.hh"
+#include "util/log/Log.hh"
+#include "json/JsonParser.hh"
+
+#include <boost/algorithm/string.hpp>
+
+#include <fstream>
+
+namespace gr {
+
+const std::string state_file = ".grive_state" ;
+const std::string ignore_file = ".griveignore" ;
+const int MAX_IGN = 65536 ;
+const char* regex_escape_chars = ".^$|()[]{}*+?\\";
+const boost::regex regex_escape_re( "[.^$|()\\[\\]{}*+?\\\\]" );
+
+inline std::string regex_escape( std::string s )
+{
+	return regex_replace( s, regex_escape_re, "\\\\&", boost::format_sed );
+}
+
+State::State( const fs::path& root, const Val& options  ) :
+	m_root		( root ),
+	m_res		( options["path"].Str() ),
+	m_cstamp	( -1 )
+{
+	Read() ;
+
+	// the "-f" option will make grive always think remote is newer
+	m_force = options.Has( "force" ) ? options["force"].Bool() : false ;
+
+	std::string m_orig_ign = m_ign;
+	if ( options.Has( "ignore" ) && options["ignore"].Str() != m_ign )
+		m_ign = options["ignore"].Str();
+	else if ( options.Has( "dir" ) )
+	{
+		const boost::regex trim_path( "^/+|/+$" );
+		std::string m_dir = regex_replace( options["dir"].Str(), trim_path, "" );
+		if ( !m_dir.empty() )
+		{
+			// "-s" is internally converted to an ignore regexp
+			m_dir = regex_escape( m_dir );
+			size_t pos = 0;
+			while ( ( pos = m_dir.find( '/', pos ) ) != std::string::npos )
+			{
+				m_dir = m_dir.substr( 0, pos ) + "$|" + m_dir;
+				pos = pos*2 + 3;
+			}
+			std::string ign = "^(?!"+m_dir+"(/|$))";
+			m_ign = ign;
+		}
+	}
+
+	m_ign_changed = m_orig_ign != "" && m_orig_ign != m_ign;
+	m_ign_re = boost::regex( m_ign.empty() ? "^\\.(grive$|grive_state$|trash)" : ( m_ign+"|^\\.(grive$|grive_state$|trash)" ) );
+}
+
+State::~State()
+{
+}
+
+/// Synchronize local directory. Build up the resource tree from files and folders
+/// of local directory.
+void State::FromLocal( const fs::path& p )
+{
+	m_res.Root()->FromLocal( m_st ) ;
+	FromLocal( p, m_res.Root(), m_st.Item( "tree" ) ) ;
+}
+
+bool State::IsIgnore( const std::string& filename )
+{
+	return regex_search( filename.c_str(), m_ign_re, boost::format_perl );
+}
+
+void State::FromLocal( const fs::path& p, Resource* folder, Val& tree )
+{
+	assert( folder != 0 ) ;
+	assert( folder->IsFolder() ) ;
+
+	Val::Object leftover = tree.AsObject();
+
+	for ( fs::directory_iterator i( p ) ; i != fs::directory_iterator() ; ++i )
+	{
+		std::string fname = i->path().filename().string() ;
+		std::string path = ( folder->IsRoot() ? fname : ( folder->RelPath() / fname ).string() );
+		
+		if ( IsIgnore( path ) )
+			Log( "file %1% is ignored by grive", path, log::verbose ) ;
+		else
+		{
+			// if the Resource object of the child already exists, it should
+			// have been so no need to do anything here
+			Resource *c = folder->FindChild( fname ), *c2 = c ;
+			if ( !c )
+			{
+				c2 = new Resource( fname, "" ) ;
+				folder->AddChild( c2 ) ;
+			}
+			leftover.erase( fname );
+			Val& rec = tree.Item( fname );
+			if ( m_force )
+				rec.Del( "srv_time" );
+			c2->FromLocal( rec ) ;
+			if ( !c )
+				m_res.Insert( c2 ) ;
+			if ( c2->IsFolder() )
+				FromLocal( *i, c2, rec.Item( "tree" ) ) ;
+		}
+	}
+
+	for( Val::Object::iterator i = leftover.begin(); i != leftover.end(); i++ )
+	{
+		std::string path = folder->IsRoot() ? i->first : ( folder->RelPath() / i->first ).string();
+		if ( IsIgnore( path ) )
+			Log( "file %1% is ignored by grive", path, log::verbose ) ;
+		else
+		{
+			// Restore state of locally deleted files
+			Resource *c = folder->FindChild( i->first ), *c2 = c ;
+			if ( !c )
+			{
+				c2 = new Resource( i->first, i->second.Has( "tree" ) ? "folder" : "file" ) ;
+				folder->AddChild( c2 ) ;
+			}
+			Val& rec = tree.Item( i->first );
+			if ( m_force || m_ign_changed )
+				rec.Del( "srv_time" );
+			c2->FromDeleted( rec );
+			if ( !c )
+				m_res.Insert( c2 ) ;
+		}
+	}
+}
+
+void State::FromRemote( const Entry& e )
+{
+	std::string fn = e.Filename() ;
+	std::string k = e.IsDir() ? "folder" : "file";
+
+	// common checkings
+	if ( !e.IsDir() && ( fn.empty() || e.ContentSrc().empty() ) )
+		Log( "%1% \"%2%\" is a google document, ignored", k, e.Name(), log::verbose ) ;
+	
+	else if ( fn.find('/') != fn.npos )
+		Log( "%1% \"%2%\" contains a slash in its name, ignored", k, e.Name(), log::verbose ) ;
+	
+	else if ( !e.IsChange() && e.ParentHrefs().size() != 1 )
+		Log( "%1% \"%2%\" has multiple parents, ignored", k, e.Name(), log::verbose ) ;
+
+	else if ( e.IsChange() )
+		FromChange( e ) ;
+
+	else if ( !Update( e ) )
+		m_unresolved.push_back( e ) ;
+}
+
+void State::ResolveEntry()
+{
+	while ( !m_unresolved.empty() )
+	{
+		if ( TryResolveEntry() == 0 )
+			break ;
+	}
+}
+
+std::size_t State::TryResolveEntry()
+{
+	assert( !m_unresolved.empty() ) ;
+
+	std::size_t count = 0 ;
+	std::list<Entry>& en = m_unresolved ;
+
+	for ( std::list<Entry>::iterator i = en.begin() ; i != en.end() ; )
+	{
+		if ( Update( *i ) )
+		{
+			i = en.erase( i ) ;
+			count++ ;
+		}
+		else
+			++i ;
+	}
+	return count ;
+}
+
+void State::FromChange( const Entry& e )
+{
+	assert( e.IsChange() ) ;
+	
+	// entries in the change feed is always treated as newer in remote,
+	// so we override the last sync time to 0
+	if ( Resource *res = m_res.FindByHref( e.SelfHref() ) )
+		m_res.Update( res, e ) ;
+}
+
+bool State::Update( const Entry& e )
+{
+	assert( !e.IsChange() ) ;
+	assert( !e.ParentHref().empty() ) ;
+
+	if ( Resource *res = m_res.FindByHref( e.SelfHref() ) )
+	{
+		std::string path = res->RelPath().string();
+		if ( IsIgnore( path ) )
+		{
+			Log( "%1% is ignored by grive", path, log::verbose ) ;
+			return true;
+		}
+		m_res.Update( res, e ) ;
+		return true;
+	}
+	else if ( Resource *parent = m_res.FindByHref( e.ParentHref() ) )
+	{
+		if ( !parent->IsFolder() )
+		{
+			// https://github.com/vitalif/grive2/issues/148
+			Log( "%1% is owned by something that's not a directory: href=%2% name=%3%", e.Name(), e.ParentHref(), parent->RelPath(), log::error );
+			return true;
+		}
+		assert( parent->IsFolder() ) ;
+
+		std::string path = parent->IsRoot() ? e.Name() : ( parent->RelPath() / e.Name() ).string();
+		if ( IsIgnore( path ) )
+		{
+			Log( "%1% is ignored by grive", path, log::verbose ) ;
+			return true;
+		}
+
+		// see if the entry already exist in local
+		std::string name = e.Name() ;
+		Resource *child = parent->FindChild( name ) ;
+		if ( child )
+		{
+			// since we are updating the ID and Href, we need to remove it and re-add it.
+			m_res.Update( child, e ) ;
+		}
+		
+		// folder entry exist in google drive, but not local. we should create
+		// the directory
+		else if ( e.IsDir() || !e.Filename().empty() )
+		{
+			// first create a dummy resource and update it later
+			child = new Resource( name, e.IsDir() ? "folder" : "file" ) ;
+			parent->AddChild( child ) ;
+			m_res.Insert( child ) ;
+			
+			// update the state of the resource
+			m_res.Update( child, e ) ;
+		}
+		
+		return true ;
+	}
+	else
+		return false ;
+}
+
+Resource* State::FindByHref( const std::string& href )
+{
+	return m_res.FindByHref( href ) ;
+}
+
+State::iterator State::begin()
+{
+	return m_res.begin() ;
+}
+
+State::iterator State::end()
+{
+	return m_res.end() ;
+}
+
+void State::Read()
+{
+	try
+	{
+		File st_file( m_root / state_file ) ;
+		m_st = ParseJson( st_file );
+		m_cstamp = m_st["change_stamp"].Int() ;
+	}
+	catch ( Exception& )
+	{
+	}
+
+	try
+	{
+		File ign_file( m_root / ignore_file ) ;
+		char ign[MAX_IGN] = { 0 };
+		int s = ign_file.Read( ign, MAX_IGN-1 ) ;
+		ParseIgnoreFile( ign, s );
+	}
+	catch ( Exception& e )
+	{
+	}
+}
+
+std::vector<std::string> split( const boost::regex& re, const char* str, int len )
+{
+	std::vector<std::string> vec;
+	boost::cregex_token_iterator i( str, str+len, re, -1, boost::format_perl );
+	boost::cregex_token_iterator j;
+	while ( i != j )
+	{
+		vec.push_back( *i++ );
+	}
+	return vec;
+}
+
+bool State::ParseIgnoreFile( const char* buffer, int size )
+{
+	const boost::regex re1( "([^\\\\]|^)[\\t\\r ]+$" );
+	const boost::regex re2( "^[\\t\\r ]+" );
+	const boost::regex re4( "([^\\\\](\\\\\\\\)*|^)\\\\\\*" );
+	const boost::regex re5( "([^\\\\](\\\\\\\\)*|^)\\\\\\?" );
+	std::string exclude_re, include_re;
+	std::vector<std::string> lines = split( boost::regex( "[\\n\\r]+" ), buffer, size );
+	for ( int i = 0; i < (int)lines.size(); i++ )
+	{
+		std::string str = regex_replace( regex_replace( lines[i], re1, "$1" ), re2, "" );
+		if ( str[0] == '#' || !str.size() )
+		{
+			continue;
+		}
+		bool inc = str[0] == '!';
+		if ( inc )
+		{
+			str = str.substr( 1 );
+		}
+		std::vector<std::string> parts = split( boost::regex( "/+" ), str.c_str(), str.size() );
+		for ( int j = 0; j < (int)parts.size(); j++ )
+		{
+			if ( parts[j] == "**" )
+			{
+				parts[j] = ".*";
+			}
+			else if ( parts[j] == "*" )
+			{
+				parts[j] = "[^/]*";
+			}
+			else
+			{
+				parts[j] = regex_escape( parts[j] );
+				std::string str1;
+				while (1)
+				{
+					str1 = regex_replace( parts[j], re5, "$1[^/]", boost::format_perl );
+					str1 = regex_replace( str1, re4, "$1[^/]*", boost::format_perl );
+					if ( str1.size() == parts[j].size() )
+						break;
+					parts[j] = str1;
+				}
+			}
+		}
+		if ( !inc )
+		{
+			str = boost::algorithm::join( parts, "/" ) + "(/|$)";
+			exclude_re = exclude_re + ( exclude_re.size() > 0 ? "|" : "" ) + str;
+		}
+		else
+		{
+			str = "";
+			std::string cur;
+			for ( int j = 0; j < (int)parts.size(); j++ )
+			{
+				cur = cur.size() > 0 ? cur + "/" + parts[j] : "^" + parts[j];
+				str = ( str.size() > 0 ? str + "|" + cur : cur ) + ( j < (int)parts.size()-1 ? "$" : "(/|$)" );
+			}
+			include_re = include_re + ( include_re.size() > 0 ? "|" : "" ) + str;
+		}
+	}
+	if ( exclude_re.size() > 0 )
+	{
+		m_ign = "^" + ( include_re.size() > 0 ? "(?!" + include_re + ")" : std::string() ) + "(" + exclude_re + ")$";
+		return true;
+	}
+	return false;
+}
+
+void State::Write()
+{
+	m_st.Set( "change_stamp", Val( m_cstamp ) ) ;
+	m_st.Set( "ignore_regexp", Val( m_ign ) ) ;
+	
+	fs::path filename = m_root / state_file ;
+	std::ofstream fs( filename.string().c_str() ) ;
+	fs << m_st ;
+}
+
+void State::Sync( Syncer *syncer, const Val& options )
+{
+	// set the last sync time to the time on the client
+	m_res.Root()->Sync( syncer, &m_res, options ) ;
+}
+
+long State::ChangeStamp() const
+{
+	return m_cstamp ;
+}
+
+void State::ChangeStamp( long cstamp )
+{
+	Log( "change stamp is set to %1%", cstamp, log::verbose ) ;
+	m_cstamp = cstamp ;
+}
+
+} // end of namespace gr

libgrive/src/base/State.hh

@@ -23,63 +23,66 @@
 
 #include "util/DateTime.hh"
 #include "util/FileSystem.hh"
+#include "json/Val.hh"
 
 #include <memory>
+#include <boost/regex.hpp>
 
 namespace gr {
 
-namespace http
-{
-	class Agent ;
-	class Header ;
-}
-
-class Json ;
-class Resource ;
 class Entry ;
 
+class Syncer ;
+
+class Resource ;
+
 class State
 {
 public :
 	typedef ResourceTree::iterator iterator ;
 
 public :
-	explicit State( const fs::path& filename, const Json& options ) ;
+	explicit State( const fs::path& root, const Val& options ) ;
 	~State() ;
 	
 	void FromLocal( const fs::path& p ) ;
 	void FromRemote( const Entry& e ) ;
 	void ResolveEntry() ;
 	
-	void Read( const fs::path& filename ) ;
-	void Write( const fs::path& filename ) const ;
+	void Read() ;
+	void Write() ;
 
 	Resource* FindByHref( const std::string& href ) ;
 	Resource* FindByID( const std::string& id ) ;
-	Resource* Find( const fs::path& path ) ;
 
-	void Sync( http::Agent *http, const http::Header& auth ) ;
+	void Sync( Syncer *syncer, const Val& options ) ;
 	
 	iterator begin() ;
 	iterator end() ;
 	
 	long ChangeStamp() const ;
 	void ChangeStamp( long cstamp ) ;
-	
+
 private :
-	void FromLocal( const fs::path& p, Resource *folder ) ;
+	bool ParseIgnoreFile( const char* buffer, int size ) ;
+	void FromLocal( const fs::path& p, Resource *folder, Val& tree ) ;
 	void FromChange( const Entry& e ) ;
 	bool Update( const Entry& e ) ;
 	std::size_t TryResolveEntry() ;
 
-	static bool IsIgnore( const std::string& filename ) ;
+	bool IsIgnore( const std::string& filename ) ;
 	
 private :
+	fs::path			m_root ;
 	ResourceTree		m_res ;
-	DateTime			m_last_sync ;
-	long				m_cstamp ;
+	int					m_cstamp ;
+	std::string			m_ign ;
+	boost::regex		m_ign_re ;
+	Val					m_st ;
+	bool				m_force ;
+	bool				m_ign_changed ;
 	
-	std::vector<Entry>	m_unresolved ;
+	std::list<Entry>	m_unresolved ;
 } ;
 
-} // end of namespace
+} // end of namespace gr

libgrive/src/base/Syncer.cc

@@ -0,0 +1,59 @@
+/*
+	grive: an GPL program to sync a local directory with Google Drive
+	Copyright (C) 2012  Wan Wai Ho
+
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License
+	as published by the Free Software Foundation version 2
+	of the License.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with this program; if not, write to the Free Software
+	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+
+#include "Syncer.hh"
+#include "Resource.hh"
+#include "Entry.hh"
+#include "http/Agent.hh"
+#include "http/Header.hh"
+#include "http/Download.hh"
+#include "util/OS.hh"
+#include "util/log/Log.hh"
+
+namespace gr {
+
+Syncer::Syncer( http::Agent *http ):
+	m_http( http )
+{
+}
+
+http::Agent* Syncer::Agent() const
+{
+	return m_http;
+}
+
+void Syncer::Download( Resource *res, const fs::path& file )
+{
+	http::Download dl( file.string(), http::Download::NoChecksum() ) ;
+	long r = m_http->Get( res->ContentSrc(), &dl, http::Header(), res->Size() ) ;
+	if ( r <= 400 )
+	{
+		if ( res->ServerTime() != DateTime() )
+			os::SetFileTime( file, res->ServerTime() ) ;
+		else
+			Log( "encountered zero date time after downloading %1%", file, log::warning ) ;
+	}
+}
+
+void Syncer::AssignIDs( Resource *res, const Entry& remote )
+{
+	res->AssignIDs( remote );
+}
+
+} // end of namespace gr

libgrive/src/base/Syncer.hh

@@ -0,0 +1,71 @@
+/*
+	grive: an GPL program to sync a local directory with Google Drive
+	Copyright (C) 2012  Wan Wai Ho
+
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License
+	as published by the Free Software Foundation version 2
+	of the License.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with this program; if not, write to the Free Software
+	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+
+#pragma once
+
+#include "util/FileSystem.hh"
+
+#include <string>
+#include <vector>
+#include <iosfwd>
+
+namespace gr {
+
+namespace http
+{
+	class Agent ;
+}
+
+class DateTime ;
+
+class Resource ;
+
+class Entry ;
+
+class Feed ;
+
+/*!	\brief	A Syncer incapsulates all resource-related upload/download/edit methods */
+class Syncer
+{
+public :
+
+	Syncer( http::Agent *http );
+
+	http::Agent* Agent() const;
+
+	virtual void DeleteRemote( Resource *res ) = 0;
+	virtual void Download( Resource *res, const fs::path& file );
+	virtual bool EditContent( Resource *res, bool new_rev ) = 0;
+	virtual bool Create( Resource *res ) = 0;
+	virtual bool Move( Resource* res, Resource* newParent, std::string newFilename ) = 0;
+
+	virtual std::unique_ptr<Feed> GetFolders() = 0;
+	virtual std::unique_ptr<Feed> GetAll() = 0;
+	virtual std::unique_ptr<Feed> GetChanges( long min_cstamp ) = 0;
+	virtual long GetChangeStamp( long min_cstamp ) = 0;
+
+protected:
+
+	http::Agent *m_http;
+
+	void AssignIDs( Resource *res, const Entry& remote );
+
+} ;
+
+} // end of namespace gr

libgrive/src/bfd/SymbolInfo.cc

@@ -22,6 +22,7 @@
 
 #include <vector>
 
+#define PACKAGE "libgrive"
 #include <bfd.h>
 #include <execinfo.h>
 #include <dlfcn.h>
@@ -48,9 +49,9 @@ SymbolInfo::SymbolInfo( )
 	m_impl->m_bfd		= 0 ;
 	m_impl->m_symbols	= 0 ;
 	m_impl->m_symbol_count	= 0 ;
-	
+
 	bfd_init( ) ;
-	
+
 	// opening itself
 	bfd *b = bfd_openr( "/proc/self/exe", 0 ) ;
 	if ( b == NULL )
@@ -59,13 +60,13 @@ SymbolInfo::SymbolInfo( )
 		          << bfd_errmsg( bfd_get_error() ) << std::endl ;
 		return ;
 	}
-	
+
 	if ( bfd_check_format( b, bfd_archive ) )
 	{
 		bfd_close( b ) ;
 		return ;
 	}
-	
+
 	char **matching ;
 	if ( !bfd_check_format_matches( b, bfd_object, &matching ) )
 	{
@@ -77,7 +78,7 @@ SymbolInfo::SymbolInfo( )
 			std::cerr << bfd_get_filename( b ) << ": Matching formats: " ;
 			for ( char **p = matching ; *p != 0 ; p++ )
 				std::cerr << " " << *p ;
-			
+
 			std::cerr << std::endl ;
 			std::free( matching ) ;
 		}
@@ -106,7 +107,7 @@ struct SymbolInfo::BacktraceInfo
 	const char			*m_func_name ;
 	unsigned int		m_lineno ;
 	unsigned int		m_is_found ;
-	
+
 	static void Callback( bfd *abfd, asection *section, void* addr ) ;
 } ;
 
@@ -116,17 +117,24 @@ void SymbolInfo::BacktraceInfo::Callback( bfd *abfd, asection *section,
 	BacktraceInfo *info = (BacktraceInfo *)data ;
 	if ((section->flags & SEC_ALLOC) == 0)
 		return ;
-	
-	bfd_vma vma = bfd_get_section_vma(abfd, section);
-	
+
+	// bfd_get_section_vma works up to 7b1cfbcf1a27951fb1b3a212995075dd6fdf985b,
+	// removed in 7c13bc8c91abf291f0206b6608b31955c5ea70d8 (binutils 2.33.1 or so)
+	// so it's substituted by its implementation to avoid checking for binutils
+	// version (which at least on Debian SID it's not that easy because the
+	// version.h is not included with the official package)
+	bfd_vma vma = section->vma;
+
 	unsigned long address = (unsigned long)(info->m_addr);
 	if ( address < vma )
 		return;
-	
-	bfd_size_type size = bfd_section_size(abfd, section);
+
+	// bfd_section_size changed between the two objects described above,
+	// same rationale applies
+	bfd_size_type size = section->size;
 	if ( address > (vma + size))
 		return ;
-	
+
 	const SymbolInfo *pthis	= info->m_pthis ;
 	info->m_is_found	=  bfd_find_nearest_line( abfd, section,
 	                                              pthis->m_impl->m_symbols,
@@ -148,7 +156,7 @@ void SymbolInfo::PrintTrace( void *addr, std::ostream& os, std::size_t idx )
 	{
 		this, addr, 0, 0, 0, false
 	} ;
-	
+
 	Dl_info sym ;
 	bfd_map_over_sections( m_impl->m_bfd,
 							&SymbolInfo::BacktraceInfo::Callback,
@@ -164,7 +172,7 @@ if ( btinfo.m_is_found )
 			filename.erase( pos, std::strlen( SRC_DIR ) ) ;
 #endif
 		os << "#"  << idx << " " << addr << " "
-			<< filename << ":" << btinfo.m_lineno 
+			<< filename << ":" << btinfo.m_lineno
 			<< " "
 			<< (btinfo.m_func_name != 0 ? Demangle(btinfo.m_func_name) : "" )
 			<< std::endl ;

libgrive/src/bfd/SymbolInfo.hh

@@ -54,7 +54,7 @@ public :
 	
 private :
 	struct Impl ;
-	const std::auto_ptr<Impl> m_impl ;
+	const std::unique_ptr<Impl> m_impl ;
 	
 	struct BacktraceInfo ;
 	friend struct BacktraceInfo ;

libgrive/src/drive/CommonUri.hh

@@ -1,36 +0,0 @@
-/*
-	grive: an GPL program to sync a local directory with Google Drive
-	Copyright (C) 2012  Wan Wai Ho
-
-	This program is free software; you can redistribute it and/or
-	modify it under the terms of the GNU General Public License
-	as published by the Free Software Foundation version 2
-	of the License.
-
-	This program is distributed in the hope that it will be useful,
-	but WITHOUT ANY WARRANTY; without even the implied warranty of
-	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-	GNU General Public License for more details.
-
-	You should have received a copy of the GNU General Public License
-	along with this program; if not, write to the Free Software
-	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
-*/
-
-#pragma once
-
-#include <string>
-
-namespace gr
-{
-	const std::string feed_base		= "https://docs.google.com/feeds/default/private/full" ;
-	const std::string feed_changes	= "https://docs.google.com/feeds/default/private/changes" ;
-	const std::string feed_metadata	= "https://docs.google.com/feeds/metadata/default" ;
-	
-	const std::string root_href =
-		"https://docs.google.com/feeds/default/private/full/folder%3Aroot" ;
-	const std::string root_create =
-		"https://docs.google.com/feeds/upload/create-session/default/private/full" ;
-	
-	std::string ChangesFeed( int changestamp ) ;
-}

libgrive/src/drive/Drive.cc

@@ -1,196 +0,0 @@
-/*
-	grive: an GPL program to sync a local directory with Google Drive
-	Copyright (C) 2012  Wan Wai Ho
-
-	This program is free software; you can redistribute it and/or
-	modify it under the terms of the GNU General Public License
-	as published by the Free Software Foundation version 2
-	of the License.
-
-	This program is distributed in the hope that it will be useful,
-	but WITHOUT ANY WARRANTY; without even the implied warranty of
-	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-	GNU General Public License for more details.
-
-	You should have received a copy of the GNU General Public License
-	along with this program; if not, write to the Free Software
-	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
-*/
-
-#include "Drive.hh"
-
-#include "CommonUri.hh"
-#include "Entry.hh"
-#include "Feed.hh"
-
-#include "http/CurlAgent.hh"
-#include "http/ResponseLog.hh"
-#include "http/XmlResponse.hh"
-#include "protocol/Json.hh"
-#include "protocol/OAuth2.hh"
-#include "util/Destroy.hh"
-#include "util/log/Log.hh"
-#include "xml/Node.hh"
-#include "xml/NodeSet.hh"
-
-#include <boost/bind.hpp>
-
-// standard C++ library
-#include <algorithm>
-#include <cassert>
-#include <cstdlib>
-#include <fstream>
-#include <map>
-#include <sstream>
-
-// for debugging only
-#include <iostream>
-
-namespace gr {
-
-namespace
-{
-	const std::string state_file = ".grive_state" ;
-}
-
-Drive::Drive( OAuth2& auth, const Json& options ) :
-	m_auth( auth ),
-	m_state( state_file, options )
-{
-	m_http_hdr.Add( "Authorization: Bearer " + m_auth.AccessToken() ) ;
-	m_http_hdr.Add( "GData-Version: 3.0" ) ;
-}
-
-void Drive::FromRemote( const Entry& entry )
-{
-	// entries from change feed does not have the parent HREF,
-	// so these checkings are done in normal entries only
-	Resource *parent = m_state.FindByHref( entry.ParentHref() ) ;
-	
-	if ( parent != 0 && !parent->IsFolder() )
-		Log( "warning: entry %1% has parent %2% which is not a folder, ignored",
-			entry.Title(), parent->Name(), log::verbose ) ;
-	
-	else if ( parent == 0 || !parent->IsInRootTree() )
-		Log( "file \"%1%\" parent doesn't exist, ignored", entry.Title(), log::verbose ) ;
-		
-	else
-		m_state.FromRemote( entry ) ;
-}
-
-void Drive::FromChange( const Entry& entry )
-{
-	if ( entry.IsRemoved() )
-		Log( "file \"%1%\" represents a deletion, ignored", entry.Title(), log::verbose ) ;
-	
-	// folders go directly
-	else
-		m_state.FromRemote( entry ) ;
-}
-
-void Drive::SaveState()
-{
-	m_state.Write( state_file ) ;
-}
-
-void Drive::SyncFolders( http::Agent *http )
-{
-	assert( http != 0 ) ;
-
-	Log( "Synchronizing folders", log::info ) ;
-
-	http::XmlResponse xml ;
-	http->Get( feed_base + "/-/folder?max-results=50&showroot=true", &xml, m_http_hdr ) ;
-	
-	Feed feed( xml.Response() ) ;
-	do
-	{
-		// first, get all collections from the query result
-		for ( Feed::iterator i = feed.begin() ; i != feed.end() ; ++i )
-		{
-			Entry e( *i ) ;
-			if ( e.Kind() == "folder" )
-			{
-				if ( e.ParentHrefs().size() != 1 )
-					Log( "folder \"%1%\" has multiple parents, ignored", e.Title(), log::verbose ) ;
-				
-				else if ( e.Title().find('/') != std::string::npos )
-					Log( "folder \"%1%\" contains a slash in its name, ignored", e.Title(), log::verbose ) ;
-				
-				else
-					m_state.FromRemote( e ) ;
-			}
-		}
-	} while ( feed.GetNext( http, m_http_hdr ) ) ;
-
-	m_state.ResolveEntry() ;
-}
-
-void Drive::DetectChanges()
-{
-	Log( "Reading local directories", log::info ) ;
-	m_state.FromLocal( "." ) ;
-	
-	http::CurlAgent http ;
-
-	long prev_stamp = m_state.ChangeStamp() ;
-	Trace( "previous change stamp is %1%", prev_stamp ) ;
-	
-	SyncFolders( &http ) ;
-
-	Log( "Reading remote server file list", log::info ) ;
-	http::XmlResponse xrsp ;
-	http.Get( feed_base + "?showfolders=true&showroot=true", &xrsp, m_http_hdr ) ;
-	xml::Node resp = xrsp.Response() ;
-
-	m_resume_link = resp["link"].
-		Find( "@rel", "http://schemas.google.com/g/2005#resumable-create-media" )["@href"] ;
-
-	Feed feed( resp ) ;
-	do
-	{
-		std::for_each( feed.begin(), feed.end(), boost::bind( &Drive::FromRemote, this, _1 ) ) ;
-	} while ( feed.GetNext( &http, m_http_hdr ) ) ;
-	
-	// pull the changes feed
-	if ( prev_stamp != -1 )
-	{
-		http::ResponseLog log( "/tmp/changes-", ".xml", &xrsp ) ;
-		Log( "Detecting changes from last sync", log::info ) ;
-		http.Get( ChangesFeed(prev_stamp+1), &log, m_http_hdr ) ;
-		
-		Feed changes( xrsp.Response() ) ;
-		std::for_each( changes.begin(), changes.end(), boost::bind( &Drive::FromChange, this, _1 ) ) ;
-	}
-}
-
-void Drive::Update()
-{
-	Log( "Synchronizing files", log::info ) ;
-	http::CurlAgent http ;
-	m_state.Sync( &http, m_http_hdr ) ;
-	
-	UpdateChangeStamp( &http ) ;
-}
-
-void Drive::DryRun()
-{
-	Log( "Synchronizing files (dry-run)", log::info ) ;
-	m_state.Sync( 0, m_http_hdr ) ;
-}
-
-void Drive::UpdateChangeStamp( http::Agent *http )
-{
-	assert( http != 0 ) ;
-
-	// get changed feed
-	http::XmlResponse xrsp ;
-	http->Get( ChangesFeed(m_state.ChangeStamp()+1), &xrsp, m_http_hdr ) ;
-	
-	// we should go through the changes to see if it was really Grive to made that change
-	// maybe by recording the updated timestamp and compare it?
-	m_state.ChangeStamp( 
-		std::atoi(xrsp.Response()["docs:largestChangestamp"]["@value"].front().Value().c_str()) ) ;
-}
-
-} // end of namespace

libgrive/src/drive/Entry.cc

@@ -1,198 +0,0 @@
-/*
-	grive: an GPL program to sync a local directory with Google Drive
-	Copyright (C) 2012  Wan Wai Ho
-
-	This program is free software; you can redistribute it and/or
-	modify it under the terms of the GNU General Public License
-	as published by the Free Software Foundation version 2
-	of the License.
-
-	This program is distributed in the hope that it will be useful,
-	but WITHOUT ANY WARRANTY; without even the implied warranty of
-	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-	GNU General Public License for more details.
-
-	You should have received a copy of the GNU General Public License
-	along with this program; if not, write to the Free Software
-	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
-*/
-
-#include "Entry.hh"
-#include "CommonUri.hh"
-
-#include "util/Crypt.hh"
-#include "util/log/Log.hh"
-#include "util/OS.hh"
-#include "xml/Node.hh"
-#include "xml/NodeSet.hh"
-
-#include <algorithm>
-#include <iterator>
-
-namespace gr {
-
-/// construct an entry for the root folder
-Entry::Entry( ) :
-	m_title			( "." ),
-	m_kind			( "folder" ),
-	m_resource_id	( "folder:root" ),
-	m_self_href		( root_href ),
-	m_create_link	( root_create ),
-	m_change_stamp	( -1 ),
-	m_is_removed	( false )
-{
-}
-
-/// construct an entry for remote
-Entry::Entry( const xml::Node& n ) :
-	m_change_stamp( -1 ),
-	m_is_removed( false )
-{
-	Update( n ) ;
-}
-
-void Entry::Update( const xml::Node& n )
-{
-	m_title			= n["title"] ;
-	m_etag			= n["@gd:etag"] ;
-	m_filename		= n["docs:suggestedFilename"] ;
-	m_content_src	= n["content"]["@src"] ;
-	m_self_href		= n["link"].Find( "@rel", "self" )["@href"] ;
-	m_alt_self		= n["link"].Find( "@rel", "http://schemas.google.com/docs/2007#alt-self" )["@href"] ;
-	m_mtime			= DateTime( n["updated"] ) ;
-
-	m_resource_id	= n["gd:resourceId"] ;
-	m_md5			= n["docs:md5Checksum"] ;
-	m_kind			= n["category"].Find( "@scheme", "http://schemas.google.com/g/2005#kind" )["@label"] ;
-	m_edit_link		= n["link"].Find( "@rel", "http://schemas.google.com/g/2005#resumable-edit-media")["@href"] ;
-	m_create_link	= n["link"].Find( "@rel", "http://schemas.google.com/g/2005#resumable-create-media")["@href"] ;
-
-	// changestamp only appear in change feed entries
-	xml::NodeSet cs	= n["docs:changestamp"]["@value"] ;
-	m_change_stamp	= cs.empty() ? -1 : std::atoi( cs.front().Value().c_str() ) ;
-
-	m_parent_hrefs.clear( ) ;
-	xml::NodeSet parents = n["link"].Find( "@rel", "http://schemas.google.com/docs/2007#parent" ) ;
-	for ( xml::NodeSet::iterator i = parents.begin() ; i != parents.end() ; ++i )
-		m_parent_hrefs.push_back( (*i)["@href"] ) ;
-
-	// convert to lower case for easy comparison
-	std::transform( m_md5.begin(), m_md5.end(), m_md5.begin(), tolower ) ;
-
-	m_is_removed = !n["gd:deleted"].empty() || !n["docs:removed"].empty() ;
-}
-
-const std::vector<std::string>& Entry::ParentHrefs() const
-{
-	return m_parent_hrefs ;
-}
-
-std::string Entry::Title() const
-{
-	return m_title ;
-}
-
-std::string Entry::Filename() const
-{
-	return m_filename ;
-}
-
-std::string Entry::Kind() const
-{
-	return m_kind ;
-}
-
-std::string Entry::MD5() const
-{
-	return m_md5 ;
-}
-
-DateTime Entry::MTime() const
-{
-	return m_mtime ;
-}
-
-std::string Entry::SelfHref() const
-{
-	return m_self_href ;
-}
-
-std::string Entry::AltSelf() const
-{
-	return m_alt_self ;
-}
-
-std::string Entry::ParentHref() const
-{
-	return m_parent_hrefs.empty() ? "" : m_parent_hrefs.front() ;
-}
-
-std::string Entry::ResourceID() const
-{
-	return m_resource_id ;
-}
-
-std::string Entry::ETag() const
-{
-	return m_etag ;
-}
-
-std::string Entry::ContentSrc() const
-{
-	return m_content_src ;
-}
-
-std::string Entry::EditLink() const
-{
-	return m_edit_link ;
-}
-
-std::string Entry::CreateLink() const
-{
-	return m_create_link ;
-}
-
-void Entry::Swap( Entry& e )
-{
-	m_title.swap( e.m_title ) ;
-	m_filename.swap( e.m_filename ) ;
-	m_kind.swap( e.m_kind ) ;
-	m_md5.swap( e.m_md5 ) ;
-	m_etag.swap( e.m_etag ) ;
-	m_resource_id.swap( e.m_resource_id ) ;
-
-	m_parent_hrefs.swap( e.m_parent_hrefs ) ;
-
-	m_self_href.swap( e.m_self_href ) ;
-	m_alt_self.swap( e.m_alt_self ) ;
-	m_content_src.swap( e.m_content_src ) ;
-	m_edit_link.swap( e.m_edit_link ) ;
-	m_create_link.swap( e.m_create_link ) ;
-
-	m_mtime.Swap( e.m_mtime ) ;
-
-	std::swap( m_change_stamp, e.m_change_stamp ) ;
-	std::swap( m_is_removed, e.m_is_removed ) ;
-}
-
-long Entry::ChangeStamp() const
-{
-	return m_change_stamp ;
-}
-
-bool Entry::IsChange() const
-{
-	return m_change_stamp != -1 ;
-}
-
-bool Entry::IsRemoved() const
-{
-	return m_is_removed ;
-}
-
-std::string Entry::Name() const
-{
-	return m_kind == "file" ? m_filename : m_title ;
-}
-
-} // end of namespace

libgrive/src/drive/Feed.cc

@@ -1,93 +0,0 @@
-/*
-	grive: an GPL program to sync a local directory with Google Drive
-	Copyright (C) 2012  Wan Wai Ho
-
-	This program is free software; you can redistribute it and/or
-	modify it under the terms of the GNU General Public License
-	as published by the Free Software Foundation version 2
-	of the License.
-
-	This program is distributed in the hope that it will be useful,
-	but WITHOUT ANY WARRANTY; without even the implied warranty of
-	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-	GNU General Public License for more details.
-
-	You should have received a copy of the GNU General Public License
-	along with this program; if not, write to the Free Software
-	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
-*/
-
-#include "Feed.hh"
-
-#include "http/Agent.hh"
-#include "http/XmlResponse.hh"
-#include "xml/NodeSet.hh"
-
-#include <cassert>
-
-namespace gr {
-
-Feed::Feed( const xml::Node& root ) :
-	m_root		( root ),
-	m_entries	( m_root["entry"] )
-{
-}
-
-void Feed::Assign( const xml::Node& root )
-{
-	m_root		= root ;
-	m_entries	= m_root["entry"] ;
-}
-
-Feed::iterator Feed::begin() const
-{
-	return iterator( m_entries.begin() ) ;
-}
-
-Feed::iterator Feed::end() const
-{
-	return iterator( m_entries.end() ) ;
-}
-
-std::string Feed::Next() const
-{
-	xml::NodeSet nss = m_root["link"].Find( "@rel", "next" ) ;
-	return nss.empty() ? "" : std::string(nss["@href"]) ;
-}
-
-bool Feed::GetNext( http::Agent *http, const http::Header& auth )
-{
-	assert( http != 0 ) ;
-
-	xml::NodeSet nss = m_root["link"].Find( "@rel", "next" ) ;
-	if ( !nss.empty() )
-	{
-		http::XmlResponse xrsp ;
-		http->Get( nss["@href"], &xrsp, auth ) ;
-		
-		m_root		= xrsp.Response() ;
-		m_entries	= m_root["entry"] ;
-		
-		return true ;
-	}
-	else
-		return false ;
-}
-
-Feed::iterator::iterator( )
-{
-}
-
-Feed::iterator::iterator( xml::Node::iterator i )
-{
-	// for some reason, gcc 4.4.4 doesn't allow me to initialize the base class
-	// in the initializer. I have no choice but to initialize here.
-	base_reference() = i ;
-}
-
-Feed::iterator::reference Feed::iterator::dereference() const
-{
-	return Entry( *base_reference() ) ;
-}
-
-} // end of namespace

libgrive/src/drive/Feed.hh

@@ -1,77 +0,0 @@
-/*
-	grive: an GPL program to sync a local directory with Google Drive
-	Copyright (C) 2012  Wan Wai Ho
-
-	This program is free software; you can redistribute it and/or
-	modify it under the terms of the GNU General Public License
-	as published by the Free Software Foundation version 2
-	of the License.
-
-	This program is distributed in the hope that it will be useful,
-	but WITHOUT ANY WARRANTY; without even the implied warranty of
-	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-	GNU General Public License for more details.
-
-	You should have received a copy of the GNU General Public License
-	along with this program; if not, write to the Free Software
-	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
-*/
-
-#pragma once
-
-#include "Entry.hh"
-
-#include "xml/Node.hh"
-#include "xml/NodeSet.hh"
-
-#include <boost/iterator_adaptors.hpp>
-
-#include <string>
-
-namespace gr {
-
-namespace http
-{
-	class Agent ;
-	class Header ;
-}
-
-class Feed
-{
-public :
-	class iterator ;
-
-public :
-	explicit Feed( const xml::Node& root ) ;
-	void Assign( const xml::Node& root ) ;
-	
-	iterator begin() const ;
-	iterator end() const ;
-	
-	std::string Next() const ;
-	bool GetNext( http::Agent *http, const http::Header& auth ) ;
-		
-private :
-	xml::Node		m_root ;
-	xml::NodeSet	m_entries ;
-} ;
-
-class Feed::iterator : public boost::iterator_adaptor<
-	Feed::iterator,
-	xml::Node::iterator,
-	Entry,
-	boost::random_access_traversal_tag,
-	Entry
->
-{
-public :
-	iterator() ;
-	explicit iterator( xml::Node::iterator i ) ;		
-
-private :
-	friend class boost::iterator_core_access;
-	
-	reference dereference() const ;
-} ;
-
-} // end of namespace

libgrive/src/drive/Resource.cc

@@ -1,671 +0,0 @@
-/*
-	grive: an GPL program to sync a local directory with Google Drive
-	Copyright (C) 2012  Wan Wai Ho
-
-	This program is free software; you can redistribute it and/or
-	modify it under the terms of the GNU General Public License
-	as published by the Free Software Foundation version 2
-	of the License.
-
-	This program is distributed in the hope that it will be useful,
-	but WITHOUT ANY WARRANTY; without even the implied warranty of
-	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-	GNU General Public License for more details.
-
-	You should have received a copy of the GNU General Public License
-	along with this program; if not, write to the Free Software
-	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
-*/
-
-#include "Resource.hh"
-#include "CommonUri.hh"
-#include "Entry.hh"
-
-#include "http/Agent.hh"
-#include "http/Download.hh"
-#include "http/Header.hh"
-// #include "http/ResponseLog.hh"
-#include "http/StringResponse.hh"
-#include "http/XmlResponse.hh"
-#include "protocol/Json.hh"
-#include "util/CArray.hh"
-#include "util/Crypt.hh"
-#include "util/log/Log.hh"
-#include "util/OS.hh"
-#include "util/StdioFile.hh"
-#include "xml/Node.hh"
-#include "xml/NodeSet.hh"
-#include "xml/String.hh"
-
-#include <boost/bind.hpp>
-#include <boost/exception/all.hpp>
-
-#include <cassert>
-
-// for debugging
-#include <iostream>
-
-namespace gr {
-
-// hard coded XML file
-const std::string xml_meta =
-	"<?xml version='1.0' encoding='UTF-8'?>\n"
-	"<entry xmlns=\"http://www.w3.org/2005/Atom\" xmlns:docs=\"http://schemas.google.com/docs/2007\">"
-		"<category scheme=\"http://schemas.google.com/g/2005#kind\" "
-		"term=\"http://schemas.google.com/docs/2007#%1%\"/>"
-		"<title>%2%</title>"
-	"</entry>" ;
-
-
-/// default constructor creates the root folder
-Resource::Resource() :
-	m_name		( "." ),
-	m_kind		( "folder" ),
-	m_id		( "folder:root" ),
-	m_href		( root_href ),
-	m_create	( root_create ),
-	m_parent	( 0 ),
-	m_state		( sync )
-{
-}
-
-Resource::Resource( const std::string& name, const std::string& kind ) :
-	m_name		( name ),
-	m_kind		( kind ),
-	m_parent	( 0 ),
-	m_state		( unknown )
-{
-}
-
-void Resource::SetState( State new_state )
-{
-	// only the new and delete states need to be set recursively
-	assert(
-		new_state == remote_new || new_state == remote_deleted ||
-		new_state == local_new  || new_state == local_deleted
-	) ;
-	
-	m_state = new_state ;
-	std::for_each( m_child.begin(), m_child.end(),
-		boost::bind( &Resource::SetState, _1, new_state ) ) ;
-}
-
-void Resource::FromRemoteFolder( const Entry& remote, const DateTime& last_sync )
-{
-	fs::path path = Path() ;
-	
-	if ( remote.CreateLink().empty() )
-		Log( "folder %1% is read-only", path, log::verbose ) ;
-		
-	// already sync
-	if ( fs::is_directory( path ) )
-	{
-		Log( "folder %1% is in sync", path, log::verbose ) ;
-		m_state = sync ;
-	}
-	
-	// remote file created after last sync, so remote is newer
-	else if ( remote.MTime() > last_sync )
-	{
-		if ( fs::exists( path ) )
-		{
-			// TODO: handle type change
-			Log( "%1% changed from folder to file", path, log::verbose ) ;
-			m_state = sync ;
-		}
-		else
-		{
-			// make all children as remote_new, if any
-			Log( "folder %1% is created in remote", path, log::verbose ) ;
-			SetState( remote_new ) ;
-		}
-	}
-	else
-	{
-		if ( fs::exists( path ) )
-		{
-			// TODO: handle type chage
-			Log( "%1% changed from file to folder", path, log::verbose ) ;
-			m_state = sync ;
-		}
-		else
-		{
-			Log( "folder %1% is deleted in local", path, log::verbose ) ;
-			SetState( local_deleted ) ;
-		}
-	}
-}
-
-/// Update the state according to information (i.e. Entry) from remote. This function
-/// compares the modification time and checksum of both copies and determine which
-/// one is newer.
-void Resource::FromRemote( const Entry& remote, const DateTime& last_sync )
-{
-	// sync folder
-	if ( remote.Kind() == "folder" && IsFolder() )
-		FromRemoteFolder( remote, last_sync ) ;
-	else
-		FromRemoteFile( remote, last_sync ) ;
-	
-	AssignIDs( remote ) ;
-	
-	assert( m_state != unknown ) ;
-	
-	if ( m_state == remote_new || m_state == remote_changed )
-	{
-		m_md5	= remote.MD5() ;
-		m_mtime	= remote.MTime() ;
-	}
-}
-
-void Resource::AssignIDs( const Entry& remote )
-{
-	// the IDs from change feed entries are different
-	if ( !remote.IsChange() )
-	{
-		m_id		= remote.ResourceID() ;
-		m_href		= remote.SelfHref() ;
-		m_edit		= remote.EditLink() ;
-		m_create	= remote.CreateLink() ;
-		m_content	= remote.ContentSrc() ;
-		m_etag		= remote.ETag() ;
-	}
-}
-
-void Resource::FromRemoteFile( const Entry& remote, const DateTime& last_sync )
-{
-	assert( m_parent != 0 ) ;
-	
-	fs::path path = Path() ;
-
-	// recursively create/delete folder
-	if ( m_parent->m_state == remote_new || m_parent->m_state == remote_deleted ||
-		 m_parent->m_state == local_new  || m_parent->m_state == local_deleted )
-	{
-		Log( "file %1% parent %2% recursively in %3% (%4%)", path,
-			( m_parent->m_state == remote_new || m_parent->m_state == local_new )      ? "created" : "deleted",
-			( m_parent->m_state == remote_new || m_parent->m_state == remote_deleted ) ? "remote"  : "local",
-			m_parent->m_state, log::verbose ) ;
-		
-		m_state = m_parent->m_state ;
-	}
-
-	// local not exists
-	else if ( !fs::exists( path ) )
-	{
-		Trace( "file %1% change stamp = %2%", Path(), remote.ChangeStamp() ) ;
-		
-		if ( remote.MTime() > last_sync || remote.ChangeStamp() > 0 )
-		{
-			Log( "file %1% is created in remote (change %2%)", path,
-				remote.ChangeStamp(), log::verbose ) ;
-			
-			m_state = remote_new ;
-		}
-		else
-		{
-			Log( "file %1% is deleted in local", path, log::verbose ) ;
-			m_state = local_deleted ;
-		}
-	}
-	
-	// if checksum is equal, no need to compare the mtime
-	else if ( remote.MD5() == m_md5 )
-	{
-		Log( "file %1% is already in sync", Path(), log::verbose ) ;
-		m_state = sync ;
-	}
-
-	// use mtime to check which one is more recent
-	else
-	{
-		assert( m_state != unknown ) ;
-
-		// if remote is modified
-		if ( remote.MTime() > m_mtime )
-		{
-			Log( "file %1% is changed in remote", path, log::verbose ) ;
-			m_state = remote_changed ;
-		}
-		
-		// remote also has the file, so it's not new in local
-		else if ( m_state == local_new || m_state == remote_deleted )
-		{
-			Log( "file %1% is changed in local", path, log::verbose ) ;
-			m_state = local_changed ;
-		}
-		else
-			Trace( "file %1% state is %2%", m_name, m_state ) ;
-	}
-}
-
-/// Update the resource with the attributes of local file or directory. This
-/// function will propulate the fields in m_entry.
-void Resource::FromLocal( const DateTime& last_sync )
-{
-	fs::path path = Path() ;
-	assert( fs::exists( path ) ) ;
-
-	// root folder is always in sync
-	if ( !IsRoot() )
-	{
-		m_mtime = os::FileCTime( path ) ;
-
-		// follow parent recursively
-		if ( m_parent->m_state == local_new || m_parent->m_state == local_deleted )
-			m_state = local_new ;
-		
-		// if the file is not created after last sync, assume file is
-		// remote_deleted first, it will be updated to sync/remote_changed
-		// in FromRemote()
-		else
-			m_state = ( m_mtime > last_sync ? local_new : remote_deleted ) ;
-		
-		m_name		= Path2Str( path.filename() ) ;
-		m_kind		= fs::is_directory(path) ? "folder"	: "file" ;
-		m_md5		= fs::is_directory(path) ? ""		: crypt::MD5::Get( path ) ;
-	}
-	
-	assert( m_state != unknown ) ;
-}
-
-std::string Resource::SelfHref() const
-{
-	return m_href ;
-}
-
-std::string Resource::Name() const
-{
-	return m_name ;
-}
-
-std::string Resource::ResourceID() const
-{
-	return m_id ;
-}
-
-const Resource* Resource::Parent() const
-{
-	assert( m_parent == 0 || m_parent->IsFolder() ) ;
-	return m_parent ;
-}
-
-Resource* Resource::Parent()
-{
-	assert( m_parent == 0 || m_parent->IsFolder() ) ;
-	return m_parent ;
-}
-
-void Resource::AddChild( Resource *child )
-{
-	assert( child != 0 ) ;
-	assert( child->m_parent == 0 || child->m_parent == this ) ;
-	assert( child != this ) ;
-
-	child->m_parent = this ;
-	m_child.push_back( child ) ;
-}
-
-void Resource::Swap( Resource& coll )
-{
-	m_name.swap( coll.m_name ) ;
-	m_kind.swap( coll.m_kind ) ;
-	m_md5.swap( coll.m_md5 ) ;
-	m_etag.swap( coll.m_etag ) ;
-	m_id.swap( coll.m_id ) ;
-
-	m_href.swap( coll.m_href ) ;
-	m_content.swap( coll.m_content ) ;	
-	m_edit.swap( coll.m_edit ) ;
-	m_create.swap( coll.m_create ) ;
-	
-	m_mtime.Swap( coll.m_mtime ) ;
-	
-	std::swap( m_parent, coll.m_parent ) ;
-	m_child.swap( coll.m_child ) ;
-	std::swap( m_state, coll.m_state ) ;
-}
-
-bool Resource::IsFolder() const
-{
-	return m_kind == "folder" ;
-}
-
-fs::path Resource::Path() const
-{
-	assert( m_parent != this ) ;
-	assert( m_parent == 0 || m_parent->IsFolder() ) ;
-
-	return m_parent != 0 ? (m_parent->Path() / m_name) : m_name ;
-}
-
-bool Resource::IsInRootTree() const
-{
-	assert( m_parent == 0 || m_parent->IsFolder() ) ;
-	return m_parent == 0 ? (SelfHref() == root_href) : m_parent->IsInRootTree() ;
-}
-
-Resource* Resource::FindChild( const std::string& name )
-{
-	for ( std::vector<Resource*>::iterator i = m_child.begin() ; i != m_child.end() ; ++i )
-	{
-		assert( (*i)->m_parent == this ) ;
-		if ( (*i)->m_name == name )
-			return *i ;
-	}
-	return 0 ;
-}
-
-// try to change the state to "sync"
-void Resource::Sync( http::Agent *http, const http::Header& auth )
-{
-	assert( m_state != unknown ) ;
-	assert( !IsRoot() || m_state == sync ) ;	// root folder is already synced
-	
-	SyncSelf( http, auth ) ;
-	
-	// if myself is deleted, no need to do the childrens
-	if ( m_state != local_deleted && m_state != remote_deleted )
-		std::for_each( m_child.begin(), m_child.end(),
-			boost::bind( &Resource::Sync, _1, http, auth ) ) ;
-}
-
-void Resource::SyncSelf( http::Agent* http, const http::Header& auth )
-{
-	assert( !IsRoot() || m_state == sync ) ;	// root is always sync
-	assert( IsRoot() || http == 0 || fs::is_directory( m_parent->Path() ) ) ;
-	assert( IsRoot() || m_parent->m_state != remote_deleted ) ;
-	assert( IsRoot() || m_parent->m_state != local_deleted ) ;
-
-	const fs::path path = Path() ;
-
-	switch ( m_state )
-	{
-	case local_new :
-		Log( "sync %1% doesn't exist in server, uploading", path, log::info ) ;
-		
-		if ( http != 0 && Create( http, auth ) )
-			m_state = sync ;
-		break ;
-	
-	case local_deleted :
-		Log( "sync %1% deleted in local. deleting remote", path, log::info ) ;
-		if ( http != 0 )
-			DeleteRemote( http, auth ) ;
-		break ;
-	
-	case local_changed :
-		Log( "sync %1% changed in local. uploading", path, log::info ) ;
-		if ( http != 0 && EditContent( http, auth ) )
-			m_state = sync ;
-		break ;
-	
-	case remote_new :
-		Log( "sync %1% created in remote. creating local", path, log::info ) ;
-		if ( http != 0 )
-		{
-			if ( IsFolder() )
-				fs::create_directories( path ) ;
-			else
-				Download( http, path, auth ) ;
-			
-			m_state = sync ;
-		}
-		break ;
-	
-	case remote_changed :
-		assert( !IsFolder() ) ;
-		Log( "sync %1% changed in remote. downloading", path, log::info ) ;
-		if ( http != 0 )
-		{
-			Download( http, path, auth ) ;
-			m_state = sync ;
-		}
-		break ;
-	
-	case remote_deleted :
-		Log( "sync %1% deleted in remote. deleting local", path, log::info ) ;
-		if ( http != 0 )
-			DeleteLocal() ;
-		break ;
-	
-	case sync :
-		Log( "sync %1% already in sync", path, log::verbose ) ;
-		break ;
-
-	// shouldn't go here
-	case unknown :
-		assert( false ) ;
-		break ;
-		
-	default :
-		break ;
-	}
-}
-
-/// this function doesn't really remove the local file. it renames it.
-void Resource::DeleteLocal()
-{
-	static const boost::format trash_file( "%1%-%2%" ) ;
-
-	assert( m_parent != 0 ) ;
-	fs::path parent = m_parent->Path() ;
-	fs::path dest	= ".trash" / parent / Name() ;
-	
-	std::size_t idx = 1 ;
-	while ( fs::exists( dest ) && idx != 0 )
-		dest = ".trash" / parent / (boost::format(trash_file) % Name() % idx++).str() ;
-	
-	// wrap around! just remove the file
-	if ( idx == 0 )
-		fs::remove_all( Path() ) ;
-	else
-	{
-		fs::create_directories( dest.parent_path() ) ;
-		fs::rename( Path(), dest ) ;
-	}
-}
-
-void Resource::DeleteRemote( http::Agent *http, const http::Header& auth )
-{
-	assert( http != 0 ) ;
-	http::StringResponse str ;
-	
-	try
-	{
-		http::Header hdr( auth ) ;
-		hdr.Add( "If-Match: " + m_etag ) ;
-		
-		// doesn't know why, but an update before deleting seems to work always
-		http::XmlResponse xml ;
-		http->Get( m_href, &xml, hdr ) ;
-		AssignIDs( Entry( xml.Response() ) ) ;
-	
-		http->Custom( "DELETE", m_href, &str, hdr ) ;
-	}
-	catch ( Exception& e )
-	{
-		// don't rethrow here. there are some cases that I don't know why
-		// the delete will fail.
-		Trace( "Exception %1% %2%",
-			boost::diagnostic_information(e),
-			str.Response() ) ;
-	}
-}
-
-
-void Resource::Download( http::Agent* http, const fs::path& file, const http::Header& auth ) const
-{
-	assert( http != 0 ) ;
-	
-	http::Download dl( file.string(), http::Download::NoChecksum() ) ;
-	long r = http->Get( m_content, &dl, auth ) ;
-	if ( r <= 400 )
-	{
-		if ( m_mtime != DateTime() )
-			os::SetFileTime( file, m_mtime ) ;
-		else
-			Log( "encountered zero date time after downloading %1%", file, log::warning ) ;
-	}
-}
-
-bool Resource::EditContent( http::Agent* http, const http::Header& auth )
-{
-	assert( http != 0 ) ;
-	assert( m_parent != 0 ) ;
-	assert( m_parent->m_state == sync ) ;
-
-	// upload link missing means that file is read only
-	if ( m_edit.empty() )
-	{
-		Log( "Cannot upload %1%: file read-only. %2%", m_name, m_state, log::warning ) ;
-		return false ;
-	}
-	
-	return Upload( http, m_edit, auth, false ) ;
-}
-
-bool Resource::Create( http::Agent* http, const http::Header& auth )
-{
-	assert( http != 0 ) ;
-	assert( m_parent != 0 ) ;
-	assert( m_parent->IsFolder() ) ;
-	assert( m_parent->m_state == sync ) ;
-	
-	if ( IsFolder() )
-	{
-		std::string uri = feed_base ;
-		if ( !m_parent->IsRoot() )
-			uri += ("/" + http->Escape(m_parent->m_id) + "/contents") ;
-		
-		std::string meta = (boost::format( xml_meta )
-			% "folder"
-			% xml::Escape(m_name)
-		).str() ;
-
-		http::Header hdr( auth ) ;
-		hdr.Add( "Content-Type: application/atom+xml" ) ;
-		
-		http::XmlResponse xml ;
-// 		http::ResponseLog log( "create", ".xml", &xml ) ;
-		http->Post( uri, meta, &xml, hdr ) ;
-		AssignIDs( Entry( xml.Response() ) ) ;
-
-		return true ;
-	}
-	else if ( !m_parent->m_create.empty() )
-	{
-		return Upload( http, m_parent->m_create + "?convert=false", auth, true ) ;
-	}
-	else
-	{
-		Log( "parent of %1% does not exist: cannot upload", Name(), log::warning ) ;
-		return false ;
-	}
-}
-
-bool Resource::Upload( http::Agent* http, const std::string& link, const http::Header& auth, bool post )
-{
-	assert( http != 0 ) ;
-	
-	StdioFile file( Path() ) ;
-	
-	// TODO: upload in chunks
-	std::string data ;
-	char buf[4096] ;
-	std::size_t count = 0 ;
-	while ( (count = file.Read( buf, sizeof(buf) )) > 0 )
-		data.append( buf, count ) ;
-
-	std::ostringstream xcontent_len ;
-	xcontent_len << "X-Upload-Content-Length: " << data.size() ;
-	
-	http::Header hdr( auth ) ;
-	hdr.Add( "Content-Type: application/atom+xml" ) ;
-	hdr.Add( "X-Upload-Content-Type: application/octet-stream" ) ;
-	hdr.Add( xcontent_len.str() ) ;
-  	hdr.Add( "If-Match: " + m_etag ) ;
-	hdr.Add( "Expect:" ) ;
-	
-	std::string meta = (boost::format( xml_meta )
-		% m_kind
-		% xml::Escape(m_name)
-	).str() ;
-	
-	http::StringResponse str ;
-	if ( post )
-		http->Post( link, meta, &str, hdr ) ;
-	else
-		http->Put( link, meta, &str, hdr ) ;
-	
-	http::Header uphdr ;
-	uphdr.Add( "Expect:" ) ;
-	uphdr.Add( "Accept:" ) ;
-
-	// the content upload URL is in the "Location" HTTP header
-	std::string uplink = http->RedirLocation() ;
-	http::XmlResponse xml ;
-	
-	http->Put( uplink, data, &xml, uphdr ) ;
-	AssignIDs( Entry( xml.Response() ) ) ;
-	
-	return true ;
-}
-
-Resource::iterator Resource::begin() const
-{
-	return m_child.begin() ;
-}
-
-Resource::iterator Resource::end() const
-{
-	return m_child.end() ;
-}
-
-std::size_t Resource::size() const
-{
-	return m_child.size() ;
-}
-
-std::ostream& operator<<( std::ostream& os, Resource::State s )
-{
-	static const char *state[] =
-	{
-		"sync",	"local_new", "local_changed", "local_deleted", "remote_new",
-		"remote_changed", "remote_deleted"
-	} ;
-	assert( s >= 0 && s < Count(state) ) ;
-	return os << state[s] ;
-}
-
-std::string Resource::StateStr() const
-{
-	std::ostringstream ss ;
-	ss << m_state ;
-	return ss.str() ;
-}
-
-std::string Resource::MD5() const
-{
-	return m_md5 ;
-}
-
-bool Resource::IsRoot() const
-{
-	return m_parent == 0 ;
-}
-
-bool Resource::HasID() const
-{
-	return !m_href.empty() && !m_id.empty() ;
-}
-
-} // end of namespace
-
-namespace std
-{
-	void swap( gr::Resource& c1, gr::Resource& c2 )
-	{
-		c1.Swap( c2 ) ;
-	}
-}

libgrive/src/drive/State.cc

@@ -1,283 +0,0 @@
-/*
-	grive: an GPL program to sync a local directory with Google Drive
-	Copyright (C) 2012  Wan Wai Ho
-
-	This program is free software; you can redistribute it and/or
-	modify it under the terms of the GNU General Public License
-	as published by the Free Software Foundation version 2
-	of the License.
-
-	This program is distributed in the hope that it will be useful,
-	but WITHOUT ANY WARRANTY; without even the implied warranty of
-	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-	GNU General Public License for more details.
-
-	You should have received a copy of the GNU General Public License
-	along with this program; if not, write to the Free Software
-	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
-*/
-
-#include "State.hh"
-
-#include "Entry.hh"
-#include "Resource.hh"
-#include "CommonUri.hh"
-
-#include "http/Agent.hh"
-#include "http/Header.hh"
-#include "util/Crypt.hh"
-#include "util/log/Log.hh"
-#include "protocol/Json.hh"
-
-#include <fstream>
-
-namespace gr {
-
-State::State( const fs::path& filename, const Json& options ) :
-	m_cstamp( -1 )
-{
-	Read( filename ) ;
-	
-	// the "-f" option will make grive always thinks remote is newer
-	Json force ;
-	if ( options.Get("force", force) && force.Bool() )
-		m_last_sync = DateTime() ;
-	
-	Log( "last sync time: %1%", m_last_sync, log::verbose ) ;
-}
-
-State::~State()
-{
-}
-
-/// Synchronize local directory. Build up the resource tree from files and folders
-/// of local directory.
-void State::FromLocal( const fs::path& p )
-{
-	FromLocal( p, m_res.Root() ) ;
-}
-
-bool State::IsIgnore( const std::string& filename )
-{
-	return filename[0] == '.' ;
-}
-
-void State::FromLocal( const fs::path& p, gr::Resource* folder )
-{
-	assert( folder != 0 ) ;
-	assert( folder->IsFolder() ) ;
-	
-	// sync the folder itself
-	folder->FromLocal( m_last_sync ) ;
-
-	for ( fs::directory_iterator i( p ) ; i != fs::directory_iterator() ; ++i )
-	{
-		std::string fname = Path2Str(i->path().filename()) ;
-// Trace( "found file %1%", i->path() ) ;
-	
-		if ( IsIgnore(fname) )
-			Log( "file %1% is ignored by grive", fname, log::verbose ) ;
-		
-		// check for broken symblic links
-		else if ( !fs::exists( i->path() ) )
-			Log( "file %1% doesn't exist (broken link?), ignored", i->path(), log::verbose ) ;
-		
-		else
-		{
-			// if the Resource object of the child already exists, it should
-			// have been so no need to do anything here
-			Resource *c = folder->FindChild( fname ) ;
-			if ( c == 0 )
-			{
-				c = new Resource( fname, fs::is_directory(i->path()) ? "folder" : "file" ) ;
-				folder->AddChild( c ) ;
-				m_res.Insert( c ) ;
-			}
-			
-			c->FromLocal( m_last_sync ) ;			
-			
-			if ( fs::is_directory( i->path() ) )
-				FromLocal( *i, c ) ;
-		}
-	}
-}
-
-void State::FromRemote( const Entry& e )
-{
-	std::string fn = e.Filename() ;				
-
-	if ( IsIgnore( e.Name() ) )
-		Log( "%1% %2% is ignored by grive", e.Kind(), e.Name(), log::verbose ) ;
-
-	// common checkings
-	else if ( e.Kind() != "folder" && (fn.empty() || e.ContentSrc().empty()) )
-		Log( "%1% \"%2%\" is a google document, ignored", e.Kind(), e.Name(), log::verbose ) ;
-	
-	else if ( fn.find('/') != fn.npos )
-		Log( "%1% \"%2%\" contains a slash in its name, ignored", e.Kind(), e.Name(), log::verbose ) ;
-	
-	else if ( !e.IsChange() && e.ParentHrefs().size() != 1 )
-		Log( "%1% \"%2%\" has multiple parents, ignored", e.Kind(), e.Name(), log::verbose ) ;
-
-	else if ( e.IsChange() )
-		FromChange( e ) ;
-
-	else if ( !Update( e ) )
-	{
-		m_unresolved.push_back( e ) ;
-	}
-}
-
-void State::ResolveEntry()
-{
-	while ( !m_unresolved.empty() )
-	{
-		if ( TryResolveEntry() == 0 )
-			break ;
-	}
-}
-
-std::size_t State::TryResolveEntry()
-{
-	assert( !m_unresolved.empty() ) ;
-
-	std::size_t count = 0 ;
-	std::vector<Entry>& en = m_unresolved ;
-	
-	for ( std::vector<Entry>::iterator i = en.begin() ; i != en.end() ; )
-	{
-		if ( Update( *i ) )
-		{
-			i = en.erase( i ) ;
-			count++ ;
-		}
-		else
-			++i ;
-	}
-	return count ;
-}
-
-void State::FromChange( const Entry& e )
-{
-	assert( e.IsChange() ) ;
-	assert( !IsIgnore( e.Name() ) ) ;
-	
-	// entries in the change feed is always treated as newer in remote,
-	// so we override the last sync time to 0
-	if ( Resource *res = m_res.FindByHref( e.AltSelf() ) )
-		m_res.Update( res, e, DateTime() ) ;
-}
-
-bool State::Update( const Entry& e )
-{
-	assert( !e.IsChange() ) ;
-	assert( !e.ParentHref().empty() ) ;
-
-	if ( Resource *res = m_res.FindByHref( e.SelfHref() ) )
-	{
-		m_res.Update( res, e, m_last_sync ) ;
-		return true ;
-	}
-	else if ( Resource *parent = m_res.FindByHref( e.ParentHref() ) )
-	{
-		assert( parent->IsFolder() ) ;
-
-		// see if the entry already exist in local
-		std::string name = e.Name() ;
-		Resource *child = parent->FindChild( name ) ;
-		if ( child != 0 )
-		{
-			// since we are updating the ID and Href, we need to remove it and re-add it.
-			m_res.Update( child, e, m_last_sync ) ;
-		}
-		
-		// folder entry exist in google drive, but not local. we should create
-		// the directory
-		else if ( e.Kind() == "folder" || !e.Filename().empty() )
-		{
-			// first create a dummy resource and update it later
-			child = new Resource( name, e.Kind() ) ;
-			parent->AddChild( child ) ;
-			m_res.Insert( child ) ;
-			
-			// update the state of the resource
-			m_res.Update( child, e, m_last_sync ) ;
-		}
-		
-		return true ;
-	}
-	else
-		return false ;
-}
-
-Resource* State::FindByHref( const std::string& href )
-{
-	return m_res.FindByHref( href ) ;
-}
-
-Resource* State::Find( const fs::path& path )
-{
-	return m_res.FindByPath( path ) ;
-}
-
-State::iterator State::begin()
-{
-	return m_res.begin() ;
-}
-
-State::iterator State::end()
-{
-	return m_res.end() ;
-}
-
-void State::Read( const fs::path& filename )
-{
-	try
-	{
-		Json json = Json::ParseFile( filename.string() ) ;
-		
-		Json last_sync = json["last_sync"] ;
-		m_last_sync.Assign(
-			last_sync["sec"].Int(),
-			last_sync["nsec"].Int() ) ;
-		
-		m_cstamp = json["change_stamp"].Int() ;
-	}
-	catch ( Exception& )
-	{
-		m_last_sync.Assign(0) ;
-	}
-}
-
-void State::Write( const fs::path& filename ) const
-{
-	Json last_sync ;
-	last_sync.Add( "sec",	Json(m_last_sync.Sec() ) );
-	last_sync.Add( "nsec",	Json(m_last_sync.NanoSec() ) );
-	
-	Json result ;
-	result.Add( "last_sync", last_sync ) ;
-	result.Add( "change_stamp", Json(m_cstamp) ) ;
-	
-	std::ofstream fs( filename.string().c_str() ) ;
-	fs << result ;
-}
-
-void State::Sync( http::Agent *http, const http::Header& auth )
-{
-	m_res.Root()->Sync( http, auth ) ;
-	m_last_sync = DateTime::Now() ;
-}
-
-long State::ChangeStamp() const
-{
-	return m_cstamp ;
-}
-
-void State::ChangeStamp( long cstamp )
-{
-	Log( "change stamp is set to %1%", cstamp, log::verbose ) ;
-	m_cstamp = cstamp ;
-}
-
-} // end of namespace

libgrive/src/drive2/CommonUri.hh

@@ -0,0 +1,39 @@
+/*
+	Common URIs for REST API
+	Copyright (C) 2015  Vitaliy Filippov
+
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License
+	as published by the Free Software Foundation version 2
+	of the License.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with this program; if not, write to the Free Software
+	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+
+#pragma once
+
+#include <string>
+
+namespace gr { namespace v2 {
+
+const std::string upload_base = "https://www.googleapis.com/upload/drive/v2/files" ;
+
+namespace feeds
+{
+	const std::string files		= "https://www.googleapis.com/drive/v2/files" ;
+	const std::string changes	= "https://www.googleapis.com/drive/v2/changes" ;
+}
+
+namespace mime_types
+{
+	const std::string folder	= "application/vnd.google-apps.folder" ;
+}
+
+} } // end of namespace gr::v2

libgrive/src/drive2/Entry2.cc

@@ -0,0 +1,96 @@
+/*
+	REST API item class implementation
+	Copyright (C) 2015  Vitaliy Filippov
+
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License
+	as published by the Free Software Foundation version 2
+	of the License.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with this program; if not, write to the Free Software
+	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+
+#include "Entry2.hh"
+#include "CommonUri.hh"
+
+#include "util/Crypt.hh"
+#include "util/log/Log.hh"
+#include "util/OS.hh"
+
+#include "json/Val.hh"
+
+#include <algorithm>
+#include <iterator>
+
+namespace gr { namespace v2 {
+
+/// construct an entry for remote, from "file" JSON object - Drive REST API
+Entry2::Entry2( const Val& item )
+{
+	Update( item ) ;
+}
+
+void Entry2::Update( const Val& item )
+{
+	bool is_chg = item["kind"].Str() == "drive#change";
+
+	// changestamp only appears in change feed entries
+	m_change_stamp	= is_chg ? item["id"].Int() : -1 ;
+	m_is_removed	= is_chg && item["deleted"].Bool() ;
+	m_size			= 0 ;
+
+	const Val& file = is_chg && !m_is_removed ? item["file"] : item;
+
+	if ( m_is_removed )
+	{
+		m_resource_id = item["fileId"];
+	}
+	else
+	{
+		m_title			= file["title"] ;
+		m_etag			= file["etag"] ;
+		Val fn;
+		m_filename		= file.Get( "title", fn ) ? fn.Str() : std::string();
+		m_self_href		= file["selfLink"] ;
+		m_mtime			= DateTime( file["modifiedDate"] ) ;
+
+		m_resource_id	= file["id"];
+		m_is_dir		= file["mimeType"].Str() == mime_types::folder ;
+		m_is_editable	= file["editable"].Bool() ;
+		m_is_removed	= file["labels"]["trashed"].Bool() ;
+		if ( !m_is_dir )
+		{
+			if ( !file.Has( "md5Checksum" ) || !file.Has("downloadUrl") )
+			{
+				// This is either a google docs document or a not-yet-uploaded file. Ignore it.
+				// FIXME: We'll need to compare timestamps to support google docs.
+				m_is_removed = true;
+			}
+			else
+			{
+				m_md5			= file["md5Checksum"] ;
+				m_size			= file["fileSize"].U64() ;
+				m_content_src	= file["downloadUrl"] ;
+				// convert to lower case for easy comparison
+				std::transform( m_md5.begin(), m_md5.end(), m_md5.begin(), tolower ) ;
+			}
+		}
+
+		m_parent_hrefs.clear( ) ;
+
+		Val::Array parents = file["parents"].AsArray() ;
+		for ( Val::Array::iterator i = parents.begin() ; i != parents.end() ; ++i )
+		{
+			m_parent_hrefs.push_back( (*i)["isRoot"].Bool() ? std::string( "root" ) : (*i)["parentLink"] ) ;
+		}
+	}
+}
+
+} } // end of namespace gr::v2

libgrive/src/drive2/Entry2.hh

@@ -0,0 +1,38 @@
+/*
+	REST API item class implementation
+	Copyright (C) 2015  Vitaliy Filippov
+
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License
+	as published by the Free Software Foundation version 2
+	of the License.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with this program; if not, write to the Free Software
+	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+
+#pragma once
+
+#include "base/Entry.hh"
+
+namespace gr {
+
+class Val ;
+
+namespace v2 {
+
+class Entry2: public Entry
+{
+public :
+	explicit Entry2( const Val& item ) ;
+private :
+	void Update( const Val& item ) ;
+} ;
+
+} } // end of namespace gr::v2

libgrive/src/drive2/Feed2.cc

@@ -0,0 +1,62 @@
+/*
+	REST API item list ("Feed") implementation
+	Copyright (C) 2015  Vitaliy Filippov
+
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License
+	as published by the Free Software Foundation version 2
+	of the License.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with this program; if not, write to the Free Software
+	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+
+#include "CommonUri.hh"
+#include "Feed2.hh"
+#include "Entry2.hh"
+
+#include "http/Agent.hh"
+#include "http/Header.hh"
+#include "json/Val.hh"
+#include "json/ValResponse.hh"
+
+#include <iostream>
+#include <boost/format.hpp>
+
+namespace gr { namespace v2 {
+
+Feed2::Feed2( const std::string& url ):
+	Feed( url )
+{
+}
+
+Feed2::~Feed2()
+{
+}
+
+bool Feed2::GetNext( http::Agent *http )
+{
+	if ( m_next.empty() )
+		return false ;
+	
+	http::ValResponse out ;
+	http->Get( m_next, &out, http::Header(), 0 ) ;
+	Val m_content = out.Response() ;
+	
+	Val::Array items = m_content["items"].AsArray() ;
+	m_entries.clear() ;
+	for ( Val::Array::iterator i = items.begin() ; i != items.end() ; ++i )
+		m_entries.push_back( Entry2( *i ) );
+	
+	Val url ;
+	m_next = m_content.Get( "nextLink", url ) ? url : std::string( "" ) ;
+	return true ;
+}
+
+} } // end of namespace gr::v2

libgrive/src/drive2/Feed2.hh

@@ -0,0 +1,38 @@
+/*
+	REST API item list ("Feed") implementation
+	Copyright (C) 2015  Vitaliy Filippov
+
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License
+	as published by the Free Software Foundation version 2
+	of the License.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with this program; if not, write to the Free Software
+	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+
+#pragma once
+
+#include "base/Feed.hh"
+
+#include "util/Exception.hh"
+
+#include <string>
+
+namespace gr { namespace v2 {
+
+class Feed2: public Feed
+{
+public :
+	Feed2( const std::string& url ) ;
+	~Feed2() ;
+	bool GetNext( http::Agent *http ) ;
+} ;
+
+} } // end of namespace gr::v2

libgrive/src/drive2/Syncer2.cc

@@ -0,0 +1,243 @@
+/*
+	REST API Syncer implementation
+	Copyright (C) 2015  Vitaliy Filippov
+
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License
+	as published by the Free Software Foundation version 2
+	of the License.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with this program; if not, write to the Free Software
+	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+
+#include "base/Resource.hh"
+#include "CommonUri.hh"
+#include "Entry2.hh"
+#include "Feed2.hh"
+#include "Syncer2.hh"
+
+#include "http/Agent.hh"
+#include "http/Download.hh"
+#include "http/Header.hh"
+#include "http/StringResponse.hh"
+#include "json/ValResponse.hh"
+#include "json/JsonWriter.hh"
+
+#include "util/OS.hh"
+#include "util/log/Log.hh"
+#include "util/StringStream.hh"
+#include "util/ConcatStream.hh"
+
+#include <boost/exception/all.hpp>
+
+#include <cassert>
+
+// for debugging
+#include <iostream>
+
+namespace gr { namespace v2 {
+
+Syncer2::Syncer2( http::Agent *http ):
+	Syncer( http )
+{
+	assert( http != 0 ) ;
+}
+
+void Syncer2::DeleteRemote( Resource *res )
+{
+	http::StringResponse str ;
+	http::Header hdr ;
+	hdr.Add( "If-Match: " + res->ETag() ) ;
+	m_http->Post( res->SelfHref() + "/trash", "", &str, hdr ) ;
+}
+
+bool Syncer2::EditContent( Resource *res, bool new_rev )
+{
+	assert( res->Parent() ) ;
+	assert( !res->ResourceID().empty() ) ;
+	assert( res->Parent()->GetState() == Resource::sync ) ;
+
+	if ( !res->IsEditable() )
+	{
+		Log( "Cannot upload %1%: file read-only. %2%", res->Name(), res->StateStr(), log::warning ) ;
+		return false ;
+	}
+
+	return Upload( res, new_rev ) ;
+}
+
+bool Syncer2::Create( Resource *res )
+{
+	assert( res->Parent() ) ;
+	assert( res->Parent()->IsFolder() ) ;
+	assert( res->Parent()->GetState() == Resource::sync ) ;
+	assert( res->ResourceID().empty() ) ;
+	
+	if ( !res->Parent()->IsEditable() )
+	{
+		Log( "Cannot upload %1%: parent directory read-only. %2%", res->Name(), res->StateStr(), log::warning ) ;
+		return false ;
+	}
+	
+	return Upload( res, false );
+}
+
+bool Syncer2::Move( Resource* res, Resource* newParentRes, std::string newFilename )
+{
+	if ( res->ResourceID().empty() )
+	{
+		Log("Can't rename file %1%, no server id found", res->Name());
+		return false;
+	}
+
+	Val meta;
+	meta.Add( "title", Val(newFilename) );
+	if ( res->IsFolder() )
+	{
+		meta.Add( "mimeType", Val( mime_types::folder ) );
+	}
+	std::string json_meta = WriteJson( meta );
+
+	Val valr ;
+
+	// Issue metadata update request
+	{
+		std::string addRemoveParents("");
+		if (res->Parent()->IsRoot() )
+			addRemoveParents += "&removeParents=root";
+		else
+			addRemoveParents += "&removeParents=" + res->Parent()->ResourceID();
+		if ( newParentRes->IsRoot() )
+			addRemoveParents += "&addParents=root";
+		else
+			addRemoveParents += "&addParents=" + newParentRes->ResourceID();
+		http::Header hdr2 ;
+		hdr2.Add( "Content-Type: application/json" );
+		http::ValResponse vrsp ;
+		// Don't change modified date because we're only moving
+		long http_code = m_http->Put(
+			feeds::files + "/" + res->ResourceID() + "?modifiedDateBehavior=noChange" + addRemoveParents,
+			json_meta, &vrsp, hdr2
+		) ;
+		valr = vrsp.Response();
+		assert( http_code == 200 && !( valr["id"].Str().empty() ) );
+	}
+
+	return true;
+}
+
+std::string to_string( uint64_t n )
+{
+	std::ostringstream s;
+	s << n;
+	return s.str();
+}
+
+bool Syncer2::Upload( Resource *res, bool new_rev )
+{
+	Val meta;
+	meta.Add( "title", Val( res->Name() ) );
+	if ( res->IsFolder() )
+		meta.Add( "mimeType", Val( mime_types::folder ) );
+	if ( !res->Parent()->IsRoot() )
+	{
+		Val parent;
+		parent.Add( "id", Val( res->Parent()->ResourceID() ) );
+		Val parents( Val::array_type );
+		parents.Add( parent );
+		meta.Add( "parents", parents );
+	}
+	std::string json_meta = WriteJson( meta );
+
+	Val valr ;
+
+	if ( res->IsFolder() )
+	{
+		// Only issue metadata update request
+		http::Header hdr2 ;
+		hdr2.Add( "Content-Type: application/json" );
+		http::ValResponse vrsp ;
+		long http_code = 0;
+		if ( res->ResourceID().empty() )
+			http_code = m_http->Post( feeds::files, json_meta, &vrsp, hdr2 ) ;
+		else
+			http_code = m_http->Put( feeds::files + "/" + res->ResourceID(), json_meta, &vrsp, hdr2 ) ;
+		valr = vrsp.Response();
+		assert( http_code == 200 && !( valr["id"].Str().empty() ) );
+	}
+	else
+	{
+		File file( res->Path() ) ;
+		uint64_t size = file.Size() ;
+		ConcatStream multipart ;
+		StringStream p1(
+			"--file_contents\r\nContent-Type: application/json; charset=utf-8\r\n\r\n" + json_meta +
+			"\r\n--file_contents\r\nContent-Type: application/octet-stream\r\nContent-Length: " + to_string( size ) +
+			"\r\n\r\n"
+		);
+		StringStream p2("\r\n--file_contents--\r\n");
+		multipart.Append( &p1 );
+		multipart.Append( &file );
+		multipart.Append( &p2 );
+
+		http::Header hdr ;
+		if ( !res->ETag().empty() )
+			hdr.Add( "If-Match: " + res->ETag() ) ;
+		hdr.Add( "Content-Type: multipart/related; boundary=\"file_contents\"" );
+		hdr.Add( "Content-Length: " + to_string( multipart.Size() ) );
+
+		http::ValResponse vrsp;
+		m_http->Request(
+			res->ResourceID().empty() ? "POST" : "PUT",
+			upload_base + ( res->ResourceID().empty() ? "" : "/" + res->ResourceID() ) +
+			"?uploadType=multipart&newRevision=" + ( new_rev ? "true" : "false" ),
+			&multipart, &vrsp, hdr
+		) ;
+		valr = vrsp.Response() ;
+		assert( !( valr["id"].Str().empty() ) );
+	}
+
+	Entry2 responseEntry = Entry2( valr ) ;
+	AssignIDs( res, responseEntry ) ;
+	res->SetServerTime( responseEntry.MTime() );
+
+	return true ;
+}
+
+std::unique_ptr<Feed> Syncer2::GetFolders()
+{
+	return std::unique_ptr<Feed>( new Feed2( feeds::files + "?maxResults=100000&q=trashed%3dfalse+and+mimeType%3d%27" + mime_types::folder + "%27" ) );
+}
+
+std::unique_ptr<Feed> Syncer2::GetAll()
+{
+	return std::unique_ptr<Feed>( new Feed2( feeds::files + "?maxResults=999999999&q=trashed%3dfalse" ) );
+}
+
+std::string ChangesFeed( long changestamp, int maxResults = 1000 )
+{
+	boost::format feed( feeds::changes + "?maxResults=%1%&includeSubscribed=false" + ( changestamp > 0 ? "&startChangeId=%2%" : "" ) ) ;
+	return ( changestamp > 0 ? feed % maxResults % changestamp : feed % maxResults ).str() ;
+}
+
+std::unique_ptr<Feed> Syncer2::GetChanges( long min_cstamp )
+{
+	return std::unique_ptr<Feed>( new Feed2( ChangesFeed( min_cstamp ) ) );
+}
+
+long Syncer2::GetChangeStamp( long min_cstamp )
+{
+	http::ValResponse res ;
+	m_http->Get( ChangesFeed( min_cstamp, 1 ), &res, http::Header(), 0 ) ;
+
+	return std::atoi( res.Response()["largestChangeId"].Str().c_str() );
+}
+
+} } // end of namespace gr::v1

libgrive/src/drive2/Syncer2.hh

@@ -0,0 +1,53 @@
+/*
+	REST API Syncer implementation
+	Copyright (C) 2015  Vitaliy Filippov
+
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License
+	as published by the Free Software Foundation version 2
+	of the License.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with this program; if not, write to the Free Software
+	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+
+#pragma once
+
+#include "base/Syncer.hh"
+
+namespace gr {
+
+class Feed;
+
+namespace v2 {
+
+class Syncer2: public Syncer
+{
+
+public :
+
+	Syncer2( http::Agent *http );
+
+	void DeleteRemote( Resource *res );
+	bool EditContent( Resource *res, bool new_rev );
+	bool Create( Resource *res );
+	bool Move( Resource* res, Resource* newParent, std::string newFilename );
+
+	std::unique_ptr<Feed> GetFolders();
+	std::unique_ptr<Feed> GetAll();
+	std::unique_ptr<Feed> GetChanges( long min_cstamp );
+	long GetChangeStamp( long min_cstamp );
+
+private :
+
+	bool Upload( Resource *res, bool new_rev );
+
+} ;
+
+} } // end of namespace gr::v2

libgrive/src/http/Agent.cc

@@ -0,0 +1,83 @@
+/*
+	Convenience wrapper methods for various kinds of HTTP requests
+	Copyright (C) 2015  Vitaliy Filippov
+
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License
+	as published by the Free Software Foundation version 2
+	of the License.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with this program; if not, write to the Free Software
+	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+
+#include "Agent.hh"
+#include "Header.hh"
+#include "util/StringStream.hh"
+
+namespace gr {
+
+namespace http {
+
+Agent::Agent()
+{
+	mMaxUpload = mMaxDownload = 0;
+}
+
+long Agent::Put(
+	const std::string&	url,
+	const std::string&	data,
+	DataStream			*dest,
+	const Header&		hdr )
+{
+	StringStream s( data );
+	return Request( "PUT", url, &s, dest, hdr );
+}
+
+long Agent::Put(
+	const std::string&	url,
+	File				*file,
+	DataStream			*dest,
+	const Header&		hdr )
+{
+	return Request( "PUT", url, (SeekStream*)file, dest, hdr );
+}
+
+long Agent::Get(
+	const std::string& 	url,
+	DataStream			*dest,
+	const Header&		hdr,
+	u64_t			downloadFileBytes )
+{
+	return Request( "GET", url, NULL, dest, hdr, downloadFileBytes );
+}
+
+long Agent::Post(
+	const std::string& 	url,
+	const std::string&	data,
+	DataStream			*dest,
+	const Header&		hdr )
+{
+	Header h( hdr ) ;
+	StringStream s( data );
+	h.Add( "Content-Type: application/x-www-form-urlencoded" );
+	return Request( "POST", url, &s, dest, h );
+}
+
+void Agent::SetUploadSpeed( unsigned kbytes )
+{
+	mMaxUpload = kbytes;
+}
+
+void Agent::SetDownloadSpeed( unsigned kbytes )
+{
+	mMaxDownload = kbytes;
+}
+
+} } // end of namespace

libgrive/src/http/Agent.hh

@@ -20,42 +20,75 @@
 #pragma once
 
 #include <string>
+#include "ResponseLog.hh"
+#include "util/Types.hh"
+#include "util/Progress.hh"
 
-namespace gr { namespace http {
+namespace gr {
+
+class SeekStream ;
+class File ;
+
+namespace http {
 
 class Header ;
-class Receivable ;
 
 class Agent
 {
+protected:
+	unsigned mMaxUpload, mMaxDownload ;
+
 public :
+	Agent() ;
+	virtual ~Agent() {}
+
+	virtual ResponseLog* GetLog() const = 0 ;
+	virtual void SetLog( ResponseLog* ) = 0 ;
+
 	virtual long Put(
 		const std::string&	url,
 		const std::string&	data,
-		Receivable			*dest,
-		const Header&		hdr ) = 0 ;
+		DataStream			*dest,
+		const Header&		hdr ) ;
 
+	virtual long Put(
+		const std::string&	url,
+		File				*file,
+		DataStream			*dest,
+		const Header&		hdr ) ;
+		
 	virtual long Get(
 		const std::string& 	url,
-		Receivable			*dest,
-		const Header&		hdr ) = 0 ;
+		DataStream			*dest,
+		const Header&		hdr,
+		u64_t			downloadFileBytes = 0 ) ;
 	
 	virtual long Post(
 		const std::string& 	url,
 		const std::string&	data,
-		Receivable			*dest,
-		const Header&		hdr ) = 0 ;
+		DataStream			*dest,
+		const Header&		hdr ) ;
 	
-	virtual long Custom(
+	virtual long Request(
 		const std::string&	method,
 		const std::string&	url,
-		Receivable			*dest,
-		const Header&		hdr ) = 0 ;
+		SeekStream			*in,
+		DataStream			*dest,
+		const Header&		hdr,
+		u64_t			downloadFileBytes = 0 ) = 0 ;
+	
+	virtual void SetUploadSpeed( unsigned kbytes ) ;
+	virtual void SetDownloadSpeed( unsigned kbytes ) ;
+	
+	virtual std::string LastError() const = 0 ;
+	virtual std::string LastErrorHeaders() const = 0 ;
 	
 	virtual std::string RedirLocation() const = 0 ;
 	
 	virtual std::string Escape( const std::string& str ) = 0 ;
 	virtual std::string Unescape( const std::string& str ) = 0 ;
+
+	virtual void SetProgressReporter( Progress* ) = 0;
 } ;
 
 } } // end of namespace

libgrive/src/http/CurlAgent.cc

@@ -19,22 +19,19 @@
 
 #include "CurlAgent.hh"
 
-#include "Download.hh"
 #include "Error.hh"
 #include "Header.hh"
-#include "Receivable.hh"
 
 #include "util/log/Log.hh"
+#include "util/DataStream.hh"
+#include "util/File.hh"
 
 #include <boost/throw_exception.hpp>
 
-// dependent libraries
-#include <curl/curl.h>
-
 #include <algorithm>
 #include <cassert>
 #include <cstring>
-#include <sstream>
+#include <limits>
 #include <streambuf>
 #include <iostream>
 
@@ -43,20 +40,17 @@
 namespace {
 
 using namespace gr::http ;
+using namespace gr ;
 
-size_t ReadCallback( void *ptr, std::size_t size, std::size_t nmemb, std::string *data )
+std::size_t ReadFileCallback( void *ptr, std::size_t size, std::size_t nmemb, SeekStream *file )
 {
 	assert( ptr != 0 ) ;
-	assert( data != 0 ) ;
+	assert( file != 0 ) ;
 
-	std::size_t count = std::min( size * nmemb, data->size() ) ;
-	if ( count > 0 )
-	{
-		std::memcpy( ptr, &(*data)[0], count ) ;
-		data->erase( 0, count ) ;
-	}
-	
-	return count ;
+	if ( size*nmemb > 0 )
+		return file->Read( static_cast<char*>(ptr), size*nmemb ) ;
+
+	return 0 ;
 }
 
 } // end of local namespace
@@ -67,10 +61,17 @@ struct CurlAgent::Impl
 {
 	CURL			*curl ;
 	std::string		location ;
+	bool			error ;
+	std::string		error_headers ;
+	std::string		error_data ;
+	DataStream		*dest ;
+	u64_t			total_download, total_upload ;
 } ;
 
-CurlAgent::CurlAgent() :
-	m_pimpl( new Impl )
+static struct curl_slist* SetHeader( CURL* handle, const Header& hdr );
+
+CurlAgent::CurlAgent() : Agent(),
+	m_pimpl( new Impl ), m_pb( 0 )
 {
 	m_pimpl->curl = ::curl_easy_init();
 }
@@ -78,11 +79,20 @@ CurlAgent::CurlAgent() :
 void CurlAgent::Init()
 {
 	::curl_easy_reset( m_pimpl->curl ) ;
-	::curl_easy_setopt( m_pimpl->curl, CURLOPT_SSL_VERIFYPEER,	0L ) ; 
+	::curl_easy_setopt( m_pimpl->curl, CURLOPT_SSL_VERIFYPEER,	0L ) ;
 	::curl_easy_setopt( m_pimpl->curl, CURLOPT_SSL_VERIFYHOST,	0L ) ;
 	::curl_easy_setopt( m_pimpl->curl, CURLOPT_HEADERFUNCTION,	&CurlAgent::HeaderCallback ) ;
-	::curl_easy_setopt( m_pimpl->curl, CURLOPT_WRITEHEADER ,	this ) ;
-	::curl_easy_setopt( m_pimpl->curl, CURLOPT_HEADER, 			0L ) ;
+	::curl_easy_setopt( m_pimpl->curl, CURLOPT_HEADERDATA,		this ) ;
+	::curl_easy_setopt( m_pimpl->curl, CURLOPT_HEADER,			0L ) ;
+	if ( mMaxUpload > 0 )
+		::curl_easy_setopt( m_pimpl->curl, CURLOPT_MAX_SEND_SPEED_LARGE, mMaxUpload ) ;
+	if ( mMaxDownload > 0 )
+		::curl_easy_setopt( m_pimpl->curl, CURLOPT_MAX_RECV_SPEED_LARGE, mMaxDownload ) ;
+	m_pimpl->error = false;
+	m_pimpl->error_headers = "";
+	m_pimpl->error_data = "";
+	m_pimpl->dest = NULL;
+	m_pimpl->total_download = m_pimpl->total_upload = 0;
 }
 
 CurlAgent::~CurlAgent()
@@ -90,31 +100,81 @@ CurlAgent::~CurlAgent()
 	::curl_easy_cleanup( m_pimpl->curl );
 }
 
+ResponseLog* CurlAgent::GetLog() const
+{
+	return m_log.get();
+}
+
+void CurlAgent::SetLog(ResponseLog *log)
+{
+	m_log.reset( log );
+}
+
+void CurlAgent::SetProgressReporter(Progress *progress)
+{
+	m_pb = progress;
+}
+
 std::size_t CurlAgent::HeaderCallback( void *ptr, size_t size, size_t nmemb, CurlAgent *pthis )
 {
-	char *str = reinterpret_cast<char*>(ptr) ;
+	char *str = static_cast<char*>(ptr) ;
 	std::string line( str, str + size*nmemb ) ;
 	
+	// Check for error (HTTP 400 and above)
+	if ( line.substr( 0, 5 ) == "HTTP/" && line[9] >= '4' )
+		pthis->m_pimpl->error = true;
+	
+	if ( pthis->m_pimpl->error )
+		pthis->m_pimpl->error_headers += line;
+	
+	if ( pthis->m_log.get() )
+		pthis->m_log->Write( str, size*nmemb );
+	
 	static const std::string loc = "Location: " ;
 	std::size_t pos = line.find( loc ) ;
 	if ( pos != line.npos )
 	{
 		std::size_t end_pos = line.find( "\r\n", pos ) ;
-		pthis->m_pimpl->location = line.substr( loc.size(), end_pos - loc.size() ) ;
+		pthis->m_pimpl->location = line.substr( pos+loc.size(), end_pos - loc.size() ) ;
 	}
 	
 	return size*nmemb ;
 }
 
-std::size_t CurlAgent::Receive( void* ptr, size_t size, size_t nmemb, Receivable *recv )
+std::size_t CurlAgent::Receive( void* ptr, size_t size, size_t nmemb, CurlAgent *pthis )
 {
-	assert( recv != 0 ) ;
-	return recv->OnData( ptr, size * nmemb ) ;
+	assert( pthis != 0 ) ;
+	if ( pthis->m_log.get() )
+		pthis->m_log->Write( (const char*)ptr, size*nmemb );
+
+	if ( pthis->m_pimpl->error && pthis->m_pimpl->error_data.size() < 65536 )
+	{
+		// Do not feed error responses to destination stream
+		pthis->m_pimpl->error_data.append( static_cast<char*>(ptr), size * nmemb ) ;
+		return size * nmemb ;
+	}
+	return pthis->m_pimpl->dest->Write( static_cast<char*>(ptr), size * nmemb ) ;
+}
+
+int CurlAgent::progress_callback( CurlAgent *pthis, curl_off_t totalDownload, curl_off_t finishedDownload, curl_off_t totalUpload, curl_off_t finishedUpload )
+{
+	// Only report download progress when set explicitly
+	if ( pthis->m_pb )
+	{
+		totalDownload = pthis->m_pimpl->total_download;
+		if ( !totalUpload )
+			totalUpload = pthis->m_pimpl->total_upload;
+		pthis->m_pb->reportProgress(
+			totalDownload > 0 ? totalDownload : totalUpload,
+			totalDownload > 0 ? finishedDownload : finishedUpload
+		);
+	}
+	return 0;
 }
 
 long CurlAgent::ExecCurl(
 	const std::string&	url,
-	Receivable			*dest,
+	DataStream			*dest,
 	const http::Header&	hdr )
 {
 	CURL *curl = m_pimpl->curl ;
@@ -124,116 +184,92 @@ long CurlAgent::ExecCurl(
 	::curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, 	error ) ;
 	::curl_easy_setopt(curl, CURLOPT_URL, 			url.c_str());
 	::curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION,	&CurlAgent::Receive ) ;
-	::curl_easy_setopt(curl, CURLOPT_WRITEDATA,		dest ) ;
+	::curl_easy_setopt(curl, CURLOPT_WRITEDATA,		this ) ;
+	m_pimpl->dest = dest ;
 
-	SetHeader( hdr ) ;
+	struct curl_slist *slist = SetHeader( m_pimpl->curl, hdr ) ;
+
+	curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L);
+    #if LIBCURL_VERSION_NUM >= 0x072000
+  	  curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, progress_callback);
+	  curl_easy_setopt(curl, CURLOPT_XFERINFODATA, this);
+    #endif
 
-	dest->Clear() ;
 	CURLcode curl_code = ::curl_easy_perform(curl);
 
-	long http_code = 0;
-	::curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code);
+	curl_slist_free_all(slist);
 
+	// get the HTTP response code
+	long http_code = 0;
+	::curl_easy_getinfo(curl,	CURLINFO_RESPONSE_CODE, &http_code);
 	Trace( "HTTP response %1%", http_code ) ;
-	::curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, 	0 ) ;
-	
-	if ( curl_code != CURLE_OK || http_code >= 400 )
+
+	// reset the curl buffer to prevent it from touching our "error" buffer
+	::curl_easy_setopt(curl,	CURLOPT_ERRORBUFFER, 	0 ) ;
+
+	m_pimpl->dest = NULL;
+
+	// only throw for libcurl errors
+	if ( curl_code != CURLE_OK )
 	{
 		BOOST_THROW_EXCEPTION(
 			Error()
 				<< CurlCode( curl_code )
-				<< HttpResponse( http_code )
 				<< Url( url )
-				<< expt::ErrMsg( error )
-				<< HttpHeader( hdr )
+				<< CurlErrMsg( error )
+				<< HttpRequestHeaders( hdr )
 		) ;
 	}
 
 	return http_code ;
 }
 
-long CurlAgent::Put(
-	const std::string&		url,
-	const std::string&		data,
-	Receivable				*dest,
-	const Header&			hdr )
+long CurlAgent::Request(
+	const std::string&	method,
+	const std::string&	url,
+	SeekStream			*in,
+	DataStream			*dest,
+	const Header&		hdr,
+	u64_t			downloadFileBytes )
 {
-	Trace("HTTP PUT \"%1%\"", url ) ;
-	
-	Init() ;
-	CURL *curl = m_pimpl->curl ;
+	Trace("HTTP %1% \"%2%\"", method, url ) ;
 
-	std::string put_data = data ;
+	Init() ;
+	m_pimpl->total_download = downloadFileBytes ;
+	CURL *curl = m_pimpl->curl ;
 
 	// set common options
-	::curl_easy_setopt(curl, CURLOPT_UPLOAD,		1L ) ;
-	::curl_easy_setopt(curl, CURLOPT_READFUNCTION,	&ReadCallback ) ;
-	::curl_easy_setopt(curl, CURLOPT_READDATA ,		&put_data ) ;
-	::curl_easy_setopt(curl, CURLOPT_INFILESIZE, 	put_data.size() ) ;
-	
-	return ExecCurl( url, dest, hdr ) ;
-}
-
-long CurlAgent::Get(
-	const std::string& 		url,
-	Receivable				*dest,
-	const Header&			hdr )
-{
-	Trace("HTTP GET \"%1%\"", url ) ;
-	Init() ;
-
-	// set get specific options
-	::curl_easy_setopt(m_pimpl->curl, CURLOPT_HTTPGET, 		1L);
-
-	return ExecCurl( url, dest, hdr ) ;
-}
-
-long CurlAgent::Post(
-	const std::string& 		url,
-	const std::string&		data,
-	Receivable				*dest,
-	const Header&			hdr )
-{
-	Trace("HTTP POST \"%1%\" with \"%2%\"", url, data ) ;
-
-	Init() ;
-	CURL *curl = m_pimpl->curl ;
-
-	// make a copy because the parameter is const
-	std::string post_data = data ;
-	
-	// set post specific options
-	::curl_easy_setopt(curl, CURLOPT_POST, 			1L);
-	::curl_easy_setopt(curl, CURLOPT_POSTFIELDS,	&post_data[0] ) ;
-	::curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, post_data.size() ) ;
-
-	return ExecCurl( url, dest, hdr ) ;
-}
-
-long CurlAgent::Custom(
-	const std::string&		method,
-	const std::string&		url,
-	Receivable				*dest,
-	const Header&			hdr )
-{
-	Trace("HTTP %2% \"%1%\"", url, method ) ;
-
-	CURL *curl = m_pimpl->curl ;
-
 	::curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, method.c_str() );
-// 	::curl_easy_setopt(curl, CURLOPT_VERBOSE,		1 );
+	if ( in )
+	{
+		::curl_easy_setopt(curl, CURLOPT_UPLOAD,			1L ) ;
+		::curl_easy_setopt(curl, CURLOPT_READFUNCTION,		&ReadFileCallback ) ;
+		::curl_easy_setopt(curl, CURLOPT_READDATA ,			in ) ;
+		::curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, 	static_cast<curl_off_t>( in->Size() ) ) ;
+	}
 
 	return ExecCurl( url, dest, hdr ) ;
 }
 
-void CurlAgent::SetHeader( const Header& hdr )
+static struct curl_slist* SetHeader( CURL *handle, const Header& hdr )
 {
 	// set headers
 	struct curl_slist *curl_hdr = 0 ;
     for ( Header::iterator i = hdr.begin() ; i != hdr.end() ; ++i )
 		curl_hdr = curl_slist_append( curl_hdr, i->c_str() ) ;
 	
-	::curl_easy_setopt( m_pimpl->curl, CURLOPT_HTTPHEADER, curl_hdr ) ;
+	::curl_easy_setopt( handle, CURLOPT_HTTPHEADER, curl_hdr ) ;
+	return curl_hdr;
+}
+
+std::string CurlAgent::LastError() const
+{
+	return m_pimpl->error_data ;
+}
+
+std::string CurlAgent::LastErrorHeaders() const
+{
+	return m_pimpl->error_headers ;
 }
 
 std::string CurlAgent::RedirLocation() const

libgrive/src/http/CurlAgent.hh

@@ -24,9 +24,13 @@
 #include <memory>
 #include <string>
 
-namespace gr { namespace http {
+#include <curl/curl.h>
 
-class Receivable ;
+namespace gr {
+
+class DataStream ;
+
+namespace http {
 
 /*!	\brief	agent to provide HTTP access
 	
@@ -38,50 +42,45 @@ class CurlAgent : public Agent
 public :
 	CurlAgent() ;
 	~CurlAgent() ;
-	
-	long Put(
-		const std::string&	url,
-		const std::string&	data,
-		Receivable			*dest,
-		const Header&		hdr ) ;
 
-	long Get(
-		const std::string& 	url,
-		Receivable			*dest,
-		const Header&		hdr ) ;
-	
-	long Post(
-		const std::string& 	url,
-		const std::string&	data,
-		Receivable			*dest,
-		const Header&		hdr ) ;
-	
-	long Custom(
+	ResponseLog* GetLog() const ;
+	void SetLog( ResponseLog *log ) ;
+	void SetProgressReporter( Progress *progress ) ;
+
+	long Request(
 		const std::string&	method,
 		const std::string&	url,
-		Receivable			*dest,
-		const Header&		hdr ) ;
+		SeekStream			*in,
+		DataStream			*dest,
+		const Header&		hdr,
+		u64_t			downloadFileBytes = 0 ) ;
+
+	std::string LastError() const ;
+	std::string LastErrorHeaders() const ;
 	
 	std::string RedirLocation() const ;
 	
 	std::string Escape( const std::string& str ) ;
 	std::string Unescape( const std::string& str ) ;
 
+	static int progress_callback( CurlAgent *pthis, curl_off_t totalDownload, curl_off_t finishedDownload, curl_off_t totalUpload, curl_off_t finishedUpload );
+
 private :
 	static std::size_t HeaderCallback( void *ptr, size_t size, size_t nmemb, CurlAgent *pthis ) ;
-	static std::size_t Receive( void* ptr, size_t size, size_t nmemb, Receivable *recv ) ;
-	
-	void SetHeader( const Header& hdr ) ;
+	static std::size_t Receive( void* ptr, size_t size, size_t nmemb, CurlAgent *pthis ) ;
+
 	long ExecCurl(
 		const std::string&	url,
-		Receivable			*dest,
+		DataStream			*dest,
 		const Header&		hdr ) ;
 
 	void Init() ;
-	
+
 private :
 	struct Impl ;
-	std::auto_ptr<Impl>	m_pimpl ;
+	std::unique_ptr<Impl> m_pimpl ;
+	std::unique_ptr<ResponseLog> m_log ;
+	Progress* m_pb ;
 } ;
 
 } } // end of namespace

libgrive/src/http/Download.cc

@@ -20,7 +20,6 @@
 #include "Download.hh"
 // #include "util/SignalHandler.hh"
 
-#include "Error.hh"
 #include "util/Crypt.hh"
 
 // boost headers
@@ -65,7 +64,7 @@ std::string Download::Finish() const
 	return m_crypt.get() != 0 ? m_crypt->Get() : "" ;
 }
 
-std::size_t Download::OnData( void *data, std::size_t count )
+std::size_t Download::Write( const char *data, std::size_t count )
 {
 	assert( data != 0 ) ;
 	
@@ -75,4 +74,10 @@ std::size_t Download::OnData( void *data, std::size_t count )
 	return m_file.Write( data, count ) ;
 }
 
+
+std::size_t Download::Read( char *data, std::size_t count )
+{
+	return count ;
+}
+
 } } // end of namespace

libgrive/src/http/Download.hh

@@ -19,8 +19,7 @@
 
 #pragma once
 
-#include "Receivable.hh"
-#include "util/StdioFile.hh"
+#include "util/File.hh"
 
 #include <string>
 
@@ -33,7 +32,7 @@ namespace crypt
 
 namespace http {
 
-class Download : public http::Receivable
+class Download : public DataStream
 {
 public :
 	struct NoChecksum {} ;
@@ -44,11 +43,12 @@ public :
 	std::string Finish() const ;
 	
 	void Clear() ;
-	std::size_t OnData( void *data, std::size_t count ) ;
+	std::size_t Write( const char *data, std::size_t count ) ;
+	std::size_t Read( char *, std::size_t ) ; 
 	
 private :
-	StdioFile					m_file ;
-	std::auto_ptr<crypt::MD5>	m_crypt ;
+	File						m_file ;
+	std::unique_ptr<crypt::MD5>	m_crypt ;
 } ;
 
 } } // end of namespace

libgrive/src/http/Error.hh

@@ -27,18 +27,24 @@ namespace gr { namespace http {
 struct Error : virtual Exception {} ;
 
 // CURL error code
-typedef boost::error_info<struct CurlCodeTag, int>			CurlCode ;
+typedef boost::error_info<struct CurlCodeTag, int>					CurlCode ;
 
-// HTTP response code
-typedef boost::error_info<struct HttpResponseTag, int>		HttpResponse ;
-
-// HTTP response body
-typedef boost::error_info<struct HttpResponseStrTag, std::string>	HttpResponseText ;
+// CURL error message
+typedef boost::error_info<struct CurlErrMsgTag, std::string>		CurlErrMsg ;
 
 // URL
-typedef boost::error_info<struct UrlTag, std::string>		Url ;
+typedef boost::error_info<struct UrlTag, std::string>				Url ;
 
-// HTTP headers
-typedef boost::error_info<struct HeaderTag, Header>		HttpHeader ;
+// HTTP request headers
+typedef boost::error_info<struct RequestHeadersTag, Header>			HttpRequestHeaders ;
+
+// HTTP response code
+typedef boost::error_info<struct ResponseCodeTag, int>				HttpResponseCode ;
+
+// HTTP response headers
+typedef boost::error_info<struct ResponseHeadersTag, std::string>	HttpResponseHeaders ;
+
+// HTTP response body
+typedef boost::error_info<struct ResponseBodyTag, std::string>		HttpResponseText ;
 
 } } // end of namespace

libgrive/src/http/Header.cc

@@ -22,6 +22,7 @@
 #include <algorithm>
 #include <iterator>
 #include <ostream>
+#include <sstream>
 
 namespace gr { namespace http {
 
@@ -34,6 +35,13 @@ void Header::Add( const std::string& str )
 	m_vec.push_back( str ) ;
 }
 
+std::string Header::Str() const
+{
+	std::ostringstream s ;
+	s << *this ;
+	return s.str() ;
+}
+
 Header::iterator Header::begin() const
 {
 	return m_vec.begin() ;
@@ -50,4 +58,11 @@ std::ostream& operator<<( std::ostream& os, const Header& h )
 	return os ;
 }
 
+Header operator+( const Header& header, const std::string& str )
+{
+	Header h( header ) ;
+	h.Add( str ) ;
+	return h ;
+}
+
 } } // end of namespace

libgrive/src/http/Header.hh

@@ -37,6 +37,7 @@ public :
 	Header() ;
 	
 	void Add( const std::string& str ) ;
+	std::string Str() const ;
 	
 	iterator begin() const ;
 	iterator end() const ;
@@ -46,5 +47,6 @@ private :
 } ;
 
 std::ostream& operator<<( std::ostream& os, const Header& h ) ;
+Header operator+( const Header& header, const std::string& str ) ;
 
 }} // end of namespace

libgrive/src/http/ResponseLog.cc

@@ -19,6 +19,7 @@
 
 #include "ResponseLog.hh"
 
+#include "util/log/Log.hh"
 #include "util/DateTime.hh"
 
 #include <cassert>
@@ -27,28 +28,55 @@ namespace gr { namespace http {
 
 ResponseLog::ResponseLog(
 	const std::string&	prefix,
-	const std::string&	suffix,
-	Receivable			*next ) :
-	m_log( Filename(prefix, suffix).c_str() ),
-	m_next( next )
+	const std::string&	suffix )
 {
+	Reset( prefix, suffix ) ;
 }
 
-std::size_t ResponseLog::OnData( void *data, std::size_t count )
+std::size_t ResponseLog::Write( const char *data, std::size_t count )
 {
-	m_log.rdbuf()->sputn( reinterpret_cast<char*>(data), count ) ;
-	return m_next->OnData( data, count ) ;
+	if ( m_enabled )
+	{
+		assert( m_log.rdbuf() != 0 ) ;
+		m_log.rdbuf()->sputn( data, count ) ;
+		m_log.flush();
+	}
+	return count;
 }
 
-void ResponseLog::Clear()
+std::size_t ResponseLog::Read( char *data, std::size_t count )
 {
-	assert( m_next != 0 ) ;
-	m_next->Clear() ;
+	return 0 ;
 }
 
 std::string ResponseLog::Filename( const std::string& prefix, const std::string& suffix )
 {
-	return prefix + DateTime::Now().Format( "%H%M%S" ) + suffix ;
+	return prefix + DateTime::Now().Format( "%F.%H%M%S" ) + suffix ;
+}
+
+void ResponseLog::Reset( const std::string& prefix, const std::string& suffix )
+{
+	if ( m_log.is_open() )
+		m_log.close() ;
+	
+	const std::string fname = Filename( prefix, suffix ) ;
+	
+	// reset previous stream state. don't care if file can be opened
+	// successfully previously
+	m_log.clear() ;
+	
+	// re-open the file
+	m_log.open( fname.c_str() ) ;
+	if ( m_log )
+	{
+		Trace( "logging HTTP response: %1%", fname ) ;
+		m_enabled = true ;
+	}
+	else
+	{
+		Trace( "cannot open log file %1%", fname ) ;
+		m_enabled = false ;
+	}
 }
 
 }} // end of namespace

libgrive/src/http/ResponseLog.hh

@@ -19,30 +19,31 @@
 
 #pragma once
 
-#include "Receivable.hh"
+#include "util/DataStream.hh"
 
 #include <fstream>
 #include <string>
 
 namespace gr { namespace http {
 
-class ResponseLog : public Receivable
+class ResponseLog : public DataStream
 {
 public :
 	ResponseLog(
 		const std::string&	prefix,
-		const std::string&	suffix,
-		Receivable			*next ) ;
-
-	std::size_t OnData( void *data, std::size_t count ) ;
-	void Clear() ;
-
+		const std::string&	suffix ) ;
+	
+	std::size_t Write( const char *data, std::size_t count ) ;
+	std::size_t Read( char *data, std::size_t count ) ;
+	
+	void Reset( const std::string& prefix, const std::string& suffix ) ;
+	
 private :
 	static std::string Filename( const std::string& prefix, const std::string& suffix ) ;
 	
 private :
+	bool	m_enabled ;
 	std::ofstream	m_log ;
-	Receivable		*m_next ;
 } ;
 
 } } // end of namespace

libgrive/src/http/StringResponse.cc

@@ -30,9 +30,14 @@ void StringResponse::Clear()
 	m_resp.clear() ;
 }
 
-std::size_t StringResponse::OnData( void *data, std::size_t count )
+std::size_t StringResponse::Write( const char *data, std::size_t count )
+{
+	m_resp.append( data, count ) ;
+	return count ;
+}
+
+std::size_t StringResponse::Read( char *data, std::size_t count )
 {
-	m_resp.append( reinterpret_cast<char*>(data), count ) ;
 	return count ;
 }
 

libgrive/src/http/StringResponse.hh

@@ -19,18 +19,19 @@
 
 #pragma once
 
-#include "Receivable.hh"
+#include "util/DataStream.hh"
 
 #include <string>
 
 namespace gr { namespace http {
 
-class StringResponse : public Receivable
+class StringResponse : public DataStream
 {
 public :
 	StringResponse() ;
 
-	std::size_t OnData( void *data, std::size_t count ) ;
+	std::size_t Write( const char *data, std::size_t count ) ;
+	std::size_t Read( char *data, std::size_t count ) ;
 	void Clear() ;
 
 	const std::string& Response() const ;

libgrive/src/http/XmlResponse.cc

@@ -28,15 +28,21 @@ XmlResponse::XmlResponse() : m_tb( new xml::TreeBuilder )
 {
 }
 
-std::size_t XmlResponse::OnData( void *data, std::size_t count )
+void XmlResponse::Clear()
 {
-	m_tb->ParseData( reinterpret_cast<char*>(data), count ) ;
+	m_tb.reset(new xml::TreeBuilder);
+}
+
+std::size_t XmlResponse::Write( const char *data, std::size_t count )
+{
+	m_tb->ParseData( data, count ) ;
 	return count ;
 }
 
-void XmlResponse::Clear()
+std::size_t XmlResponse::Read( char *, std::size_t )
 {
-	m_tb.reset( new xml::TreeBuilder ) ;
+	// throw something better
+	throw -1 ;
 }
 
 void XmlResponse::Finish()

libgrive/src/http/XmlResponse.hh

@@ -19,31 +19,27 @@
 
 #pragma once
 
-#include "Receivable.hh"
+#include "util/DataStream.hh"
+#include "xml/TreeBuilder.hh"
 
 #include <memory>
 
-namespace gr { namespace xml
-{
-	class Node ;
-	class TreeBuilder ;
-} }
-
 namespace gr { namespace http {
 
-class XmlResponse : public Receivable
+class XmlResponse : public DataStream
 {
 public :
 	XmlResponse() ;
 
 	void Clear() ;
-	std::size_t OnData( void *data, std::size_t count ) ;
+	std::size_t Write( const char *data, std::size_t count ) ;
+	std::size_t Read( char *data, std::size_t count ) ;
 	void Finish() ;
 
 	xml::Node Response() const ;
 	
 private :
-	std::auto_ptr<xml::TreeBuilder>	m_tb ;
+	std::unique_ptr<xml::TreeBuilder>	m_tb ;
 } ;
 
 } } // end of namespace

libgrive/src/json/JsonParser.cc

@@ -0,0 +1,196 @@
+/*
+	grive: an GPL program to sync a local directory with Google Drive
+	Copyright (C) 2013 Wan Wai Ho
+
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License
+	as published by the Free Software Foundation version 2
+	of the License.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with this program; if not, write to the Free Software
+	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+	MA  02110-1301, USA.
+*/
+
+#include "JsonParser.hh"
+
+#include "Val.hh"
+#include "ValBuilder.hh"
+
+#include <yajl/yajl_parse.h>
+
+namespace gr {
+
+namespace
+{
+	int OnNull( void *ctx )
+	{
+		ValVisitor *b = reinterpret_cast<ValVisitor*>(ctx) ;
+		b->VisitNull() ;
+		return true ;
+	}
+	
+	int OnBool( void *ctx, int value )
+	{
+		ValVisitor *b = reinterpret_cast<ValVisitor*>(ctx) ;
+		b->Visit( static_cast<bool>(value) ) ;
+		return true ;
+	}
+	
+	int OnInt( void *ctx, long long value )
+	{
+		ValVisitor *b = reinterpret_cast<ValVisitor*>(ctx) ;
+		b->Visit(value) ;
+		return true ;
+	}
+	
+	int OnDouble( void *ctx, double value )
+	{
+		ValVisitor *b = reinterpret_cast<ValVisitor*>(ctx) ;
+		b->Visit(value) ;
+		return true ;
+	}
+	
+	int OnStr( void *ctx, const unsigned char *str, std::size_t len )
+	{
+		ValVisitor *b = reinterpret_cast<ValVisitor*>(ctx) ;
+		b->Visit( std::string(reinterpret_cast<const char*>(str), len) ) ;
+		return true ;
+	}
+	
+	int StartMap( void *ctx )
+	{
+		ValVisitor *b = reinterpret_cast<ValVisitor*>(ctx) ;
+		b->StartObject() ;
+		return true ;
+	}
+	
+	int OnMapKey( void *ctx, const unsigned char *str, std::size_t len )
+	{
+		ValVisitor *b = reinterpret_cast<ValVisitor*>(ctx) ;
+		b->VisitKey( std::string(reinterpret_cast<const char*>(str), len) ) ;
+		return true ;
+	}
+	
+	int EndMap( void *ctx )
+	{
+		ValVisitor *b = reinterpret_cast<ValVisitor*>(ctx) ;
+		b->EndObject() ;
+		return true ;
+	}
+	
+	int StartArray( void *ctx )
+	{
+		ValVisitor *b = reinterpret_cast<ValVisitor*>(ctx) ;
+		b->StartArray() ;
+		return true ;
+	}
+	
+	int EndArray( void *ctx )
+	{
+		ValVisitor *b = reinterpret_cast<ValVisitor*>(ctx) ;
+		b->EndArray() ;
+		return true ;
+	}
+
+	const yajl_callbacks callbacks = {  
+		OnNull,  
+		OnBool,  
+		OnInt,  
+		OnDouble,  
+		0,  
+		OnStr,  
+		StartMap,  
+		OnMapKey,  
+		EndMap,  
+		StartArray,  
+		EndArray,  
+	};  
+}
+
+Val ParseJson( const std::string& json )
+{
+	ValBuilder b;
+	JsonParser parser( &b ) ;
+	parser.Parse( json.c_str(), json.size() ) ;
+	parser.Finish() ;
+	return b.Result();
+}
+
+Val ParseJson( DataStream &in )
+{
+	ValBuilder b;
+	JsonParser parser( &b ) ;
+	parser.Parse( in ) ;
+	parser.Finish() ;
+	return b.Result();
+}
+
+struct JsonParser::Impl
+{
+	ValVisitor	*callback ;
+	yajl_handle	hand ;
+} ;
+
+JsonParser::JsonParser( ValVisitor *callback ) :
+	m_impl( new Impl )
+{
+	m_impl->callback	= callback ;
+	m_impl->hand		= yajl_alloc( &callbacks, 0, m_impl->callback ) ;
+}
+
+JsonParser::~JsonParser()
+{
+	yajl_free( m_impl->hand ) ;
+}
+
+void JsonParser::Parse( const char *str, std::size_t size )
+{
+	const unsigned char *ustr = reinterpret_cast<unsigned const char*>(str) ;
+	
+	yajl_status r = yajl_parse( m_impl->hand, ustr, size ) ;
+	
+	if ( r != yajl_status_ok )
+	{
+		unsigned char *msg = yajl_get_error( m_impl->hand, true, ustr, size ) ;
+		std::string msg_str(reinterpret_cast<char*>(msg)) ;
+		yajl_free_error(m_impl->hand, msg) ;
+		
+		BOOST_THROW_EXCEPTION(
+			Error()
+				<< ParseErr_(msg_str)
+				<< JsonText_(std::string(str,size))
+		);
+	}
+}
+
+void JsonParser::Parse( DataStream &in )
+{
+	char buf[1024] ;
+	std::size_t count = 0 ;
+
+	while ( (count = in.Read( buf, sizeof(buf) ) ) > 0 )
+	{
+		Parse( buf, count );
+	}
+}
+
+void JsonParser::Finish()
+{
+	if ( yajl_complete_parse(m_impl->hand) != yajl_status_ok )
+	{
+		unsigned char *msg = yajl_get_error( m_impl->hand, false, 0, 0 ) ;
+		std::string msg_str(reinterpret_cast<char*>(msg)) ;
+		yajl_free_error(m_impl->hand, msg) ;
+		
+		BOOST_THROW_EXCEPTION( Error() << ParseErr_(msg_str) ) ;
+	}
+}
+
+} // end of namespace gr::json

libgrive/src/json/JsonParser.hh

@@ -0,0 +1,57 @@
+/*
+	grive: an GPL program to sync a local directory with Google Drive
+	Copyright (C) 2013 Wan Wai Ho
+
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License
+	as published by the Free Software Foundation version 2
+	of the License.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with this program; if not, write to the Free Software
+	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+	MA  02110-1301, USA.
+*/
+
+#pragma once
+
+#include "Val.hh"
+#include "util/Exception.hh"
+#include "util/DataStream.hh"
+
+#include <string>
+#include <memory>
+
+namespace gr {
+
+class ValVisitor ;
+
+Val ParseJson( const std::string& json ) ;
+Val ParseJson( DataStream &in ) ;
+
+class JsonParser
+{
+public :
+	struct Error : virtual Exception {} ;
+	typedef boost::error_info<struct ParseErr,	std::string> ParseErr_ ;
+	typedef boost::error_info<struct JsonText,	std::string> JsonText_ ;
+
+	explicit JsonParser( ValVisitor *callback ) ;
+	~JsonParser() ;
+	
+	void Parse( const char *str, std::size_t size ) ;
+	void Parse( DataStream &in ) ;
+	void Finish() ;
+	
+private :
+	struct Impl ;
+	std::unique_ptr<Impl>	m_impl ;
+} ;
+
+} // end of namespace
+

libgrive/src/json/JsonWriter.cc

@@ -0,0 +1,119 @@
+/*
+	grive: an GPL program to sync a local directory with Google Drive
+	Copyright (C) 2013 Wan Wai Ho
+
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License
+	as published by the Free Software Foundation version 2
+	of the License.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with this program; if not, write to the Free Software
+	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+	MA  02110-1301, USA.
+*/
+
+#include "JsonWriter.hh"
+#include "util/StringStream.hh"
+
+#include <yajl/yajl_gen.h>
+
+#include <cassert>
+
+namespace gr {
+
+struct JsonWriter::Impl
+{
+	yajl_gen	gen ;
+	DataStream 	*out ;
+} ;
+
+JsonWriter::JsonWriter( DataStream *out ) :
+	m_impl( new Impl )
+{
+	assert( out != 0 ) ;
+
+	m_impl->out = out ;
+	m_impl->gen = yajl_gen_alloc(0) ;
+	yajl_gen_config( m_impl->gen, yajl_gen_print_callback, &JsonWriter::WriteCallback, this ) ;
+}
+
+JsonWriter::~JsonWriter()
+{
+	yajl_gen_free( m_impl->gen ) ;
+}
+
+void JsonWriter::Visit( long long t )
+{
+	yajl_gen_integer( m_impl->gen, t ) ;
+}
+
+void JsonWriter::Visit( double t )
+{
+	yajl_gen_double( m_impl->gen, t ) ;
+}
+
+void JsonWriter::Visit( const std::string& t )
+{
+	yajl_gen_string( m_impl->gen,
+		reinterpret_cast<const unsigned char*>(t.c_str()), t.size() ) ;
+}
+
+void JsonWriter::Visit( bool t )
+{
+	yajl_gen_bool( m_impl->gen, t ) ;
+}
+
+void JsonWriter::VisitNull()
+{
+	yajl_gen_null( m_impl->gen ) ;
+}
+
+void JsonWriter::StartArray()
+{
+	yajl_gen_array_open( m_impl->gen ) ;
+}
+
+void JsonWriter::EndArray()
+{
+	yajl_gen_array_close( m_impl->gen ) ;
+}
+
+void JsonWriter::StartObject()
+{
+	yajl_gen_map_open( m_impl->gen ) ;
+}
+
+void JsonWriter::VisitKey( const std::string& t )
+{
+	Visit(t) ;
+}
+
+void JsonWriter::EndObject()
+{
+	yajl_gen_map_close( m_impl->gen ) ;
+}
+
+void JsonWriter::WriteCallback( void *ctx, const char *str, std::size_t size )
+{
+	JsonWriter *pthis = reinterpret_cast<JsonWriter*>(ctx) ;
+	assert( pthis != 0 ) ;
+	assert( pthis->m_impl->out != 0 ) ;
+	
+	pthis->m_impl->out->Write( str, size ) ;
+}
+
+std::string WriteJson( const Val& val )
+{
+	StringStream ss ;
+	JsonWriter wr( &ss ) ;
+	val.Visit( &wr ) ;
+	return ss.Str() ;
+}
+
+} // end of namespace

libgrive/src/json/JsonWriter.hh

@@ -0,0 +1,60 @@
+/*
+	grive: an GPL program to sync a local directory with Google Drive
+	Copyright (C) 2013 Wan Wai Ho
+
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License
+	as published by the Free Software Foundation version 2
+	of the License.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with this program; if not, write to the Free Software
+	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+	MA  02110-1301, USA.
+*/
+
+#pragma once
+
+#include "Val.hh"
+#include "ValVisitor.hh"
+#include <memory>
+
+namespace gr {
+
+class DataStream ;
+
+class JsonWriter : public ValVisitor
+{
+public :
+	JsonWriter( DataStream *out ) ;
+	~JsonWriter() ;
+
+	void Visit( long long t ) ;
+	void Visit( double t ) ;
+	void Visit( const std::string& t ) ;
+	void Visit( bool t ) ;
+	void VisitNull() ;
+
+	void StartArray() ;
+	void EndArray() ;
+	void StartObject() ;
+	void VisitKey( const std::string& t ) ;
+	void EndObject() ;
+
+private :
+	static void WriteCallback( void *ctx, const char *str, std::size_t size ) ;
+
+private :
+	struct Impl ;
+	std::unique_ptr<Impl>	m_impl ;
+} ;
+
+std::string WriteJson( const Val& val );
+
+} // end of namespace
+

libgrive/src/json/Val.cc

@@ -0,0 +1,315 @@
+/*
+	grive: an GPL program to sync a local directory with Google Drive
+	Copyright (C) 2013 Wan Wai Ho
+
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License
+	as published by the Free Software Foundation version 2
+	of the License.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with this program; if not, write to the Free Software
+	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+	MA  02110-1301, USA.
+*/
+
+#include "Val.hh"
+#include "JsonWriter.hh"
+#include "ValVisitor.hh"
+#include "util/StdStream.hh"
+
+#include <iostream>
+
+namespace gr {
+
+const Val& Val::Null()
+{
+	static const Val null( null_type ) ;
+	return null ;
+}
+
+Val::Val( ) :
+	m_base( new Impl<Object> )
+{
+}
+
+Val::Val( TypeEnum type )
+{
+	switch ( type )
+	{
+		case int_type:		m_base.reset( new Impl<long long> ) ;	break ;
+		case bool_type:		m_base.reset( new Impl<bool> ) ; 		break ;
+		case double_type:	m_base.reset( new Impl<double> ) ; 		break ;
+		case string_type:	m_base.reset( new Impl<std::string> ) ;	break ;
+		case array_type:	m_base.reset( new Impl<Array> ) ; break ;
+		case object_type:	m_base.reset( new Impl<Object> ) ; break ;
+		case null_type:	
+		default:			m_base.reset( new Impl<void> ) ; break ;
+	}
+}
+
+Val::Val( const Val& v ) :
+	m_base( v.m_base->Clone() )
+{
+}
+
+Val::~Val()
+{
+}
+
+void Val::Swap( Val& val )
+{
+	std::swap( m_base, val.m_base ) ;
+}
+
+Val& Val::operator=( const Val& val )
+{
+	Val tmp(val) ;
+	Swap(tmp) ;
+	return *this ;
+}
+
+Val::TypeEnum Val::Type() const
+{
+	return m_base->Type() ;
+}
+
+const Val& Val::operator[]( const std::string& key ) const
+{
+	const Object& obj = As<Object>() ;
+	Object::const_iterator i = obj.find(key) ;
+	if ( i != obj.end() )
+		return i->second ;
+	
+	// shut off compiler warning
+	BOOST_THROW_EXCEPTION(Error() << NoKey_(key)) ;
+	throw ;
+}
+
+Val& Val::operator[]( const std::string& key )
+{
+	Object& obj = As<Object>() ;
+	Object::iterator i = obj.find(key) ;
+	if ( i != obj.end() )
+		return i->second ;
+	
+	// shut off compiler warning
+	BOOST_THROW_EXCEPTION(Error() << NoKey_(key)) ;
+	throw ;
+}
+
+const Val& Val::operator[]( std::size_t index ) const
+{
+	const Array& ar = As<Array>() ;
+	if ( index < ar.size() )
+		return ar[index] ;
+	
+	// shut off compiler warning
+	BOOST_THROW_EXCEPTION(Error() << OutOfRange_(index)) ;
+	throw ;
+}
+
+std::string Val::Str() const
+{
+	if ( Type() == int_type )
+		return boost::to_string( As<long long>() );
+	return As<std::string>() ;
+}
+
+Val::operator std::string() const
+{
+	return Str();
+}
+
+int Val::Int() const
+{
+	if ( Type() == string_type )
+		return std::atoi( As<std::string>().c_str() );
+	return static_cast<int>(As<long long>()) ;
+}
+
+unsigned long long Val::U64() const
+{
+	if ( Type() == string_type )
+		return strtoull( As<std::string>().c_str(), NULL, 10 );
+	return static_cast<unsigned long long>(As<long long>()) ;
+}
+
+double Val::Double() const
+{
+	if ( Type() == string_type )
+		return std::atof( As<std::string>().c_str() );
+	return As<double>() ;
+}
+
+bool Val::Bool() const
+{
+	return As<bool>() ;
+}
+
+const Val::Array& Val::AsArray() const
+{
+	return As<Array>() ;
+}
+
+Val::Array& Val::AsArray()
+{
+	return As<Array>() ;
+}
+
+const Val::Object& Val::AsObject() const
+{
+	return As<Object>() ;
+}
+
+Val::Object& Val::AsObject()
+{
+	return As<Object>() ;
+}
+
+bool Val::Has( const std::string& key ) const
+{
+	const Object& obj = As<Object>() ;
+	return obj.find(key) != obj.end() ;
+}
+
+bool Val::Del( const std::string& key )
+{
+	Object& obj = As<Object>() ;
+	return obj.erase(key) > 0 ;
+}
+
+Val& Val::Item( const std::string& key )
+{
+	return As<Object>()[key];
+}
+
+bool Val::Get( const std::string& key, Val& val ) const
+{
+	const Object& obj = As<Object>() ;
+	Object::const_iterator i = obj.find(key) ;
+	if ( i != obj.end() )
+	{
+		val = i->second ;
+		return true ;
+	}
+	else
+		return false ;
+}
+
+void Val::Add( const std::string& key, const Val& value )
+{
+	As<Object>().insert( std::make_pair(key, value) ) ;
+}
+
+void Val::Set( const std::string& key, const Val& value )
+{
+	Object& obj = As<Object>();
+	Object::iterator i = obj.find(key);
+	if (i == obj.end())
+		obj.insert(std::make_pair(key, value));
+	else
+		i->second = value;
+}
+
+void Val::Add( const Val& json )
+{
+	As<Array>().push_back( json ) ;
+}
+
+void Val::Visit( ValVisitor *visitor ) const
+{
+	switch ( Type() )
+	{
+		case null_type:		visitor->VisitNull() ;					break ;
+		case int_type:		visitor->Visit( As<long long>() ) ;		break ;
+		case double_type:	visitor->Visit( As<double>() ) ;		break ;
+		case string_type:	visitor->Visit( As<std::string>() ) ;	break ;
+		case bool_type:		visitor->Visit( As<bool>() ) ;			break ;
+		
+		case object_type:
+		{
+			visitor->StartObject() ;
+			
+			const Object& obj = As<Object>() ;
+			for ( Object::const_iterator i = obj.begin() ; i != obj.end() ; ++i )
+			{
+				visitor->VisitKey( i->first ) ;
+				i->second.Visit( visitor ) ;
+			}
+			
+			visitor->EndObject() ;
+			break ;
+		}
+
+		case array_type:
+		{
+			visitor->StartArray() ;
+			
+			const Array& arr = As<Array>() ;
+			for ( Array::const_iterator i = arr.begin() ; i != arr.end() ; ++i )
+				i->Visit( visitor ) ;
+			
+			visitor->EndArray() ;
+			break ;
+		}
+	}
+}
+
+void Val::Select( const Object& obj, const std::string& key, std::vector<Val>& result ) const
+{
+	Object::const_iterator i = obj.find(key) ;
+	if ( i != obj.end() )
+		result.push_back(i->second) ;
+}
+
+/**	If \a this is an array of objects, this function returns all values of
+	the objects in the array with the key \a key. If \a this is an object,
+	just return the value with the key \a key. 
+*/
+std::vector<Val> Val::Select( const std::string& key ) const
+{
+	std::vector<Val> result ;
+	if ( Is<Object>() )
+		Select( As<Object>(), key, result ) ;
+	
+	else if ( Is<Array>() )
+	{
+		const Array& array = As<Array>() ;
+		for ( Array::const_iterator i = array.begin() ; i != array.end() ; ++i )
+		{
+			if ( i->Is<Object>() )
+				Select( i->As<Object>(), key, result ) ;
+		}
+	}
+	return result ;
+}
+
+std::ostream& operator<<( std::ostream& os, const Val& val )
+{
+	StdStream ss( os.rdbuf() ) ;
+	JsonWriter wr( &ss ) ;
+	val.Visit( &wr ) ;
+
+	return os ;
+}
+
+} // end of namespace
+
+namespace std
+{
+	void swap( gr::Val& v1, gr::Val& v2 )
+	{
+		v1.Swap( v2 ) ;
+	}
+	
+	ostream& operator<<( ostream& os, gr::Val::TypeEnum t )
+	{
+		return os << static_cast<int>(t) ;
+	}
+}

libgrive/src/json/Val.hh

@@ -0,0 +1,234 @@
+/*
+	grive: an GPL program to sync a local directory with Google Drive
+	Copyright (C) 2013 Wan Wai Ho
+
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License
+	as published by the Free Software Foundation version 2
+	of the License.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with this program; if not, write to the Free Software
+	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+	MA  02110-1301, USA.
+*/
+
+#pragma once
+
+#include "util/Exception.hh"
+
+#include <cstddef>
+#include <iosfwd>
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace gr {
+
+class ValVisitor ;
+
+class Val
+{
+public :
+	enum TypeEnum { null_type, bool_type, double_type, int_type, object_type, array_type, string_type } ;
+
+	struct Error : virtual Exception {} ;
+	typedef boost::error_info<struct SrcType,	TypeEnum> SrcType_ ;
+	typedef boost::error_info<struct DestType,	TypeEnum> DestType_ ;
+	typedef boost::error_info<struct NoKey,		std::string> NoKey_ ;
+	typedef boost::error_info<struct OutOfRange,std::size_t> OutOfRange_ ;
+	
+private :
+	template <typename T>
+	struct Type2Enum ;
+
+	template <typename T>
+	struct SupportType ;
+
+public :
+	typedef std::vector<Val>			Array ;
+	typedef std::map<std::string, Val>	Object ;
+
+public :
+	Val() ;
+	Val( const Val& v ) ;
+	explicit Val( TypeEnum type ) ;
+	~Val() ;
+
+	static const Val& Null() ;
+
+	template <typename T>
+	explicit Val( const T& t )
+	{
+		Assign(t) ;
+	}
+
+	template <typename T>
+	Val& Assign( const T& t ) ;
+	
+	void Swap( Val& val ) ;
+	Val& operator=( const Val& val ) ;
+		
+	template <typename T>
+	Val& operator=( const T& t )
+	{
+		return Assign(t) ;
+	}
+
+	operator std::string() const ;
+
+	template <typename T>
+	const T& As() const ;
+
+	template <typename T>
+	T& As() ;
+
+	template <typename T>
+	bool Is() const ;
+
+	TypeEnum Type() const ;
+
+	// shortcuts for As<>()
+	std::string Str() const ;
+	int		Int() const ;
+	unsigned long long U64() const ;
+	double	Double() const ;
+	bool	Bool() const ;
+	const Array&	AsArray() const ;
+	Array&	AsArray() ;
+	const Object&	AsObject() const ;
+	Object&	AsObject() ;
+
+	// shortcuts for objects
+	Val& operator[]( const std::string& key ) ; // get updatable ref or throw
+	const Val& operator[]( const std::string& key ) const ; // get const ref or throw
+	Val& Item( const std::string& key ) ; // insert if not exists and get
+	bool Has( const std::string& key ) const ; // check if exists
+	bool Get( const std::string& key, Val& val ) const ; // get or return false
+	void Add( const std::string& key, const Val& val ) ; // insert or do nothing
+	void Set( const std::string& key, const Val& val ) ; // insert or update
+	bool Del( const std::string& key ); // delete or do nothing
+	
+	// shortcuts for array (and array of objects)
+	const Val& operator[]( std::size_t index ) const ;
+	void Add( const Val& json ) ;
+	
+	std::vector<Val> Select( const std::string& key ) const ;
+	
+	friend std::ostream& operator<<( std::ostream& os, const Val& val ) ;
+	void Visit( ValVisitor *visitor ) const ;
+
+private :
+	struct Base ;
+
+	template <typename T>
+	struct Impl ;
+	
+	std::unique_ptr<Base>	m_base ;
+
+private :
+	void Select( const Object& obj, const std::string& key, std::vector<Val>& result ) const ;
+} ;
+
+template <> struct Val::Type2Enum<void>			{ static const TypeEnum type = null_type ; } ;
+template <> struct Val::Type2Enum<long long>	{ static const TypeEnum type = int_type ;  } ;
+template <> struct Val::Type2Enum<bool>			{ static const TypeEnum type = bool_type ; } ;
+template <> struct Val::Type2Enum<double>		{ static const TypeEnum type = double_type ;} ;
+template <> struct Val::Type2Enum<std::string>	{ static const TypeEnum type = string_type ; } ;
+template <> struct Val::Type2Enum<Val::Array>	{ static const TypeEnum type = array_type ; } ;
+template <> struct Val::Type2Enum<Val::Object>	{ static const TypeEnum type = object_type ; } ;
+
+template <> struct Val::SupportType<int>			{ typedef long long	Type ; } ;
+template <> struct Val::SupportType<unsigned>		{ typedef long long	Type ; } ;
+template <> struct Val::SupportType<long>			{ typedef long long	Type ; } ;
+template <> struct Val::SupportType<unsigned long>	{ typedef long long	Type ; } ;
+template <> struct Val::SupportType<short>			{ typedef long long	Type ; } ;
+template <> struct Val::SupportType<unsigned short>	{ typedef long long	Type ; } ;
+template <> struct Val::SupportType<long long>			{ typedef long long	Type ; } ;
+template <> struct Val::SupportType<unsigned long long>	{ typedef long long	Type ; } ;
+
+template <> struct Val::SupportType<bool>			{ typedef bool		Type ; } ;
+template <> struct Val::SupportType<double>			{ typedef double	Type ; } ;
+template <> struct Val::SupportType<std::string>	{ typedef std::string	Type ; } ;
+template <> struct Val::SupportType<const char*>	{ typedef std::string	Type ; } ;
+template <> struct Val::SupportType<Val::Array>		{ typedef Val::Array	Type ; } ;
+template <> struct Val::SupportType<Val::Object>	{ typedef Val::Object	Type ; } ;
+
+struct Val::Base
+{
+	virtual ~Base() {}
+	virtual Base* Clone() const = 0 ;
+	virtual TypeEnum Type() const = 0 ;
+} ;
+
+template <typename T>
+struct Val::Impl : public Base
+{
+	T val ;
+	Impl( ) : val() {}
+	Impl( const T& t ) : val(t) {}
+	Impl<T>* Clone() const { return new Impl<T>(val); }
+	TypeEnum Type() const { return Type2Enum<T>::type ; }
+} ;
+
+template <>
+struct Val::Impl<void>	: public Base
+{
+	Impl<void>* Clone() const { return new Impl<void>; }
+	TypeEnum Type() const { return null_type ; }
+} ;
+
+template <typename T>
+Val& Val::Assign( const T& t )
+{
+	m_base.reset( new Impl<typename SupportType<T>::Type>(t) ) ;
+	return *this ;
+}
+
+template <typename T>
+const T& Val::As() const
+{
+	const Impl<T> *impl = dynamic_cast<const Impl<T> *>( m_base.get() ) ;
+	if ( !impl )
+	{
+		TypeEnum dest = Type2Enum<T>::type ;
+		BOOST_THROW_EXCEPTION(
+			Error() << SrcType_( Type() ) << DestType_( dest )
+		) ;
+	}
+	return impl->val ;
+}
+
+template <typename T>
+T& Val::As()
+{
+	Impl<T> *impl = dynamic_cast<Impl<T> *>( m_base.get() ) ;
+	if ( !impl )
+	{
+		TypeEnum dest = Type2Enum<T>::type ;
+		BOOST_THROW_EXCEPTION(
+			Error() << SrcType_( Type() ) << DestType_( dest )
+		) ;
+	}
+	return impl->val ;
+}
+
+template <typename T>
+bool Val::Is() const
+{
+	return Type() == Type2Enum<T>::type ;
+}
+
+} // end of namespace
+
+namespace std
+{
+	void swap( gr::Val& v1, gr::Val& v2 ) ;
+	ostream& operator<<( ostream& os, gr::Val::TypeEnum t ) ;
+}

libgrive/src/json/ValBuilder.cc

@@ -0,0 +1,140 @@
+/*
+	grive: an GPL program to sync a local directory with Google Drive
+	Copyright (C) 2013 Wan Wai Ho
+
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License
+	as published by the Free Software Foundation version 2
+	of the License.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with this program; if not, write to the Free Software
+	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+	MA  02110-1301, USA.
+*/
+
+#include "ValBuilder.hh"
+
+namespace gr {
+
+ValBuilder::ValBuilder( )
+{
+}
+
+ValBuilder::~ValBuilder()
+{
+}
+
+void ValBuilder::Visit( long long t )
+{
+	Build(Val(t)) ;
+}
+
+void ValBuilder::Visit( double t )
+{
+	Build(Val(t)) ;
+}
+
+void ValBuilder::Visit( const std::string& t )
+{
+	Build(Val(t)) ;
+}
+
+void ValBuilder::Visit( bool t )
+{
+	Build(Val(t)) ;
+}
+
+void ValBuilder::VisitNull()
+{
+	Build(Val()) ;
+}
+
+void ValBuilder::Build( const Val& t )
+{
+	if ( m_ctx.empty() )
+	{
+		Level l = { Val::Null(), t } ;
+		m_ctx.push( l ) ;
+	}
+	
+	else if ( m_ctx.top().val.Is<Val::Array>() )
+	{
+		Val::Array& ar = m_ctx.top().val.As<Val::Array>() ;
+		ar.push_back( t ) ;
+	}
+	else if ( m_ctx.top().val.Is<Val::Object>() )
+	{
+		if ( !m_ctx.top().key.Is<std::string>() )
+			BOOST_THROW_EXCEPTION( Error() << NoKey_(t) ) ;
+	
+		else
+		{
+			Val::Object& obj = m_ctx.top().val.As<Val::Object>() ;
+			obj.insert( std::make_pair( m_ctx.top().key.Str(), t ) ) ;
+			m_ctx.top().key = Val::Null() ;
+		}
+	}
+	else
+		BOOST_THROW_EXCEPTION( Error() << Unexpected_(m_ctx.top().val) ) ;
+}
+
+void ValBuilder::VisitKey( const std::string& t )
+{
+	m_ctx.top().key = t ;
+}
+
+void ValBuilder::StartArray()
+{
+	Level l = { Val::Null(), Val(Val::Array()) } ;
+	m_ctx.push(l) ;
+}
+
+void ValBuilder::EndArray()
+{
+	End( Val::array_type ) ;
+}
+
+void ValBuilder::End( Val::TypeEnum type )
+{
+	if ( m_ctx.top().val.Type() == type )
+	{
+		if( !m_ctx.top().key.Is<void>() )
+			BOOST_THROW_EXCEPTION( Error() << Unexpected_(m_ctx.top().key) ) ;
+	
+		// get top Val from stack
+		Val current ;
+		current.Swap( m_ctx.top().val ) ;
+		m_ctx.pop() ;
+		
+		Build(current) ;
+	}
+}
+
+void ValBuilder::StartObject()
+{
+	Level l = { Val::Null(), Val( Val::Object() ) } ;
+	m_ctx.push(l) ;
+}
+
+void ValBuilder::EndObject()
+{
+	End( Val::object_type ) ;
+}
+
+Val ValBuilder::Result() const
+{
+	if ( !m_ctx.size() )
+		BOOST_THROW_EXCEPTION( Error() << NoKey_( Val(std::string("")) ) ) ;
+	Val r = m_ctx.top().val;
+	if ( m_ctx.size() > 1 )
+		BOOST_THROW_EXCEPTION( Error() << Unexpected_(m_ctx.top().val) ) ;
+	return r;
+}
+
+} // end of namespace

libgrive/src/json/ValBuilder.hh

@@ -0,0 +1,74 @@
+/*
+	grive: an GPL program to sync a local directory with Google Drive
+	Copyright (C) 2013 Wan Wai Ho
+
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License
+	as published by the Free Software Foundation version 2
+	of the License.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with this program; if not, write to the Free Software
+	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+	MA  02110-1301, USA.
+*/
+
+#pragma once
+
+#include "ValVisitor.hh"
+
+#include "Val.hh"
+#include "util/Exception.hh"
+
+#include <stack>
+#include <memory>
+
+namespace gr {
+
+class ValBuilder : public ValVisitor
+{
+public :
+	struct Error : virtual Exception {} ;
+	typedef boost::error_info<struct Mismatch,		Val> Mismatch_ ;
+	typedef boost::error_info<struct Unexpected,	Val> Unexpected_ ;
+	typedef boost::error_info<struct NoKey,			Val> NoKey_ ;
+	
+public :
+	ValBuilder( ) ;
+	~ValBuilder() ;
+
+	void Visit( long long t ) ;
+	void Visit( double t ) ;
+	void Visit( const std::string& t ) ;
+	void Visit( bool t ) ;
+	void VisitNull() ;
+	void Build( const Val& t ) ;
+
+	void StartArray() ;
+	void EndArray() ;
+	void StartObject() ;
+	void VisitKey( const std::string& t ) ;
+	void EndObject() ;
+
+	Val Result() const ;
+
+private :
+	void End( Val::TypeEnum type ) ;
+
+private :
+	struct Level
+	{
+		Val	key ;
+		Val	val ;
+	} ;
+
+	std::stack<Level> 	m_ctx ;
+} ;
+
+} // end of namespace
+

libgrive/src/json/ValResponse.cc

@@ -1,6 +1,6 @@
 /*
 	grive: an GPL program to sync a local directory with Google Drive
-	Copyright (C) 2012  Wan Wai Ho
+	Copyright (C) 2013 Wan Wai Ho
 
 	This program is free software; you can redistribute it and/or
 	modify it under the terms of the GNU General Public License
@@ -14,32 +14,40 @@
 
 	You should have received a copy of the GNU General Public License
 	along with this program; if not, write to the Free Software
-	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+	MA  02110-1301, USA.
 */
 
-#include "JsonResponse.hh"
+#include "ValResponse.hh"
 
-#include "protocol/Json.hh"
+#include "Val.hh"
 
 namespace gr { namespace http {
 
-JsonResponse::JsonResponse()
+ValResponse::ValResponse( ) :
+	m_parser( &m_val )
 {
 }
 
-void JsonResponse::Clear()
+std::size_t ValResponse::Write( const char *data, std::size_t count )
 {
-	m_resp.Clear() ;
+	m_parser.Parse( data, count ) ;
+	return count ;
 }
 
-std::size_t JsonResponse::OnData( void *data, std::size_t count )
+std::size_t ValResponse::Read( char *data, std::size_t count )
 {
-	return m_resp.OnData( data, count ) ;
+	return count ;
 }
 
-Json JsonResponse::Response() const
+Val ValResponse::Response() const
 {
-	return Json::Parse( m_resp.Response() ) ;
+	return m_val.Result() ;
 }
 
-} } // end of namespace
+void ValResponse::Finish()
+{
+	m_parser.Finish() ;
+}
+
+} } // end of namespace gr::http

libgrive/src/json/ValResponse.hh

@@ -0,0 +1,52 @@
+/*
+	grive: an GPL program to sync a local directory with Google Drive
+	Copyright (C) 2013 Wan Wai Ho
+
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License
+	as published by the Free Software Foundation version 2
+	of the License.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with this program; if not, write to the Free Software
+	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+	MA  02110-1301, USA.
+*/
+
+#pragma once
+
+#include "util/DataStream.hh"
+
+#include "JsonParser.hh"
+#include "ValBuilder.hh"
+
+namespace gr
+{
+	class Val ;
+}
+
+namespace gr { namespace http {
+
+class ValResponse : public DataStream
+{
+public :
+	ValResponse() ;
+
+	std::size_t Write( const char *data, std::size_t count ) ;
+	std::size_t Read( char *data, std::size_t count ) ;
+
+	void Finish() ;
+	Val Response() const ;
+	
+private :
+	ValBuilder	m_val ;
+	JsonParser	m_parser ;
+} ;
+
+} } // end of namespace gr::http
+

libgrive/src/json/ValVisitor.hh

@@ -0,0 +1,46 @@
+/*
+	grive: an GPL program to sync a local directory with Google Drive
+	Copyright (C) 2013 Wan Wai Ho
+
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License
+	as published by the Free Software Foundation version 2
+	of the License.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with this program; if not, write to the Free Software
+	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+	MA  02110-1301, USA.
+*/
+
+#pragma once
+
+#include <string>
+
+namespace gr {
+
+class ValVisitor
+{
+public :
+	virtual ~ValVisitor() {}
+
+	virtual void Visit( long long t ) = 0 ;
+	virtual void Visit( double t ) = 0 ;
+	virtual void Visit( const std::string& t ) = 0 ;
+	virtual void Visit( bool t ) = 0 ;
+	virtual void VisitNull() = 0 ;
+
+	virtual void StartArray() = 0 ;
+	virtual void EndArray() = 0 ;
+	virtual void StartObject() = 0 ;
+	virtual void VisitKey( const std::string& t ) = 0 ;
+	virtual void EndObject() = 0 ;
+} ;
+
+} // end of namespace
+

libgrive/src/protocol/AuthAgent.cc

@@ -0,0 +1,176 @@
+/*
+	grive: an GPL program to sync a local directory with Google Drive
+	Copyright (C) 2012  Wan Wai Ho
+
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License
+	as published by the Free Software Foundation version 2
+	of the License.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with this program; if not, write to the Free Software
+	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+
+#include "AuthAgent.hh"
+
+#include "http/Error.hh"
+#include "http/Header.hh"
+#include "util/log/Log.hh"
+#include "util/OS.hh"
+#include "util/File.hh"
+
+#include <cassert>
+
+namespace gr {
+
+using namespace http ;
+
+AuthAgent::AuthAgent( OAuth2& auth, Agent *real_agent ) :
+	Agent(),
+	m_auth	( auth ),
+	m_agent	( real_agent )
+{
+}
+
+http::ResponseLog* AuthAgent::GetLog() const
+{
+	return m_agent->GetLog();
+}
+
+void AuthAgent::SetLog( http::ResponseLog *log )
+{
+	return m_agent->SetLog( log );
+}
+
+void AuthAgent::SetProgressReporter( Progress *progress )
+{
+	m_agent->SetProgressReporter( progress );
+}
+
+void AuthAgent::SetUploadSpeed( unsigned kbytes )
+{
+	m_agent->SetUploadSpeed( kbytes );
+}
+
+void AuthAgent::SetDownloadSpeed( unsigned kbytes )
+{
+	m_agent->SetDownloadSpeed( kbytes );
+}
+
+http::Header AuthAgent::AppendHeader( const http::Header& hdr ) const
+{
+	http::Header h(hdr) ;
+	h.Add( "Authorization: Bearer " + m_auth.AccessToken() ) ;
+	h.Add( "GData-Version: 3.0" ) ;
+	return h ;
+}
+
+long AuthAgent::Request(
+	const std::string&	method,
+	const std::string&	url,
+	SeekStream			*in,
+	DataStream			*dest,
+	const http::Header&	hdr,
+	u64_t			downloadFileBytes )
+{
+	long response;
+	Header auth;
+	m_interval = 0;
+	do
+	{
+		auth = AppendHeader( hdr );
+		if ( in )
+			in->Seek( 0, 0 );
+		response = m_agent->Request( method, url, in, dest, auth, downloadFileBytes );
+	} while ( CheckRetry( response ) );
+	return CheckHttpResponse( response, url, auth );
+}
+
+std::string AuthAgent::LastError() const
+{
+	return m_agent->LastError() ;
+}
+
+std::string AuthAgent::LastErrorHeaders() const
+{
+	return m_agent->LastErrorHeaders() ;
+}
+
+std::string AuthAgent::RedirLocation() const
+{
+	return m_agent->RedirLocation() ;
+}
+
+std::string AuthAgent::Escape( const std::string& str )
+{
+	return m_agent->Escape( str ) ;
+}
+
+std::string AuthAgent::Unescape( const std::string& str )
+{
+	return m_agent->Unescape( str ) ;
+}
+
+bool AuthAgent::CheckRetry( long response )
+{
+	// HTTP 500 and 503 should be temporary. just wait a bit and retry
+	if ( response == 500 || response == 503 )
+	{
+		Log( "request failed due to temporary error: %1% (body: %2%). retrying in 5 seconds",
+			response, m_agent->LastError(), log::warning ) ;
+			
+		os::Sleep( 5 ) ;
+		return true ;
+	}
+	// HTTP 403 is the result of API rate limiting. attempt exponential backoff and try again
+	else if ( response == 429 || ( response == 403 && (
+		m_agent->LastError().find("\"reason\": \"userRateLimitExceeded\",") != std::string::npos ||
+		m_agent->LastError().find("\"reason\": \"rateLimitExceeded\",") != std::string::npos ) ) )
+	{
+		m_interval = m_interval <= 0 ? 1 : ( m_interval < 64 ? m_interval*2 : 120 );
+		Log( "request failed due to rate limiting: %1% (body: %2%). retrying in %3% seconds",
+			response, m_agent->LastError(), m_interval, log::warning ) ;
+		os::Sleep( m_interval ) ;
+		return true ;
+	}
+	// HTTP 401 Unauthorized. the auth token has been expired. refresh it
+	else if ( response == 401 )
+	{
+		Log( "request failed due to auth token expired: %1% (body: %2%). refreshing token",
+			response, m_agent->LastError(), log::warning ) ;
+			
+		os::Sleep( 5 ) ;
+		m_auth.Refresh() ;
+		return true ;
+	}
+	else
+		return false ;
+}
+
+long AuthAgent::CheckHttpResponse(
+		long 				response,
+		const std::string&	url,
+		const http::Header&	hdr  )
+{
+	// throw for other HTTP errors
+	if ( response >= 400 )
+	{
+		BOOST_THROW_EXCEPTION(
+			Error()
+				<< HttpResponseCode( response )
+				<< HttpResponseHeaders( m_agent->LastErrorHeaders() )
+				<< HttpResponseText( m_agent->LastError() )
+				<< Url( url )
+				<< HttpRequestHeaders( hdr ) ) ;
+	}
+	
+	return response ;
+}
+
+} // end of namespace

libgrive/src/protocol/AuthAgent.hh

@@ -0,0 +1,77 @@
+/*
+	grive: an GPL program to sync a local directory with Google Drive
+	Copyright (C) 2012  Wan Wai Ho
+
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License
+	as published by the Free Software Foundation version 2
+	of the License.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with this program; if not, write to the Free Software
+	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+
+#pragma once
+
+#include "http/Agent.hh"
+#include "OAuth2.hh"
+
+#include <memory>
+
+namespace gr {
+
+/*!	\brief	An HTTP agent with support OAuth2
+	
+	This is a HTTP agent that provide support for OAuth2. It will also perform retries on
+	certain HTTP errors.
+*/
+class AuthAgent : public http::Agent
+{
+public :
+	AuthAgent( OAuth2& auth, http::Agent* real_agent ) ;
+
+	http::ResponseLog* GetLog() const ;
+	void SetLog( http::ResponseLog *log ) ;
+
+	long Request(
+		const std::string&	method,
+		const std::string&	url,
+		SeekStream			*in,
+		DataStream			*dest,
+		const http::Header&	hdr,
+		u64_t			downloadFileBytes = 0 ) ;
+	
+	std::string LastError() const ;
+	std::string LastErrorHeaders() const ;
+	
+	std::string RedirLocation() const ;
+	
+	std::string Escape( const std::string& str ) ;
+	std::string Unescape( const std::string& str ) ;
+
+	void SetUploadSpeed( unsigned kbytes ) ;
+	void SetDownloadSpeed( unsigned kbytes ) ;
+
+	void SetProgressReporter( Progress *progress ) ;
+
+private :
+	http::Header AppendHeader( const http::Header& hdr ) const ;
+	bool CheckRetry( long response ) ;
+	long CheckHttpResponse(
+		long 				response,
+		const std::string&	url,
+		const http::Header&	hdr  ) ;
+	
+private :
+	OAuth2&		m_auth ;
+	http::Agent*	m_agent ;
+	int		m_interval ;
+} ;
+
+} // end of namespace

libgrive/src/protocol/Json.cc

@@ -1,366 +0,0 @@
-/*
-	grive: an GPL program to sync a local directory with Google Drive
-	Copyright (C) 2012  Wan Wai Ho
-
-	This program is free software; you can redistribute it and/or
-	modify it under the terms of the GNU General Public License
-	as published by the Free Software Foundation version 2
-	of the License.
-
-	This program is distributed in the hope that it will be useful,
-	but WITHOUT ANY WARRANTY; without even the implied warranty of
-	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-	GNU General Public License for more details.
-
-	You should have received a copy of the GNU General Public License
-	along with this program; if not, write to the Free Software
-	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
-*/
-
-#include "Json.hh"
-
-#include "util/StdioFile.hh"
-
-#include <json/json_tokener.h>
-#include <json/linkhash.h>
-
-#include <cassert>
-#include <cstring>
-#include <iostream>
-#include <sstream>
-
-namespace gr {
-
-Json::Json( ) :
-	m_json( ::json_object_new_object() )
-{
-	if ( m_json == 0 )
-		BOOST_THROW_EXCEPTION( Error() << expt::ErrMsg( "cannot create json object" ) ) ;
-}
-
-Json::Json( const char *str ) :
-	m_json( ::json_object_new_string( str ) )
-{
-	if ( m_json == 0 )
-		BOOST_THROW_EXCEPTION( 
-			Error() << expt::ErrMsg( "cannot create json string \"" + std::string(str) + "\"" ) ) ;
-}
-
-template <>
-Json::Json( const std::string& str ) :
-	m_json( ::json_object_new_string( str.c_str() ) )
-{
-	if ( m_json == 0 )
-		BOOST_THROW_EXCEPTION( 
-			Error() << expt::ErrMsg( "cannot create json string \"" + str + "\"" ) ) ;
-
-	// paranoid check
-	assert( ::json_object_get_string( m_json ) == str ) ;
-}
-
-template <>
-Json::Json( const int& l ) :
-	m_json( ::json_object_new_int( l ) )
-{
-	if ( m_json == 0 )
-		BOOST_THROW_EXCEPTION( Error() << expt::ErrMsg( "cannot create json int" ) ) ;
-}
-
-template <>
-Json::Json( const long& l ) :
-	m_json( ::json_object_new_int( static_cast<int>(l) ) )
-{
-	if ( m_json == 0 )
-		BOOST_THROW_EXCEPTION( Error() << expt::ErrMsg( "cannot create json int" ) ) ;
-}
-
-template <>
-Json::Json( const unsigned long& l ) :
-	m_json( ::json_object_new_int( static_cast<int>(l) ) )
-{
-	if ( m_json == 0 )
-		BOOST_THROW_EXCEPTION( Error() << expt::ErrMsg( "cannot create json int" ) ) ;
-}
-
-template <>
-Json::Json( const std::vector<Json>& arr ) :
-	m_json( ::json_object_new_array( ) )
-{
-	if ( m_json == 0 )
-		BOOST_THROW_EXCEPTION( Error() << expt::ErrMsg( "cannot create json int" ) ) ;
-	
-	for ( std::vector<Json>::const_iterator i = arr.begin() ; i != arr.end() ; ++i )
-		Add( *i ) ;
-}
-
-template <>
-Json::Json( const bool& b ) :
-	m_json( ::json_object_new_boolean( b ) )
-{
-	if ( m_json == 0 )
-		BOOST_THROW_EXCEPTION( Error() << expt::ErrMsg( "cannot create json bool" ) ) ;
-}
-
-Json::Json( struct json_object *json, NotOwned ) :
-	m_json( json )
-{
-	assert( m_json != 0 ) ;
-}
-
-Json::Json( struct json_object *json ) :
-	m_json( json )
-{
-	assert( json != 0 ) ;
-	::json_object_get( m_json ) ;
-}
-
-Json::Json( const Json& rhs ) :
-	m_json( rhs.m_json )
-{
-	assert( m_json != 0 ) ;
-	::json_object_get( m_json ) ;
-}
-
-Json::~Json( )
-{
-	assert( m_json != 0 ) ;
-	if ( m_json != 0 )
-		::json_object_put( m_json ) ;
-}
-
-Json& Json::operator=( const Json& rhs )
-{
-	Json tmp( rhs ) ;
-	Swap( tmp ) ;
-	return *this ;
-}
-
-void Json::Swap( Json& other )
-{
-	assert( m_json != 0 ) ;
-	assert( other.m_json != 0 ) ;
-	std::swap( m_json, other.m_json ) ;
-}
-
-Json Json::operator[]( const std::string& key ) const
-{
-	assert( m_json != 0 ) ;
-	
-	struct json_object *j = ::json_object_object_get( m_json, key.c_str() ) ;
-	if ( j == 0 )
-		BOOST_THROW_EXCEPTION(
-			Error()
-				<< expt::ErrMsg( "key: " + key + " is not found in object" )
-				<< JsonInfo( *this ) ) ;
-	
-	return Json( j ) ;
-}
-
-Json Json::operator[]( const std::size_t& idx ) const
-{
-	assert( m_json != 0 ) ;
-
-	struct json_object *j = ::json_object_array_get_idx( m_json, idx ) ;
-	if ( j == 0 )
-	{
-		std::ostringstream ss ;
-		ss << "index " << idx << " is not found in array" ;
-		BOOST_THROW_EXCEPTION(
-			Error()
-				<< expt::ErrMsg( ss.str() )
-				<< JsonInfo( *this ) ) ;
-	}
-	
-	return Json( j ) ;
-}
-
-bool Json::Has( const std::string& key ) const
-{
-	assert( m_json != 0 ) ;
-	return ::json_object_object_get( m_json, key.c_str() ) != 0 ;
-}
-
-bool Json::Get( const std::string& key, Json& json ) const
-{
-	assert( m_json != 0 ) ;
-	struct json_object *j = ::json_object_object_get( m_json, key.c_str() ) ;
-	if ( j != 0 )
-	{
-		Json tmp( j, NotOwned() ) ;
-		json.Swap( tmp ) ;
-		return true ;
-	}
-	else
-		return false ;
-}
-
-void Json::Add( const std::string& key, const Json& json )
-{
-	assert( m_json != 0 ) ;
-	assert( json.m_json != 0 ) ;
-	
-	::json_object_get( json.m_json ) ;
-	::json_object_object_add( m_json, key.c_str(), json.m_json ) ;
-}
-
-void Json::Add( const Json& json )
-{
-	assert( m_json != 0 ) ;
-	assert( json.m_json != 0 ) ;
-	
-	::json_object_get( json.m_json ) ;
-	::json_object_array_add( m_json, json.m_json ) ;
-}
-
-bool Json::Bool() const
-{
-	assert( m_json != 0 ) ;
-	return ::json_object_get_boolean( m_json ) ;
-}
-
-template <>
-bool Json::Is<bool>() const
-{
-	assert( m_json != 0 ) ;
-	return ::json_object_is_type( m_json, json_type_boolean ) ;
-}
-
-std::string Json::Str() const
-{
-	assert( m_json != 0 ) ;
-	return ::json_object_get_string( m_json ) ;
-}
-
-template <>
-bool Json::Is<std::string>() const
-{
-	assert( m_json != 0 ) ;
-	return ::json_object_is_type( m_json, json_type_string ) ;
-}
-
-int Json::Int() const
-{
-	assert( m_json != 0 ) ;
-	return ::json_object_get_int( m_json ) ;
-}
-
-template <>
-bool Json::Is<int>() const
-{
-	assert( m_json != 0 ) ;
-	return ::json_object_is_type( m_json, json_type_int ) ;
-}
-
-std::ostream& operator<<( std::ostream& os, const Json& json )
-{
-	assert( json.m_json != 0 ) ;
-	return os << ::json_object_to_json_string( json.m_json ) ;
-}
-
-void Json::Write( StdioFile& file ) const
-{
-	const char *str = ::json_object_to_json_string( m_json ) ;
-	file.Write( str, std::strlen(str) ) ;
-}
-
-Json::Type Json::DataType() const
-{
-	assert( m_json != 0 ) ;
-	return static_cast<Type>( ::json_object_get_type( m_json ) ) ;
-}
-
-Json::Object Json::AsObject() const
-{
-	Object result ;
-	
-	json_object_object_foreach( m_json, key, val )
-	{
-		result.insert( Object::value_type( key, Json( val ) ) ) ;
-	}
-	
-	return result ;
-}
-
-template <>
-bool Json::Is<Json::Object>() const
-{
-	assert( m_json != 0 ) ;
-	return ::json_object_is_type( m_json, json_type_object ) ;
-}
-
-Json::Array Json::AsArray() const
-{
-	std::size_t count = ::json_object_array_length( m_json ) ;
-	Array result ;
-	
-	for ( std::size_t i = 0 ; i < count ; ++i )
-		result.push_back( Json( ::json_object_array_get_idx( m_json, i ) ) ) ;
-	
-	return result ;
-}
-
-template <>
-bool Json::Is<Json::Array>() const
-{
-	assert( m_json != 0 ) ;
-	return ::json_object_is_type( m_json, json_type_array ) ;
-}
-
-Json Json::FindInArray( const std::string& key, const std::string& value ) const
-{
-	std::size_t count = ::json_object_array_length( m_json ) ;
-	
-	for ( std::size_t i = 0 ; i < count ; ++i )
-	{
-		Json item( ::json_object_array_get_idx( m_json, i ) ) ;
-		if ( item.Has(key) && item[key].Str() == value )
-			return item ;
-	}
-	BOOST_THROW_EXCEPTION(
-		Error() << expt::ErrMsg( "cannot find " + key + " = " + value + " in array" ) ) ;
-}
-
-bool Json::FindInArray( const std::string& key, const std::string& value, Json& result ) const
-{
-	try
-	{
-		result = FindInArray( key, value ) ;
-		return true ;
-	}
-	catch ( Error& )
-	{
-		return false ;
-	}
-}
-
-Json Json::Parse( const std::string& str )
-{
-	struct json_object *json = ::json_tokener_parse( str.c_str() ) ;
-	if ( json == 0 )
-		BOOST_THROW_EXCEPTION( Error() << expt::ErrMsg( "json parse error" ) ) ;
-	
-	return Json( json, NotOwned() ) ;
-}
-
-Json Json::ParseFile( const std::string& filename )
-{
-	StdioFile file( filename ) ;
-	struct json_tokener *tok = ::json_tokener_new() ;
-	
-	struct json_object *json = 0 ;
-	
-	char buf[1024] ;
-	std::size_t count = 0 ;
-
-	while ( (count = file.Read( buf, sizeof(buf) ) ) > 0 )
-		json = ::json_tokener_parse_ex( tok, buf, count ) ;
-	
-	if ( json == 0 )
-		BOOST_THROW_EXCEPTION( Error() << expt::ErrMsg( ::json_tokener_errors[tok->err] ) ) ;
-	
-	::json_tokener_free( tok ) ;
-	
-	return Json( json, NotOwned() ) ;
-}
-
-}

libgrive/src/protocol/Json.hh

@@ -1,96 +0,0 @@
-/*
-	grive: an GPL program to sync a local directory with Google Drive
-	Copyright (C) 2012  Wan Wai Ho
-
-	This program is free software; you can redistribute it and/or
-	modify it under the terms of the GNU General Public License
-	as published by the Free Software Foundation version 2
-	of the License.
-
-	This program is distributed in the hope that it will be useful,
-	but WITHOUT ANY WARRANTY; without even the implied warranty of
-	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-	GNU General Public License for more details.
-
-	You should have received a copy of the GNU General Public License
-	along with this program; if not, write to the Free Software
-	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
-*/
-
-#pragma once
-
-#include "util/Exception.hh"
-
-#include <string>
-#include <map>
-#include <vector>
-
-struct json_object ;
-
-namespace gr {
-
-class StdioFile ;
-
-class Json
-{
-public :
-	typedef std::map<std::string, Json>	Object ;
-	typedef std::vector<Json>			Array ;
-
-	struct Error : virtual Exception {} ;
-	typedef boost::error_info<struct JsonTag, Json>	JsonInfo ;
-	
-public :
-	template <typename T>
-	explicit Json( const T& val ) ;
-	
-	Json() ;
-	Json( const Json& rhs ) ;
-	Json( const char *str ) ;
-	~Json( ) ;
-	
-	static Json Parse( const std::string& str ) ;
-	static Json ParseFile( const std::string& filename ) ;
-	
-	Json operator[]( const std::string& key ) const ;
-	Json operator[]( const std::size_t& idx ) const ;
-	Json& operator=( const Json& rhs ) ;
-	
-	void Swap( Json& other ) ;
-
-	std::string Str() const ;
-	int		Int() const ;
-	double	Double() const ;
-	bool	Bool() const ;
-	Array	AsArray() const ;
-	Object	AsObject() const ;
-
-	template <typename T>
-	bool Is() const ;
-	
-	bool Has( const std::string& key ) const ;
-	bool Get( const std::string& key, Json& json ) const ;
-	void Add( const std::string& key, const Json& json ) ;
-	void Add( const Json& json ) ;
-	Json FindInArray( const std::string& key, const std::string& value ) const ;
-	bool FindInArray( const std::string& key, const std::string& value, Json& result ) const ;
-	
-	friend std::ostream& operator<<( std::ostream& os, const Json& json ) ;
-	void Write( StdioFile& file ) const ;
-
-	enum Type { null_type, bool_type, double_type, int_type, object_type, array_type, string_type } ;
-	
-	Type DataType() const ;
-	
-private :
-	Json( struct json_object *json ) ;
-	
-	struct NotOwned {} ;
-	Json( struct json_object *json, NotOwned ) ;
-	
-private :
-public :
-	struct json_object	*m_json ;
-} ;
-
-}

libgrive/src/protocol/OAuth2.cc

@@ -19,13 +19,19 @@
 
 #include "OAuth2.hh"
 
-#include "JsonResponse.hh"
-#include "Json.hh"
+#include "json/ValResponse.hh"
 
 #include "http/CurlAgent.hh"
 #include "http/Header.hh"
 #include "util/log/Log.hh"
 
+#include <netinet/in.h>
+#include <sys/socket.h>
+#include <string.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <poll.h>
+
 // for debugging
 #include <iostream>
 
@@ -34,10 +40,12 @@ namespace gr {
 const std::string token_url		= "https://accounts.google.com/o/oauth2/token" ;
 
 OAuth2::OAuth2(
+	http::Agent* agent,
 	const std::string& refresh_code,
 	const std::string&	client_id,
 	const std::string&	client_secret ) :
 	m_refresh( refresh_code ),
+	m_agent( agent ),
 	m_client_id( client_id ),
 	m_client_secret( client_secret )
 {
@@ -45,49 +53,160 @@ OAuth2::OAuth2(
 }
 
 OAuth2::OAuth2(
+	http::Agent* agent,
 	const std::string&	client_id,
 	const std::string&	client_secret ) :
+	m_agent( agent ),
+	m_port( 0 ),
+	m_socket( -1 ),
 	m_client_id( client_id ),
 	m_client_secret( client_secret )
 {
 }
 
-void OAuth2::Auth( const std::string&	auth_code )
+OAuth2::~OAuth2()
+{
+	if ( m_socket >= 0 )
+	{
+		close( m_socket );
+		m_socket = -1;
+	}
+}
+
+bool OAuth2::Auth( const std::string&	auth_code )
 {
 	std::string post =
 		"code="				+ auth_code +
 		"&client_id="		+ m_client_id +
 		"&client_secret="	+ m_client_secret +
-		"&redirect_uri="	+ "urn:ietf:wg:oauth:2.0:oob" +
+		"&redirect_uri=http%3A%2F%2Flocalhost:" + std::to_string( m_port ) + "%2Fauth" +
 		"&grant_type=authorization_code" ;
 
-	http::JsonResponse  resp ;
-	http::CurlAgent		http ;
+	http::ValResponse  resp ;
 
-	DisableLog dlog( log::debug ) ;
-	http.Post( token_url, post, &resp, http::Header() ) ;
+	long code = m_agent->Post( token_url, post, &resp, http::Header() ) ;
+	if ( code >= 200 && code < 300 )
+	{
+		Val jresp	= resp.Response() ;
+		m_access	= jresp["access_token"].Str() ;
+		m_refresh	= jresp["refresh_token"].Str() ;
+	}
+	else
+	{
+		Log( "Failed to obtain auth token: HTTP %1%, body: %2%",
+			code, m_agent->LastError(), log::error ) ;
+		return false;
+	}
 
-	Json jresp	= resp.Response() ;
-	m_access	= jresp["access_token"].Str() ;
-	m_refresh	= jresp["refresh_token"].Str() ;
+	return true;
 }
 
-std::string OAuth2::MakeAuthURL(
-	const std::string&	client_id,
-	const std::string&	state )
+std::string OAuth2::MakeAuthURL()
 {
-	http::CurlAgent h ;
-
+	if ( !m_port )
+	{
+		sockaddr_storage addr = { 0 };
+		addr.ss_family = AF_INET;
+		m_socket = socket( AF_INET, SOCK_STREAM, 0 );
+		if ( m_socket < 0 )
+			throw std::runtime_error( std::string("socket: ") + strerror(errno) );
+		if ( bind( m_socket, (sockaddr*)&addr, sizeof( addr ) ) < 0 )
+		{
+			close( m_socket );
+			m_socket = -1;
+			throw std::runtime_error( std::string("bind: ") + strerror(errno) );
+		}
+		socklen_t len = sizeof( addr );
+		if ( getsockname( m_socket, (sockaddr *)&addr, &len ) == -1 )
+		{
+			close( m_socket );
+			m_socket = -1;
+			throw std::runtime_error( std::string("getsockname: ") + strerror(errno) );
+		}
+		m_port = ntohs(((sockaddr_in*)&addr)->sin_port);
+		if ( listen( m_socket, 128 ) < 0 )
+		{
+			close( m_socket );
+			m_socket = -1;
+			m_port = 0;
+			throw std::runtime_error( std::string("listen: ") + strerror(errno) );
+		}
+	}
 	return "https://accounts.google.com/o/oauth2/auth"
-		"?scope=" +
-			h.Escape( "https://www.googleapis.com/auth/userinfo.email" )	+ "+" + 
-			h.Escape( "https://www.googleapis.com/auth/userinfo.profile" )	+ "+" +
-			h.Escape( "https://docs.google.com/feeds/" )					+ "+" + 
-			h.Escape( "https://docs.googleusercontent.com/" )				+ "+" + 
-			h.Escape( "https://spreadsheets.google.com/feeds/" )			+
-		"&redirect_uri=urn:ietf:wg:oauth:2.0:oob"
+		"?scope=" + m_agent->Escape( "https://www.googleapis.com/auth/drive" ) +
+		"&redirect_uri=http%3A%2F%2Flocalhost:" + std::to_string( m_port ) + "%2Fauth" +
 		"&response_type=code"
-		"&client_id=" + client_id ;
+		"&client_id=" + m_client_id ;
+}
+
+bool OAuth2::GetCode( )
+{
+	sockaddr_storage addr = { 0 };
+	int peer_fd = -1;
+	while ( peer_fd < 0 )
+	{
+		socklen_t peer_addr_size = sizeof( addr );
+		peer_fd = accept( m_socket, (sockaddr*)&addr, &peer_addr_size );
+		if ( peer_fd == -1 && errno != EAGAIN && errno != EINTR )
+			throw std::runtime_error( std::string("accept: ") + strerror(errno) );
+	}
+	fcntl( peer_fd, F_SETFL, fcntl( peer_fd, F_GETFL, 0 ) | O_NONBLOCK );
+	struct pollfd pfd = (struct pollfd){
+		.fd = peer_fd,
+		.events = POLLIN|POLLRDHUP,
+	};
+	char buf[4096];
+	std::string request;
+	while ( true )
+	{
+		pfd.revents = 0;
+		poll( &pfd, 1, -1 );
+		if ( pfd.revents & POLLRDHUP )
+			break;
+		int r = 1;
+		while ( r > 0 )
+		{
+			r = read( peer_fd, buf, sizeof( buf ) );
+			if ( r > 0 )
+				request += std::string( buf, r );
+			else if ( r == 0 )
+				break;
+			else if ( errno != EAGAIN && errno != EINTR )
+				throw std::runtime_error( std::string("read: ") + strerror(errno) );
+		}
+		if ( r == 0 || ( r < 0 && request.find( "\n" ) > 0 ) ) // GET ... HTTP/1.1\r\n
+			break;
+	}
+	bool ok = false;
+	if ( request.substr( 0, 10 ) == "GET /auth?" )
+	{
+		std::string line = request;
+		int p = line.find( "\n" );
+		if ( p > 0 )
+			line = line.substr( 0, p );
+		p = line.rfind( " " );
+		if ( p > 0 )
+			line = line.substr( 0, p );
+		p = line.find( "code=" );
+		if ( p > 0 )
+			line = line.substr( p+5 );
+		p = line.find( "&" );
+		if ( p > 0 )
+			line = line.substr( 0, p );
+		ok = Auth( line );
+	}
+	std::string response = ( ok
+		? "Authenticated successfully. Please close the page"
+		: "Authentication error. Please try again" );
+	response = "HTTP/1.1 200 OK\r\n"
+		"Content-Type: text/html; charset=utf-8\r\n"
+		"Connection: close\r\n"
+		"\r\n"+
+		response+
+		"\r\n";
+	write( peer_fd, response.c_str(), response.size() );
+	close( peer_fd );
+	return ok;
 }
 
 void OAuth2::Refresh( )
@@ -98,13 +217,18 @@ void OAuth2::Refresh( )
 		"&client_secret="	+ m_client_secret +
 		"&grant_type=refresh_token" ;
 
-	http::JsonResponse  resp ;
-	http::CurlAgent		http ;
-    
-	DisableLog dlog( log::debug ) ;
-	http.Post( token_url, post, &resp, http::Header() ) ;
+	http::ValResponse  resp ;
 
-	m_access	= resp.Response()["access_token"].Str() ;
+	long code = m_agent->Post( token_url, post, &resp, http::Header() ) ;
+
+	if ( code >= 200 && code < 300 )
+		m_access = resp.Response()["access_token"].Str() ;
+	else
+	{
+		Log( "Failed to refresh auth token: HTTP %1%, body: %2%",
+			code, m_agent->LastError(), log::error ) ;
+		BOOST_THROW_EXCEPTION( AuthFailed() );
+	}
 }
 
 std::string OAuth2::RefreshToken( ) const

libgrive/src/protocol/OAuth2.hh

@@ -19,42 +19,53 @@
 
 #pragma once
 
+#include "http/Agent.hh"
+#include "util/Exception.hh"
 #include <string>
+#include <memory>
 
 namespace gr {
 
 class OAuth2
 {
+public :
+	struct AuthFailed : virtual Exception {} ;
+
 public :
 	OAuth2(
+		http::Agent* agent,
 		const std::string&	client_id,
 		const std::string&	client_secret ) ;
 	OAuth2(
+		http::Agent* agent,
 		const std::string&	refresh_code,
 		const std::string&	client_id,
 		const std::string&	client_secret ) ;
+	~OAuth2( ) ;
 
 	std::string Str() const ;
-	
-	static std::string MakeAuthURL(
-		const std::string&	client_id,
-		const std::string&	state = std::string() ) ;
 
-	void Auth( const std::string& auth_code ) ;
+	std::string MakeAuthURL() ;
+
+	bool Auth( const std::string& auth_code ) ;
 	void Refresh( ) ;
-		
+	bool GetCode( ) ;
+
 	std::string RefreshToken( ) const ;
 	std::string AccessToken( ) const ;
-	
+
 	// adding HTTP auth header
 	std::string HttpHeader( ) const ;
-	
+
 private :
 	std::string m_access ;
 	std::string m_refresh ;
-	
+	http::Agent* m_agent ;
+	int m_port ;
+	int m_socket ;
+
 	const std::string	m_client_id ;
 	const std::string	m_client_secret ;
 } ;
-	
+
 } // end of namespace

libgrive/src/util/ConcatStream.cc

@@ -0,0 +1,105 @@
+/*
+	A stream representing a concatenation of several underlying streams
+	Copyright (C) 2015 Vitaliy Filippov
+
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License
+	as published by the Free Software Foundation version 2
+	of the License.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with this program; if not, write to the Free Software
+	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+
+#include <string.h>
+#include "ConcatStream.hh"
+
+namespace gr {
+
+ConcatStream::ConcatStream() :
+	m_size( 0 ), m_pos( 0 ), m_cur( 0 )
+{
+}
+
+std::size_t ConcatStream::Read( char *data, std::size_t size )
+{
+	std::size_t done = 0, l;
+	while ( done < size && m_cur < m_streams.size() )
+	{
+		l = m_streams[m_cur]->Read( data+done, ( size-done < m_sizes[m_cur]-m_pos ? size-done : m_sizes[m_cur]-m_pos ) );
+		done += l;
+		m_pos += l;
+		if ( !l )
+		{
+			// current stream was truncated in the meantime, pad output with zeros
+			memset( data+done, 0, m_sizes[m_cur]-m_pos );
+			done += m_sizes[m_cur]-m_pos;
+			m_pos = m_sizes[m_cur];
+		}
+		if ( m_pos >= m_sizes[m_cur] )
+		{
+			m_cur++;
+			if ( m_cur < m_streams.size() )
+				m_streams[m_cur]->Seek( 0, 0 );
+		}
+	}
+	return done ;
+}
+
+std::size_t ConcatStream::Write( const char *data, std::size_t size )
+{
+	return 0 ;
+}
+
+off_t ConcatStream::Seek( off_t offset, int whence )
+{
+	if ( whence == 1 )
+		offset += m_pos;
+	else if ( whence == 2 )
+		offset += Size();
+	if ( (u64_t)offset > Size() )
+		offset = Size();
+	m_cur = 0;
+	m_pos = offset;
+	if ( m_streams.size() )
+	{
+		while ( (u64_t)offset > m_sizes[m_cur] )
+			m_cur++;
+		m_streams[m_cur]->Seek( offset - ( m_cur > 0 ? m_sizes[m_cur-1] : 0 ), 0 );
+	}
+	return m_pos ;
+}
+
+off_t ConcatStream::Tell() const
+{
+	return m_pos ;
+}
+
+u64_t ConcatStream::Size() const
+{
+	return m_size ;
+}
+
+void ConcatStream::Append( SeekStream *stream )
+{
+	if ( stream )
+	{
+		u64_t size = stream->Size();
+		if ( size > 0 )
+		{
+			// "fix" stream size at the moment of adding so further changes of underlying files
+			// don't affect the total size of resulting stream...
+			m_size += size;
+			m_streams.push_back( stream );
+			m_sizes.push_back( m_size );
+		}
+	}
+}
+
+} // end of namespace

libgrive/src/util/ConcatStream.hh

@@ -0,0 +1,49 @@
+/*
+	A stream representing a concatenation of several underlying streams
+	Copyright (C) 2015 Vitaliy Filippov
+
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License
+	as published by the Free Software Foundation version 2
+	of the License.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with this program; if not, write to the Free Software
+	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+
+#pragma once
+
+#include "DataStream.hh"
+
+#include <vector>
+
+namespace gr {
+
+class ConcatStream : public SeekStream
+{
+public :
+	ConcatStream() ;
+
+	std::size_t Read( char *data, std::size_t size ) ;
+	std::size_t Write( const char *data, std::size_t size ) ;
+
+	off_t Seek( off_t offset, int whence ) ;
+	off_t Tell() const ;
+	u64_t Size() const ;
+
+	void Append( SeekStream *stream ) ;
+
+private :
+	std::vector<SeekStream*> m_streams ;
+	std::vector<u64_t> m_sizes ;
+	u64_t m_size, m_pos ;
+	size_t m_cur ;
+} ;
+
+} // end of namespace

libgrive/src/util/Config.cc

@@ -0,0 +1,119 @@
+/*
+	grive: an GPL program to sync a local directory with Google Drive
+	Copyright (C) 2012  Wan Wai Ho
+
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License
+	as published by the Free Software Foundation version 2
+	of the License.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with this program; if not, write to the Free Software
+	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+
+#include "Config.hh"
+
+#include "util/File.hh"
+#include "json/JsonWriter.hh"
+#include "json/JsonParser.hh"
+
+#include <boost/program_options.hpp>
+
+#include <iostream>
+#include <iterator>
+
+namespace po = boost::program_options;
+
+namespace gr {
+
+const std::string	default_filename	= ".grive";
+const char			*env_name			= "GR_CONFIG";
+const std::string	default_root_folder = ".";
+
+Config::Config( const po::variables_map& vm )
+{
+	if ( vm.count( "id" ) > 0 )
+		m_cmd.Add( "id",	Val( vm["id"].as<std::string>() ) ) ;
+	if ( vm.count( "secret" ) > 0 )
+		m_cmd.Add( "secret",	Val( vm["secret"].as<std::string>() ) ) ;
+	m_cmd.Add( "new-rev",	Val(vm.count("new-rev") > 0) ) ;
+	m_cmd.Add( "force",		Val(vm.count("force") > 0 ) ) ;
+	m_cmd.Add( "path",		Val(vm.count("path") > 0
+		? vm["path"].as<std::string>()
+		: default_root_folder ) ) ;
+	m_cmd.Add( "dir",		Val(vm.count("dir") > 0
+		? vm["dir"].as<std::string>()
+		: "" ) ) ;
+	if ( vm.count( "ignore" ) > 0 )
+		m_cmd.Add( "ignore",	Val( vm["ignore"].as<std::string>() ) );
+	m_cmd.Add( "no-remote-new", Val( vm.count( "no-remote-new" ) > 0 || vm.count( "upload-only" ) > 0 ) );
+	m_cmd.Add( "upload-only", Val( vm.count( "upload-only" ) > 0 ) );
+	m_cmd.Add( "no-delete-remote", Val( vm.count( "no-delete-remote" ) > 0 ) );
+	
+	m_path	= GetPath( fs::path(m_cmd["path"].Str()) ) ;
+	m_file	= Read( ) ;
+}
+
+fs::path Config::GetPath( const fs::path& root_path )
+{
+	// config file will be (in order of preference)
+	// value specified in environment string
+	// value specified in defaultConfigFileName in path from commandline --path
+	// value specified in defaultConfigFileName in current directory
+	const char *env = ::getenv( env_name ) ;
+	return root_path / (env ? env : default_filename) ;
+}
+
+const fs::path Config::Filename() const
+{
+	return m_path ;
+}
+
+void Config::Save( )
+{
+	gr::File file( m_path.string(), 0600 ) ;
+	JsonWriter wr( &file ) ;
+	m_file.Visit( &wr ) ;
+}
+
+void Config::Set( const std::string& key, const Val& value )
+{
+	m_file.Set( key, value ) ;
+}
+
+Val Config::Get( const std::string& key ) const
+{
+	return m_cmd.Has(key) ? m_cmd[key] : m_file[key] ;
+}
+
+Val Config::GetAll() const
+{
+	Val::Object obj		= m_file.AsObject() ;
+	Val::Object cmd_obj	= m_cmd.AsObject() ;
+	
+	for ( Val::Object::iterator i = cmd_obj.begin() ; i != cmd_obj.end() ; ++i )
+		obj[i->first] = i->second ;
+	
+	return Val( obj ) ;
+}
+
+Val Config::Read()
+{
+	try
+	{
+		gr::File file(m_path) ;
+		return ParseJson( file ) ;
+	}
+	catch ( Exception& e )
+	{
+		return Val() ;
+	}
+}
+
+} // end of namespace

libgrive/src/util/Config.hh

@@ -21,40 +21,47 @@
 
 #include "Exception.hh"
 #include "FileSystem.hh"
+#include "json/Val.hh"
 
-#include <string>
+namespace boost
+{
+	namespace program_options
+	{
+		class variables_map ;
+	}
+}
 
 namespace gr {
 
-class StdioFile
+class Config
 {
 public :
 	struct Error : virtual Exception {} ;
+	typedef boost::error_info<struct FileTag, std::string>	File ;
 
-public :
-	StdioFile() ;
-	StdioFile( const fs::path& path ) ;
-	StdioFile( const fs::path& path, int mode ) ;
-	~StdioFile( ) ;
+	Config( const boost::program_options::variables_map& vm ) ;
 
-	void OpenForRead( const fs::path& path ) ;
-	void OpenForWrite( const fs::path& path, int mode = 0600 ) ;
-	void Close() ;
-	bool IsOpened() const ;
+	const fs::path Filename() const ;
 	
-	std::size_t Read( void *ptr, std::size_t size ) ;
-	std::size_t Write( const void *ptr, std::size_t size ) ;
+	void Set( const std::string& key, const Val& value ) ;
+	Val Get( const std::string& key ) const ;
 
-	long Seek( long offset, int whence ) ;
-	long Tell() const ;
-	
-	void Chmod( int mode ) ;
+	Val GetAll() const ;
+	void Save() ;
 
 private :
-	void Open( const fs::path& path, int flags, int mode ) ;
-	
+	Val Read( ) ;
+	static fs::path GetPath( const fs::path& root_path ) ;
+
 private :
-	int	m_fd ;
+	//! config file path
+	fs::path	m_path;
+	
+	//! config values loaded from config file
+	Val		m_file ;
+	
+	//! config values from command line
+	Val		m_cmd ;
 } ;
 	
 } // end of namespace

libgrive/src/util/Crypt.cc

@@ -19,11 +19,11 @@
 
 #include "Crypt.hh"
 
-#include "StdioFile.hh"
+#include "File.hh"
 #include "Exception.hh"
+#include "MemMap.hh"
 
 #include <iomanip>
-#include <sstream>
 
 // dependent libraries
 #include <gcrypt.h>
@@ -31,7 +31,8 @@
 
 namespace gr { namespace crypt {
 
-const std::size_t read_size = 8 * 1024 ;
+// map 4MB of data at a time
+const u64_t read_size = 1024 * 4096 ;
 
 struct MD5::Impl
 {
@@ -43,7 +44,10 @@ MD5::MD5() : m_impl( new Impl )
 	::gcry_error_t err = ::gcry_md_open( &m_impl->hd, GCRY_MD_MD5, 0 ) ;
 	if ( err != GPG_ERR_NO_ERROR )
 	{
-		BOOST_THROW_EXCEPTION( Exception() << expt::ErrMsg( ::gcry_strerror(err) ) ) ;
+		BOOST_THROW_EXCEPTION( Exception()
+			<< GCryptErr_( ::gcry_strerror(err) )
+			<< GCryptApi_( "gcry_md_open" )
+		) ;
 	}
 }
 
@@ -74,24 +78,25 @@ std::string MD5::Get( const fs::path& file )
 {
 	try
 	{
-		StdioFile sfile( file ) ;
+		File sfile( file ) ;
 		return Get( sfile ) ;
 	}
-	catch ( StdioFile::Error& )
+	catch ( File::Error& )
 	{
 		return "" ;
 	}
 }
 
-std::string MD5::Get( StdioFile& file )
+std::string MD5::Get( File& file )
 {
-	char buf[read_size] ;
-
 	MD5 crypt ;
 	
-	std::size_t count = 0 ;
-	while ( (count = file.Read( buf, sizeof(buf) )) > 0 )
-		crypt.Write( buf, count ) ;
+	u64_t size = file.Size() ;
+	for ( u64_t i = 0 ; i < size ; i += read_size )
+	{
+		MemMap map( file, i, static_cast<std::size_t>(std::min(read_size, size-i)) ) ;
+		crypt.Write( map.Addr(), map.Length() ) ;
+	}
 
 	return crypt.Get() ;
 }

libgrive/src/util/Crypt.hh

@@ -19,6 +19,8 @@
 
 #pragma once
 
+#include "util/Exception.hh"
+
 #include <string>
 #include <memory>
 
@@ -26,17 +28,21 @@
 
 namespace gr {
 
-class StdioFile ;
+class File ;
 
 namespace crypt {
 
 class MD5
 {
+public :
+	typedef boost::error_info<struct GCryptErr,	std::string>	GCryptErr_ ;
+	typedef boost::error_info<struct GCryptApi,	std::string>	GCryptApi_ ;
+
 public :
 	MD5() ;
 	~MD5() ;
 
-	static std::string Get( StdioFile& file ) ;
+	static std::string Get( File& file ) ;
 	static std::string Get( const boost::filesystem::path& file ) ;
 	
 	void Write( const void *data, std::size_t size ) ;
@@ -44,7 +50,7 @@ public :
 
 private :
 	struct Impl ;
-	std::auto_ptr<Impl>	m_impl ;
+	std::unique_ptr<Impl>	m_impl ;
 } ;
 
 } } // end of namespace gr

libgrive/src/util/DataStream.hh

@@ -0,0 +1,60 @@
+/*
+	webwrite: an GPL program to sync a local directory with Google Drive
+	Copyright (C) 2012  Wan Wai Ho
+
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License
+	as published by the Free Software Foundation version 2
+	of the License.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with this program; if not, write to the Free Software
+	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+
+#pragma once
+
+#include <cstddef>
+#include "util/Types.hh"
+
+namespace gr {
+
+/**	\brief	Encapsulation of data streams. Useful for unit tests.
+	This class provides two functions: Read() and Write().
+*/
+class DataStream
+{
+protected :
+	virtual ~DataStream() {}
+	
+public :
+	/**	Reading from the stream. The caller indicates that it wants
+		to read `size` bytes and must provide enough space pointed
+		by `data`.
+		\param	data	Buffer to hold the data read from the stream
+						Must have at least `size` bytes.
+		\param	size	Number of bytes the caller wants to read.
+		\throw	wb::Exception	In case of any error.
+		\return			The number of byte actually read from the stream.
+						0 indicates the end of stream, i.e. you will
+						still get 0 if you call again.
+	*/
+	virtual std::size_t Read( char *data, std::size_t size ) = 0 ;
+	virtual std::size_t Write( const char *data, std::size_t size ) = 0 ;
+} ;
+
+class SeekStream: public DataStream
+{
+public :
+	virtual off_t Seek( off_t offset, int whence ) = 0 ;
+	virtual off_t Tell() const = 0 ;
+	virtual u64_t Size() const = 0 ;
+} ;
+
+
+} // end of namespace

libgrive/src/util/DateTime.cc

@@ -33,7 +33,6 @@
 #include <cassert>
 #include <cstdlib>
 #include <cstring>
-#include <iostream>
 #include <iomanip>
 
 #include <time.h>

libgrive/src/util/Exception.cc

@@ -17,7 +17,7 @@
 	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */
 
-#include "util/Exception.hh"
+#include "Exception.hh"
 
 #include "bfd/Backtrace.hh"
 #include "bfd/Debug.hh"
@@ -26,7 +26,6 @@
 
 #include <cstdlib>
 #include <iterator>
-#include <sstream>
 
 namespace gr {
 
@@ -35,8 +34,13 @@ class Backtrace ;
 Exception::Exception( )
 {
 #ifdef HAVE_BFD
-	*this << expt::BacktraceInfo( Backtrace() ) ;
+	*this << expt::Backtrace_( Backtrace() ) ;
 #endif
 }
 
+const char* Exception::what() const throw()
+{
+	return boost::diagnostic_information_what( *this ) ;
+}
+
 } // end of namespace

libgrive/src/util/Exception.hh

@@ -1,5 +1,5 @@
 /*
-	grive: an GPL program to sync a local directory with Google Drive
+	webwrite: an GPL program to sync a local directory with Google Drive
 	Copyright (C) 2012  Wan Wai Ho
 
 	This program is free software; you can redistribute it and/or
@@ -32,47 +32,25 @@ class Backtrace ;
 /**	\defgroup	exception	Exception Classes
 */
 
-///	base class for exception in libpdfdoc
-/**	\ingroup exception
-	This class is the base class for all exception class in libpdfdoc.
+/**	\brief	base class for exception in WebWrite
+	\ingroup exception
+	This class is the base class for all exception class in WebWrite.
+	It allows us to catch all WebWrite exception with one catch clause.
 */
 struct Exception :
 	virtual public std::exception,
 	virtual public boost::exception
 {
 	Exception( ) ;
+	
+	virtual const char* what() const throw() ;
 } ;
 
-struct FileError	: virtual Exception {} ;
-
-///	Parse error exception.
-/**	\ingroup exception
-	This exception will be thrown when there is a parse error when reading
-	a PDF file.
-*/
-struct ParseError	: virtual Exception {} ;
-
-///	Invalid type exception.
-/**	\ingroup exception
-	This exception will be thrown when the Object cannot convert its
-	underlying data to a specific type. The what() member function will
-	describe the expected and actual type of the data.
-*/
-struct BadType 		: virtual Exception {} ;
-
-struct Unsupported	: virtual Exception {} ;
-
-// Exception informations
+/// Exception informations
 namespace expt
 {
 	// back-trace information. should be present for all exceptions
-	typedef boost::error_info<struct BacktraceTag, Backtrace>	BacktraceInfo ;
-
-	// generic error message
-	typedef boost::error_info<struct MsgTag, std::string>		ErrMsg ;
-	
-	// nested exception
-	typedef boost::error_info<struct ExceptionTag, Exception>	Nested ;
+	typedef boost::error_info<struct BacktraceTag, Backtrace>	Backtrace_ ;
 }
 
 } // end of namespace

libgrive/src/util/File.cc

@@ -0,0 +1,288 @@
+/*
+	grive: an GPL program to sync a local directory with Google Drive
+	Copyright (C) 2012  Wan Wai Ho
+
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License
+	as published by the Free Software Foundation version 2
+	of the License.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with this program; if not, write to the Free Software
+	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+
+#include "File.hh"
+
+#include <cassert>
+
+// boost headers
+#include <boost/throw_exception.hpp>
+#include <boost/exception/errinfo_api_function.hpp>
+#include <boost/exception/errinfo_errno.hpp>
+#include <boost/exception/errinfo_file_name.hpp>
+#include <boost/exception/errinfo_file_open_mode.hpp>
+#include <boost/exception/info.hpp>
+
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <fcntl.h>
+
+#if defined(__FreeBSD__) || defined(__OpenBSD__)
+#include <unistd.h>
+#endif
+
+#ifdef WIN32
+	#include <io.h>
+	typedef int ssize_t ;
+#else
+	#include <sys/mman.h>
+#endif
+
+// local functions
+namespace {
+
+using namespace gr ;
+
+off_t LSeek( int fd, off_t offset, int whence )
+{
+	assert( fd >= 0 ) ;
+	
+	off_t r = ::lseek( fd, offset, whence ) ;
+	if ( r == static_cast<off_t>(-1) )
+	{
+		BOOST_THROW_EXCEPTION(
+			File::Error()
+				<< boost::errinfo_api_function("lseek")
+				<< boost::errinfo_errno(errno)
+		) ;
+	}
+	
+	return r ;
+}
+
+struct stat FStat( int fd )
+{
+	struct stat s = {} ;
+	if ( ::fstat( fd, &s ) != 0 )
+	{
+		BOOST_THROW_EXCEPTION(
+			File::Error()
+				<< boost::errinfo_api_function("fstat")
+				<< boost::errinfo_errno(errno)
+		) ;
+	}
+	return s ;
+}
+
+} // end of local functions
+
+namespace gr {
+
+File::File( ) : m_fd( -1 )
+{
+}
+
+/**	Opens the file for reading.
+	\param	path	Path to the file to be opened.
+	\throw	Error	When the file cannot be openned.
+*/
+File::File( const fs::path& path ) : m_fd( -1 )
+{
+	OpenForRead( path ) ;
+}
+
+/**	Opens the file for writing.
+	\param	path	Path to the file to be opened.
+	\param	mode	Mode of the file to be created, e.g. 0600 for user
+					readable/writable.
+	\throw	Error	When the file cannot be opened.
+*/
+File::File( const fs::path& path, int mode ) : m_fd( -1 )
+{
+	OpenForWrite( path, mode ) ;
+}
+
+/**	The destructor will close the file. 
+*/
+File::~File( )
+{
+	Close() ;
+}
+
+void File::Open( const fs::path& path, int flags, int mode )
+{
+	if ( IsOpened() )
+		Close() ;
+	
+	assert( m_fd == -1 ) ;
+	m_fd = ::open( path.string().c_str(), flags, mode ) ;
+	if ( m_fd == -1 )
+	{
+		BOOST_THROW_EXCEPTION(
+			Error()
+				<< boost::errinfo_api_function("open")
+				<< boost::errinfo_errno(errno)
+				<< boost::errinfo_file_name(path.string())
+		) ;
+	}
+}
+
+void File::OpenForRead( const fs::path& path )
+{
+	int flags = O_RDONLY ;
+#ifdef WIN32
+	flags |= O_BINARY ;
+#endif
+	Open( path, flags, 0 ) ;
+}
+
+void File::OpenForWrite( const fs::path& path, int mode )
+{
+	int flags = O_CREAT|O_RDWR|O_TRUNC ;
+#ifdef WIN32
+	flags |= O_BINARY ;
+#endif
+	Open( path, flags, mode ) ;
+}
+
+void File::Close()
+{
+	if ( IsOpened() )
+	{
+		close( m_fd ) ;
+		m_fd = -1 ;
+	}
+}
+
+bool File::IsOpened() const
+{
+	return m_fd != -1 ;
+}
+
+/**	Read bytes from file. See DataStream::Read() for details.
+	\throw	Error	In case of any error.
+*/
+std::size_t File::Read( char *ptr, std::size_t size )
+{
+	assert( IsOpened() ) ;
+	ssize_t count = ::read( m_fd, ptr, size ) ;
+	if ( count == -1 )
+	{
+		BOOST_THROW_EXCEPTION(
+			Error()
+				<< boost::errinfo_api_function("read")
+				<< boost::errinfo_errno(errno)
+		) ;
+	}
+	return count ;
+}
+
+std::size_t File::Write( const char *ptr, std::size_t size )
+{
+	assert( IsOpened() ) ;
+	ssize_t count = ::write( m_fd, ptr, size ) ;
+	if ( count == -1 )
+	{
+		BOOST_THROW_EXCEPTION(
+			Error()
+				<< boost::errinfo_api_function("write")
+				<< boost::errinfo_errno(errno)
+		) ;
+	}
+	return count ;
+}
+
+off_t File::Seek( off_t offset, int whence )
+{
+	assert( IsOpened() ) ;
+	return LSeek( m_fd, offset, whence ) ;
+}
+
+off_t File::Tell() const
+{
+	assert( IsOpened() ) ;
+	return LSeek( m_fd, 0, SEEK_CUR ) ;
+}
+
+u64_t File::Size() const
+{
+	assert( IsOpened() ) ;
+	
+	struct stat s = FStat(m_fd) ;
+	
+	assert( s.st_size >= 0 ) ;
+	return static_cast<uint64_t>( s.st_size ) ;
+}
+
+void File::Chmod( int mode )
+{
+	assert( IsOpened() ) ;
+#ifndef WIN32	
+	if ( ::fchmod( m_fd, mode ) != 0 )
+	{
+		BOOST_THROW_EXCEPTION(
+			Error()
+				<< boost::errinfo_api_function("fchmod")
+				<< boost::errinfo_errno(errno)
+		) ;
+	}
+#endif
+}
+
+/// This function is not implemented in win32 yet.
+void* File::Map( off_t offset, std::size_t length )
+{
+	assert( IsOpened() ) ;
+	
+#ifdef WIN32
+	assert( false ) ;
+	return 0 ;
+#else
+	void *addr = ::mmap( 0, length, PROT_READ, MAP_PRIVATE, m_fd, offset ) ;
+	if ( addr == reinterpret_cast<void*>( -1 ) )
+	{
+		BOOST_THROW_EXCEPTION(
+			Error()
+				<< boost::errinfo_api_function("mmap")
+				<< boost::errinfo_errno(errno)
+		) ;
+	}
+	return addr ;
+#endif
+}
+
+void File::UnMap( void *addr, std::size_t length )
+{
+#ifndef WIN32
+	if ( ::munmap( addr, length ) != 0 )
+	{
+		BOOST_THROW_EXCEPTION(
+			Error()
+				<< boost::errinfo_api_function("munmap")
+				<< boost::errinfo_errno(errno)
+		) ;
+	}
+#endif
+}
+
+struct stat File::Stat() const
+{
+	struct stat result = {} ;
+	if ( ::fstat( m_fd, &result ) != 0 )
+	{
+		BOOST_THROW_EXCEPTION(
+			Error()
+				<< boost::errinfo_api_function("fstat")
+				<< boost::errinfo_errno(errno)
+		) ;
+	}
+	return result ;
+}
+
+} // end of namespace

libgrive/src/util/File.hh

@@ -0,0 +1,82 @@
+/*
+	grive: an GPL program to sync a local directory with Google Drive
+	Copyright (C) 2012  Wan Wai Ho
+
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License
+	as published by the Free Software Foundation version 2
+	of the License.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with this program; if not, write to the Free Software
+	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+
+#pragma once
+
+#include "DataStream.hh"
+#include "Exception.hh"
+#include "FileSystem.hh"
+#include "Types.hh"
+
+#include <string>
+
+struct stat ;
+
+namespace gr {
+
+/**	\brief	A wrapper class for file read/write.
+
+	It is a simple wrapper around the UNIX file descriptor. It will
+	throw exceptions (i.e. Error) when it encounters errors.
+*/
+class File : public SeekStream
+{
+public :
+	/// File specific errors. It often includes
+	/// boost::errinfo_api_function and boost::errinfo_errno for the
+	/// detail information.
+	struct Error : virtual Exception {} ;
+
+public :
+	File() ;
+	File( const fs::path& path ) ;
+	File( const fs::path& path, int mode ) ;
+	~File( ) ;
+
+	File( const File& rhs ) ;
+	File& operator=( const File& rhs ) ;
+	void Swap( File& other ) ;
+
+	void OpenForRead( const fs::path& path ) ;
+	void OpenForWrite( const fs::path& path, int mode = 0600 ) ;
+	void Close() ;
+	bool IsOpened() const ;
+	
+	std::size_t Read( char *ptr, std::size_t size ) ;
+	std::size_t Write( const char *ptr, std::size_t size ) ;
+
+	off_t Seek( off_t offset, int whence ) ;
+	off_t Tell() const ;
+	u64_t Size() const ;
+	
+	void Chmod( int mode ) ;
+
+	void* Map( off_t offset, std::size_t length ) ;
+	static void UnMap( void *addr, std::size_t length ) ;
+
+	struct stat Stat() const ;
+
+private :
+	void Open( const fs::path& path, int flags, int mode ) ;
+	
+private :
+	int	m_fd ;
+} ;
+	
+} // end of namespace

libgrive/src/util/FileSystem.hh

@@ -25,17 +25,4 @@
 namespace gr
 {
 	namespace fs = boost::filesystem ;
-	
-	// these two functions are for ancient distro which does not have boost v1.44 or later
-	// will be removed once people upgrade
-	
-	inline std::string Path2Str( const fs::path& p )
-	{
-		return p.string() ;
-	}
-	
-	inline std::string Path2Str( const std::string& s )
-	{
-		return s ;
-	}
 }

libgrive/src/util/Function.hh

@@ -178,7 +178,7 @@ public :
 
 private :
 	typedef impl::FuncImpl<Type> Impl ;
-	std::auto_ptr<Impl>	m_pimpl ;
+	std::unique_ptr<Impl>	m_pimpl ;
 } ;
 
 } // end of namespace

libgrive/src/util/MemMap.cc

@@ -0,0 +1,46 @@
+/*
+	grive: an GPL program to sync a local directory with Google Drive
+	Copyright (C) 2012  Wan Wai Ho
+
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License
+	as published by the Free Software Foundation version 2
+	of the License.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with this program; if not, write to the Free Software
+	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+
+#include "MemMap.hh"
+#include "File.hh"
+
+namespace gr {
+
+MemMap::MemMap( File& file, off_t offset, std::size_t length ) :
+	m_addr	( file.Map( offset, length ) ),
+	m_length( length )
+{
+}
+
+MemMap::~MemMap()
+{
+	File::UnMap( m_addr, m_length ) ;
+}
+	
+void* MemMap::Addr() const
+{
+	return m_addr ;
+}
+
+std::size_t MemMap::Length() const
+{
+	return m_length ;
+}
+
+} // end of namespace

libgrive/src/util/MemMap.hh

@@ -19,29 +19,29 @@
 
 #pragma once
 
-#include "util/Exception.hh"
-#include "protocol/Json.hh"
+#include "Exception.hh"
+#include "Types.hh"
+#include <cstddef>
 
 namespace gr {
 
-class Config
+class File ;
+
+class MemMap
 {
 public :
 	struct Error : virtual Exception {} ;
-	typedef boost::error_info<struct FileTag, std::string>	File ;
 
-	static const std::string& Filename() ;
-
-	Config() ;
+public :
+	MemMap( File& file, off_t offset, std::size_t length ) ;
+	~MemMap() ;
 	
-	Json& Get() ;
-	void Save() ;
+	void* Addr() const ;
+	std::size_t Length() const ;
 
 private :
-	Json Read( const std::string& filename ) ;
-
-private :
-	Json m_cfg ;
+	void		*m_addr ;
+	std::size_t	m_length ;
 } ;
-	
+
 } // end of namespace

libgrive/src/util/OS.cc

@@ -39,12 +39,12 @@
 
 namespace gr { namespace os {
 
-DateTime FileCTime( const fs::path& filename )
+void Stat( const fs::path& filename, DateTime *t, off_t *size, FileType *ft )
 {
-	return FileCTime( filename.string() ) ;
+	Stat( filename.string(), t, size, ft ) ;
 }
 
-DateTime FileCTime( const std::string& filename )
+void Stat( const std::string& filename, DateTime *t, off64_t *size, FileType *ft )
 {
 	struct stat s = {} ;
 	if ( ::stat( filename.c_str(), &s ) != 0 )
@@ -57,11 +57,18 @@ DateTime FileCTime( const std::string& filename )
 		) ;
 	}
 	
-#if defined __APPLE__ && defined __DARWIN_64_BIT_INO_T
-	return DateTime( s.st_ctimespec.tv_sec, s.st_ctimespec.tv_nsec ) ;
+	if ( t )
+	{
+#if defined __NetBSD__ || ( defined __APPLE__ && defined __DARWIN_64_BIT_INO_T )
+		*t = DateTime( s.st_ctimespec.tv_sec, s.st_ctimespec.tv_nsec ) ;
 #else
-	return DateTime( s.st_ctim.tv_sec, s.st_ctim.tv_nsec);
+		*t = DateTime( s.st_ctim.tv_sec, s.st_ctim.tv_nsec);
 #endif
+	}
+	if ( size )
+		*size = s.st_size;
+	if ( ft )
+		*ft = S_ISDIR( s.st_mode ) ? FT_DIR : ( S_ISREG( s.st_mode ) ? FT_FILE : FT_UNKNOWN ) ;
 }
 
 void SetFileTime( const fs::path& filename, const DateTime& t )
@@ -81,4 +88,17 @@ void SetFileTime( const std::string& filename, const DateTime& t )
 		) ;
 }
 
+void Sleep( unsigned int sec )
+{
+	struct timespec ts = { sec, 0 } ;
+	
+	int result = 0 ;
+	do
+	{
+		struct timespec rem ;
+		nanosleep( &ts, &rem ) ;
+		ts = rem ;
+	} while ( result == -1 && errno == EINTR ) ;
+}
+
 } } // end of namespaces