Experimental: "cacheless restore" using a seekable stream

Do two passes instead of just sequentially writing all chunks to the standard output.
On the first pass, all "chunk emit" instructions are remembered together with their output
positions indexed by bundle id, and all "byte emit" instructions are executed using seeks.
On the second pass, all remembered "chunk emit" instructions are executed in the bundle
order. This makes zbackup decompress every used bundle only ONCE instead of doing
it (basically the same work) many times while reading different chunks.

This allows for bigger bundle sizes (I use 32M), which reduces the number of files
in the repository and makes it more cloud-storage-sync friendly, and further improves
the compression ratio.

.gitignore

@@ -0,0 +1,9 @@
+*.o
+CMakeFiles/
+CMakeCache.txt
+Makefile
+cmake_install.cmake
+/zbackup.pb.cc
+/zbackup.pb.h
+/zbackup
+/zbackup.exe

.travis.yml

@@ -0,0 +1,18 @@
+sudo: false
+language: cpp
+cache: ccache
+addons:
+  apt:
+    packages:
+    - cmake
+    - libssl-dev
+    - libprotobuf-dev
+    - protobuf-compiler
+    - liblzma-dev
+    - zlib1g-dev
+    - liblzo2-dev
+script:
+  - mkdir objdir
+  - cd objdir
+  - cmake ../
+  - make

CMakeLists.txt

@@ -1,10 +1,14 @@
-# Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-# Part of ZBackup. Licensed under GNU GPLv2 or later
+# Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+# Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
-cmake_minimum_required( VERSION 2.8.9 )
+cmake_minimum_required( VERSION 2.8.2 )
 project( zbackup )
 
-set( CMAKE_BUILD_TYPE Release )
+list( APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" )
+
+if( NOT CMAKE_BUILD_TYPE )
+  set( CMAKE_BUILD_TYPE Release )
+endif( NOT CMAKE_BUILD_TYPE )
 
 find_package( ZLIB REQUIRED )
 include_directories( ${ZLIB_INCLUDE_DIRS} )
@@ -15,6 +19,13 @@ include_directories( ${OPENSSL_INCLUDE_DIR} )
 find_package( Protobuf REQUIRED )
 include_directories( ${PROTOBUF_INCLUDE_DIRS} )
 include_directories( ${CMAKE_CURRENT_BINARY_DIR} )
+
+find_program( PROTOBUF_PROTOC_CHECK NAMES protoc DOC "Protobuf compiler binary" )
+
+IF( ${PROTOBUF_PROTOC_CHECK} STREQUAL "PROTOBUF_PROTOC_CHECK-NOTFOUND" )
+  MESSAGE( FATAL_ERROR "Could not find protobuf compiler. Make sure protobuf-compiler package is installed." )
+ENDIF( ${PROTOBUF_PROTOC_CHECK} STREQUAL "PROTOBUF_PROTOC_CHECK-NOTFOUND" )
+
 PROTOBUF_GENERATE_CPP( protoSrcs protoHdrs zbackup.proto )
 
 find_package( Threads REQUIRED )
@@ -22,6 +33,31 @@ find_package( Threads REQUIRED )
 find_package( LibLZMA REQUIRED )
 include_directories( ${LIBLZMA_INCLUDE_DIRS} )
 
+find_package( LibLZO COMPONENTS LIBLZO_HAS_LZO1X_DECOMPRESS_SAFE LIBLZO_HAS_LZO1X_1_COMPRESS )
+if ( LIBLZO_FOUND )
+  ADD_DEFINITIONS( -DHAVE_LIBLZO )
+  include_directories( ${LIBLZO_INCLUDE_DIRS} )
+else ( LIBLZO_FOUND )
+  set( LIBLZO_LIBRARIES )
+endif( LIBLZO_FOUND )
+
+find_package( LibUnwind COMPONENTS LIBUNWIND_HAS_UNW_GETCONTEXT LIBUNWIND_HAS_INIT_LOCAL )
+if ( LIBUNWIND_FOUND )
+  ADD_DEFINITIONS( -DHAVE_LIBUNWIND )
+  include_directories( ${LIBUNWIND_INCLUDE_DIRS} )
+else ( LIBUNWIND_FOUND )
+  set( LIBUNWIND_LIBRARIES )
+endif( LIBUNWIND_FOUND )
+
+add_custom_target( invalidate_files ALL
+  COMMAND ${CMAKE_COMMAND} -E touch "${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt" )
+execute_process( OUTPUT_VARIABLE ZBACKUP_VERSION
+  COMMAND git describe --tags --always
+  OUTPUT_STRIP_TRAILING_WHITESPACE )
+if ( ZBACKUP_VERSION )
+  ADD_DEFINITIONS( -DZBACKUP_VERSION="${ZBACKUP_VERSION}" )
+endif( ZBACKUP_VERSION )
+
 file( GLOB sourceFiles "*.cc" )
 add_executable( zbackup ${sourceFiles} ${protoSrcs} ${protoHdrs} )
 
@@ -31,6 +67,8 @@ target_link_libraries( zbackup
   ${CMAKE_THREAD_LIBS_INIT}
   ${ZLIB_LIBRARIES}
   ${LIBLZMA_LIBRARIES}
+  ${LIBLZO_LIBRARIES}
+  ${LIBUNWIND_LIBRARIES}
 )
 
 install( TARGETS zbackup DESTINATION bin )

CONTRIBUTORS

@@ -0,0 +1,21 @@
+This file contains a list of people who have made contributions
+to the ZBackup.
+
+Original design and implementation:
+  Konstantin Isakov <ikm@zbackup.org>
+
+Core maintainers:
+  Konstantin Isakov <ikm@zbackup.org>
+  Vladimir Stackov <amigo.elite@gmail.com>
+
+Code contributions:
+  Benjamin Koch <bbbsnowball@gmail.com>
+  Gleb Golubitsky <sectoid@gnolltech.org>
+  Igor Katson <igor.katson@gmail.com>
+  Vitaliy Filippov <vitalif@yourcmc.ru>
+  Eugene Agafonov <e.a.agafonov@gmail.com>
+  Antonia Stevens <a@antevens.com>
+  Frank Groeneveld <frank@frankgroeneveld.nl>
+
+Feel free to add yourself to this list in your pull-request.
+Please modify this file instead of source headers.

README.md

@@ -1,6 +1,8 @@
+[![Build Status](https://travis-ci.org/zbackup/zbackup.svg)](https://travis-ci.org/zbackup/zbackup) [![Coverity Scan Build Status](https://scan.coverity.com/projects/4229/badge.svg)](https://scan.coverity.com/projects/4229)
+
 # Introduction
 
-**zbackup** is globally-deduplicating backup tool, based on the ideas found in [rsync](http://rsync.samba.org/).  Feed a large `.tar` into it, and it will store duplicate regions of it only once, then compress and optionally encrypt the result. Feed another `.tar` file, and it will also re-use any data found in any previous backups. This way only new changes are stored, and as long as the files are not very different, the amount of storage required is very low. Any of the backup files stored previously can be read back in full at any time. The program is format-agnostic, so you can feed virtually any files to it (any types of archives, proprietary formats, even raw disk images -- but see [Caveats](#caveats)).
+**zbackup** is a globally-deduplicating backup tool, based on the ideas found in [rsync](http://rsync.samba.org/).  Feed a large `.tar` into it, and it will store duplicate regions of it only once, then compress and optionally encrypt the result. Feed another `.tar` file, and it will also re-use any data found in any previous backups. This way only new changes are stored, and as long as the files are not very different, the amount of storage required is very low. Any of the backup files stored previously can be read back in full at any time. The program is format-agnostic, so you can feed virtually any files to it (any types of archives, proprietary formats, even raw disk images -- but see [Caveats](#caveats)).
 
 This is achieved by sliding a window with a rolling hash over the input at a byte granularity and checking whether the block in focus was ever met already. If a rolling hash matches, an additional full cryptographic hash is calculated to ensure the block is indeed the same. The deduplication happens then.
 
@@ -8,25 +10,27 @@ This is achieved by sliding a window with a rolling hash over the input at a byt
 
 The program has the following features:
 
- * Parallel LZMA compression of the stored data
+ * Parallel LZMA or LZO compression of the stored data
  * Built-in AES encryption of the stored data
- * Possibility to delete old backup data in the future
+ * Possibility to delete old backup data
  * Use of a 64-bit rolling hash, keeping the amount of soft collisions to zero
  * Repository consists of immutable files. No existing files are ever modified
  * Written in C++ only with only modest library dependencies
  * Safe to use in production (see [below](#safety))
+ * Possibility to exchange data between repos without recompression
 
 # Build dependencies
 
- * `cmake` >= 6.8.9 (though it should not be too hard to compile the sources by hand if needed)
+ * `cmake` >= 2.8.2 (though it should not be too hard to compile the sources by hand if needed)
  * `libssl-dev` for all encryption, hashing and random numbers
- * `libprotobuf-dev` for data serialization
+ * `libprotobuf-dev` and `protobuf-compiler` for data serialization
  * `liblzma-dev` for compression
- * `zlib1g-dev` for adler32 calcuation
+ * `liblzo2-dev` for compression (optional)
+ * `zlib1g-dev` for adler32 calculation
 
 # Quickstart
 
-To build:
+To build and install:
 
 ```bash
 cd zbackup
@@ -36,10 +40,12 @@ sudo make install
 # or just run as ./zbackup
 ```
 
+`zbackup` is also part of the [Fedora/EPEL](https://apps.fedoraproject.org/packages/zbackup), [Debian](https://packages.debian.org/search?keywords=zbackup), [Ubuntu](http://packages.ubuntu.com/search?keywords=zbackup), [Arch Linux](https://aur.archlinux.org/packages/zbackup/) and [FreeBSD](http://www.freshports.org/sysutils/zbackup/).
+
 To use:
 
 ```bash
-zbackup init --non-enrypted /my/backup/repo
+zbackup init --non-encrypted /my/backup/repo
 tar c /my/precious/data | zbackup backup /my/backup/repo/backups/backup-`date '+%Y-%m-%d'`
 zbackup restore /my/backup/repo/backups/backup-`date '+%Y-%m-%d'` > /my/precious/backup-restored.tar
 ```
@@ -49,7 +55,7 @@ If you have a lot of RAM to spare, you can use it to speed-up the restore proces
 If encryption is wanted, create a file with your password:
 
 ``` bash
-# more secure to to use an editor
+# more secure to use an editor
 echo mypassword > ~/.my_backup_password
 chmod 600 ~/.my_backup_password
 ```
@@ -73,17 +79,14 @@ If you have a 32-bit system and a lot of cores, consider lowering the number of
  * While you can pipe any data into the program, the data should be uncompressed and unencrypted -- otherwise no deduplication could be performed on it. `zbackup` would compress and encrypt the data itself, so there's no need to do that yourself. So just run `tar c` and pipe it into `zbackup` directly. If backing up disk images employing encryption, pipe the unencrypted version (the one you normally mount). If you create `.zip` or `.rar` files, use no compression (`-0` or `-m0`) and no encryption.
  * Parallel LZMA compression uses a lot of RAM (several hundreds of megabytes, depending on the number of threads used), and ten times more virtual address space. The latter is only relevant on 32-bit architectures where it's limited to 2 or 3 GB. If you hit the ceiling, lower the number of threads with `--threads`.
  * Since the data is deduplicated, there's naturally no redundancy in it. A loss of a single file can lead to a loss of virtually all data. Make sure you store it on a redundant storage (RAID1, a cloud provider etc).
- * The encryption key, if used, is stored in the `info` file in the root of the repo. It is encrypted with your password. Technically thus you can change your password without re-encrypting any data, and as long as no one possesses the old `info` file and knows your old password, you would be safe (even though the actual option to change password is not implemented yet -- someone who needs this is welcome to create a pull request -- the possibility is all there).  Also note that it is crucial you don't lose your `info` file, as otherwise the whole backup would be lost. 
+ * The encryption key, if used, is stored in the `info` file in the root of the repo. It is encrypted with your password. Technically thus you can change your password without re-encrypting any data, and as long as no one possesses the old `info` file and knows your old password, you would be safe (note that ability to change repo type between encrypted and non-encrypted is not implemented yet -- someone who needs this is welcome to create a pull request -- the possibility is all there). Also note that it is crucial you don't lose your `info` file, as otherwise the whole backup would be lost. 
 
 # Limitations
 
  * Right now the only modes supported are reading from standard input and writing to standard output. FUSE mounts and NBD servers may be added later if someone contributes the code.
  * The program keeps all known blocks in an in-RAM hash table, which may create scalability problems for very large repos (see [below](#scalability)).
  * The only encryption mode currently implemented is `AES-128` in `CBC` mode with `PKCS#7` padding. If you believe that this is not secure enough, patches are welcome. Before you jump to conclusions however, read [this article](http://www.schneier.com/blog/archives/2009/07/another_new_aes.html).
- * The only compression mode supported is LZMA, which suits backups very nicely.
  * It's only possible to fully restore the backup in order to get to a required file, without any option to quickly pick it out. `tar` would not allow to do it anyway, but e.g. for `zip` files it could have been possible. This is possible to implement though, e.g. by exposing the data over a FUSE filesystem.
- * There's no option to delete old backup data yet. The possibility is all there, though. Someone needs to implement it (see [below](#improvements)).
- * There's no option to specify block and bundle sizes other than the default ones (currently `64k` and `2MB` respectively), though it's trivial to add command-line switches for those.
 
 Most of those limitations can be lifted by implementing the respective features.
 
@@ -111,7 +114,7 @@ The repository has the following directory structure:
  * The `backups` directory contain your backups. Those are very small files which are needed for restoration. They are encrypted if encryption is enabled. The names can be arbitrary. It is possible to arrange files in subdirectories, too. Free renaming is also allowed.
  * The `bundles` directory contains the bulk of data. Each bundle internally contains multiple small chunks, compressed together and encrypted. Together all those chunks account for all deduplicated data stored.
  * The `index` directory contains the full index of all chunks in the repository, together with their bundle names. A separate index file is created for each backup session. Technically those files are redundant, all information is contained in the bundles themselves. However, having a separate `index` is nice for two reasons: 1) it's faster to read as it incurs less seeks, and 2) it allows making backups while storing bundles elsewhere. Bundles are only needed when restoring -- otherwise it's sufficient to only have `index`. One could then move all newly created bundles into another machine after each backup.
- * `info` is a very important file, which contains all global repository metadata, such as chunk and bundle sizes, and an encryption key encrypted with the user password. It is paramount not to lose it, so backing it up separately somewhere might be a good idea. On the other hand, if you absolutely don't trust your remote storage provider, you might consider not storing it with the rest of the data. It would then be impossible to decrypt it at all, even if your password gets known later.
+ * `info` is a very important file which contains all global repository metadata, such as chunk and bundle sizes, and an encryption key encrypted with the user password. It is paramount not to lose it, so backing it up separately somewhere might be a good idea. On the other hand, if you absolutely don't trust your remote storage provider, you might consider not storing it with the rest of the data. It would then be impossible to decrypt it at all, even if your password gets known later.
 
 The program does not have any facilities for sending your backup over the network. You can `rsync` the repo to another computer or use any kind of cloud storage capable of storing files. Since `zbackup` never modifies any existing files, the latter is especially easy -- just tell the upload tool you use not to upload any files which already exist on the remote side (e.g. with `gsutil` it's `gsutil cp -R -n /my/backup gs:/mybackup/`).
 
@@ -120,7 +123,7 @@ To aid with creating backups, there's an utility called `tartool` included with
 # Scalability
 
 This section tries do address the question on the maximum amount of data which can be held in a backup repository. What is meant here is the deduplicated data. The number of bytes in all source files ever fed into the repository doesn't matter, but the total size of the resulting repository does.
-Internally all input data is split into small blocks called chunks (up to `64k` each by default). Blocks are collected into bundles (up to `2MB` each by default), and those bundles are then compressed and encrypted.
+Internally all input data is split into small blocks called chunks (up to `64k` each by default). Chunks are collected into bundles (up to `2MB` each by default), and those bundles are then compressed and encrypted.
 
 There are then two problems with the total number of chunks in the repository:
 
@@ -136,13 +139,29 @@ All in all, as long as the amount of RAM permits, one can go up to several terab
   * `AES-128` in `CBC` mode with `PKCS#7` padding is used for encryption. This seems to be a reasonbly safe classic solution. Each encrypted file has a random IV as its first 16 bytes.
   * We use Google's [protocol buffers](https://developers.google.com/protocol-buffers/) to represent data structures in binary form. They are very efficient and relatively simple to use.
 
+# Compression
+
+`zbackup` uses LZMA to compress stored data. It compresses very well, but it will slow down your backup
+(unless you have a very fast CPU).
+
+LZO is much faster, but the files will be bigger. If you don't
+want your backup process to be cpu-bound, you should consider using LZO. However, there are some caveats:
+
+* LZO is so fast that other parts of `zbackup` consume significant portions of the CPU. In fact, it is only using one core on my machine because compression is the only thing that can run in parallel.
+* I've hacked the LZO support in a day. You shouldn't trust it. Please make sure that restore works before you assume that your data is safe. That may still be faster than a backup with LZMA ;-)
+* LZMA is still the default, so make sure that you use the `-o bundle.compression_method=lzo` argument when you init the repo or whenever you do a backup.
+
+You can mix LZMA and LZO in a repository. Each bundle file has a field that says how it was compressed, so
+`zbackup` will use the right method to decompress it. You could use an old `zbackup` respository with only LZMA
+bundles and start using LZO. However, please think twice before you do that because old versions of `zbackup`
+won't be able to read those bundles.
+
 # Improvements
 
 There's a lot to be improved in the program. It was released with the minimum amount of functionality to be useful. It is also stable. This should hopefully stimulate people to join the development and add all those other fancy features. Here's a list of ideas:
 
- * Additional options, such as configurable chunk and bundle sizes etc.
- * A command to change password.
- * A command to perform garbage collection. The program should skim through all backups and note which chunks are used by all of them. Then it should skim through all bundles and see which chunks among the ones stored were never used by the backups. If a bundle has more than *X%* of unused chunks, the remaining chunks should be transferred into brand new bundles. The old bundles should be deleted then. Once the process finishes, a new single index file with all existing chunk ids should be written, replacing all previous index files. With this command, it would become possible to remove old backups.
+ * Ability to change bundle type (between encrypted and non-encrypted).
+ * Improved garbage collection. The program should support ability to specify maximum index file size / maximum index file count (for better compatibility with cloud storages as well) or something like retention policy.
  * A command to fsck the repo by doing something close to what garbage collection does, but also checking all hashes and so on.
  * Parallel decompression. Right now decompression is single-threaded, but it is possible to look ahead in the stream and perform prefetching.
  * Support for mounting the repo over FUSE. Random access to data would then be possible.
@@ -164,14 +183,15 @@ The author is reachable over email at <ikm@zbackup.org>. Please be constructive
 `zbackup` is certainly not the first project to embrace the idea of using a rolling hash for deduplication. Here's a list of other projects the author found on the web:
 
  * [bup](https://github.com/bup/bup), based on storing data in `git` packs. No possibility of removing old data. This program was the initial inspiration for `zbackup`.
- * [ddar](http://www.synctus.com/ddar/), seems to be a little bit outdated. Contains a nice list of alternatives with comparisons.
+ * [ddar](https://github.com/basak/ddar), seems to be a little bit outdated. Contains a nice list of alternatives with comparisons.
  * [rdiff-backup](http://www.nongnu.org/rdiff-backup/), based on the original `rsync` algorithm. Does not do global deduplication, only working over the files with the same file name.
  * [duplicity](http://duplicity.nongnu.org/), which looks similar to `rdiff-backup` with regards to mode of operation.
  * Some filesystems (most notably [ZFS](http://en.wikipedia.org/wiki/ZFS) and [Btrfs](http://en.wikipedia.org/wiki/Btrfs)) provide deduplication features. They do so only at block level though, without a sliding window, so they can not accomodate to arbitrary byte insertion/deletion in the middle of data.
+ * [Attic](https://attic-backup.org/), which looks very similar to `zbackup`.
 
 # Credits
 
-Copyright (c) 2013-2013 Konstantin Isakov (<ikm@zbackup.org>). Licensed under GNU GPLv2 or later.
+Copyright (c) 2012-2014 Konstantin Isakov (<ikm@zbackup.org>) and ZBackup contributors, see CONTRIBUTORS. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE.
 
 This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version.
 

adler32.hh

@@ -1,8 +1,8 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
-#ifndef ADLER32_HH_INCLUDED__
-#define ADLER32_HH_INCLUDED__
+#ifndef ADLER32_HH_INCLUDED
+#define ADLER32_HH_INCLUDED
 
 #include <zlib.h>
 #include <stdint.h>

appendallocator.cc

@@ -1,5 +1,5 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
 #include <stdlib.h>
 #include <new>

appendallocator.hh

@@ -1,8 +1,8 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
-#ifndef APPENDALLOCATOR_HH_INCLUDED__
-#define APPENDALLOCATOR_HH_INCLUDED__
+#ifndef APPENDALLOCATOR_HH_INCLUDED
+#define APPENDALLOCATOR_HH_INCLUDED
 
 #include <stdlib.h>
 #include <limits>

backup_collector.cc

@@ -0,0 +1,145 @@
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
+
+#include "backup_collector.hh"
+
+using std::string;
+
+void BundleCollector::startIndex( string const & indexFn )
+{
+  indexModified = indexNecessary = false;
+  indexTotalChunks = indexUsedChunks = 0;
+  indexModifiedBundles = indexKeptBundles = indexRemovedBundles = 0;
+}
+
+void BundleCollector::finishIndex( string const & indexFn )
+{
+  verbosePrintf( "Chunks used: %d/%d, bundles: %d kept, %d modified, %d removed\n",
+                 indexUsedChunks, indexTotalChunks, indexKeptBundles,
+                 indexModifiedBundles, indexRemovedBundles );
+  if ( indexModified )
+  {
+    filesToUnlink.push_back( indexFn );
+    commit();
+  }
+  else
+  {
+    chunkStorageWriter->reset();
+    if ( gcDeep && !indexNecessary )
+      // this index was a complete copy so we don't need it
+      filesToUnlink.push_back( indexFn );
+  }
+}
+
+void BundleCollector::startBundle( Bundle::Id const & bundleId )
+{
+  savedId = bundleId;
+  totalChunks = 0;
+  usedChunks = 0;
+}
+
+void BundleCollector::processChunk( ChunkId const & chunkId, uint32_t size )
+{
+  if ( gcDeep )
+  {
+    if ( overallChunkSet.find ( chunkId ) == overallChunkSet.end() )
+      overallChunkSet.insert( chunkId );
+    else
+      return;
+  }
+
+  totalChunks++;
+  if ( usedChunkSet.find( chunkId ) != usedChunkSet.end() )
+  {
+    usedChunks++;
+    indexNecessary = true;
+  }
+}
+
+void BundleCollector::finishBundle( Bundle::Id const & bundleId, BundleInfo const & info )
+{
+  string i = Bundle::generateFileName( savedId, "", false );
+  indexTotalChunks += totalChunks;
+  indexUsedChunks += usedChunks;
+  if ( 0 == usedChunks && 0 != totalChunks )
+  {
+    dPrintf( "Deleting %s bundle\n", i.c_str() );
+    filesToUnlink.push_back( Dir::addPath( bundlesPath, i ) );
+    indexModified = true;
+    indexRemovedBundles++;
+  }
+  else if ( usedChunks < totalChunks )
+  {
+    dPrintf( "%s: used %d/%d chunks\n", i.c_str(), usedChunks, totalChunks );
+    filesToUnlink.push_back( Dir::addPath( bundlesPath, i ) );
+    indexModified = true;
+    copyUsedChunks( info );
+    indexModifiedBundles++;
+  }
+  else
+  {
+    if ( gcRepack )
+    {
+      filesToUnlink.push_back( Dir::addPath( bundlesPath, i ) );
+      indexModified = true;
+      copyUsedChunks( info );
+      indexModifiedBundles++;
+    }
+    else
+    {
+      if ( gcDeep && 0 == totalChunks )
+      {
+        if ( overallBundleSet.find ( bundleId ) == overallBundleSet.end() )
+        {
+          overallBundleSet.insert( bundleId );
+          dPrintf( "Deleting %s bundle\n", i.c_str() );
+          filesToUnlink.push_back( Dir::addPath( bundlesPath, i ) );
+          indexModified = true;
+          indexRemovedBundles++;
+        }
+        else
+        {
+          // trigger index update
+          indexModified = true;
+        }
+      }
+      else
+      {
+        if ( gcDeep && overallBundleSet.find ( bundleId ) == overallBundleSet.end() )
+          overallBundleSet.insert( bundleId );
+
+        chunkStorageWriter->addBundle( info, savedId );
+        dPrintf( "Keeping %s bundle\n", i.c_str() );
+        indexKeptBundles++;
+      }
+    }
+  }
+}
+
+void BundleCollector::copyUsedChunks( BundleInfo const & info )
+{
+  // Copy used chunks to the new index
+  string chunk;
+  size_t chunkSize;
+  for ( int x = info.chunk_record_size(); x--; )
+  {
+    BundleInfo_ChunkRecord const & record = info.chunk_record( x );
+    ChunkId id( record.id() );
+    if ( usedChunkSet.find( id ) != usedChunkSet.end() )
+    {
+      chunkStorageReader->get( id, chunk, chunkSize );
+      chunkStorageWriter->add( id, chunk.data(), chunkSize );
+    }
+  }
+}
+
+void BundleCollector::commit()
+{
+  for ( int i = filesToUnlink.size(); i--; )
+  {
+    dPrintf( "Unlinking %s\n", filesToUnlink[i].c_str() );
+    unlink( filesToUnlink[i].c_str() );
+  }
+  filesToUnlink.clear();
+  chunkStorageWriter->commit();
+}

backup_collector.hh

@@ -0,0 +1,50 @@
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
+
+#ifndef BACKUP_COLLECTOR_HH_INCLUDED
+#define BACKUP_COLLECTOR_HH_INCLUDED
+
+#include <string>
+#include <vector>
+
+#include "backup_restorer.hh"
+#include "backup_file.hh"
+
+#include "debug.hh"
+
+using std::string;
+
+class BundleCollector: public IndexProcessor
+{
+private:
+  Bundle::Id savedId;
+  int totalChunks, usedChunks, indexTotalChunks, indexUsedChunks;
+  int indexModifiedBundles, indexKeptBundles, indexRemovedBundles;
+  bool indexModified, indexNecessary;
+  vector< string > filesToUnlink;
+  BackupRestorer::ChunkSet overallChunkSet;
+  std::set< Bundle::Id > overallBundleSet;
+
+  void copyUsedChunks( BundleInfo const & info );
+
+public:
+  string bundlesPath;
+  ChunkStorage::Reader *chunkStorageReader;
+  ChunkStorage::Writer *chunkStorageWriter;
+  BackupRestorer::ChunkSet usedChunkSet;
+  bool gcRepack, gcDeep;
+
+  void startIndex( string const & indexFn );
+
+  void finishIndex( string const & indexFn );
+
+  void startBundle( Bundle::Id const & bundleId );
+
+  void processChunk( ChunkId const & chunkId, uint32_t size );
+
+  void finishBundle( Bundle::Id const & bundleId, BundleInfo const & info );
+
+  void commit();
+};
+
+#endif

backup_creator.cc

@@ -1,5 +1,5 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
 #include <openssl/sha.h>
 #include <string.h>
@@ -15,10 +15,10 @@ namespace {
   unsigned const MinChunkSize = 256;
 }
 
-BackupCreator::BackupCreator( StorageInfo const & info,
+BackupCreator::BackupCreator( Config const & config,
                               ChunkIndex & chunkIndex,
                               ChunkStorage::Writer & chunkStorageWriter ):
-  chunkMaxSize( info.chunk_max_size() ),
+  chunkMaxSize( config.GET_STORABLE( chunk, max_size ) ),
   chunkIndex( chunkIndex ), chunkStorageWriter( chunkStorageWriter ),
   ringBufferFill( 0 ),
   chunkToSaveFill( 0 ),

backup_creator.hh

@@ -1,8 +1,8 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
-#ifndef BACKUP_CREATOR_HH_INCLUDED__
-#define BACKUP_CREATOR_HH_INCLUDED__
+#ifndef BACKUP_CREATOR_HH_INCLUDED
+#define BACKUP_CREATOR_HH_INCLUDED
 
 #include <google/protobuf/io/zero_copy_stream_impl_lite.h>
 #include <stddef.h>
@@ -17,6 +17,7 @@
 #include "rolling_hash.hh"
 #include "sptr.hh"
 #include "zbackup.pb.h"
+#include "config.hh"
 
 using std::vector;
 using std::string;
@@ -69,7 +70,7 @@ class BackupCreator: ChunkIndex::ChunkInfoInterface, NoCopy
   virtual ChunkId const & getChunkId();
 
 public:
-  BackupCreator( StorageInfo const &, ChunkIndex &, ChunkStorage::Writer & );
+  BackupCreator( Config const &, ChunkIndex &, ChunkStorage::Writer & );
 
   /// The data is fed the following way: the user fills getInputBuffer() with
   /// up to getInputBufferSize() bytes, then calls handleMoreData() with the

backup_exchanger.cc

@@ -0,0 +1,49 @@
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
+
+#include "backup_exchanger.hh"
+#include "dir.hh"
+#include "debug.hh"
+
+namespace BackupExchanger {
+
+vector< string > findOrRebuild( string const & src, string const & dst, string const & relativePath )
+{
+  vector< string > files;
+
+  Dir::Listing lst ( Dir::addPath( src, relativePath ) );
+
+  Dir::Entry entry;
+
+  while ( lst.getNext( entry ) )
+  {
+    string currentRelativePath ( relativePath );
+    if ( currentRelativePath.empty() )
+      currentRelativePath.assign( entry.getFileName() );
+    else
+      currentRelativePath.assign( Dir::addPath( relativePath, entry.getFileName() ) );
+
+    if ( entry.isDir() )
+    {
+      verbosePrintf( "Found directory %s...\n", currentRelativePath.c_str() );
+      string srcFullPath( Dir::addPath( src, currentRelativePath ) );
+      string dstFullPath( Dir::addPath( dst, currentRelativePath ) );
+      if ( !dst.empty() && !Dir::exists( dstFullPath.c_str() ) )
+      {
+        verbosePrintf( "Directory %s not found in destination, creating...\n",
+            currentRelativePath.c_str() );
+        Dir::create( dstFullPath.c_str() );
+      }
+      vector< string > subFiles( findOrRebuild( src, dst, currentRelativePath ) );
+      files.insert( files.end(), subFiles.begin(), subFiles.end() );
+    }
+    else
+    {
+      verbosePrintf( "Found file %s...\n", currentRelativePath.c_str() );
+      files.push_back( currentRelativePath );
+    }
+  }
+
+  return files;
+}
+}

backup_exchanger.hh

@@ -0,0 +1,31 @@
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
+
+#ifndef BACKUP_EXCHANGER_HH_INCLUDED
+#define BACKUP_EXCHANGER_HH_INCLUDED
+
+#include <string>
+#include <vector>
+#include "tmp_mgr.hh"
+
+namespace BackupExchanger {
+
+using std::string;
+using std::vector;
+using std::pair;
+
+enum {
+  backups,
+  bundles,
+  indexes,
+  Flags
+};
+
+/// Recreate source directory structure in destination
+vector< string > findOrRebuild( string const & src,
+    string const & dst = std::string(),
+    string const & relativePath = std::string() );
+typedef pair< sptr< TemporaryFile >, string > PendingExchangeRename;
+}
+
+#endif

backup_file.cc

@@ -1,5 +1,5 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
 #include "backup_file.hh"
 

backup_file.hh

@@ -1,8 +1,8 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
-#ifndef BACKUP_FILE_HH_INCLUDED__
-#define BACKUP_FILE_HH_INCLUDED__
+#ifndef BACKUP_FILE_HH_INCLUDED
+#define BACKUP_FILE_HH_INCLUDED
 
 #include <exception>
 #include <string>

backup_restorer.cc

@@ -1,5 +1,5 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
 #include <google/protobuf/io/coded_stream.h>
 #include <google/protobuf/io/zero_copy_stream_impl_lite.h>
@@ -15,40 +15,123 @@ namespace BackupRestorer {
 using std::vector;
 using google::protobuf::io::CodedInputStream;
 
+void restoreMap( ChunkStorage::Reader & chunkStorageReader,
+              ChunkMap const * chunkMap, SeekableSink *output )
+{
+  string chunk;
+  size_t chunkSize;
+  for ( ChunkMap::const_iterator it = chunkMap->begin(); it != chunkMap->end(); it++ )
+  {
+    for ( ChunkPosition::const_iterator pi = (*it).second.begin(); pi != (*it).second.end(); pi++ )
+    {
+      if ( output )
+      {
+        // Need to emit a chunk, reading it from the store
+        chunkStorageReader.get( (*pi).first, chunk, chunkSize );
+        output->saveData( (*pi).second, chunk.data(), chunkSize );
+      }
+    }
+  }
+}
+
 void restore( ChunkStorage::Reader & chunkStorageReader,
               std::string const & backupData,
-              DataSink & output )
+              DataSink * output, ChunkSet * chunkSet,
+              ChunkMap * chunkMap, SeekableSink * seekOut )
 {
   google::protobuf::io::ArrayInputStream is( backupData.data(),
                                              backupData.size() );
   CodedInputStream cis( &is );
   CodedInputStream::Limit limit = cis.PushLimit( backupData.size() );
+  // The following line prevents it from barfing on large backupData.
+  // TODO: this disables size checks for each separate message. Figure a better
+  // way to do this while keeping them enabled. It seems we need to create an
+  // instance of CodedInputStream for each message, but it might be expensive
+  cis.SetTotalBytesLimit( backupData.size(), -1 );
 
   // Used when emitting chunks
   string chunk;
 
   BackupInstruction instr;
+  int64_t position = 0;
   while ( cis.BytesUntilLimit() > 0 )
   {
     Message::parse( instr, cis );
 
     if ( instr.has_chunk_to_emit() )
     {
-      // Need to emit a chunk, reading it from the store
+      ChunkId id( instr.chunk_to_emit() );
       size_t chunkSize;
-      chunkStorageReader.get( ChunkId( instr.chunk_to_emit() ), chunk,
-                              chunkSize );
-      output.saveData( chunk.data(), chunkSize );
+      if ( output )
+      {
+        // Need to emit a chunk, reading it from the store
+        chunkStorageReader.get( id, chunk, chunkSize );
+        output->saveData( chunk.data(), chunkSize );
+      }
+      if ( chunkMap )
+      {
+        Bundle::Id const *bundleId = chunkStorageReader.getBundleId( id, chunkSize );
+        ChunkMap::iterator it = chunkMap->find( *bundleId );
+        if ( it == chunkMap->end() )
+        {
+          ChunkPosition v;
+          std::pair< ChunkMap::iterator, bool > r = chunkMap->insert( std::make_pair( *bundleId, v ) );
+          it = r.first;
+        }
+        (*it).second.push_back( std::make_pair( id, position ) );
+        position += chunkSize;
+      }
+      if ( chunkSet )
+      {
+        chunkSet->insert( id );
+      }
     }
 
-    if ( instr.has_bytes_to_emit() )
+    if ( ( output || chunkMap ) && instr.has_bytes_to_emit() )
     {
       // Need to emit the bytes directly
       string const & bytes = instr.bytes_to_emit();
-      output.saveData( bytes.data(), bytes.size() );
+      if ( output )
+        output->saveData( bytes.data(), bytes.size() );
+      if ( chunkMap )
+      {
+        if ( seekOut )
+          seekOut->saveData( position, bytes.data(), bytes.size() );
+        position += bytes.size();
+      }
     }
   }
 
   cis.PopLimit( limit );
 }
+
+void restoreIterations( ChunkStorage::Reader & chunkStorageReader,
+  BackupInfo & backupInfo, std::string & backupData, ChunkSet * chunkSet )
+{
+  // Perform the iterations needed to get to the actual user backup data
+  for ( ; ; )
+  {
+    backupData.swap( *backupInfo.mutable_backup_data() );
+
+    if ( backupInfo.iterations() )
+    {
+      struct StringWriter: public DataSink
+      {
+        string result;
+
+        virtual void saveData( void const * data, size_t size )
+        {
+          result.append( ( char const * ) data, size );
+        }
+      } stringWriter;
+
+      restore( chunkStorageReader, backupData, &stringWriter, chunkSet, NULL, NULL );
+      backupInfo.mutable_backup_data()->swap( stringWriter.result );
+      backupInfo.set_iterations( backupInfo.iterations() - 1 );
+    }
+    else
+      break;
+  }
+}
+
 }

backup_restorer.hh

@@ -1,12 +1,16 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
-#ifndef BACKUP_RESTORER_HH_INCLUDED__
-#define BACKUP_RESTORER_HH_INCLUDED__
+#ifndef BACKUP_RESTORER_HH_INCLUDED
+#define BACKUP_RESTORER_HH_INCLUDED
 
 #include <stddef.h>
 #include <exception>
 #include <string>
+#include <set>
+
+#undef __DEPRECATED
+#include <ext/hash_map>
 
 #include "chunk_storage.hh"
 #include "ex.hh"
@@ -19,15 +23,44 @@ public:
   virtual ~DataSink() {}
 };
 
+/// Generic interface to seekable data output
+class SeekableSink
+{
+public:
+  virtual void saveData( int64_t position, void const * data, size_t size )=0;
+};
+
+namespace __gnu_cxx
+{
+  template<>
+  struct hash< Bundle::Id >
+  {
+    size_t operator()( Bundle::Id v ) const
+    { return *((size_t*)(v.blob)); }
+  };
+}
+
 /// Restores the backup
 namespace BackupRestorer {
 
 DEF_EX( Ex, "Backup restorer exception", std::exception )
 DEF_EX( exTooManyBytesToEmit, "A backup record asks to emit too many bytes", Ex )
+DEF_EX( exBytesToMap, "Can't restore bytes to ChunkMap", Ex )
+
+typedef std::set< ChunkId > ChunkSet;
+typedef std::vector< std::pair < ChunkId, int64_t > > ChunkPosition;
+typedef __gnu_cxx::hash_map< Bundle::Id, ChunkPosition > ChunkMap;
 
 /// Restores the given backup
 void restore( ChunkStorage::Reader &, std::string const & backupData,
-              DataSink & );
+              DataSink *, ChunkSet *, ChunkMap *, SeekableSink * );
+
+/// Restores ChunkMap using seekable output
+void restoreMap( ChunkStorage::Reader & chunkStorageReader,
+              ChunkMap const * chunkMap, SeekableSink *output );
+
+/// Performs restore iterations on backupData
+void restoreIterations( ChunkStorage::Reader &, BackupInfo &, std::string &, ChunkSet * );
 }
 
 #endif

bundle.cc

@@ -1,22 +1,30 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
-#include <lzma.h>
 #include <stdint.h>
 
 #include "bundle.hh"
 #include "check.hh"
 #include "dir.hh"
-#include "encrypted_file.hh"
 #include "encryption.hh"
 #include "hex.hh"
 #include "message.hh"
+#include "adler32.hh"
+#include "compression.hh"
 
 namespace Bundle {
 
 enum
 {
-  FileFormatVersion = 1
+  FileFormatVersion = 1,
+
+  // This means, we don't use LZMA in this file.
+  FileFormatVersionNotLZMA,
+
+  // <- add more versions here
+
+  // This is the first version, we do not support.
+  FileFormatVersionFirstUnsupported
 };
 
 void Creator::addChunk( string const & id, void const * data, size_t size )
@@ -27,14 +35,84 @@ void Creator::addChunk( string const & id, void const * data, size_t size )
   payload.append( ( char const * ) data, size );
 }
 
-void Creator::write( std::string const & fileName, EncryptionKey const & key )
+void Creator::write( std::string const & fileName, EncryptionKey const & key,
+    Reader & reader )
 {
   EncryptedFile::OutputStream os( fileName.c_str(), key, Encryption::ZeroIv );
 
   os.writeRandomIv();
 
-  FileHeader header;
-  header.set_version( FileFormatVersion );
+  Message::serialize( reader.getBundleHeader(), os );
+
+  Message::serialize( reader.getBundleInfo(), os );
+  os.writeAdler32();
+
+  void * bufPrev = NULL;
+  const void * bufCurr = NULL;
+  int sizePrev = 0, sizeCurr = 0;
+  bool readPrev = false, readCurr = false;
+
+  for ( ; ; )
+  {
+    bool readCurr = reader.is->Next( &bufCurr, &sizeCurr );
+
+    if ( readCurr )
+    {
+      if ( readPrev )
+      {
+        os.write( bufPrev, sizePrev );
+
+        readPrev = readCurr;
+        free( bufPrev );
+        bufPrev = malloc( sizeCurr );
+        memcpy( bufPrev, bufCurr, sizeCurr );
+        sizePrev = sizeCurr;
+      }
+      else
+      {
+        readPrev = readCurr;
+        bufPrev = malloc( sizeCurr );
+        memcpy( bufPrev, bufCurr, sizeCurr );
+        sizePrev = sizeCurr;
+      }
+    }
+    else
+    {
+      if ( readPrev )
+      {
+        sizePrev -= sizeof( Adler32::Value );
+        os.write( bufPrev, sizePrev );
+        os.writeAdler32();
+        free ( bufPrev );
+        break;
+      }
+    }
+  }
+
+  if ( reader.is.get() )
+    reader.is.reset();
+}
+
+void Creator::write( Config const & config, std::string const & fileName,
+    EncryptionKey const & key )
+{
+  EncryptedFile::OutputStream os( fileName.c_str(), key, Encryption::ZeroIv );
+
+  os.writeRandomIv();
+
+  BundleFileHeader header;
+
+  const_sptr<Compression::CompressionMethod> compression =
+    Compression::CompressionMethod::selectedCompression;
+  header.set_compression_method( compression->getName() );
+
+  // The old code only support lzma, so we will bump up the version, if we're
+  // using lzma. This will make it fail cleanly.
+  if ( compression->getName() == "lzma" )
+    header.set_version( FileFormatVersion );
+  else
+    header.set_version( FileFormatVersionNotLZMA );
+
   Message::serialize( header, os );
 
   Message::serialize( info, os );
@@ -42,16 +120,10 @@ void Creator::write( std::string const & fileName, EncryptionKey const & key )
 
   // Compress
 
-  uint32_t preset = 6; // TODO: make this customizable, although 6 seems to be
-                       // the best option
-	lzma_stream strm = LZMA_STREAM_INIT;
-	lzma_ret ret;
+  sptr<Compression::EnDecoder> encoder = compression->createEncoder(
+      config );
 
-  ret = lzma_easy_encoder( &strm, preset, LZMA_CHECK_CRC64 );
-  CHECK( ret == LZMA_OK, "lzma_easy_encoder error: %d", (int) ret );
-
-  strm.next_in = ( uint8_t const * ) payload.data();
-  strm.avail_in = payload.size();
+  encoder->setInput( payload.data(), payload.size() );
 
   for ( ; ; )
   {
@@ -60,48 +132,40 @@ void Creator::write( std::string const & fileName, EncryptionKey const & key )
       int size;
       if ( !os.Next( &data, &size ) )
       {
-        lzma_end( &strm );
+        encoder.reset();
         throw exBundleWriteFailed();
       }
       if ( !size )
         continue;
-      strm.next_out = ( uint8_t * ) data;
-      strm.avail_out = size;
+      encoder->setOutput( data, size );
     }
 
     // Perform the compression
-    ret = lzma_code( &strm, LZMA_FINISH );
-
-    if ( ret == LZMA_STREAM_END )
+    if ( encoder->process( true ) )
     {
-      if ( strm.avail_out )
-        os.BackUp( strm.avail_out );
+      if ( encoder->getAvailableOutput() )
+        os.BackUp( encoder->getAvailableOutput() );
       break;
     }
-
-    CHECK( ret == LZMA_OK, "lzma_code error: %d", (int) ret );
   }
 
-	lzma_end( &strm );
+  encoder.reset();
 
   os.writeAdler32();
 }
 
-Reader::Reader( string const & fileName, EncryptionKey const & key )
+Reader::Reader( string const & fileName, EncryptionKey const & key, bool keepStream )
 {
-  EncryptedFile::InputStream is( fileName.c_str(), key, Encryption::ZeroIv );
+  is = new EncryptedFile::InputStream( fileName.c_str(), key, Encryption::ZeroIv );
+  is->consumeRandomIv();
 
-  is.consumeRandomIv();
+  Message::parse( header, *is );
 
-  FileHeader header;
-  Message::parse( header, is );
-
-  if ( header.version() != FileFormatVersion )
+  if ( header.version() >= FileFormatVersionFirstUnsupported )
     throw exUnsupportedVersion();
 
-  BundleInfo info;
-  Message::parse( info, is );
-  is.checkAdler32();
+  Message::parse( info, *is );
+  is->checkAdler32();
 
   size_t payloadSize = 0;
   for ( int x = info.chunk_record_size(); x--; )
@@ -109,54 +173,49 @@ Reader::Reader( string const & fileName, EncryptionKey const & key )
 
   payload.resize( payloadSize );
 
-  lzma_stream strm = LZMA_STREAM_INIT;
+  if ( keepStream )
+    return;
 
-  lzma_ret ret;
+  sptr<Compression::EnDecoder> decoder = Compression::CompressionMethod::findCompression(
+                                           header.compression_method() )->createDecoder();
 
-  ret = lzma_stream_decoder( &strm, UINT64_MAX, 0 );
-  CHECK( ret == LZMA_OK,"lzma_stream_decoder error: %d", (int) ret );
-
-  strm.next_out = ( uint8_t * ) &payload[ 0 ];
-  strm.avail_out = payload.size();
+  decoder->setOutput( &payload[ 0 ], payload.size() );
 
   for ( ; ; )
   {
     {
       void const * data;
       int size;
-      if ( !is.Next( &data, &size ) )
+      if ( !is->Next( &data, &size ) )
       {
-        lzma_end( &strm );
+        decoder.reset();
         throw exBundleReadFailed();
       }
       if ( !size )
         continue;
-      strm.next_in = ( uint8_t const * ) data;
-      strm.avail_in = size;
+      decoder->setInput( data, size );
     }
 
-    ret = lzma_code( &strm, LZMA_RUN );
-
-    if ( ret == LZMA_STREAM_END )
+    if ( decoder->process( false ) )
     {
-      if ( strm.avail_in )
-        is.BackUp( strm.avail_in );
+      if ( decoder->getAvailableInput() )
+        is->BackUp( decoder->getAvailableInput() );
       break;
     }
 
-    CHECK( ret == LZMA_OK, "lzma_code error: %d", (int) ret );
-
-    if ( !strm.avail_out && strm.avail_in )
+    if ( !decoder->getAvailableOutput() && decoder->getAvailableInput() )
     {
       // Apparently we have more data than we were expecting
-      lzma_end( &strm );
+      decoder.reset();
       throw exTooMuchData();
     }
   }
 
-  lzma_end( &strm );
+  decoder.reset();
 
-  is.checkAdler32();
+  is->checkAdler32();
+  if ( is.get() )
+    is.reset();
 
   // Populate the map
   char const * next = payload.data();

bundle.hh

@@ -1,8 +1,8 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
-#ifndef BUNDLE_HH_INCLUDED__
-#define BUNDLE_HH_INCLUDED__
+#ifndef BUNDLE_HH_INCLUDED
+#define BUNDLE_HH_INCLUDED
 
 #include <stddef.h>
 #include <string.h>
@@ -16,6 +16,8 @@
 #include "nocopy.hh"
 #include "static_assert.hh"
 #include "zbackup.pb.h"
+#include "encrypted_file.hh"
+#include "config.hh"
 
 namespace Bundle {
 
@@ -40,10 +42,47 @@ struct Id
   { return memcmp( blob, other.blob, sizeof( blob ) ) == 0; }
   bool operator != ( Id const & other ) const
   { return ! operator == ( other ); }
+  bool operator < ( Id const & other ) const
+  { return memcmp( blob, other.blob, sizeof( blob ) ) < 0; }
 };
 
 STATIC_ASSERT( sizeof( Id ) == IdSize );
 
+/// Reads the bundle and allows accessing chunks
+class Reader: NoCopy
+{
+  BundleInfo info;
+  BundleFileHeader header;
+  /// Unpacked payload
+  string payload;
+  /// Maps chunk id blob to its contents and size
+  typedef map< string, pair< char const *, size_t > > Chunks;
+  Chunks chunks;
+
+public:
+  DEF_EX( Ex, "Bundle reader exception", std::exception )
+  DEF_EX( exBundleReadFailed, "Bundle read failed", Ex )
+  DEF_EX( exUnsupportedVersion, "Unsupported version of the index file format", Ex )
+  DEF_EX( exTooMuchData, "More data than expected in a bundle", Ex )
+  DEF_EX( exDuplicateChunks, "Chunks with the same id found in a bundle", Ex )
+
+  Reader( string const & fileName, EncryptionKey const & key,
+      bool keepStream = false );
+
+  /// Reads the chunk into chunkData and returns true, or returns false if there
+  /// was no such chunk in the bundle. chunkData may be enlarged but won't
+  /// be shrunk. The size of the actual chunk would be stored in chunkDataSize
+  bool get( string const & chunkId, string & chunkData, size_t & chunkDataSize );
+  BundleInfo getBundleInfo()
+  { return info; }
+  BundleFileHeader getBundleHeader()
+  { return header; }
+  string getPayload()
+  { return payload; }
+
+  sptr< EncryptedFile::InputStream > is;
+};
+
 /// Creates a bundle by adding chunks to it until it's full, then compressing
 /// it and writing out to disk
 class Creator
@@ -65,37 +104,15 @@ public:
   /// Compresses and writes the bundle to the given file. The operation is
   /// time-consuming - calling this function from a worker thread could be
   /// warranted
-  void write( string const & fileName, EncryptionKey const & );
+  void write( Config const &, string const & fileName, EncryptionKey const & );
+  void write( string const & fileName, EncryptionKey const &,
+      Bundle::Reader & reader );
 
   /// Returns the current BundleInfo record - this is used for index files
   BundleInfo const & getCurrentBundleInfo() const
   { return info; }
 };
 
-/// Reads the bundle and allows accessing chunks
-class Reader: NoCopy
-{
-  /// Unpacked payload
-  string payload;
-  /// Maps chunk id blob to its contents and size
-  typedef map< string, pair< char const *, size_t > > Chunks;
-  Chunks chunks;
-
-public:
-  DEF_EX( Ex, "Bundle reader exception", std::exception )
-  DEF_EX( exBundleReadFailed, "Bundle read failed", Ex )
-  DEF_EX( exUnsupportedVersion, "Unsupported version of the index file format", Ex )
-  DEF_EX( exTooMuchData, "More data than expected in a bundle", Ex )
-  DEF_EX( exDuplicateChunks, "Chunks with the same id found in a bundle", Ex )
-
-  Reader( string const & fileName, EncryptionKey const & );
-
-  /// Reads the chunk into chunkData and returns true, or returns false if there
-  /// was no such chunk in the bundle. chunkData may be enlarged but won't
-  /// be shrunk. The size of the actual chunk would be stored in chunkDataSize
-  bool get( string const & chunkId, string & chunkData, size_t & chunkDataSize );
-};
-
 /// Generates a full file name for a bundle with the given id. If createDirs
 /// is true, any intermediate directories will be created if they don't exist
 /// already

check.hh

@@ -1,8 +1,8 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
-#ifndef CHECK_HH_INCLUDED__
-#define CHECK_HH_INCLUDED__
+#ifndef CHECK_HH_INCLUDED
+#define CHECK_HH_INCLUDED
 
 #include <inttypes.h>
 #include <stdio.h>

chunk_id.cc

@@ -1,5 +1,5 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
 #include "chunk_id.hh"
 
@@ -38,9 +38,17 @@ void ChunkId::setFromBlob( void const * data )
   rollingHash = fromLittleEndian( v );
 }
 
+bool operator <( const ChunkId &lhs, const ChunkId &rhs )
+{
+  int r = memcmp( &lhs.cryptoHash, &rhs.cryptoHash, sizeof( lhs.cryptoHash ) );
+  if ( r != 0 )
+    return r < 0;
+  return memcmp( &lhs.rollingHash, &rhs.rollingHash, sizeof( lhs.rollingHash ) ) < 0;
+}
+
 ChunkId::ChunkId( string const & blob )
 {
-  CHECK( blob.size() == BlobSize, "incorrect blob sise: %zu", blob.size() );
+  CHECK( blob.size() == BlobSize, "incorrect blob size: %zu", blob.size() );
 
   setFromBlob( blob.data() );
 }

chunk_id.hh

@@ -1,8 +1,8 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
-#ifndef CHUNK_ID_HH_INCLUDED__
-#define CHUNK_ID_HH_INCLUDED__
+#ifndef CHUNK_ID_HH_INCLUDED
+#define CHUNK_ID_HH_INCLUDED
 
 #include <string>
 #include "rolling_hash.hh"
@@ -35,4 +35,6 @@ struct ChunkId
   ChunkId( string const & blob );
 };
 
+bool operator <( const ChunkId &lhs, const ChunkId &rhs );
+
 #endif

chunk_index.cc

@@ -1,5 +1,5 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
 #include <stdio.h>
 #include <string.h>
@@ -12,18 +12,18 @@
 #include "index_file.hh"
 #include "zbackup.pb.h"
 
-ChunkIndex::Chain::Chain( ChunkId const & id, Bundle::Id const * bundleId ):
-  next( 0 ), bundleId( bundleId )
+ChunkIndex::Chain::Chain( ChunkId const & id, uint32_t size, Bundle::Id const * bundleId ):
+  next( 0 ), size( size ), bundleId( bundleId )
 {
   memcpy( cryptoHash, id.cryptoHash, sizeof( cryptoHash ) );
 }
 
 bool ChunkIndex::Chain::equalsTo( ChunkId const & id )
 {
-  return memcmp( cryptoHash, id.cryptoHash, sizeof ( cryptoHash ) ) == 0;
+  return memcmp( cryptoHash, id.cryptoHash, sizeof( cryptoHash ) ) == 0;
 }
 
-void ChunkIndex::loadIndex()
+void ChunkIndex::loadIndex( IndexProcessor & ip )
 {
   Dir::Listing lst( indexPath );
 
@@ -34,47 +34,85 @@ void ChunkIndex::loadIndex()
   while( lst.getNext( entry ) )
   {
     verbosePrintf( "Loading index file %s...\n", entry.getFileName().c_str() );
-
-    IndexFile::Reader reader( key,
-                             Dir::addPath( indexPath, entry.getFileName() ) );
-
-    BundleInfo info;
-    Bundle::Id bundleId;
-    while( reader.readNextRecord( info, bundleId ) )
+    try
     {
-      Bundle::Id * savedId = storage.allocateObjects< Bundle::Id >( 1 );
-      memcpy( savedId, &bundleId, sizeof( bundleId ) );
+      string indexFn = Dir::addPath( indexPath, entry.getFileName() );
+      IndexFile::Reader reader( key, indexFn );
 
-      lastBundleId = savedId;
+      ip.startIndex( indexFn );
 
-      ChunkId id;
-
-      for ( int x = info.chunk_record_size(); x--; )
+      BundleInfo info;
+      Bundle::Id bundleId;
+      while( reader.readNextRecord( info, bundleId ) )
       {
-        BundleInfo_ChunkRecord const & record = info.chunk_record( x );
+        Bundle::Id * savedId = storage.allocateObjects< Bundle::Id >( 1 );
+        memcpy( savedId, &bundleId, sizeof( bundleId ) );
 
-        if ( record.id().size() != ChunkId::BlobSize )
-          throw exIncorrectChunkIdSize();
+        ChunkId id;
 
-        id.setFromBlob( record.id().data() );
-        registerNewChunkId( id, savedId );
+        ip.startBundle( *savedId );
+
+        for ( int x = info.chunk_record_size(); x--; )
+        {
+          BundleInfo_ChunkRecord const & record = info.chunk_record( x );
+
+          if ( record.id().size() != ChunkId::BlobSize )
+            throw exIncorrectChunkIdSize();
+
+          id.setFromBlob( record.id().data() );
+          ip.processChunk( id, record.size() );
+        }
+
+        ip.finishBundle( *savedId, info );
       }
+
+      ip.finishIndex( indexFn );
+    }
+    catch( std::exception & e )
+    {
+      verbosePrintf( "error: %s\n", e.what() );
+      continue;
     }
   }
 
   verbosePrintf( "Index loaded.\n" );
 }
 
+void ChunkIndex::startIndex( string const & )
+{
+}
+
+void ChunkIndex::startBundle( Bundle::Id const & bundleId )
+{
+  lastBundleId = &bundleId;
+}
+
+void ChunkIndex::processChunk( ChunkId const & chunkId, uint32_t size )
+{
+  registerNewChunkId( chunkId, size, lastBundleId );
+}
+
+void ChunkIndex::finishBundle( Bundle::Id const &, BundleInfo const & )
+{
+}
+
+void ChunkIndex::finishIndex( string const & )
+{
+}
+
 ChunkIndex::ChunkIndex( EncryptionKey const & key, TmpMgr & tmpMgr,
-                        string const & indexPath ):
+                        string const & indexPath, bool prohibitChunkIndexLoading ):
   key( key ), tmpMgr( tmpMgr ), indexPath( indexPath ), storage( 65536, 1 ),
   lastBundleId( NULL )
 {
-  loadIndex();
+  if ( !prohibitChunkIndexLoading )
+    loadIndex( *this );
+  dPrintf( "%s for %s is instantiated and initialized, hasKey: %s\n",
+      __CLASS, indexPath.c_str(), key.hasKey() ? "true" : "false" );
 }
 
 Bundle::Id const * ChunkIndex::findChunk( ChunkId::RollingHashPart rollingHash,
-                                          ChunkInfoInterface & chunkInfo )
+                                          ChunkInfoInterface & chunkInfo, uint32_t *size )
 {
   HashTable::iterator i = hashTable.find( rollingHash );
 
@@ -86,8 +124,14 @@ Bundle::Id const * ChunkIndex::findChunk( ChunkId::RollingHashPart rollingHash,
       id = &chunkInfo.getChunkId();
     // Check the chains
     for ( Chain * chain = i->second; chain; chain = chain->next )
+    {
       if ( chain->equalsTo( *id ) )
+      {
+        if ( size )
+          *size = chain->size;
         return chain->bundleId;
+      }
+    }
   }
 
   return NULL;
@@ -105,37 +149,37 @@ struct ChunkInfoImmediate: public ChunkIndex::ChunkInfoInterface
 };
 }
 
-Bundle::Id const * ChunkIndex::findChunk( ChunkId const & chunkId )
+Bundle::Id const * ChunkIndex::findChunk( ChunkId const & chunkId, uint32_t *size )
 {
   ChunkInfoImmediate chunkInfo( chunkId );
-  return findChunk( chunkId.rollingHash, chunkInfo );
+  return findChunk( chunkId.rollingHash, chunkInfo, size );
 }
 
-ChunkIndex::Chain * ChunkIndex::registerNewChunkId( ChunkId const & id,
+ChunkIndex::Chain * ChunkIndex::registerNewChunkId( ChunkId const & id, uint32_t size,
                                                     Bundle::Id const * bundleId )
 {
   HashTable::iterator i =
     hashTable.insert( std::make_pair( id.rollingHash, ( Chain *) 0 ) ).first;
 
-  Chain * & chain = i->second;
+  Chain ** chain = &i->second;
 
   // Check the chains
-  for ( ; chain; chain = chain->next )
-    if ( chain->equalsTo( id ) )
+  for ( ; *chain; chain = &( ( *chain )->next ) )
+    if ( ( *chain )->equalsTo( id ) )
     {
       return NULL; // The entry existed already
     }
 
   // Create a new chain
-  chain = new ( storage.allocateObjects< Chain >( 1 ) ) Chain( id, bundleId );
+  *chain = new ( storage.allocateObjects< Chain >( 1 ) ) Chain( id, size, bundleId );
 
-  return chain;
+  return *chain;
 }
 
 
-bool ChunkIndex::addChunk( ChunkId const & id, Bundle::Id const & bundleId )
+bool ChunkIndex::addChunk( ChunkId const & id, uint32_t size, Bundle::Id const & bundleId )
 {
-  if ( Chain * chain = registerNewChunkId( id, NULL ) )
+  if ( Chain * chain = registerNewChunkId( id, size, NULL ) )
   {
     // Allocate or re-use bundle id
     if ( !lastBundleId || *lastBundleId != bundleId )

chunk_index.hh

@@ -1,8 +1,8 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
-#ifndef CHUNK_INDEX_HH_INCLUDED__
-#define CHUNK_INDEX_HH_INCLUDED__
+#ifndef CHUNK_INDEX_HH_INCLUDED
+#define CHUNK_INDEX_HH_INCLUDED
 
 // <ext/hash_map> is obsolete, but <unordered_map> requires C++11. Make up your
 // mind, GNU people!
@@ -29,31 +29,43 @@
 
 using std::vector;
 
-// 32-bit specific hash function for unsigned long long which is what uint64_t
-// is on 32-bit platforms
-#if SIZE_MAX == UINT32_MAX
+/// __gnu_cxx::hash is not defined for unsigned long long. As uint64_t is
+/// typedefed as unsigned long long on all 32-bit architectures and on some
+/// 64-bit ones, we need to define this. Our keys should have more or less
+/// uniform bit distribution, so on 32-bit systems returning the lower 32 bits
+/// should be fine
 namespace __gnu_cxx
 {
   template<>
   struct hash< unsigned long long >
   {
     size_t operator()( unsigned long long v ) const
-    { return v ^ ( v >> 32 ); }
+    { return v; }
   };
 }
-#endif
+
+class IndexProcessor
+{
+public:
+  virtual void startIndex( string const & ) = 0;
+  virtual void startBundle( Bundle::Id const & ) = 0;
+  virtual void processChunk( ChunkId const &, uint32_t ) = 0;
+  virtual void finishBundle( Bundle::Id const &, BundleInfo const & ) = 0;
+  virtual void finishIndex( string const & ) = 0;
+};
 
 /// Maintains an in-memory hash table allowing to check whether we have a
 /// specific chunk or not, and if we do, get the bundle id it's in
-class ChunkIndex: NoCopy
+class ChunkIndex: NoCopy, IndexProcessor
 {
   struct Chain
   {
     ChunkId::CryptoHashPart cryptoHash;
+    uint32_t size;
     Chain * next;
     Bundle::Id const * bundleId;
 
-    Chain( ChunkId const &, Bundle::Id const * bundleId );
+    Chain( ChunkId const &, uint32_t, Bundle::Id const * bundleId );
 
     bool equalsTo( ChunkId const & id );
   };
@@ -76,7 +88,7 @@ public:
   DEF_EX( Ex, "Chunk index exception", std::exception )
   DEF_EX( exIncorrectChunkIdSize, "Incorrect chunk id size encountered", Ex )
 
-  ChunkIndex( EncryptionKey const &, TmpMgr &, string const & indexPath );
+  ChunkIndex( EncryptionKey const &, TmpMgr &, string const & indexPath, bool );
 
   struct ChunkInfoInterface
   {
@@ -89,21 +101,27 @@ public:
 
   /// If the given chunk exists, its bundle id is returned, otherwise NULL
   Bundle::Id const * findChunk( ChunkId::RollingHashPart,
-                                ChunkInfoInterface & );
+                                ChunkInfoInterface &, uint32_t *size = NULL );
 
   /// If the given chunk exists, its bundle id is returned, otherwise NULL
-  Bundle::Id const * findChunk( ChunkId const & );
+  Bundle::Id const * findChunk( ChunkId const &, uint32_t *size = NULL );
 
   /// Adds a new chunk to the index if it did not exist already. Returns true
   /// if added, false if existed already
-  bool addChunk( ChunkId const &, Bundle::Id const & );
+  bool addChunk( ChunkId const &, uint32_t, Bundle::Id const & );
+
+  void startIndex( string const & );
+  void startBundle( Bundle::Id const & );
+  void processChunk( ChunkId const &, uint32_t );
+  void finishBundle( Bundle::Id const &, BundleInfo const & );
+  void finishIndex( string const & );
+
+  void loadIndex( IndexProcessor & );
 
 private:
-  void loadIndex();
-
   /// Inserts new chunk id into the in-memory hash table. Returns the created
   /// Chain if it was inserted, NULL if it existed before
-  Chain * registerNewChunkId( ChunkId const & id, Bundle::Id const * );
+  Chain * registerNewChunkId( ChunkId const & id, uint32_t, Bundle::Id const * );
 };
 
 #endif

chunk_storage.cc

@@ -1,5 +1,5 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
 #include "check.hh"
 #include "chunk_storage.hh"
@@ -10,11 +10,11 @@
 
 namespace ChunkStorage {
 
-Writer::Writer( StorageInfo const & storageInfo,
+Writer::Writer( Config const & configIn,
                 EncryptionKey const & encryptionKey,
                 TmpMgr & tmpMgr, ChunkIndex & index, string const & bundlesDir,
                 string const & indexDir, size_t maxCompressorsToRun ):
-  storageInfo( storageInfo ), encryptionKey( encryptionKey ),
+  config( configIn ), encryptionKey( encryptionKey ),
   tmpMgr( tmpMgr ), index( index ), bundlesDir( bundlesDir ),
   indexDir( indexDir ), hasCurrentBundleId( false ),
   maxCompressorsToRun( maxCompressorsToRun ), runningCompressors( 0 )
@@ -30,11 +30,11 @@ Writer::~Writer()
 
 bool Writer::add( ChunkId const & id, void const * data, size_t size )
 {
-  if ( index.addChunk( id, getCurrentBundleId() ) )
+  if ( index.addChunk( id, size, getCurrentBundleId() ) )
   {
     // Added to the index? Emit to the bundle then
     if ( getCurrentBundle().getPayloadSize() + size >
-         storageInfo.bundle_max_payload_size() )
+         config.GET_STORABLE( bundle, max_payload_size ) )
       finishCurrentBundle();
 
     getCurrentBundle().addChunk( id.toBlob(), data, size );
@@ -45,6 +45,19 @@ bool Writer::add( ChunkId const & id, void const * data, size_t size )
     return false;
 }
 
+void Writer::addBundle( BundleInfo const & bundleInfo, Bundle::Id const & bundleId )
+{
+  if ( !indexFile.get() )
+  {
+    // Create a new index file
+    indexTempFile = tmpMgr.makeTemporaryFile();
+    indexFile = new IndexFile::Writer( encryptionKey,
+                                       indexTempFile->getFileName() );
+  }
+
+  indexFile->add( bundleInfo, bundleId );
+}
+
 void Writer::commit()
 {
   finishCurrentBundle();
@@ -68,7 +81,7 @@ void Writer::commit()
     // Generate a random filename
     unsigned char buf[ 24 ]; // Same comments as for Bundle::IdSize
 
-    Random::genaratePseudo( buf, sizeof( buf ) );
+    Random::generatePseudo( buf, sizeof( buf ) );
 
     indexTempFile->moveOverTo( Dir::addPath( indexDir,
                                              toHex( buf, sizeof( buf ) ) ) );
@@ -76,6 +89,20 @@ void Writer::commit()
   }
 }
 
+void Writer::reset()
+{
+  finishCurrentBundle();
+
+  waitForAllCompressorsToFinish();
+
+  pendingBundleRenames.clear();
+
+  if ( indexFile.get() )
+  {
+    indexFile.reset();
+  }
+}
+
 Bundle::Creator & Writer::getCurrentBundle()
 {
   if ( !currentBundle.get() )
@@ -90,15 +117,7 @@ void Writer::finishCurrentBundle()
 
   Bundle::Id const & bundleId = getCurrentBundleId();
 
-  if ( !indexFile.get() )
-  {
-    // Create a new index file
-    indexTempFile = tmpMgr.makeTemporaryFile();
-    indexFile = new IndexFile::Writer( encryptionKey,
-                                       indexTempFile->getFileName() );
-  }
-
-  indexFile->add( currentBundle->getCurrentBundleInfo(), bundleId );
+  addBundle( currentBundle->getCurrentBundleInfo(), bundleId );
 
   sptr< TemporaryFile > file = tmpMgr.makeTemporaryFile();
 
@@ -111,7 +130,8 @@ void Writer::finishCurrentBundle()
   while ( runningCompressors >= maxCompressorsToRun )
     runningCompressorsCondition.wait( runningCompressorsMutex );
 
-  Compressor * compressor = new Compressor( *this, currentBundle,
+  Compressor * compressor = new Compressor( config,
+                                            *this, currentBundle,
                                             file->getFileName() );
 
   currentBundle.reset();
@@ -133,17 +153,18 @@ Bundle::Id const & Writer::getCurrentBundleId()
   if ( !hasCurrentBundleId )
   {
     // Generate a new one
-    Random::genaratePseudo( &currentBundleId, sizeof( currentBundleId ) );
+    Random::generatePseudo( &currentBundleId, sizeof( currentBundleId ) );
     hasCurrentBundleId = true;
   }
 
   return currentBundleId;
 }
 
-Writer::Compressor::Compressor( Writer & writer,
+Writer::Compressor::Compressor( Config const & configIn, Writer & writer,
                                 sptr< Bundle::Creator > const & bundleCreator,
                                 string const & fileName ):
-  writer( writer ), bundleCreator( bundleCreator ), fileName( fileName )
+  writer( writer ), bundleCreator( bundleCreator ), fileName( fileName ),
+  config( configIn )
 {
 }
 
@@ -151,11 +172,11 @@ void * Writer::Compressor::Compressor::threadFunction() throw()
 {
   try
   {
-    bundleCreator->write( fileName, writer.encryptionKey );
+    bundleCreator->write( config, fileName, writer.encryptionKey );
   }
   catch( std::exception & e )
   {
-    FAIL( "Bunding writing failed: %s", e.what() );
+    FAIL( "Bundle writing failed: %s", e.what() );
   }
 
   {
@@ -173,22 +194,39 @@ void * Writer::Compressor::Compressor::threadFunction() throw()
   return NULL;
 }
 
-Reader::Reader( StorageInfo const & storageInfo,
+Reader::Reader( Config const & configIn,
                 EncryptionKey const & encryptionKey,
                 ChunkIndex & index, string const & bundlesDir,
                 size_t maxCacheSizeBytes ):
-  storageInfo( storageInfo ), encryptionKey( encryptionKey ),
+  config( configIn ), encryptionKey( encryptionKey ),
   index( index ), bundlesDir( bundlesDir ),
   // We need to have at least one cached reader, otherwise we would have to
   // unpack a bundle each time a chunk is read, even for consecutive chunks
   // in the same bundle
-  cachedReaders( maxCacheSizeBytes < storageInfo.bundle_max_payload_size() ?
-    1 : maxCacheSizeBytes / storageInfo.bundle_max_payload_size() )
+  cachedReaders(
+      maxCacheSizeBytes < config.GET_STORABLE( bundle, max_payload_size ) ?
+      1 : maxCacheSizeBytes / config.GET_STORABLE( bundle, max_payload_size ) )
 {
   verbosePrintf( "Using up to %zu MB of RAM as cache\n",
                  maxCacheSizeBytes / 1048576 );
 }
 
+Bundle::Id const * Reader::getBundleId( ChunkId const & chunkId, size_t & size )
+{
+  uint32_t s;
+  if ( Bundle::Id const * bundleId = index.findChunk( chunkId, &s ) )
+  {
+    size = s;
+    return bundleId;
+  }
+  else
+  {
+    string blob = chunkId.toBlob();
+    throw exNoSuchChunk( toHex( ( unsigned char const * ) blob.data(),
+                                blob.size() ) );
+  }
+}
+
 void Reader::get( ChunkId const & chunkId, string & data, size_t & size )
 {
   if ( Bundle::Id const * bundleId = index.findChunk( chunkId ) )

chunk_storage.hh

@@ -1,8 +1,8 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
-#ifndef CHUNK_STORAGE_HH_INCLUDED__
-#define CHUNK_STORAGE_HH_INCLUDED__
+#ifndef CHUNK_STORAGE_HH_INCLUDED
+#define CHUNK_STORAGE_HH_INCLUDED
 
 #include <stddef.h>
 #include <exception>
@@ -23,6 +23,7 @@
 #include "sptr.hh"
 #include "tmp_mgr.hh"
 #include "zbackup.pb.h"
+#include "config.hh"
 
 namespace ChunkStorage {
 
@@ -40,7 +41,7 @@ public:
   /// All new bundles and index files are created as temp files. Call commit()
   /// to move them to their permanent locations. commit() is never called
   /// automatically!
-  Writer( StorageInfo const &, EncryptionKey const &,
+  Writer( Config const &, EncryptionKey const &,
           TmpMgr &, ChunkIndex & index, string const & bundlesDir,
           string const & indexDir, size_t maxCompressorsToRun );
 
@@ -48,10 +49,16 @@ public:
   /// in the index, does nothing and returns false
   bool add( ChunkId const &, void const * data, size_t size );
 
+  /// Adds an existing bundle to the index
+  void addBundle( BundleInfo const &, Bundle::Id const & bundleId );
+
   /// Commits all newly created bundles. Must be called before destroying the
   /// object -- otherwise all work will be removed from the temp dir and lost
   void commit();
 
+  /// Throw away all current changes.
+  void reset();
+
   ~Writer();
 
 private:
@@ -61,8 +68,9 @@ private:
     Writer & writer;
     sptr< Bundle::Creator > bundleCreator;
     string fileName;
+    Config const & config;
   public:
-    Compressor( Writer &, sptr< Bundle::Creator > const &,
+    Compressor( Config const &, Writer &, sptr< Bundle::Creator > const &,
                 string const & fileName );
   protected:
     virtual void * threadFunction() throw();
@@ -84,7 +92,7 @@ private:
   /// Wait for all compressors to finish
   void waitForAllCompressorsToFinish();
 
-  StorageInfo const & storageInfo;
+  Config const & config;
   EncryptionKey const & encryptionKey;
   TmpMgr & tmpMgr;
   ChunkIndex & index;
@@ -113,9 +121,11 @@ class Reader: NoCopy
 public:
   DEF_EX_STR( exNoSuchChunk, "no such chunk found:", Ex )
 
-  Reader( StorageInfo const &, EncryptionKey const &, ChunkIndex & index,
+  Reader( Config const &, EncryptionKey const &, ChunkIndex & index,
           string const & bundlesDir, size_t maxCacheSizeBytes );
 
+  Bundle::Id const * getBundleId( ChunkId const &, size_t & size );
+
   /// Loads the given chunk from the store into the given buffer. May throw file
   /// and decompression exceptions. 'data' may be enlarged but won't be shrunk.
   /// The size of the actual chunk would be stored in 'size'
@@ -125,7 +135,7 @@ public:
   Bundle::Reader & getReaderFor( Bundle::Id const & );
 
 private:
-  StorageInfo const & storageInfo;
+  Config const & config;
   EncryptionKey const & encryptionKey;
   ChunkIndex & index;
   string bundlesDir;

cmake/FindLibLZMA.cmake

@@ -0,0 +1,80 @@
+#.rst:
+# FindLibLZMA
+# -----------
+#
+# Find LibLZMA
+#
+# Find LibLZMA headers and library
+#
+# ::
+#
+#   LIBLZMA_FOUND             - True if liblzma is found.
+#   LIBLZMA_INCLUDE_DIRS      - Directory where liblzma headers are located.
+#   LIBLZMA_LIBRARIES         - Lzma libraries to link against.
+#   LIBLZMA_HAS_AUTO_DECODER  - True if lzma_auto_decoder() is found (required).
+#   LIBLZMA_HAS_EASY_ENCODER  - True if lzma_easy_encoder() is found (required).
+#   LIBLZMA_HAS_LZMA_PRESET   - True if lzma_lzma_preset() is found (required).
+#   LIBLZMA_VERSION_MAJOR     - The major version of lzma
+#   LIBLZMA_VERSION_MINOR     - The minor version of lzma
+#   LIBLZMA_VERSION_PATCH     - The patch version of lzma
+#   LIBLZMA_VERSION_STRING    - version number as a string (ex: "5.0.3")
+
+#=============================================================================
+# Copyright 2008 Per Øyvind Karlsen <peroyvind@mandriva.org>
+# Copyright 2009 Alexander Neundorf <neundorf@kde.org>
+# Copyright 2009 Helio Chissini de Castro <helio@kde.org>
+# Copyright 2012 Mario Bensi <mbensi@ipsquad.net>
+#
+# Distributed under the OSI-approved BSD License (the "License");
+# see accompanying file Copyright.txt for details.
+#
+# This software is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the License for more information.
+#=============================================================================
+# (To distribute this file outside of CMake, substitute the full
+#  License text for the above reference.)
+
+
+find_path(LIBLZMA_INCLUDE_DIR lzma.h )
+find_library(LIBLZMA_LIBRARY lzma)
+
+if(LIBLZMA_INCLUDE_DIR AND EXISTS "${LIBLZMA_INCLUDE_DIR}/lzma/version.h")
+    file(STRINGS "${LIBLZMA_INCLUDE_DIR}/lzma/version.h" LIBLZMA_HEADER_CONTENTS REGEX "#define LZMA_VERSION_[A-Z]+ [0-9]+")
+
+    string(REGEX REPLACE ".*#define LZMA_VERSION_MAJOR ([0-9]+).*" "\\1" LIBLZMA_VERSION_MAJOR "${LIBLZMA_HEADER_CONTENTS}")
+    string(REGEX REPLACE ".*#define LZMA_VERSION_MINOR ([0-9]+).*" "\\1" LIBLZMA_VERSION_MINOR "${LIBLZMA_HEADER_CONTENTS}")
+    string(REGEX REPLACE ".*#define LZMA_VERSION_PATCH ([0-9]+).*" "\\1" LIBLZMA_VERSION_PATCH "${LIBLZMA_HEADER_CONTENTS}")
+
+    set(LIBLZMA_VERSION_STRING "${LIBLZMA_VERSION_MAJOR}.${LIBLZMA_VERSION_MINOR}.${LIBLZMA_VERSION_PATCH}")
+    unset(LIBLZMA_HEADER_CONTENTS)
+endif()
+
+# We're using new code known now as XZ, even library still been called LZMA
+# it can be found in http://tukaani.org/xz/
+# Avoid using old codebase
+if (LIBLZMA_LIBRARY)
+   include(CheckLibraryExists)
+   set(CMAKE_REQUIRED_QUIET_SAVE ${CMAKE_REQUIRED_QUIET})
+   set(CMAKE_REQUIRED_QUIET ${LibLZMA_FIND_QUIETLY})
+   CHECK_LIBRARY_EXISTS(${LIBLZMA_LIBRARY} lzma_auto_decoder "" LIBLZMA_HAS_AUTO_DECODER)
+   CHECK_LIBRARY_EXISTS(${LIBLZMA_LIBRARY} lzma_easy_encoder "" LIBLZMA_HAS_EASY_ENCODER)
+   CHECK_LIBRARY_EXISTS(${LIBLZMA_LIBRARY} lzma_lzma_preset "" LIBLZMA_HAS_LZMA_PRESET)
+   set(CMAKE_REQUIRED_QUIET ${CMAKE_REQUIRED_QUIET_SAVE})
+endif ()
+
+include(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(LibLZMA  REQUIRED_VARS  LIBLZMA_INCLUDE_DIR
+                                                          LIBLZMA_LIBRARY
+                                                          LIBLZMA_HAS_AUTO_DECODER
+                                                          LIBLZMA_HAS_EASY_ENCODER
+                                                          LIBLZMA_HAS_LZMA_PRESET
+                                           VERSION_VAR    LIBLZMA_VERSION_STRING
+                                 )
+
+if (LIBLZMA_FOUND)
+    set(LIBLZMA_LIBRARIES ${LIBLZMA_LIBRARY})
+    set(LIBLZMA_INCLUDE_DIRS ${LIBLZMA_INCLUDE_DIR})
+endif ()
+
+mark_as_advanced( LIBLZMA_INCLUDE_DIR LIBLZMA_LIBRARY )

cmake/FindLibLZO.cmake

@@ -0,0 +1,70 @@
+#.rst:
+# FindLibLZO
+# -----------
+#
+# Find LibLZO
+#
+# Find LibLZO headers and library
+#
+# ::
+#
+#   LIBLZO_FOUND                     - True if liblzo is found.
+#   LIBLZO_INCLUDE_DIRS              - Directory where liblzo headers are located.
+#   LIBLZO_LIBRARIES                 - Lzo libraries to link against.
+#   LIBLZO_HAS_LZO1X_DECOMPRESS_SAFE - True if lzo1x_decompress_safe() is found (required).
+#   LIBLZO_HAS_LZO1X_1_COMPRESS      - True if lzo1x_1_compress() is found (required).
+#   LIBLZO_VERSION_STRING            - version number as a string (ex: "5.0.3")
+
+#=============================================================================
+# Copyright 2008 Per Øyvind Karlsen <peroyvind@mandriva.org>
+# Copyright 2009 Alexander Neundorf <neundorf@kde.org>
+# Copyright 2009 Helio Chissini de Castro <helio@kde.org>
+# Copyright 2012 Mario Bensi <mbensi@ipsquad.net>
+# Copyright 2012-2014 Konstantin Isakov <ikm@zbackup.org>
+# Copyright 2013 Benjamin Koch <bbbsnowball@gmail.com> (from lzma to lzo)
+# Copyright 2014 Vladimir Stackov <amigo.elite@gmail.com>
+#
+# Distributed under the OSI-approved BSD License (the "License");
+# see accompanying file Copyright.txt for details.
+#
+# This software is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the License for more information.
+#=============================================================================
+# (To distribute this file outside of CMake, substitute the full
+#  License text for the above reference.)
+
+
+find_path(LIBLZO_INCLUDE_DIR lzo/lzo1x.h )
+find_library(LIBLZO_LIBRARY lzo2)
+
+if(LIBLZO_INCLUDE_DIR AND EXISTS "${LIBLZO_INCLUDE_DIR}/lzo/lzoconf.h")
+    file(STRINGS "${LIBLZO_INCLUDE_DIR}/lzo/lzoconf.h" LIBLZO_HEADER_CONTENTS REGEX "#define LZO_VERSION_STRING.+\"[^\"]+\"")
+    string(REGEX REPLACE ".*#define LZO_VERSION_STRING.+\"([^\"]+)\".*" "\\1" LIBLZO_VERSION_STRING "${LIBLZO_HEADER_CONTENTS}")
+    unset(LIBLZO_HEADER_CONTENTS)
+endif()
+
+# We're just using two functions.
+if (LIBLZO_LIBRARY)
+   include(CheckLibraryExists)
+   set(CMAKE_REQUIRED_QUIET_SAVE ${CMAKE_REQUIRED_QUIET})
+   set(CMAKE_REQUIRED_QUIET ${LibLZMA_FIND_QUIETLY})
+   CHECK_LIBRARY_EXISTS(${LIBLZO_LIBRARY} lzo1x_decompress_safe "" LIBLZO_HAS_LZO1X_DECOMPRESS_SAFE)
+   CHECK_LIBRARY_EXISTS(${LIBLZO_LIBRARY} lzo1x_1_compress "" LIBLZO_HAS_LZO1X_1_COMPRESS)
+   set(CMAKE_REQUIRED_QUIET ${CMAKE_REQUIRED_QUIET_SAVE})
+endif ()
+
+include(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(LibLZO  REQUIRED_VARS  LIBLZO_INCLUDE_DIR
+                                                         LIBLZO_LIBRARY
+                                                         LIBLZO_HAS_LZO1X_DECOMPRESS_SAFE
+                                                         LIBLZO_HAS_LZO1X_1_COMPRESS
+                                          VERSION_VAR    LIBLZO_VERSION_STRING
+                                 )
+
+if (LIBLZO_FOUND)
+    set(LIBLZO_LIBRARIES ${LIBLZO_LIBRARY})
+    set(LIBLZO_INCLUDE_DIRS ${LIBLZO_INCLUDE_DIR})
+endif ()
+
+mark_as_advanced( LIBLZO_INCLUDE_DIR LIBLZO_LIBRARY )

cmake/FindLibUnwind.cmake

@@ -0,0 +1,77 @@
+#.rst:
+# FindLibUnwind
+# -----------
+#
+# Find LibUnwind
+#
+# Find LibUnwind headers and library
+#
+# ::
+#
+#   LIBUNWIND_FOUND                     - True if libunwind is found.
+#   LIBUNWIND_INCLUDE_DIRS              - Directory where libunwind headers are located.
+#   LIBUNWIND_LIBRARIES                 - Unwind libraries to link against.
+#   LIBUNWIND_HAS_UNW_GETCONTEXT        - True if unw_getcontext() is found (required).
+#   LIBUNWIND_HAS_UNW_INIT_LOCAL        - True if unw_init_local() is found (required).
+#   LIBUNWIND_VERSION_STRING            - version number as a string (ex: "5.0.3")
+
+#=============================================================================
+# Copyright 2014 ZBackup contributors
+#
+# Distributed under the OSI-approved BSD License (the "License");
+# see accompanying file Copyright.txt for details.
+#
+# This software is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the License for more information.
+#=============================================================================
+# (To distribute this file outside of CMake, substitute the full
+#  License text for the above reference.)
+
+
+find_path(LIBUNWIND_INCLUDE_DIR libunwind.h )
+if(NOT EXISTS "${LIBUNWIND_INCLUDE_DIR}/unwind.h")
+  MESSAGE("Found libunwind.h but corresponding unwind.h is absent!")
+  SET(LIBUNWIND_INCLUDE_DIR "")
+endif()
+
+find_library(LIBUNWIND_LIBRARY unwind)
+
+if(LIBUNWIND_INCLUDE_DIR AND EXISTS "${LIBUNWIND_INCLUDE_DIR}/libunwind-common.h")
+  file(STRINGS "${LIBUNWIND_INCLUDE_DIR}/libunwind-common.h" LIBUNWIND_HEADER_CONTENTS REGEX "#define UNW_VERSION_[A-Z]+\t[0-9]*")
+
+  string(REGEX REPLACE ".*#define UNW_VERSION_MAJOR\t([0-9]*).*" "\\1" LIBUNWIND_VERSION_MAJOR "${LIBUNWIND_HEADER_CONTENTS}")
+  string(REGEX REPLACE ".*#define UNW_VERSION_MINOR\t([0-9]*).*" "\\1" LIBUNWIND_VERSION_MINOR "${LIBUNWIND_HEADER_CONTENTS}")
+  string(REGEX REPLACE ".*#define UNW_VERSION_EXTRA\t([0-9]*).*" "\\1" LIBUNWIND_VERSION_EXTRA "${LIBUNWIND_HEADER_CONTENTS}")
+
+  if(LIBUNWIND_VERSION_EXTRA)
+    set(LIBUNWIND_VERSION_STRING "${LIBUNWIND_VERSION_MAJOR}.${LIBUNWIND_VERSION_MINOR}.${LIBUNWIND_VERSION_EXTRA}")
+  else(not LIBUNWIND_VERSION_EXTRA)
+    set(LIBUNWIND_VERSION_STRING "${LIBUNWIND_VERSION_MAJOR}.${LIBUNWIND_VERSION_MINOR}")
+  endif()
+  unset(LIBUNWIND_HEADER_CONTENTS)
+endif()
+
+if (LIBUNWIND_LIBRARY)
+  include(CheckSymbolExists)
+  set(CMAKE_REQUIRED_QUIET_SAVE ${CMAKE_REQUIRED_QUIET})
+  set(CMAKE_REQUIRED_QUIET ${LibUnwind_FIND_QUIETLY})
+  CHECK_SYMBOL_EXISTS(unw_getcontext "${LIBUNWIND_INCLUDE_DIR}/libunwind.h" LIBUNWIND_HAS_UNW_GETCONTEXT)
+  CHECK_SYMBOL_EXISTS(unw_init_local "${LIBUNWIND_INCLUDE_DIR}/libunwind.h" LIBUNWIND_HAS_UNW_INIT_LOCAL)
+  set(CMAKE_REQUIRED_QUIET ${CMAKE_REQUIRED_QUIET_SAVE})
+endif ()
+
+include(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(LibUnwind  REQUIRED_VARS  LIBUNWIND_INCLUDE_DIR
+                                                            LIBUNWIND_LIBRARY
+                                                            LIBUNWIND_HAS_UNW_GETCONTEXT
+                                                            LIBUNWIND_HAS_UNW_INIT_LOCAL
+                                             VERSION_VAR    LIBUNWIND_VERSION_STRING
+                                 )
+
+if (LIBUNWIND_FOUND)
+  set(LIBUNWIND_LIBRARIES ${LIBUNWIND_LIBRARY})
+  set(LIBUNWIND_INCLUDE_DIRS ${LIBUNWIND_INCLUDE_DIR})
+endif ()
+
+mark_as_advanced( LIBUNWIND_INCLUDE_DIR LIBUNWIND_LIBRARY )

compression.cc

@@ -0,0 +1,704 @@
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
+
+#include <string>
+
+#include "compression.hh"
+#include "check.hh"
+#include "endian.hh"
+
+namespace Compression {
+
+EnDecoder::EnDecoder()
+{
+}
+
+EnDecoder::~EnDecoder()
+{
+}
+
+CompressionMethod::~CompressionMethod()
+{
+}
+
+// LZMA
+
+#include <lzma.h>
+
+class LZMAEnDecoder : public EnDecoder
+{
+protected:
+  static lzma_stream initValue;
+  lzma_stream strm;
+public:
+  LZMAEnDecoder()
+  {
+    strm = initValue;
+  }
+
+  void setInput( const void* data, size_t size )
+  {
+    strm.next_in  = (const uint8_t *) data;
+    strm.avail_in = size;
+  }
+
+  void setOutput( void* data, size_t size )
+  {
+    strm.next_out  = (uint8_t *) data;
+    strm.avail_out = size;
+  }
+
+  size_t getAvailableInput()
+  {
+    return strm.avail_in;
+  }
+
+  size_t getAvailableOutput()
+  {
+    return strm.avail_out;
+  }
+
+  bool process( bool finish )
+  {
+    lzma_ret ret = lzma_code( &strm, ( finish ? LZMA_FINISH : LZMA_RUN ) );
+
+    CHECK( ret == LZMA_OK || ret == LZMA_STREAM_END, "lzma_code error: %d", (int) ret );
+
+    return ( ret == LZMA_STREAM_END );
+  }
+
+  ~LZMAEnDecoder()
+  {
+    lzma_end( &strm );
+  }
+};
+lzma_stream LZMAEnDecoder::initValue = LZMA_STREAM_INIT;
+
+class LZMAEncoder : public LZMAEnDecoder
+{
+public:
+  LZMAEncoder()
+  {
+    uint32_t preset = 6;
+    lzma_ret ret = lzma_easy_encoder( &strm, preset, LZMA_CHECK_CRC64 );
+    CHECK( ret == LZMA_OK, "lzma_easy_encoder error: %d", (int) ret );
+  }
+
+  LZMAEncoder( Config const & config )
+  {
+    uint32_t compressionLevel = config.GET_STORABLE( lzma, compression_level );
+    uint32_t preset = ( compressionLevel > 9 ) ?
+      ( compressionLevel - 10 ) | LZMA_PRESET_EXTREME :
+      compressionLevel;
+    lzma_ret ret = lzma_easy_encoder( &strm, preset, LZMA_CHECK_CRC64 );
+    CHECK( ret == LZMA_OK, "lzma_easy_encoder error: %d", (int) ret );
+  }
+};
+
+class LZMADecoder : public LZMAEnDecoder
+{
+public:
+  LZMADecoder()
+  {
+    lzma_ret ret = lzma_stream_decoder( &strm, UINT64_MAX, 0 );
+    CHECK( ret == LZMA_OK,"lzma_stream_decoder error: %d", (int) ret );
+  }
+};
+
+class LZMACompression : public CompressionMethod
+{
+public:
+  sptr<EnDecoder> createEncoder( Config const & config ) const
+  {
+    return new LZMAEncoder( config );
+  }
+
+  sptr<EnDecoder> createEncoder() const
+  {
+    return new LZMAEncoder();
+  }
+
+  sptr<EnDecoder> createDecoder() const
+  {
+    return new LZMADecoder();
+  }
+
+  std::string getName() const { return "lzma"; }
+};
+
+// LZO
+
+// liblzo implements a lot of algorithms "for unlimited backward compatibility"
+
+// The web site says:
+// "My experiments have shown that LZO1B is good with a large blocksize
+//  or with very redundant data, LZO1F is good with a small blocksize or
+//  with binary data and that LZO1X is often the best choice of all.
+//  LZO1Y and LZO1Z are almost identical to LZO1X - they can achieve a
+//  better compression ratio on some files.
+//  Beware, your mileage may vary."
+// => I'm using LZO1X, as suggested
+
+#include <string.h>
+
+// Unfortunately, liblzo always works with the whole data, so it doesn't support
+// the streaming approach that most other libraries use. This means that we have
+// to use a big buffer for the data. The class NoStreamEnDecoder implements this
+// so we can use it, if there is another library like liblzo.
+
+// Collect all data and process it in one pass
+class NoStreamEnDecoder : public EnDecoder
+{
+  std::string accDataIn, accDataOut;
+  const char* dataIn;
+  char* dataOut;
+  size_t availIn, availOut;
+  bool processed;
+  size_t posInAccDataOut;
+protected:
+  // you must implement these:
+
+  // Should we try with the existing output buffer which has availOut
+  // bytes of free space? If you know that this will fail, return false.
+  // You may peek into dataIn which contains the complete compressed data.
+  virtual bool shouldTryWith( const char* dataIn, size_t availIn, size_t availOut ) =0;
+
+  // We will allocate a buffer for the output data. How big should it be?
+  // You may peek into dataIn which contains the complete compressed data.
+  virtual size_t suggestOutputSize( const char* dataIn, size_t availIn ) =0;
+
+  // Is this input complete?
+  // An encoder should return false.
+  virtual bool isCompleteInput( const char* dataIn, size_t availIn ) =0;
+
+  // Process the data in dataIn and put the result into dataOut. You musn't
+  // write more than availOut bytes! If the output buffer is big enough,
+  // process the data and store the output size in outputSize. If the output
+  // buffer is too small, return false and we will give you a bigger one. If
+  // any other error occurrs, abort the program. We don't have any better
+  // error handling. Sorry. Do NOT return false for errors that won't be
+  // remedied by a bigger buffer!
+  virtual bool doProcess( const char* dataIn, size_t availIn,
+    char* dataOut, size_t availOut, size_t& outputSize ) =0;
+
+  void setUnusedInput( size_t unused )
+  {
+    this->dataIn += availIn - unused;
+    this->availIn = unused;
+  }
+public:
+  NoStreamEnDecoder()
+  {
+    dataIn = dataOut = NULL;
+    availIn = availOut = posInAccDataOut = 0;
+    processed = false;
+  }
+
+  void setInput( const void* data, size_t size )
+  {
+    dataIn  = (const char *) data;
+    availIn = size;
+  }
+
+  void setOutput( void* data, size_t size )
+  {
+    dataOut  = (char *) data;
+    availOut = size;
+  }
+
+  size_t getAvailableInput()
+  {
+    return availIn;
+  }
+
+  size_t getAvailableOutput()
+  {
+    return availOut;
+  }
+
+  bool process( bool finish )
+  {
+    // try to process the input, if we haven't done it, yet
+    if ( !processed )
+    {
+      // data has not been encoded
+      if ( accDataIn.empty() )
+      {
+        // this is the first piece of data
+        if ( finish || isCompleteInput( dataIn, availIn ) )
+        {
+          // special case: all the data has been passed at once
+          // -> process it without using accDataIn
+          processFinish( dataIn, availIn );
+        }
+      }
+
+      // if we didn't process the data, put it into accumulator
+      if ( !processed )
+      {
+        // accumulate data in accDataIn
+        accDataIn.append( dataIn, availIn );
+
+        // If this was the last bit of data, we process it, now.
+        if ( finish || isCompleteInput( accDataIn.data(), accDataIn.size() ) )
+        {
+          processFinish( accDataIn.data(), accDataIn.size() );
+        }
+      }
+    }
+
+    // If the input has been processed, try to copy some of it to the output buffer.
+    if ( processed )
+    {
+      // data has been encoded or decoded, remaining output is in accDataOut
+      // -> copy to output
+      if ( availOut > 0 && accDataOut.size() - posInAccDataOut > 0 )
+      {
+        size_t sz = availOut;
+        if ( sz > accDataOut.size() - posInAccDataOut )
+          sz = accDataOut.size() - posInAccDataOut;
+
+        memcpy( dataOut, accDataOut.data() + posInAccDataOut, sz );
+        dataOut  += sz;
+        availOut -= sz;
+        posInAccDataOut += sz;
+      }
+
+      // no more data left? -> return true
+      return ( accDataOut.size() - posInAccDataOut == 0 );
+    }
+    else
+    {
+      // not yet processed, so we cannot be done
+      return false;
+    }
+  }
+
+private:
+  void processFinish( const char* dataIn, size_t availIn )
+  {
+    // should we try with the existing output buffer?
+    if ( shouldTryWith( dataIn, availIn, availOut ) )
+    {
+      size_t outputSize;
+      if ( doProcess( dataIn, availIn, dataOut, availOut, outputSize ) )
+      {
+        // it worked :-)
+        processed = true;
+        availOut -= outputSize;
+        return ;
+      }
+    }
+
+    // we use our own buffer
+    size_t bufferSize = suggestOutputSize( dataIn, availIn );
+    do {
+      accDataOut.resize( bufferSize );
+
+      size_t outputSize;
+      //TODO doc says we mustn't modify the pointer returned by data()...
+      if ( doProcess( dataIn, availIn,
+          (char*) accDataOut.data(), bufferSize, outputSize ) )
+      {
+        // buffer is big enough
+        accDataOut.resize( outputSize );
+        processed = true;
+        return ;
+      }
+
+      // try a bigger one
+      bufferSize *= 2;
+    } while (true);
+  }
+};
+
+#ifdef __APPLE__
+#include <machine/endian.h>
+#elif __FreeBSD__
+#include <sys/endian.h>
+#else
+#include <endian.h>
+#endif
+
+// like NoStreamEnDecoder, but also adds the uncompressed size before the stream
+//NOTE You should make sure that the compression function doesn't overwrite any
+//     memory, if this information is corrupted! This could be exploited by a
+//     malicious person and there is nothing I can do about it. I could check for
+//     an overflow, but when control gets back to this class, it is already too
+//     late, as one 'ret' instruction is enough to do harm.
+class NoStreamAndUnknownSizeDecoder : public NoStreamEnDecoder
+{
+protected:
+  // You implement this one:
+  // If you don't know the real decoded size, don't change outputSize.
+  virtual bool doProcessNoSize( const char* dataIn, size_t availIn,
+      char* dataOut, size_t availOut, size_t& outputSize ) =0;
+
+  bool shouldTryWith( const char* dataIn, size_t availIn, size_t availOut )
+  {
+    return suggestOutputSize( dataIn, availIn ) <= availOut;
+  }
+
+  // Is this input complete?
+  bool isCompleteInput( const char* dataIn, size_t availIn )
+  {
+    if ( availIn < 2*sizeof(uint64_t) )
+      return false;
+
+    dataIn += sizeof(uint64_t);
+
+    size_t inputSize = le32toh( *(uint32_t*) dataIn );
+
+    return ( availIn >= inputSize + 2*sizeof(uint64_t) );
+  }
+
+  size_t suggestOutputSize( const char* dataIn, size_t availIn )
+  {
+    CHECK( availIn >= sizeof(uint64_t), "not enough input data" );
+    // We're not using size_t because we need a type that has the same size on all
+    // architectures. A 32-bit host won't be able to open files with more than
+    // 4GB (actually much less), so 4 byte are enough. Even a 64-bit host would
+    // have some trouble with allocating 8GB of RAM just for our buffers ;-)
+    //NOTE If your compiler doesn't accept this cast, your size_t is smaller than
+    //     uint32_t. In that case, you are in trouble...
+    size_t outputSize = le32toh( *(uint32_t*) dataIn );
+    return outputSize;
+  }
+
+  bool doProcess( const char* dataIn, size_t availIn,
+      char* dataOut, size_t availOut, size_t& outputSize )
+  {
+    if ( availIn < 2*sizeof( uint64_t ) )
+      return false;
+
+    //NOTE We skip 8 bytes. If we later decide to drop compatibility with 32-bit
+    //     hosts, we can save a 64-bit size. Well, that will be much later, when
+    //     we can easily hold two copies of a 4GB file in main memory :-D
+
+    size_t neededOutputSize = le32toh( *(uint32_t*) dataIn );
+    dataIn += sizeof(uint64_t);
+    size_t inputSize = le32toh( *(uint32_t*) dataIn );
+    dataIn += sizeof(uint64_t);
+    /*if ( outputSize < neededOutputSize )
+      return false;*/
+
+    outputSize = neededOutputSize;
+
+    availIn -= 2*sizeof( uint64_t );
+
+    // We might not need all of our input data.
+    setUnusedInput( availIn - inputSize );
+    availIn = inputSize;
+
+    size_t reportedOutputSize = neededOutputSize;
+    if ( !doProcessNoSize( dataIn, availIn, dataOut, availOut, reportedOutputSize ) )
+      return false;
+
+    CHECK( reportedOutputSize == neededOutputSize,
+      "Size of decoded data is different than expected" );
+
+    return true;
+  }
+};
+
+// encoder for NoStreamAndUnknownSizeDecoder
+class NoStreamAndUnknownSizeEncoder : public NoStreamEnDecoder
+{
+protected:
+  // You implement this one:
+  virtual bool doProcessNoSize( const char* dataIn, size_t availIn,
+      char* dataOut, size_t availOut, size_t& outputSize ) =0;
+
+  bool shouldTryWith( const char*, size_t, size_t availOut )
+  {
+    // If the compression doesn't use any spaces...
+    return availOut > sizeof( uint64_t );
+  }
+
+  bool isCompleteInput( const char* dataIn, size_t availIn )
+  {
+    // We cannot know whether the user wants to send more data.
+    // -> return false; user must use finish=true to signal end of data
+    return false;
+  }
+
+  size_t getOverhead()
+  {
+    return 2*sizeof( uint64_t );
+  }
+
+  size_t suggestOutputSize( const char*, size_t availIn )
+  {
+    // We assume that the compression won't make the data any bigger.
+    return availIn + getOverhead();
+  }
+
+  bool doProcess( const char* dataIn, size_t availIn,
+      char* dataOut, size_t availOut, size_t& outputSize )
+  {
+    CHECK( availIn <= UINT32_MAX,
+      "You want to compress more than 4GB of data?! Sorry, we don't support that, yet." );
+
+    memcpy(dataOut, "ABCDEFGHIJKLMNOP", 16);
+
+    // store size
+    *(uint32_t*)dataOut = htole32( availIn );
+    uint32_t* compressedSize = (uint32_t*) ( dataOut + sizeof( uint64_t ) );
+
+    // compressed data goes after the size
+    // We skip more than we actually use; see NoStreamAndUnknownSizeDecoder::doProcess(...).
+    dataOut  += getOverhead();
+    availOut -= getOverhead();
+
+    if ( !doProcessNoSize( dataIn, availIn, dataOut, availOut, outputSize ) )
+      return false;
+
+    CHECK( outputSize <= UINT32_MAX,
+      "The compressed data is more than 4GB?! Sorry, we don't support that, yet." );
+    *compressedSize = htole32( (uint32_t) outputSize );
+
+    outputSize += getOverhead();
+
+    return true;
+  }
+};
+
+#ifdef HAVE_LIBLZO
+
+#include <lzo/lzo1x.h>
+
+// finally, we can implement lzo
+class LZO1X_1_Decoder : public NoStreamAndUnknownSizeDecoder
+{
+protected:
+  bool doProcessNoSize( const char* dataIn, size_t availIn,
+      char* dataOut, size_t availOut, size_t& outputSize )
+  {
+    // same argument is used for available output size and size of decompressed data
+    outputSize = availOut;
+    int ret = lzo1x_decompress_safe( (const lzo_bytep) dataIn, availIn,
+      (lzo_bytep) dataOut, (lzo_uintp) &outputSize, NULL );
+
+  if ( ret == LZO_E_OUTPUT_OVERRUN )
+    return false;
+
+  CHECK( ret >= LZO_E_OK, "lzo1x_decompress_safe failed (code %d)", ret );
+
+  return true;
+  }
+};
+class LZO1X_1_Compression;
+class LZO1X_1_Encoder : public NoStreamAndUnknownSizeEncoder
+{
+  const LZO1X_1_Compression* compression;
+  static size_t calcMaxCompressedSize( size_t availIn );
+public:
+  LZO1X_1_Encoder( const LZO1X_1_Compression* compression )
+  {
+    this->compression = compression;
+  }
+
+protected:
+  bool doProcessNoSize( const char* dataIn, size_t availIn,
+      char* dataOut, size_t availOut, size_t& outputSize );
+  bool shouldTryWith( const char*, size_t, size_t availOut );
+  size_t suggestOutputSize( const char*, size_t availIn );
+};
+class LZO1X_1_Compression : public CompressionMethod
+{
+  static bool initialized;
+  static void init()
+  {
+    //TODO This is not thread-safe. Does it have to be?
+    if (!initialized)
+    {
+      int ret = lzo_init();
+      CHECK( ret == LZO_E_OK, "lzo_init failed (%d)", ret );
+      initialized = true;
+    }
+  }
+public:
+  sptr< EnDecoder > createEncoder( Config const & config ) const
+  {
+    init();
+    return new LZO1X_1_Encoder(this);
+  }
+
+  sptr< EnDecoder > createEncoder() const
+  {
+    init();
+    return new LZO1X_1_Encoder(this);
+  }
+
+  sptr< EnDecoder > createDecoder() const
+  {
+    init();
+    return new LZO1X_1_Decoder();
+  }
+
+  std::string getName() const { return "lzo1x_1"; }
+
+  lzo_voidp getWorkmem( size_t size ) const
+  {
+    return new char[size];
+  }
+
+  void giveBackWorkmem( lzo_voidp wrkmem ) const
+  {
+    //TODO I think we should keep the memory around and reuse it. After all
+    //     it is only a few kilobytes and we will need it a lot. However, I
+    //     won't risk anything here because I don't know whether this will be
+    //     called by more than one thread.
+    delete[] (char*)wrkmem;
+  }
+};
+
+bool LZO1X_1_Compression::initialized = false;
+
+size_t LZO1X_1_Encoder::calcMaxCompressedSize( size_t availIn )
+{
+  // It seems that lzo1x_1_compress does NOT check whether the buffer is big enough.
+  // The documentation refers to example/simple.c which says:
+  // "Because the input block may be incompressible, we must provide a little more
+  //  output space in case that compression is not possible."
+  // -> We use the same formula.
+  return (availIn + availIn / 16 + 64 + 3);
+}
+
+bool LZO1X_1_Encoder::shouldTryWith( const char* dataIn, size_t availIn, size_t availOut )
+{
+  return availOut >= suggestOutputSize( dataIn, availIn );
+}
+
+size_t LZO1X_1_Encoder::suggestOutputSize( const char*, size_t availIn )
+{
+  // It seems that lzo1x_1_compress does NOT check whether the buffer is big enough.
+  // The documentation refers to example/simple.c which says:
+  // "Because the input block may be incompressible, we must provide a little more
+  //  output space in case that compression is not possible."
+  // -> We use the same formula.
+  return calcMaxCompressedSize( availIn ) + getOverhead();
+}
+
+bool LZO1X_1_Encoder::doProcessNoSize( const char* dataIn, size_t availIn,
+    char* dataOut, size_t availOut, size_t& outputSize )
+{
+  // It seems that lzo1x_1_compress does NOT check whether the buffer is big enough.
+  // Therefore, we won't try it unless we are sure that the buffer is big enough.
+  if ( availOut < calcMaxCompressedSize( availIn ) )
+    return false;
+
+  // same argument is used for available output size (haha, see above)
+  // and size of decompressed data
+  outputSize = availOut;
+
+  lzo_voidp wrkmem = compression->getWorkmem(LZO1X_1_MEM_COMPRESS);
+  int ret = lzo1x_1_compress( (const lzo_bytep) dataIn, availIn,
+    (lzo_bytep) dataOut, (lzo_uintp) &outputSize, wrkmem );
+  compression->giveBackWorkmem(wrkmem);
+
+  if ( ret == LZO_E_OUTPUT_OVERRUN )
+    return false;
+
+  CHECK( ret >= LZO_E_OK, "lzo1x_1_compress failed (code %d)", ret );
+
+  return true;
+}
+
+#endif  // HAVE_LIBLZO
+
+// register them
+
+const_sptr< CompressionMethod > const CompressionMethod::compressions[] = {
+  new LZMACompression(),
+# ifdef HAVE_LIBLZO
+  new LZO1X_1_Compression(),
+# endif
+  // NULL entry marks end of list. Don't remove it!
+  NULL
+};
+
+const_sptr< CompressionMethod > CompressionMethod::selectedCompression =
+  compressions[ 0 ];
+
+const_sptr< CompressionMethod > CompressionMethod::findCompression(
+    const std::string& name, bool optional )
+{
+  for ( const const_sptr<CompressionMethod>* c = compressions + 0; *c; ++c )
+  {
+    if ( (*c)->getName() == name )
+    {
+      return (*c);
+    }
+  }
+  if ( !optional )
+  {
+    throw exUnsupportedCompressionMethod( name );
+  }
+  return NULL;
+}
+
+// iterator over compressions
+CompressionMethod::iterator::iterator( const const_sptr< CompressionMethod > * ptr ):
+  ptr( ptr )
+{
+}
+
+CompressionMethod::iterator::iterator( const iterator & it ):
+  ptr( it.ptr )
+{
+}
+
+CompressionMethod::iterator& CompressionMethod::iterator::operator =( const iterator& it )
+{
+  this->ptr = it.ptr;
+  return *this;
+}
+
+bool CompressionMethod::iterator::operator ==( const iterator& other ) const
+{
+  // special case: one has ptr==NULL (end iterator returned by end()) and the
+  // other has *ptr==NULL (end iterator obtained by calling ++)
+  if ( !ptr && ( !other.ptr || !*other.ptr ) )
+    return true;
+  else if ( !other.ptr && ( !ptr || !*ptr ) )
+    return true;
+  else
+    return (ptr == other.ptr);
+}
+bool CompressionMethod::iterator::operator !=( const iterator& other ) const
+{
+  return !( *this == other );
+}
+
+bool CompressionMethod::iterator::atEnd() const
+{
+  return !ptr || !*ptr;
+}
+
+CompressionMethod::iterator& CompressionMethod::iterator::operator ++()
+{
+  CHECK( ptr && *ptr, "Cannot increment the end iterator" );
+
+  ++ptr;
+
+  return *this;
+}
+
+const_sptr<CompressionMethod> CompressionMethod::iterator::operator *()
+{
+  CHECK( ptr && *ptr, "Cannot dereference the end iterator" );
+
+  return *ptr;
+}
+
+CompressionMethod::iterator CompressionMethod::begin()
+{
+  return iterator(compressions);
+}
+CompressionMethod::iterator CompressionMethod::end()
+{
+  return iterator(NULL);
+}
+
+}

compression.hh

@@ -0,0 +1,92 @@
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
+
+#ifndef COMPRESSION_HH_INCLUDED
+#define COMPRESSION_HH_INCLUDED
+
+#include "sptr.hh"
+#include "ex.hh"
+#include "nocopy.hh"
+#include "config.hh"
+
+namespace Compression {
+
+DEF_EX( Ex, "Compression exception", std::exception )
+DEF_EX_STR( exUnsupportedCompressionMethod, "Unsupported compression method:", Ex )
+
+// used for encoding or decoding
+class EnDecoder: NoCopy
+{
+protected:
+  EnDecoder();
+public:
+  virtual ~EnDecoder();
+
+  // encoder can read up to size bytes from data
+  virtual void setInput ( const void * data, size_t size ) = 0;
+  // how many bytes of the last input haven't been used, yet?
+  virtual size_t getAvailableInput() = 0;
+
+  // encoder can write up to size bytes to output
+  virtual void setOutput( void * data, size_t size ) = 0;
+  // how many bytes of free space are remaining in the output buffer
+  virtual size_t getAvailableOutput() = 0;
+
+  // process some bytes
+  // finish: will you pass more data to the encoder via setOutput?
+  // NOTE You must eventually set finish to true.
+  // returns, whether all output bytes have been written
+  virtual bool process( bool finish ) = 0;
+};
+
+// compression method
+class CompressionMethod
+{
+public:
+  virtual ~CompressionMethod();
+
+  // returns name of compression method
+  // This name is saved in the file header of the compressed file.
+  virtual std::string getName() const = 0;
+
+  virtual sptr< EnDecoder > createEncoder( Config const & ) const = 0;
+  virtual sptr< EnDecoder > createEncoder() const = 0;
+  virtual sptr< EnDecoder > createDecoder() const = 0;
+
+  // find a compression by name
+  // If optional is false, it will either return a valid CompressionMethod
+  // object or abort the program. If optional is true, it will return
+  // NULL, if it cannot find the a compression with that name.
+  static const_sptr< CompressionMethod > findCompression(
+    const std::string & name, bool optional = false );
+
+  static const_sptr< CompressionMethod > selectedCompression;
+
+  static const_sptr< CompressionMethod > const compressions[];
+
+  class iterator
+  {
+    friend class CompressionMethod;
+
+    const const_sptr< CompressionMethod > * ptr;
+    iterator( const const_sptr< CompressionMethod > * ptr );
+  public:
+    iterator( const iterator & it );
+    iterator & operator =( const iterator & it );
+
+    bool operator == ( const iterator & other ) const;
+    bool operator != ( const iterator & other ) const;
+
+    bool atEnd() const;
+
+    iterator & operator ++();
+
+    const_sptr< CompressionMethod > operator * ();
+  };
+  static iterator begin();
+  static iterator end();
+};
+
+}
+
+#endif

config.cc

@@ -0,0 +1,609 @@
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
+
+#include <algorithm>
+#include "config.hh"
+#include "ex.hh"
+#include "debug.hh"
+#include "utils.hh"
+#include "compression.hh"
+
+#define VALID_SUFFIXES "Valid suffixes:\n" \
+                       "B - multiply by 1 (bytes)\n" \
+                       "KiB - multiply by 1024 (kibibytes)\n" \
+                       "MiB - multiply by 1024*1024 (mebibytes)\n" \
+                       "GiB - multiply by 1024*1024*1024 (gibibytes)\n" \
+                       "KB - multiply by 1000 (kilobytes)\n" \
+                       "MB - multiply by 1000*1000 (megabytes)\n" \
+                       "GB - multiply by 1000*1000*1000 (gigabytes)\n" \
+
+#define SKIP_ON_VALIDATION \
+{ \
+  if ( validate ) \
+    return true; \
+}
+
+// Some configurables could be just a switch
+// So we introducing a macros that would indicate
+// that this configurable is not a switch
+#define REQUIRE_VALUE \
+{ \
+  if ( !hasValue && !validate ) \
+    return false; \
+}
+
+#define PARSE_OR_VALIDATE( parse_src, validate_src ) \
+  ( !validate && ( parse_src ) ) || ( validate && ( validate_src ) )
+
+DEF_EX_STR( exInvalidThreadsValue, "Invalid threads value specified:", std::exception )
+
+void Config::prefillKeywords()
+{
+  /* Textual representations of the tokens. */
+
+  Keyword defaultKeywords[] = {
+    // Storable options
+    {
+      "chunk.max_size",
+      Config::oChunk_max_size,
+      Config::Storable,
+      "Maximum chunk size used when storing chunks\n"
+      "Affects deduplication ratio directly\n"
+      "Default is %s",
+      Utils::numberToString( GET_STORABLE( chunk, max_size ) )
+    },
+    {
+      "bundle.max_payload_size",
+      Config::oBundle_max_payload_size,
+      Config::Storable,
+      "Maximum number of bytes a bundle can hold. Only real chunk bytes are\n"
+      "counted, not metadata. Any bundle should be able to contain at least\n"
+      "one arbitrary single chunk, so this should not be smaller than\n"
+      "chunk.max_size\n"
+      "Default is %s",
+      Utils::numberToString( GET_STORABLE( bundle, max_payload_size ) )
+    },
+    {
+      "bundle.compression_method",
+      Config::oBundle_compression_method,
+      Config::Storable,
+      "Compression method for new bundles\n"
+      "Default is %s",
+      GET_STORABLE( bundle, compression_method )
+    },
+    {
+      "lzma.compression_level",
+      Config::oLZMA_compression_level,
+      Config::Storable,
+      "Compression level for new LZMA-compressed files\n"
+      "Valid values: 0-19 (values over 9 enables extreme mode)\n"
+      "Default is %s",
+      Utils::numberToString( GET_STORABLE( lzma, compression_level ) )
+    },
+
+    // Shortcuts for storable options
+    {
+      "compression",
+      Config::oBundle_compression_method,
+      Config::Storable,
+      "Shortcut for bundle.compression_method\n"
+      "Default is %s",
+      GET_STORABLE( bundle, compression_method )
+    },
+
+    // Runtime options
+    {
+      "threads",
+      Config::oRuntime_threads,
+      Config::Runtime,
+      "Maximum number of compressor threads to use in backup process\n"
+      "Default is %s on your system",
+      Utils::numberToString( runtime.threads )
+    },
+    {
+      "cache-size",
+      Config::oRuntime_cacheSize,
+      Config::Runtime,
+      "Cache size to use in restore process.\n"
+      "Affects restore process speed directly.\n"
+      VALID_SUFFIXES
+      "Default is %sMiB",
+      Utils::numberToString( runtime.cacheSize / 1024 / 1024 )
+    },
+    {
+      "exchange",
+      Config::oRuntime_exchange,
+      Config::Runtime,
+      "Data to exchange between repositories in import/export process.\n"
+      "Can be specified multiple times.\n"
+      "Valid values:\n"
+      "backups - exchange backup instructions (files in backups/ directory)\n"
+      "bundles - exchange bundles with data (files in bunles/ directory)\n"
+      "indexes - exchange indexes of chunks (files in index/ directory)\n"
+      "No default value, you should specify it explicitly."
+    },
+
+    {
+      "gc.repack",
+      Config::oRuntime_gcRepack,
+      Config::Runtime,
+      "Repack indexes and bundles during garbage collection.\n"
+      "Normally you would not need this.\n"
+      "Beware that this options causes very intensive IO!\n"
+      "Not default, you should specify it explicitly."
+    },
+
+    {
+      "paths.respect_tmp",
+      Config::oRuntime_pathsRespectTmp,
+      Config::Runtime,
+      "ZBackup will use TMPDIR environment variable\n"
+      "for temporary files if set.\n"
+      "Not default, you should specify it explicitly."
+    },
+
+    { "", Config::oBadOption, Config::None }
+  };
+
+  keywords = new Keyword[ sizeof( defaultKeywords) / sizeof( Keyword ) ];
+  std::copy( defaultKeywords, defaultKeywords +
+      sizeof( defaultKeywords) / sizeof( Keyword ), keywords );
+  cleanup_keywords = true;
+}
+
+Config::~Config()
+{
+  // prevent memleaks
+  // TODO: use sptr
+  if ( cleanup_storable )
+    delete storable;
+  if ( cleanup_keywords )
+    delete[] keywords;
+}
+
+Config::Config():
+  cleanup_storable( true )
+{
+  storable = new ConfigInfo;
+  prefillKeywords();
+  dPrintf( "%s is instantiated and initialized with default values\n",
+      __CLASS );
+}
+
+Config::Config( const Config & configIn )
+{
+  *this = configIn;
+  cleanup_storable = false;
+  cleanup_keywords = false;
+}
+
+Config::Config( ConfigInfo * configInfo ):
+  cleanup_storable( false )
+{
+  storable = configInfo;
+  prefillKeywords();
+  dPrintf( "%s is instantiated and initialized with supplied ConfigInfo\n",
+      __CLASS );
+}
+
+Config::Config( const Config & configIn, ConfigInfo * configInfo )
+{
+  configInfo->MergeFrom( *configIn.storable );
+  *this = configIn;
+  storable = configInfo;
+  cleanup_storable = false;
+  cleanup_keywords = false;
+  dPrintf( "%s is instantiated and initialized with supplied values\n",
+      __CLASS );
+}
+
+Config::OpCodes Config::parseToken( const char * option, const OptionType type )
+{
+  for ( u_int i = 0; !keywords[ i ].name.empty(); i++ )
+  {
+    if ( strcasecmp( option, keywords[ i ].name.c_str() ) == 0 )
+    {
+      if ( keywords[ i ].type != type )
+      {
+        fprintf( stderr, "Invalid option type specified for %s\n", option );
+        break;
+      }
+
+      return keywords[ i ].opcode;
+    }
+  }
+
+  return Config::oBadOption;
+}
+
+bool Config::parseOrValidate( const string & option, const OptionType type,
+   bool validate )
+{
+  string prefix;
+  if ( type == Runtime )
+    prefix.assign( "runtime" );
+  else
+  if ( type == Storable )
+    prefix.assign( "storable" );
+
+  dPrintf( "%s %s option \"%s\"...\n", ( validate ? "Validating" : "Parsing" ),
+      prefix.c_str(), option.c_str() );
+
+  bool hasValue = false;
+  size_t optionLength = option.length() + 1;
+  char optionName[ optionLength ], optionValue[ optionLength ];
+
+  if ( sscanf( option.c_str(), "%[^=]=%s", optionName, optionValue ) == 2 )
+  {
+    dPrintf( "%s option %s: %s\n", prefix.c_str(), optionName, optionValue );
+    hasValue = true;
+  }
+  else
+    dPrintf( "%s option %s\n", prefix.c_str(), option.c_str() );
+
+  int opcode = parseToken( hasValue ? optionName : option.c_str(), type );
+
+  size_t sizeValue;
+  char suffix[ 16 ];
+  int n;
+  unsigned int scale, scaleBase = 1;
+  uint32_t uint32Value;
+
+  switch ( opcode )
+  {
+    case oChunk_max_size:
+      SKIP_ON_VALIDATION;
+      REQUIRE_VALUE;
+
+      if ( sscanf( optionValue, "%u %n", &uint32Value, &n ) == 1
+          && !optionValue[ n ] )
+      {
+        SET_STORABLE( chunk, max_size, uint32Value );
+        dPrintf( "storable[chunk][max_size] = %u\n",
+            GET_STORABLE( chunk, max_size ) );
+
+        return true;
+      }
+
+      return false;
+      /* NOTREACHED */
+      break;
+
+    case oBundle_max_payload_size:
+      SKIP_ON_VALIDATION;
+      REQUIRE_VALUE;
+
+      if ( sscanf( optionValue, "%u %n", &uint32Value, &n ) == 1
+          && !optionValue[ n ] )
+      {
+        SET_STORABLE( bundle, max_payload_size, uint32Value );
+        dPrintf( "storable[bundle][max_payload_size] = %u\n",
+            GET_STORABLE( bundle, max_payload_size ) );
+
+        return true;
+      }
+
+      return false;
+      /* NOTREACHED */
+      break;
+
+    case oLZMA_compression_level:
+      REQUIRE_VALUE;
+
+      if ( PARSE_OR_VALIDATE(
+            sscanf( optionValue, "%u %n", &uint32Value, &n ) != 1 ||
+            optionValue[ n ] || uint32Value > 19,
+            GET_STORABLE( lzma, compression_level ) > 19 )
+         )
+        return false;
+
+      SKIP_ON_VALIDATION;
+      SET_STORABLE( lzma, compression_level, uint32Value );
+      dPrintf( "storable[lzma][compression_level] = %u\n",
+          GET_STORABLE( lzma, compression_level ) );
+
+      return true;
+      /* NOTREACHED */
+      break;
+
+    case oBundle_compression_method:
+      REQUIRE_VALUE;
+
+      if ( PARSE_OR_VALIDATE( strcmp( optionValue, "lzma" ) == 0,
+           GET_STORABLE( bundle, compression_method ) == "lzma" ) )
+      {
+        const_sptr< Compression::CompressionMethod > lzma =
+          Compression::CompressionMethod::findCompression( "lzma" );
+        if ( !lzma )
+        {
+          fprintf( stderr, "zbackup is compiled without LZMA support, but the code "
+            "would support it. If you install liblzma (including development files) "
+            "and recompile zbackup, you can use LZMA.\n" );
+          return false;
+        }
+        Compression::CompressionMethod::selectedCompression = lzma;
+      }
+      else
+      if ( PARSE_OR_VALIDATE(
+            strcmp( optionValue, "lzo1x_1" ) == 0 || strcmp( optionValue, "lzo" ) == 0,
+            GET_STORABLE( bundle, compression_method ) == "lzo1x_1" ) )
+      {
+        const_sptr< Compression::CompressionMethod > lzo =
+          Compression::CompressionMethod::findCompression( "lzo1x_1" );
+        if ( !lzo )
+        {
+          fprintf( stderr, "zbackup is compiled without LZO support, but the code "
+            "would support it. If you install liblzo2 (including development files) "
+            "and recompile zbackup, you can use LZO.\n" );
+          return false;
+        }
+        Compression::CompressionMethod::selectedCompression = lzo;
+      }
+      else
+      {
+        fprintf( stderr,
+            "ZBackup doesn't support %s compression.\n"
+            "You probably need a newer version.\n", validate ?
+            GET_STORABLE( bundle, compression_method ).c_str() : optionValue );
+        fprintf( stderr, "Supported compression methods:\n" );
+        for ( const const_sptr< Compression::CompressionMethod > * c =
+            Compression::CompressionMethod::compressions; *c; ++c )
+        {
+          fprintf( stderr, "%s\n", (*c)->getName().c_str() );
+        }
+        fprintf( stderr, "\n" );
+
+        return false;
+      }
+
+      SKIP_ON_VALIDATION;
+      SET_STORABLE( bundle, compression_method,
+          Compression::CompressionMethod::selectedCompression->getName() );
+      dPrintf( "storable[bundle][compression_method] = %s\n",
+          GET_STORABLE( bundle, compression_method ).c_str() );
+
+      return true;
+      /* NOTREACHED */
+      break;
+
+    case oRuntime_threads:
+      REQUIRE_VALUE;
+
+      sizeValue = runtime.threads;
+      if ( sscanf( optionValue, "%zu %n", &sizeValue, &n ) != 1 ||
+           optionValue[ n ] || sizeValue < 1 )
+        throw exInvalidThreadsValue( optionValue );
+      runtime.threads = sizeValue;
+
+      dPrintf( "runtime[threads] = %zu\n", runtime.threads );
+
+      return true;
+      /* NOTREACHED */
+      break;
+
+    case oRuntime_cacheSize:
+      REQUIRE_VALUE;
+
+      sizeValue = runtime.cacheSize;
+      if ( sscanf( optionValue, "%zu %15s %n",
+                   &sizeValue, suffix, &n ) == 2 && !optionValue[ n ] )
+      {
+        // Check the suffix
+        for ( char * c = suffix; *c; ++c )
+          *c = tolower( *c );
+
+        if ( strcmp( suffix, "b" ) == 0 )
+        {
+          scale = 1;
+        }
+        else
+        if ( strcmp( suffix, "kib" ) == 0 )
+        {
+          scaleBase = 1024;
+          scale = scaleBase;
+        }
+        else
+        if ( strcmp( suffix, "mib" ) == 0 )
+        {
+          scaleBase = 1024;
+          scale = scaleBase * scaleBase;
+        }
+        else
+        if ( strcmp( suffix, "gib" ) == 0 )
+        {
+          scaleBase = 1024;
+          scale = scaleBase * scaleBase * scaleBase;
+        }
+        else
+        if ( strcmp( suffix, "kb" ) == 0 )
+        {
+          scaleBase = 1000;
+          scale = scaleBase;
+        }
+        else
+        if ( strcmp( suffix, "mb" ) == 0 )
+        {
+          scaleBase = 1000;
+          scale = scaleBase * scaleBase;
+        }
+        else
+        if ( strcmp( suffix, "gb" ) == 0 )
+        {
+          scaleBase = 1000;
+          scale = scaleBase * scaleBase * scaleBase;
+        }
+        else
+        {
+          // SI or IEC
+          fprintf( stderr, "Invalid suffix specified in cache size (%s): %s.\n"
+                   VALID_SUFFIXES, optionValue, suffix );
+          return false;
+        }
+        runtime.cacheSize = sizeValue * scale;
+
+        dPrintf( "runtime[cacheSize] = %zu\n", runtime.cacheSize );
+
+        return true;
+      }
+      return false;
+      /* NOTREACHED */
+      break;
+
+    case oRuntime_exchange:
+      REQUIRE_VALUE;
+
+      if ( strcmp( optionValue, "backups" ) == 0 )
+        runtime.exchange.set( BackupExchanger::backups );
+      else
+      if ( strcmp( optionValue, "bundles" ) == 0 )
+        runtime.exchange.set( BackupExchanger::bundles );
+      else
+      if ( strcmp( optionValue, "indexes" ) == 0 ||
+           strcmp( optionValue, "index" ) == 0 )
+        runtime.exchange.set( BackupExchanger::indexes );
+      else
+      {
+        fprintf( stderr, "Invalid exchange value specified: %s\n"
+                 "Must be one of the following: backups, bundles, indexes.\n",
+                 optionValue );
+        return false;
+      }
+
+      dPrintf( "runtime[exchange] = %s\n", runtime.exchange.to_string().c_str() );
+
+      return true;
+      /* NOTREACHED */
+      break;
+
+    case oRuntime_gcRepack:
+      runtime.gcRepack = true;
+
+      dPrintf( "runtime[gcRepack] = true\n" );
+
+      return true;
+      /* NOTREACHED */
+      break;
+
+    case oRuntime_pathsRespectTmp:
+      runtime.pathsRespectTmp = true;
+
+      dPrintf( "runtime[pathsRespectTmp] = true\n" );
+
+      return true;
+      /* NOTREACHED */
+      break;
+
+    case oBadOption:
+    default:
+      return false;
+      /* NOTREACHED */
+      break;
+  }
+
+  /* NOTREACHED */
+  return false;
+}
+
+void Config::showHelp( const OptionType type )
+{
+  fprintf( stderr,
+"Available %s options overview:\n\n"
+"== help ==\n"
+"show this message\n"
+"", ( type == Runtime ? "runtime" : ( type == Storable ? "storable" : "" ) ) );
+
+  for ( u_int i = 0; !keywords[ i ].name.empty(); i++ )
+  {
+    if ( keywords[ i ].type != type )
+      continue;
+
+    fprintf( stderr, "\n== %s ==\n", keywords[ i ].name.c_str() );
+    fprintf( stderr, keywords[ i ].description.c_str(),
+       keywords[ i ].defaultValue.c_str() );
+    fprintf( stderr, "\n" );
+  }
+}
+
+bool Config::parseProto( const string & str, google::protobuf::Message * mutable_message )
+{
+  return google::protobuf::TextFormat::ParseFromString( str, mutable_message );
+}
+
+string Config::toString( google::protobuf::Message const & message )
+{
+  std::string str;
+  google::protobuf::TextFormat::PrintToString( message, &str );
+
+  return str;
+}
+
+bool Config::validateProto( const string & oldConfigData, const string & configData )
+{
+  Config config;
+  dPrintf( "Validating proto...\n" );
+  if ( !parseProto( configData, config.storable ) )
+    return false;
+
+  const ::google::protobuf::Descriptor * configDescriptor =
+    config.storable->descriptor();
+  for ( int i = 0; i < configDescriptor->field_count(); i++ )
+  {
+    const ::google::protobuf::FieldDescriptor * storage =
+      configDescriptor->field( i );
+    dPrintf( "Storage: %s - %d - %d\n", storage->name().c_str(),
+        storage->label(), storage->type());
+
+    // TODO: support for top-level fields
+    if ( storage->type() == ::google::protobuf::FieldDescriptor::TYPE_MESSAGE )
+    {
+      const ::google::protobuf::Descriptor * storageDescriptor =
+        storage->message_type();
+
+      for ( int j = 0; j < storageDescriptor->field_count(); j++ )
+      {
+        const ::google::protobuf::FieldDescriptor * field =
+          storageDescriptor->field( j );
+
+        dPrintf( "Field: %s - %d - %d\n", field->name().c_str(),
+            field->label(), field->type());
+
+        string option = storage->name() + "." + field->name();
+
+        if ( !config.parseOrValidate( option.c_str(), Storable, true ) )
+        {
+          fprintf( stderr, "Invalid option specified: %s\n",
+                   option.c_str() );
+          return false;
+        }
+      }
+    }
+  }
+
+  return true;
+}
+
+void Config::reset_storable()
+{
+  // TODO: Use protobuf introspection
+  // to fill messages in loop with default values
+  // without explicit declaration
+  Config defaultConfig;
+
+  SET_STORABLE( chunk, max_size, defaultConfig.GET_STORABLE( chunk, max_size ) );
+  SET_STORABLE( bundle, max_payload_size, defaultConfig.GET_STORABLE(
+        bundle, max_payload_size ) );
+  SET_STORABLE( bundle, compression_method, defaultConfig.GET_STORABLE(
+        bundle, compression_method ) );
+  SET_STORABLE( lzma, compression_level, defaultConfig.GET_STORABLE(
+        lzma, compression_level ) );
+}
+
+void Config::show()
+{
+  printf( "%s", toString( *storable ).c_str() );
+}
+
+void Config::show( const ConfigInfo & config )
+{
+  printf( "%s", toString( config ).c_str() );
+}

config.hh

@@ -0,0 +1,115 @@
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
+
+#ifndef CONFIG_HH_INCLUDED
+#define CONFIG_HH_INCLUDED
+
+#include <string>
+#include <bitset>
+#include <google/protobuf/text_format.h>
+#include "zbackup.pb.h"
+#include "mt.hh"
+#include "backup_exchanger.hh"
+
+// TODO: make *_storable to be variadic
+#define SET_STORABLE( storage, property, value ) \
+  storable->mutable_##storage()->set_##property( value )
+
+#define GET_STORABLE( storage, property ) \
+  storable->storage().property()
+
+using std::string;
+using std::bitset;
+
+class Config
+{
+public:
+  struct RuntimeConfig
+  {
+    size_t threads;
+    size_t cacheSize;
+    bitset< BackupExchanger::Flags > exchange;
+    bool gcRepack;
+    bool pathsRespectTmp;
+
+    // Default runtime config
+    RuntimeConfig():
+      threads( getNumberOfCpus() ),
+      cacheSize( 40 * 1024 * 1024 ), // 40 MB
+      gcRepack ( false ),
+      pathsRespectTmp( false )
+    {
+    }
+  };
+
+  enum OptionType
+  {
+    Runtime,
+    Storable,
+    None
+  };
+
+  /* Keyword tokens. */
+  typedef enum
+  {
+    oBadOption,
+
+    oChunk_max_size,
+    oBundle_max_payload_size,
+    oBundle_compression_method,
+    oLZMA_compression_level,
+
+    oRuntime_threads,
+    oRuntime_cacheSize,
+    oRuntime_exchange,
+    oRuntime_gcRepack,
+    oRuntime_pathsRespectTmp,
+
+    oDeprecated, oUnsupported
+  } OpCodes;
+
+  // Validator for user-supplied storable configuration
+  static bool validateProto( const string &, const string & );
+
+  static bool parseProto( const string &, google::protobuf::Message * );
+
+  static string toString( google::protobuf::Message const & );
+
+  // Print configuration to screen
+  static void show( const ConfigInfo & );
+  void show();
+
+  void showHelp( const OptionType );
+
+  OpCodes parseToken( const char *, const OptionType );
+  bool parseOrValidate( const string &, const OptionType, bool validate = false );
+
+  Config( const Config & );
+  Config( const Config &, ConfigInfo * );
+  Config( ConfigInfo * );
+  Config();
+  ~Config();
+
+  void reset_storable();
+
+  RuntimeConfig runtime;
+  ConfigInfo * storable;
+private:
+  struct Keyword
+  {
+    string name;
+    Config::OpCodes opcode;
+    Config::OptionType type;
+    string description;
+    string defaultValue;
+  };
+
+  Keyword * keywords;
+
+  bool cleanup_storable;
+  bool cleanup_keywords;
+
+  void prefillKeywords();
+};
+
+#endif

debug.cc

@@ -1,4 +1,11 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
 bool verboseMode = true;
+
+#ifndef NDEBUG
+#ifdef HAVE_LIBUNWIND
+#include "debug.hh"
+
+#endif
+#endif

debug.hh

@@ -1,20 +1,38 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
-#ifndef DEBUG_HH_INCLUDED__
-#define DEBUG_HH_INCLUDED__
+#ifndef DEBUG_HH_INCLUDED
+#define DEBUG_HH_INCLUDED
 
 #include <stdio.h>
+#include <typeinfo>
 
 // Macros we use to output debugging information
 
+#define __CLASS typeid( *this ).name()
+
 #ifndef NDEBUG
 
-#define dPrintf( ... ) (fprintf( stderr, __VA_ARGS__ ))
+#define __FILE_BASE (strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : __FILE__)
+
+#define dPrintf( ... ) ({ fprintf( stderr, "[DEBUG] at %s( %s:%d ): ", __func__,\
+      __FILE_BASE, __LINE__ );\
+    fprintf( stderr, __VA_ARGS__ ); })
+
+#ifdef HAVE_LIBUNWIND
+#define UNW_LOCAL_ONLY
+#include <libunwind.h>
+
+// TODO: pretty backtraces
+#define dPrintBacktrace( ... ) ()
+#else
+#define dPrintBacktrace( ... ) ()
+#endif
 
 #else
 
 #define dPrintf( ... )
+#define dPrintBacktrace( ... ) ()
 
 #endif
 

dir.cc

@@ -1,5 +1,5 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
 #include <fcntl.h>
 #include <libgen.h>
@@ -66,6 +66,13 @@ string getDirName( string const & path )
   return dirname( copy.data() );
 }
 
+bool isDirEmpty( string const & path )
+{
+  Listing lst(path);
+  Entry tmp;
+  return !lst.getNext(tmp);
+}
+
 Listing::Listing( string const & dirName ): dirName( dirName )
 {
   dir = opendir( dirName.c_str() );
@@ -96,8 +103,13 @@ bool Listing::getNext( Entry & result )
     if ( !entryPtr )
       return false;
 
+#ifndef __APPLE__
     if ( fstatat( dirfd( dir ), entry.d_name, &entryStats,
                   AT_SYMLINK_NOFOLLOW ) != 0 )
+#else
+    if ( lstat( addPath( dirName, entry.d_name ).c_str(),
+                &entryStats ) != 0)
+#endif
       throw exCantList( dirName );
 
     bool isDir = S_ISDIR( entryStats.st_mode );

dir.hh

@@ -1,8 +1,8 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
-#ifndef DIR_HH_INCLUDED__
-#define DIR_HH_INCLUDED__
+#ifndef DIR_HH_INCLUDED
+#define DIR_HH_INCLUDED
 
 #include <dirent.h>
 #include <sys/types.h>
@@ -42,6 +42,9 @@ string getRealPath( string const & );
 /// Returns the directory part of the given path
 string getDirName( string const & );
 
+/// Checkes whether directory is empty
+bool isDirEmpty( string const & );
+
 /// A separator used to separate names in the path.
 inline char separator()
 { return '/'; }

encrypted_file.cc

@@ -1,5 +1,5 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
 #include <string.h>
 #include <algorithm>
@@ -9,6 +9,7 @@
 #include "endian.hh"
 #include "page_size.hh"
 #include "random.hh"
+#include "debug.hh"
 
 namespace EncryptedFile {
 
@@ -22,6 +23,7 @@ InputStream::InputStream( char const * fileName, EncryptionKey const & key,
   buffer( std::max( getPageSize(), ( unsigned ) BlockSize * 2 ) ),
   fill( 0 ), remainder( 0 ), backedUp( false )
 {
+  dPrintf( "Loading %s, hasKey: %s\n", fileName, key.hasKey() ? "true" : "false" );
   if ( key.hasKey() )
   {
     memcpy( iv, iv_, sizeof( iv ) );
@@ -239,6 +241,7 @@ OutputStream::OutputStream( char const * fileName, EncryptionKey const & key,
   file( fileName, UnbufferedFile::WriteOnly ), filePos( 0 ), key( key ),
   buffer( getPageSize() ), start( buffer.data() ), avail( 0 ), backedUp( false )
 {
+  dPrintf( "Saving %s, hasKey: %s\n", fileName, key.hasKey() ? "true" : "false" );
   if ( key.hasKey() )
     memcpy( iv, iv_, sizeof( iv ) );
 }
@@ -341,7 +344,7 @@ void OutputStream::writeRandomIv()
   if ( key.hasKey() )
   {
     char iv[ Encryption::IvSize ];
-    Random::genaratePseudo( iv, sizeof( iv ) );
+    Random::generatePseudo( iv, sizeof( iv ) );
     write( iv, sizeof( iv ) );
   }
 }

encrypted_file.hh

@@ -1,8 +1,8 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
-#ifndef ENCRYPTED_FILE_HH_INCLUDED__
-#define ENCRYPTED_FILE_HH_INCLUDED__
+#ifndef ENCRYPTED_FILE_HH_INCLUDED
+#define ENCRYPTED_FILE_HH_INCLUDED
 
 #include <google/protobuf/io/zero_copy_stream.h>
 #include <stddef.h>

encryption.cc

@@ -1,5 +1,5 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
 #include <openssl/aes.h>
 

encryption.hh

@@ -1,8 +1,8 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
-#ifndef ENCRYPTION_HH_INCLUDED__
-#define ENCRYPTION_HH_INCLUDED__
+#ifndef ENCRYPTION_HH_INCLUDED
+#define ENCRYPTION_HH_INCLUDED
 
 #include <stddef.h>
 #include <exception>

encryption_key.cc

@@ -1,5 +1,5 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
 #include <openssl/aes.h>
 #include <openssl/evp.h>
@@ -66,12 +66,13 @@ EncryptionKey::~EncryptionKey()
 }
 
 void EncryptionKey::generate( string const & password,
-                              EncryptionKeyInfo & info )
+                              EncryptionKeyInfo & info,
+                              EncryptionKey & encryptionkey )
 {
   // Use this buf for salts
-  char buf[ 16 ];
+  char buf[ KeySize ];
 
-  Random::genaratePseudo( buf, sizeof( buf ) );
+  Random::generatePseudo( buf, sizeof( buf ) );
   info.set_salt( buf, sizeof( buf ) );
   info.set_rounds( 10000 ); // TODO: make this configurable
 
@@ -79,11 +80,13 @@ void EncryptionKey::generate( string const & password,
   deriveKey( password, info, derivedKey, sizeof( derivedKey ) );
 
   char key[ KeySize ];
-
-  Random::genarateTrue( key, sizeof( key ) );
+  if ( encryptionkey.hasKey() )
+    memcpy( key, encryptionkey.getKey(), KeySize );
+  else
+    Random::generateTrue( key, sizeof( key ) );
 
   // Fill in the HMAC verification part
-  Random::genaratePseudo( buf, sizeof( buf ) );
+  Random::generatePseudo( buf, sizeof( buf ) );
   info.set_key_check_input( buf, sizeof( buf ) );
   info.set_key_check_hmac( calculateKeyHmac( key, sizeof( key ),
                                              info.key_check_input() ) );

encryption_key.hh

@@ -1,8 +1,8 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
-#ifndef ENCRYPTION_KEY_HH_INCLUDED__
-#define ENCRYPTION_KEY_HH_INCLUDED__
+#ifndef ENCRYPTION_KEY_HH_INCLUDED
+#define ENCRYPTION_KEY_HH_INCLUDED
 
 #include <exception>
 #include <string>
@@ -41,7 +41,8 @@ public:
   { return sizeof( key ); }
 
   /// Generates new key info using the given password
-  static void generate( string const & password, EncryptionKeyInfo & );
+  static void generate( string const & password, EncryptionKeyInfo &,
+                        EncryptionKey & encryptionkey );
 
   /// Returns a static instance without any key set
   static EncryptionKey const & noKey();

endian.hh

@@ -1,17 +1,41 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
-#ifndef ENDIAN_HH_INCLUDED__
-#define ENDIAN_HH_INCLUDED__
+#ifndef ENDIAN_HH_INCLUDED
+#define ENDIAN_HH_INCLUDED
 
 #include <stdint.h>
 #include <arpa/inet.h>
-#include <endian.h>
 
-#if __BYTE_ORDER != __LITTLE_ENDIAN
-#error Please add support for architectures different from little-endian.
+#ifdef __APPLE__
+#include <machine/endian.h>
+#include <libkern/OSByteOrder.h>
+
+#define htobe16(x) OSSwapHostToBigInt16(x)
+#define htole16(x) OSSwapHostToLittleInt16(x)
+#define be16toh(x) OSSwapBigToHostInt16(x)
+#define le16toh(x) OSSwapLittleToHostInt16(x)
+
+#define htobe32(x) OSSwapHostToBigInt32(x)
+#define htole32(x) OSSwapHostToLittleInt32(x)
+#define be32toh(x) OSSwapBigToHostInt32(x)
+#define le32toh(x) OSSwapLittleToHostInt32(x)
+
+#define htobe64(x) OSSwapHostToBigInt64(x)
+#define htole64(x) OSSwapHostToLittleInt64(x)
+#define be64toh(x) OSSwapBigToHostInt64(x)
+#define le64toh(x) OSSwapLittleToHostInt64(x)
+//__APPLE__
+
+#elif __FreeBSD__
+#include <sys/endian.h>
+
+#else
+#include <endian.h>
 #endif
 
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+
 /// Converts the given host-order value to big-endian value
 inline uint32_t toBigEndian( uint32_t v ) { return htonl( v ); }
 /// Converts the given host-order value to little-endian value
@@ -21,4 +45,24 @@ inline uint64_t toLittleEndian( uint64_t v ) { return v; }
 inline uint32_t fromLittleEndian( uint32_t v ) { return v; }
 inline uint64_t fromLittleEndian( uint64_t v ) { return v; }
 
+#elif __BYTE_ORDER == __BIG_ENDIAN
+
+// Note: the functions used are non-standard. Add more ifdefs if needed
+
+/// Converts the given host-order value to big-endian value
+inline uint32_t toBigEndian( uint32_t v ) { return v; }
+/// Converts the given host-order value to little-endian value
+inline uint32_t toLittleEndian( uint32_t v ) { return htole32( v ); }
+inline uint64_t toLittleEndian( uint64_t v ) { return htole64( v ); }
+/// Converts the given little-endian value to host-order value
+inline uint32_t fromLittleEndian( uint32_t v ) { return le32toh( v ); }
+inline uint64_t fromLittleEndian( uint64_t v ) { return le64toh( v ); }
+
+#else
+
+#error Please add support for architectures different from little-endian and\
+ big-endian.
+
+#endif
+
 #endif

ex.hh

@@ -1,8 +1,8 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
-#ifndef EX_HH_INCLUDED__
-#define EX_HH_INCLUDED__
+#ifndef EX_HH_INCLUDED
+#define EX_HH_INCLUDED
 
 #include <exception>
 #include <stdio.h>

file.cc

@@ -1,13 +1,21 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
 #include <limits.h>
 #include <sys/stat.h>
 #include <unistd.h>
 #include <cerrno>
 #include <cstring>
+#if defined( __APPLE__ ) || defined( __OpenBSD__ ) || defined(__FreeBSD__) || defined(__CYGWIN__)
+  #include <sys/socket.h>
+#else
+  #include <sys/sendfile.h>
+#endif
+#include <sys/types.h>
+#include <fcntl.h>
 
 #include "file.hh"
+#include "utils.hh"
 
 enum
 {
@@ -36,10 +44,56 @@ void File::erase( std::string const & filename ) throw( exCantErase )
 }
 
 void File::rename( std::string const & from,
-                   std::string const & to ) throw( exCantRename )
+                   std::string const & to ) throw( exCantRename,
+                                                   exCantErase )
 {
-  if ( ::rename( from.c_str(), to.c_str() ) != 0 )
-    throw exCantRename( from + " to " + to );
+  int res = 0;
+  res = ::rename( from.c_str(), to.c_str() );
+  if ( 0 != res )
+  {
+    if ( EXDEV == errno )
+    {
+      int read_fd;
+      int write_fd;
+      struct stat stat_buf;
+      off_t offset = 0;
+
+      /* Open the input file. */
+      read_fd = ::open( from.c_str(), O_RDONLY );
+      /* Stat the input file to obtain its size. */
+      if ( fstat( read_fd, &stat_buf ) != 0 )
+        throw exCantRename( from + " to " + to );
+      /* Open the output file for writing, with the same permissions as the
+       source file. */
+      write_fd = ::open( to.c_str(), O_WRONLY | O_CREAT, stat_buf.st_mode );
+      /* Blast the bytes from one file to the other. */
+      #if defined( __APPLE__ )
+      if ( -1 == sendfile( write_fd, read_fd, offset, &stat_buf.st_size, NULL, 0 ) )
+         throw exCantRename( from + " to " + to );
+      #elif defined( __OpenBSD__ ) || defined(__FreeBSD__) || defined(__CYGWIN__)
+
+      size_t BUFSIZE = 4096, size;
+      char buf[BUFSIZE];
+
+      while ( ( size = ::read( read_fd, buf, BUFSIZE ) ) != -1 && size != 0 )
+        ::write( write_fd, buf, size );
+
+      if ( size == -1 )
+         throw exCantRename( from + " to " + to );
+
+      #else
+      if ( -1 == sendfile( write_fd, read_fd, &offset, stat_buf.st_size ) )
+         throw exCantRename( from + " to " + to );
+      #endif
+
+      /* Close up. */
+      ::close( read_fd );
+      ::close( write_fd );
+      File::erase ( from );
+    }
+    else
+      throw exCantRename( from + " to " + to );
+  }
 }
 
 void File::open( char const * filename, OpenMode mode ) throw( exCantOpen )
@@ -64,6 +118,28 @@ void File::open( char const * filename, OpenMode mode ) throw( exCantOpen )
     throw exCantOpen( std::string( filename ) + ": " + strerror( errno ) );
 }
 
+void File::open( int fd, OpenMode mode ) throw( exCantOpen )
+{
+  char const * m;
+
+  switch( mode )
+  {
+    case Update:
+      m = "r+b";
+      break;
+    case WriteOnly:
+      m = "wb";
+      break;
+    default:
+      m = "rb";
+  }
+
+  f = fdopen( fd, m );
+
+  if ( !f )
+    throw exCantOpen( "fd#" + Utils::numberToString( fd ) + ": " + strerror( errno ) );
+}
+
 File::File( char const * filename, OpenMode mode ) throw( exCantOpen ):
   writeBuffer( 0 )
 {
@@ -76,6 +152,12 @@ File::File( std::string const & filename, OpenMode mode )
   open( filename.c_str(), mode );
 }
 
+File::File( int fd, OpenMode mode )
+  throw( exCantOpen ): writeBuffer( 0 )
+{
+  open( fd, mode );
+}
+
 void File::read( void * buf, size_t size ) throw( exReadError, exWriteError )
 {
   if ( !size )
@@ -167,7 +249,7 @@ char * File::gets( char * s, int size, bool stripNl )
   if ( result && stripNl )
   {
     size_t len = strlen( result );
-    
+
     char * last = result + len;
 
     while( len-- )
@@ -262,6 +344,13 @@ bool File::eof() throw( exWriteError )
   return feof( f );
 }
 
+int File::error() throw( exReadError )
+{
+  int result = ferror( f );
+
+  return result;
+}
+
 FILE * File::file() throw( exWriteError )
 {
   flushWriteBuffer();

file.hh

@@ -1,8 +1,8 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
-#ifndef FILE_HH_INCLUDED__
-#define FILE_HH_INCLUDED__
+#ifndef FILE_HH_INCLUDED
+#define FILE_HH_INCLUDED
 
 #include <stddef.h>
 #include <cstdio>
@@ -45,6 +45,9 @@ public:
   File( std::string const & filename, OpenMode )
     throw( exCantOpen );
 
+  File( int fd, OpenMode )
+    throw( exCantOpen );
+
   /// Reads the number of bytes to the buffer, throws an error if it
   /// failed to fill the whole buffer (short read, i/o error etc)
   void read( void * buf, size_t size ) throw( exReadError, exWriteError );
@@ -108,6 +111,9 @@ public:
   /// Returns true if end-of-file condition is set
   bool eof() throw( exWriteError );
 
+  /// Returns ferror
+  int error() throw( exReadError );
+
   /// Returns the underlying FILE * record, so other operations can be
   /// performed on it
   FILE * file() throw( exWriteError );
@@ -132,7 +138,8 @@ public:
 
   /// Renames the given file
   static void rename( std::string const & from,
-                      std::string const & to ) throw( exCantRename );
+                      std::string const & to ) throw( exCantRename,
+                                                      exCantErase );
 
   /// Throwing this class instead of exReadError will make the description
   /// include the file name
@@ -153,6 +160,7 @@ public:
 private:
 
   void open( char const * filename, OpenMode ) throw( exCantOpen );
+  void open( int fd, OpenMode ) throw( exCantOpen );
   void flushWriteBuffer() throw( exWriteError );
   void releaseWriteBuffer() throw( exWriteError );
 };

hex.cc

@@ -1,5 +1,5 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
 #include "hex.hh"
 

hex.hh

@@ -1,8 +1,8 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
-#ifndef HEX_HH_INCLUDED__
-#define HEX_HH_INCLUDED__
+#ifndef HEX_HH_INCLUDED
+#define HEX_HH_INCLUDED
 
 #include <string>
 

index_file.cc

@@ -1,5 +1,5 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
 #include <string.h>
 

index_file.hh

@@ -1,8 +1,8 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
-#ifndef INDEX_FILE_HH_INCLUDED__
-#define INDEX_FILE_HH_INCLUDED__
+#ifndef INDEX_FILE_HH_INCLUDED
+#define INDEX_FILE_HH_INCLUDED
 
 #include <exception>
 #include <string>

licenses/LICENSE

@@ -0,0 +1,58 @@
+ZBackup, a versatile deduplicating backup tool
+Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup
+contributors, see CONTRIBUTORS
+
+This program is free software; you can redistribute it and/or modify it
+under the terms of the GNU General Public License (GPL) as published by
+the Free Software Foundation; either version 2 of the License, or (at
+your option) any later version. The full text of versions 2 and 3 of
+the GPL can be found respectively in the files LICENSE-GPLV2 and
+LICENSE-GPLV3.
+
+EXCEPTION: This distribution of ZBackup may be linked against OpenSSL
+according to the terms of the section below entitled "OpenSSL Exception."
+
+ADDITION: This distribution of ZBackup uses sources from CMake project
+which is distributed under the OSI-approved BSD 3-clause License.
+See LICENSE-CMAKE for details.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+
+
+                       _OpenSSL Exception_
+
+0. Definitions
+
+"ZBackup" means ZBackup software licensed under version 2 or any later
+version of the GNU General Public License (collectively, "GPL"), or a
+work based on such software and licensed under the GPL.
+
+"OpenSSL" means OpenSSL toolkit software distributed by the OpenSSL
+Project and licensed under the OpenSSL Licenses, or a work based on such
+software and licensed under the OpenSSL Licenses.
+
+"OpenSSL Licenses" means the OpenSSL License and Original SSLeay License
+under which the OpenSSL Project distributes the OpenSSL toolkit software,
+as those licenses appear in the file LICENSE-OPENSSL.
+
+1. Exception
+
+You have permission to copy, modify, propagate, and distribute a work
+formed by combining OpenSSL with ZBackup, or a work derivative of such a
+combination, even if such copying, modification, propagation, or
+distribution would otherwise violate the terms of the GPL. You must
+comply with the GPL in all respects for all of the code used other than
+OpenSSL.
+
+You may include this OpenSSL Exception and its grant of permissions when
+you distribute ZBackup. Inclusion of this notice with such a distribution
+constitutes a grant of such permission. If you do not wish to grant these
+permissions, remove this section entitled "OpenSSL Exception" from your
+distribution.

licenses/LICENSE-CMAKE

@@ -0,0 +1,57 @@
+CMake - Cross Platform Makefile Generator
+Copyright 2000-2014 Kitware, Inc.
+Copyright 2000-2011 Insight Software Consortium
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+* Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright
+  notice, this list of conditions and the following disclaimer in the
+  documentation and/or other materials provided with the distribution.
+
+* Neither the names of Kitware, Inc., the Insight Software Consortium,
+  nor the names of their contributors may be used to endorse or promote
+  products derived from this software without specific prior written
+  permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+------------------------------------------------------------------------------
+
+The above copyright and license notice applies to distributions of
+CMake in source and binary form.  Some source files contain additional
+notices of original copyright by their contributors; see each source
+for details.  Third-party software packages supplied with CMake under
+compatible licenses provide their own copyright notices documented in
+corresponding subdirectories.
+
+------------------------------------------------------------------------------
+
+CMake was initially developed by Kitware with the following sponsorship:
+
+ * National Library of Medicine at the National Institutes of Health
+   as part of the Insight Segmentation and Registration Toolkit (ITK).
+
+ * US National Labs (Los Alamos, Livermore, Sandia) ASC Parallel
+   Visualization Initiative.
+
+ * National Alliance for Medical Image Computing (NAMIC) is funded by the
+   National Institutes of Health through the NIH Roadmap for Medical Research,
+   Grant U54 EB005149.
+
+ * Kitware, Inc.

licenses/LICENSE-GPLV2

@@ -1,4 +1,4 @@
-GNU GENERAL PUBLIC LICENSE
+                    GNU GENERAL PUBLIC LICENSE
                        Version 2, June 1991
 
  Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
@@ -290,8 +290,8 @@ to attach them to the start of each source file to most effectively
 convey the exclusion of warranty; and each file should have at least
 the "copyright" line and a pointer to where the full notice is found.
 
-    ZBackup, a versatile deduplicating backup tool
-    Copyright (C) 2013  zbackup
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -329,7 +329,7 @@ necessary.  Here is a sample; alter the names:
   Yoyodyne, Inc., hereby disclaims all copyright interest in the program
   `Gnomovision' (which makes passes at compilers) written by James Hacker.
 
-  {signature of Ty Coon}, 1 April 1989
+  <signature of Ty Coon>, 1 April 1989
   Ty Coon, President of Vice
 
 This General Public License does not permit incorporating your program into

licenses/LICENSE-GPLV3

@@ -0,0 +1,674 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU General Public License is a free, copyleft license for
+software and other kinds of works.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+the GNU General Public License is intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.  We, the Free Software Foundation, use the
+GNU General Public License for most of our software; it applies also to
+any other work released this way by its authors.  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  To protect your rights, we need to prevent others from denying you
+these rights or asking you to surrender the rights.  Therefore, you have
+certain responsibilities if you distribute copies of the software, or if
+you modify it: responsibilities to respect the freedom of others.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must pass on to the recipients the same
+freedoms that you received.  You must make sure that they, too, receive
+or can get the source code.  And you must show them these terms so they
+know their rights.
+
+  Developers that use the GNU GPL protect your rights with two steps:
+(1) assert copyright on the software, and (2) offer you this License
+giving you legal permission to copy, distribute and/or modify it.
+
+  For the developers' and authors' protection, the GPL clearly explains
+that there is no warranty for this free software.  For both users' and
+authors' sake, the GPL requires that modified versions be marked as
+changed, so that their problems will not be attributed erroneously to
+authors of previous versions.
+
+  Some devices are designed to deny users access to install or run
+modified versions of the software inside them, although the manufacturer
+can do so.  This is fundamentally incompatible with the aim of
+protecting users' freedom to change the software.  The systematic
+pattern of such abuse occurs in the area of products for individuals to
+use, which is precisely where it is most unacceptable.  Therefore, we
+have designed this version of the GPL to prohibit the practice for those
+products.  If such problems arise substantially in other domains, we
+stand ready to extend this provision to those domains in future versions
+of the GPL, as needed to protect the freedom of users.
+
+  Finally, every program is threatened constantly by software patents.
+States should not allow patents to restrict development and use of
+software on general-purpose computers, but in those that do, we wish to
+avoid the special danger that patents applied to a free program could
+make it effectively proprietary.  To prevent this, the GPL assures that
+patents cannot be used to render the program non-free.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Use with the GNU Affero General Public License.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU Affero General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the special requirements of the GNU Affero General Public License,
+section 13, concerning interaction through a network will apply to the
+combination as such.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If the program does terminal interaction, make it output a short
+notice like this when it starts in an interactive mode:
+
+    <program>  Copyright (C) <year>  <name of author>
+    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, your program's commands
+might be different; for a GUI interface, you would use an "about box".
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU GPL, see
+<http://www.gnu.org/licenses/>.
+
+  The GNU General Public License does not permit incorporating your program
+into proprietary programs.  If your program is a subroutine library, you
+may consider it more useful to permit linking proprietary applications with
+the library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.  But first, please read
+<http://www.gnu.org/philosophy/why-not-lgpl.html>.

licenses/LICENSE-OPENSSL

@@ -0,0 +1,127 @@
+
+  LICENSE ISSUES
+  ==============
+
+  The OpenSSL toolkit stays under a dual license, i.e. both the conditions of
+  the OpenSSL License and the original SSLeay license apply to the toolkit.
+  See below for the actual license texts. Actually both licenses are BSD-style
+  Open Source licenses. In case of any license issues related to OpenSSL
+  please contact openssl-core@openssl.org.
+
+  OpenSSL License
+  ---------------
+
+/* ====================================================================
+ * Copyright (c) 1998-2011 The OpenSSL Project.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * 3. All advertising materials mentioning features or use of this
+ *    software must display the following acknowledgment:
+ *    "This product includes software developed by the OpenSSL Project
+ *    for use in the OpenSSL Toolkit. (http://www.openssl.org/)"
+ *
+ * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
+ *    endorse or promote products derived from this software without
+ *    prior written permission. For written permission, please contact
+ *    openssl-core@openssl.org.
+ *
+ * 5. Products derived from this software may not be called "OpenSSL"
+ *    nor may "OpenSSL" appear in their names without prior written
+ *    permission of the OpenSSL Project.
+ *
+ * 6. Redistributions of any form whatsoever must retain the following
+ *    acknowledgment:
+ *    "This product includes software developed by the OpenSSL Project
+ *    for use in the OpenSSL Toolkit (http://www.openssl.org/)"
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
+ * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
+ * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ * ====================================================================
+ *
+ * This product includes cryptographic software written by Eric Young
+ * (eay@cryptsoft.com).  This product includes software written by Tim
+ * Hudson (tjh@cryptsoft.com).
+ *
+ */
+
+ Original SSLeay License
+ -----------------------
+
+/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
+ * All rights reserved.
+ *
+ * This package is an SSL implementation written
+ * by Eric Young (eay@cryptsoft.com).
+ * The implementation was written so as to conform with Netscapes SSL.
+ * 
+ * This library is free for commercial and non-commercial use as long as
+ * the following conditions are aheared to.  The following conditions
+ * apply to all code found in this distribution, be it the RC4, RSA,
+ * lhash, DES, etc., code; not just the SSL code.  The SSL documentation
+ * included with this distribution is covered by the same copyright terms
+ * except that the holder is Tim Hudson (tjh@cryptsoft.com).
+ * 
+ * Copyright remains Eric Young's, and as such any Copyright notices in
+ * the code are not to be removed.
+ * If this package is used in a product, Eric Young should be given attribution
+ * as the author of the parts of the library used.
+ * This can be in the form of a textual message at program startup or
+ * in documentation (online or textual) provided with the package.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *    "This product includes cryptographic software written by
+ *     Eric Young (eay@cryptsoft.com)"
+ *    The word 'cryptographic' can be left out if the rouines from the library
+ *    being used are not cryptographic related :-).
+ * 4. If you include any Windows specific code (or a derivative thereof) from 
+ *    the apps directory (application code) you must include an acknowledgement:
+ *    "This product includes software written by Tim Hudson (tjh@cryptsoft.com)"
+ * 
+ * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ * 
+ * The licence and distribution terms for any publically available version or
+ * derivative of this code cannot be changed.  i.e. this code cannot simply be
+ * copied and put under another distribution licence
+ * [including the GNU Public Licence.]
+ */
+

message.cc

@@ -1,5 +1,5 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
 #include "message.hh"
 

message.hh

@@ -1,8 +1,8 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
-#ifndef MESSAGE_HH_INCLUDED__
-#define MESSAGE_HH_INCLUDED__
+#ifndef MESSAGE_HH_INCLUDED
+#define MESSAGE_HH_INCLUDED
 
 #include <google/protobuf/io/coded_stream.h>
 #include <google/protobuf/io/zero_copy_stream.h>

mt.cc

@@ -1,5 +1,5 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
 #include "mt.hh"
 

mt.hh

@@ -1,8 +1,8 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
-#ifndef MT_HH_INCLUDED__
-#define MT_HH_INCLUDED__
+#ifndef MT_HH_INCLUDED
+#define MT_HH_INCLUDED
 
 #include <pthread.h>
 #include <stddef.h>

nocopy.hh

@@ -1,8 +1,8 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
-#ifndef NOCOPY_HH_INCLUDED__
-#define NOCOPY_HH_INCLUDED__
+#ifndef NOCOPY_HH_INCLUDED
+#define NOCOPY_HH_INCLUDED
 
 /// A simple class to disallow copying of the class objects. Inherit from it to
 /// use it

objectcache.cc

@@ -1,5 +1,5 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
 #include "objectcache.hh"
 

objectcache.hh

@@ -1,8 +1,8 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
-#ifndef OBJECTCACHE_HH_INCLUDED__
-#define OBJECTCACHE_HH_INCLUDED__
+#ifndef OBJECTCACHE_HH_INCLUDED
+#define OBJECTCACHE_HH_INCLUDED
 
 #include <string>
 #include <list>

page_size.cc

@@ -1,5 +1,5 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
 #include "page_size.hh"
 

page_size.hh

@@ -1,8 +1,8 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
-#ifndef PAGE_SIZE_HH_INCLUDED__
-#define PAGE_SIZE_HH_INCLUDED__
+#ifndef PAGE_SIZE_HH_INCLUDED
+#define PAGE_SIZE_HH_INCLUDED
 
 /// Returns the page size used by this system
 unsigned getPageSize();

random.cc

@@ -1,17 +1,17 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
 #include "random.hh"
 #include <openssl/rand.h>
 
 namespace Random {
-void genarateTrue( void * buf, unsigned size )
+void generateTrue( void * buf, unsigned size )
 {
   if ( RAND_bytes( (unsigned char *) buf, size ) != 1 )
     throw exCantGenerate();
 }
 
-void genaratePseudo( void * buf, unsigned size )
+void generatePseudo( void * buf, unsigned size )
 {
   if ( RAND_pseudo_bytes( (unsigned char *) buf, size ) < 0 )
     throw exCantGenerate();

random.hh

@@ -1,8 +1,8 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
-#ifndef RANDOM_HH_INCLUDED__
-#define RANDOM_HH_INCLUDED__
+#ifndef RANDOM_HH_INCLUDED
+#define RANDOM_HH_INCLUDED
 
 #include <exception>
 
@@ -12,10 +12,10 @@ namespace Random {
 DEF_EX( exCantGenerate, "Error generating random sequence, try later", std::exception )
 
 /// This one fills the buffer with true randomness, suitable for a key
-void genarateTrue( void * buf, unsigned size );
+void generateTrue( void * buf, unsigned size );
 /// This one fills the buffer with pseudo randomness, suitable for salts but not
 /// keys
-void genaratePseudo( void * buf, unsigned size );
+void generatePseudo( void * buf, unsigned size );
 }
 
 #endif

rolling_hash.cc

@@ -1,5 +1,5 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
 #include "rolling_hash.hh"
 

rolling_hash.hh

@@ -1,8 +1,8 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
-#ifndef ROLLING_HASH_HH_INCLUDED__
-#define ROLLING_HASH_HH_INCLUDED__
+#ifndef ROLLING_HASH_HH_INCLUDED
+#define ROLLING_HASH_HH_INCLUDED
 
 #include <stdint.h>
 #include <stddef.h>

sha256.cc

@@ -1,5 +1,5 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
 #include "sha256.hh"
 

sha256.hh

@@ -1,8 +1,8 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
-#ifndef SHA256_HH_INCLUDED__
-#define SHA256_HH_INCLUDED__
+#ifndef SHA256_HH_INCLUDED
+#define SHA256_HH_INCLUDED
 
 #include <string>
 #include <openssl/sha.h>

sptr.hh

@@ -1,8 +1,8 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
-#ifndef SPTR_HH_INCLUDED__
-#define SPTR_HH_INCLUDED__
+#ifndef SPTR_HH_INCLUDED
+#define SPTR_HH_INCLUDED
 
 /// A generic non-intrusive smart-pointer template. We could use boost::, tr1::
 /// or whatever, but since there's no standard solution yet, it isn't worth
@@ -74,6 +74,9 @@ public:
   { if ( &other != this ) { reset(); p = other.p; count = other.count; increment(); }
     return * this; }
 
+  operator bool( void ) const
+  { return !!p; }
+
   bool operator ! ( void ) const
   { return !p; }
 

static_assert.hh

@@ -1,8 +1,8 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
-#ifndef STATIC_ASSERT_HH_INCLUDED__
-#define STATIC_ASSERT_HH_INCLUDED__
+#ifndef STATIC_ASSERT_HH_INCLUDED
+#define STATIC_ASSERT_HH_INCLUDED
 
 // Based on the one from the Boost library. It wouldn't make sense to depend on
 // boost just for that

storage_info_file.cc

@@ -1,11 +1,12 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
 #include <stddef.h>
 
 #include "encrypted_file.hh"
 #include "message.hh"
 #include "storage_info_file.hh"
+#include "debug.hh"
 
 namespace StorageInfoFile {
 
@@ -16,6 +17,7 @@ enum
 
 void save( string const & fileName, StorageInfo const & storageInfo )
 {
+  dPrintf( "Saving storage info...\n" );
   EncryptedFile::OutputStream os( fileName.c_str(), EncryptionKey::noKey(),
                                   NULL );
   FileHeader header;
@@ -28,6 +30,7 @@ void save( string const & fileName, StorageInfo const & storageInfo )
 
 void load( string const & fileName, StorageInfo & storageInfo )
 {
+  dPrintf( "Loading storage info...\n" );
   EncryptedFile::InputStream is( fileName.c_str(), EncryptionKey::noKey(),
                                  NULL );
   FileHeader header;
@@ -40,3 +43,47 @@ void load( string const & fileName, StorageInfo & storageInfo )
 }
 
 }
+
+namespace ExtendedStorageInfoFile {
+
+enum
+{
+  FileFormatVersion = 1
+};
+
+void save( string const & fileName, EncryptionKey const & encryptionKey,
+           ExtendedStorageInfo const & extendedStorageInfo )
+{
+  dPrintf( "Saving extended storage info, hasKey: %s\n",
+      encryptionKey.hasKey() ? "true" : "false" );
+  EncryptedFile::OutputStream os( fileName.c_str(), encryptionKey,
+                                  Encryption::ZeroIv );
+  os.writeRandomIv();
+
+  FileHeader header;
+  header.set_version( FileFormatVersion );
+  Message::serialize( header, os );
+
+  Message::serialize( extendedStorageInfo, os );
+  os.writeAdler32();
+}
+
+void load( string const & fileName, EncryptionKey const & encryptionKey,
+           ExtendedStorageInfo & extendedStorageInfo )
+{
+  dPrintf( "Loading extended storage info, hasKey: %s\n",
+      encryptionKey.hasKey() ? "true" : "false" );
+  EncryptedFile::InputStream is( fileName.c_str(), encryptionKey,
+                                 Encryption::ZeroIv );
+  is.consumeRandomIv();
+
+  FileHeader header;
+  Message::parse( header, is );
+  if ( header.version() != FileFormatVersion )
+    throw exUnsupportedVersion();
+
+  Message::parse( extendedStorageInfo, is );
+  is.checkAdler32();
+}
+
+}

storage_info_file.hh

@@ -1,8 +1,8 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
-#ifndef STORAGE_INFO_FILE_HH_INCLUDED__
-#define STORAGE_INFO_FILE_HH_INCLUDED__
+#ifndef STORAGE_INFO_FILE_HH_INCLUDED
+#define STORAGE_INFO_FILE_HH_INCLUDED
 
 #include <exception>
 #include <string>
@@ -25,4 +25,18 @@ void save( string const & fileName, StorageInfo const & );
 void load( string const & fileName, StorageInfo & );
 }
 
+namespace ExtendedStorageInfoFile {
+
+using std::string;
+
+DEF_EX( Ex, "Extended storage info file exception", std::exception )
+DEF_EX( exUnsupportedVersion, "Unsupported version of the extended storage info file format", Ex )
+
+/// Saves the given ExtendedStorageInfo data into the given file
+void save( string const & fileName, EncryptionKey const &, ExtendedStorageInfo const & );
+
+/// Loads the given ExtendedStorageInfo data from the given file
+void load( string const & fileName, EncryptionKey const &, ExtendedStorageInfo & );
+}
+
 #endif

tartool/CMakeLists.txt

@@ -1,5 +1,5 @@
-# Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-# Part of ZBackup. Licensed under GNU GPLv2 or later
+# Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+# Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
 cmake_minimum_required( VERSION 2.6.0 )
 project( tartool )

tartool/tartool.cc

@@ -1,5 +1,5 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
 #include <stdio.h>
 #include <stdlib.h>

tests/bundle/bundle.pro

@@ -0,0 +1,50 @@
+######################################################################
+# Automatically generated by qmake (2.01a) Sun Jul 14 20:54:52 2013
+######################################################################
+
+TEMPLATE = app
+TARGET = 
+DEPENDPATH += .
+INCLUDEPATH += .
+
+CONFIG = debug
+
+LIBS += -lcrypto -lprotobuf -lz -lprotobuf -llzma -llzo2
+DEFINES += __STDC_FORMAT_MACROS
+DEFINES += HAVE_LIBLZO
+
+# Input
+SOURCES += test_bundle.cc \
+    ../../unbuffered_file.cc \
+    ../../tmp_mgr.cc \
+    ../../page_size.cc \
+    ../../random.cc \
+    ../../encryption_key.cc \
+    ../../encryption.cc \
+    ../../encrypted_file.cc \
+    ../../file.cc \
+    ../../dir.cc \
+    ../../bundle.cc \
+    ../../message.cc \
+    ../../hex.cc \
+    ../../compression.cc \
+    ../../zbackup.pb.cc
+
+HEADERS += \
+    ../../unbuffered_file.hh \
+    ../../tmp_mgr.hh \
+    ../../adler32.hh \
+    ../../page_size.hh \
+    ../../random.hh \
+    ../../encryption_key.hh \
+    ../../encrypted_file.hh \
+    ../../encryption.hh \
+    ../../ex.hh \
+    ../../file.hh \
+    ../../dir.hh \
+    ../../bundle.hh \
+    ../../message.hh \
+    ../../hex.hh \
+    ../../compression.hh \
+    ../../message.hh \
+    ../../zbackup.pb.h

tests/bundle/test_bundle.cc

@@ -0,0 +1,171 @@
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <vector>
+#include "../../encrypted_file.hh"
+#include "../../encryption_key.hh"
+#include "../../random.hh"
+#include "../../tmp_mgr.hh"
+#include "../../check.hh"
+#include "../../adler32.hh"
+#include "../../bundle.hh"
+#include "../../compression.hh"
+#include "../../message.hh"
+
+using namespace Compression;
+
+
+char tmpbuf[100];
+
+void testCompatibility()
+{
+  // The LZO code uses a different file header than the previous code
+  // because it adds the compression_method field. Nevertheless, it
+  // must be compatible with previous code.
+
+  TmpMgr tmpMgr( "/dev/shm" );
+  sptr< TemporaryFile > tempFile = tmpMgr.makeTemporaryFile();
+  std::string fileName = tempFile->getFileName();
+
+  EncryptionKey noKey( std::string(), NULL );
+
+
+  // Write old header, read as new header
+  {
+    {
+      EncryptedFile::OutputStream os( fileName.c_str(), noKey, Encryption::ZeroIv );
+
+      FileHeader header;
+      header.set_version( 42 );
+      Message::serialize( header, os );
+    }
+
+    {
+      EncryptedFile::InputStream is( fileName.c_str(), noKey, Encryption::ZeroIv );
+
+      BundleFileHeader header;
+      Message::parse( header, is );
+
+      CHECK( header.version()            == 42,     "version is wrong when reading old header with new program" );
+      CHECK( header.compression_method() == "lzma", "compression_method is wrong when reading old header with new program" );
+    }
+  }
+
+  // Write new header, read as old header
+  //NOTE In the real code, this will only work, if the file uses LZMA. If it doesn't, the version
+  //     field is increased and the old code will refuse to read the file.
+  {
+    {
+      EncryptedFile::OutputStream os( fileName.c_str(), noKey, Encryption::ZeroIv );
+
+      BundleFileHeader header;
+      header.set_version( 42 );
+      Message::serialize( header, os );
+    }
+
+    {
+      EncryptedFile::InputStream is( fileName.c_str(), noKey, Encryption::ZeroIv );
+
+      FileHeader header;
+      Message::parse( header, is );
+
+      CHECK( header.version() == 42,     "version is wrong when reading new header with old program" );
+      // cannot check compression_method because the field doesn't exist
+    }
+  }
+
+  printf("compatibility test successful.\n");
+}
+
+void readAndWrite( EncryptionKey const & key,
+  const_sptr<CompressionMethod> compression1, const_sptr<CompressionMethod> compression2 )
+{
+  // temporary file for the bundle
+  TmpMgr tmpMgr( "/dev/shm" );
+  sptr< TemporaryFile > tempFile = tmpMgr.makeTemporaryFile();
+
+  // some chunk data
+  int     chunkCount = rand() % 30;
+  size_t  chunkSize  = rand() % 20 ? 64*1024 : 10;
+  char**  chunks      = new char*[chunkCount];
+  string* chunkIds    = new string[chunkCount];
+
+  CompressionMethod::defaultCompression = compression1;
+
+  // write bundle
+  {
+    Bundle::Creator bundle;
+
+    for (int i=0;i<chunkCount;i++) {
+      chunks[i] = new char[chunkSize];
+      Random::genaratePseudo( chunks[i], chunkSize );
+
+      //TODO make it look like a real Id (or even let it match the data)
+      //TODO make sure we don't have any duplicate Ids
+      sprintf(tmpbuf, "0x%08x", rand());
+      chunkIds[i] = string(tmpbuf);
+
+      bundle.addChunk( chunkIds[i], chunks[i], chunkSize );
+    }
+
+    bundle.write( tempFile->getFileName().c_str(), key );
+  }
+
+  CompressionMethod::defaultCompression = compression2;
+
+  // read it and compare
+  {
+    Bundle::Reader bundle( tempFile->getFileName().c_str(), key );
+
+    for (int i=0;i<chunkCount;i++) {
+      string data;
+      size_t size;
+      bool ret = bundle.get( chunkIds[i], data, size );
+      CHECK( ret, "bundle.get returned false for chunk %d (%s)", i, chunkIds[i].c_str() );
+      CHECK( size == chunkSize, "wrong chunk size for chunk %d (%s)", i, chunkIds[i].c_str() );
+      CHECK( memcmp(data.c_str(), chunks[i], chunkSize) == 0, "wrong chunk data for chunk %d (%s)", i, chunkIds[i].c_str() );
+    }
+  }
+
+  // clean up
+  for (int i=0;i<chunkCount;i++)
+    delete[] chunks[i];
+  delete[] chunks;
+  //TODO does that call the destructors?
+  delete[] chunkIds;
+
+  printf(".");
+  fflush(stdout);
+}
+
+int main()
+{
+  EncryptionKeyInfo keyInfo;
+  EncryptionKey::generate( "blah", keyInfo );
+  EncryptionKey key( "blah", &keyInfo );
+  EncryptionKey noKey( std::string(), NULL );
+
+  testCompatibility();
+
+  std::vector< const_sptr<CompressionMethod> > compressions;
+  for ( CompressionMethod::iterator it = CompressionMethod::begin(); it!=CompressionMethod::end(); ++it ) {
+    printf( "supported compression: %s\n", (*it)->getName().c_str() );
+    compressions.push_back( *it );
+  }
+
+  for ( size_t iteration = 100; iteration--; ) {
+    // default compression while writing the file
+    const_sptr<CompressionMethod> compression1 = compressions[ rand() % compressions.size() ];
+    // default compression while reading the file
+    // The reader should ignore it and always use the compression that was used for the file.
+    const_sptr<CompressionMethod> compression2 = compressions[ rand() % compressions.size() ];
+
+    readAndWrite( ( rand() & 1 ) ? key : noKey, compression1, compression2 );
+  }
+
+  printf("\n");
+
+  return 0;
+}

tests/encrypted_file/test_encrypted_file.cc

@@ -1,5 +1,5 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
 #include <stdlib.h>
 #include "../../encrypted_file.hh"

tests/rolling_hash/test_rolling_hash.cc

@@ -1,5 +1,5 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
 #include <stdlib.h>
 #include <stdio.h>

tmp_mgr.cc

@@ -1,8 +1,9 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
 #include "tmp_mgr.hh"
 
+#include <sys/stat.h>
 #include <stdlib.h>
 #include <unistd.h>
 #include "dir.hh"
@@ -44,6 +45,8 @@ sptr< TemporaryFile > TmpMgr::makeTemporaryFile()
   string name( Dir::addPath( path, "XXXXXX") );
 
   int fd = mkstemp( &name[ 0 ] );
+  if ( fchmod ( fd, S_IRUSR | S_IWUSR | S_IRGRP ) != 0 )
+    throw exCantCreate( path );
 
   if ( fd == -1 || close( fd ) != 0 )
     throw exCantCreate( path );

tmp_mgr.hh

@@ -1,8 +1,8 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
-#ifndef TMP_MGR_HH_INCLUDED__
-#define TMP_MGR_HH_INCLUDED__
+#ifndef TMP_MGR_HH_INCLUDED
+#define TMP_MGR_HH_INCLUDED
 
 #include <exception>
 #include <string>

unbuffered_file.cc

@@ -1,5 +1,5 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
 #define _LARGEFILE64_SOURCE
 
@@ -12,12 +12,21 @@
 #include "check.hh"
 #include "unbuffered_file.hh"
 
+
+#if defined( __APPLE__ ) || defined( __OpenBSD__ ) || defined(__FreeBSD__) || defined(__CYGWIN__)
+#define lseek64 lseek
+#endif
+
+
 UnbufferedFile::UnbufferedFile( char const * fileName, Mode mode )
   throw( exCantOpen )
 {
-  int flags = O_LARGEFILE |
-              ( mode == WriteOnly ? ( O_WRONLY | O_CREAT | O_TRUNC ) :
-                                    O_RDONLY );
+
+  int flags = ( mode == ReadWrite ? ( O_RDWR | O_CREAT ) :
+     ( mode == WriteOnly ? ( O_WRONLY | O_CREAT | O_TRUNC ) : O_RDONLY ) );
+#if !defined( __APPLE__ ) && !defined( __OpenBSD__ ) && !defined(__FreeBSD__) && !defined(__CYGWIN__)
+  flags |= O_LARGEFILE;
+#endif
   fd = open( fileName, flags, 0666 );
   if ( fd < 0 )
     throw exCantOpen( fileName );
@@ -91,6 +100,12 @@ void UnbufferedFile::seekCur( Offset offset ) throw( exSeekError )
     throw exSeekError();
 }
 
+void UnbufferedFile::seek( Offset offset ) throw( exSeekError )
+{
+  if ( lseek64( fd, offset, SEEK_SET ) < 0 )
+    throw exSeekError();
+}
+
 UnbufferedFile::~UnbufferedFile() throw()
 {
   close( fd );

unbuffered_file.hh

@@ -1,8 +1,8 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
-#ifndef UNBUFFERED_FILE_HH_INCLUDED__
-#define UNBUFFERED_FILE_HH_INCLUDED__
+#ifndef UNBUFFERED_FILE_HH_INCLUDED
+#define UNBUFFERED_FILE_HH_INCLUDED
 
 #include <stddef.h>
 #include <stdint.h>
@@ -31,7 +31,8 @@ public:
   enum Mode
   {
     ReadOnly,
-    WriteOnly
+    WriteOnly,
+    ReadWrite
   };
 
   typedef int64_t Offset;
@@ -53,6 +54,9 @@ public:
   /// Seeks to the given offset, relative to the current file offset
   void seekCur( Offset ) throw( exSeekError );
 
+  /// Seeks to the given offset, relative to the beginning
+  void seek( Offset ) throw( exSeekError );
+
   ~UnbufferedFile() throw();
 
 private:

utils.hh

@@ -0,0 +1,21 @@
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
+
+#ifndef UTILS_HH_INCLUDED
+#define UTILS_HH_INCLUDED
+
+#include <sstream>
+
+namespace Utils {
+
+template <typename T>
+std::string numberToString( T pNumber )
+{
+  std::ostringstream oOStrStream;
+  oOStrStream << pNumber;
+  return oOStrStream.str();
+}
+
+}
+
+#endif

version.cc

@@ -0,0 +1,9 @@
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
+
+#include <string>
+#ifndef ZBACKUP_VERSION
+std::string zbackup_version( "1.4" );
+#else
+std::string zbackup_version( ZBACKUP_VERSION );
+#endif

version.hh

@@ -0,0 +1,10 @@
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
+
+#ifndef VERSION_HH_INCLUDED
+#define VERSION_HH_INCLUDED
+#include <string>
+
+extern std::string zbackup_version;
+
+#endif

zbackup.cc

@@ -1,422 +1,217 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
-#include <ctype.h>
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <time.h>
-#include <unistd.h>
-#include <vector>
-
-#include "backup_creator.hh"
-#include "backup_file.hh"
-#include "backup_restorer.hh"
+#include "zutils.hh"
 #include "debug.hh"
-#include "dir.hh"
-#include "encryption_key.hh"
-#include "ex.hh"
-#include "file.hh"
-#include "mt.hh"
-#include "sha256.hh"
-#include "sptr.hh"
-#include "storage_info_file.hh"
-#include "zbackup.hh"
-
-using std::vector;
-
-Paths::Paths( string const & storageDir ): storageDir( storageDir )
-{
-}
-
-string Paths::getTmpPath()
-{
-  return string( Dir::addPath( storageDir, "tmp" ) );
-}
-
-string Paths::getBundlesPath()
-{
-  return string( Dir::addPath( storageDir, "bundles" ) );
-}
-
-string Paths::getStorageInfoPath()
-{
-  return string( Dir::addPath( storageDir, "info" ) );
-}
-
-string Paths::getIndexPath()
-{
-  return string( Dir::addPath( storageDir, "index" ) );
-}
-
-string Paths::getBackupsPath()
-{
-  return string( Dir::addPath( storageDir, "backups" ) );
-}
-
-ZBackupBase::ZBackupBase( string const & storageDir, string const & password ):
-  Paths( storageDir ), storageInfo( loadStorageInfo() ),
-  encryptionkey( password, storageInfo.has_encryption_key() ?
-                   &storageInfo.encryption_key() : 0 ),
-  tmpMgr( getTmpPath() ),
-  chunkIndex( encryptionkey, tmpMgr, getIndexPath() )
-{
-}
-
-StorageInfo ZBackupBase::loadStorageInfo()
-{
-  StorageInfo storageInfo;
-
-  StorageInfoFile::load( getStorageInfoPath(), storageInfo );
-
-  return storageInfo;
-}
-
-void ZBackupBase::initStorage( string const & storageDir,
-                               string const & password,
-                               bool isEncrypted )
-{
-  StorageInfo storageInfo;
-  // TODO: make the following configurable
-  storageInfo.set_chunk_max_size( 65536 );
-  storageInfo.set_bundle_max_payload_size( 0x200000 );
-
-  if ( isEncrypted )
-    EncryptionKey::generate( password,
-                             *storageInfo.mutable_encryption_key() );
-
-  Paths paths( storageDir );
-
-  if ( !Dir::exists( storageDir ) )
-    Dir::create( storageDir );
-
-  if ( !Dir::exists( paths.getBundlesPath() ) )
-    Dir::create( paths.getBundlesPath() );
-
-  if ( !Dir::exists( paths.getBackupsPath() ) )
-    Dir::create( paths.getBackupsPath() );
-
-  if ( !Dir::exists( paths.getIndexPath() ) )
-    Dir::create( paths.getIndexPath() );
-
-  string storageInfoPath( paths.getStorageInfoPath() );
-
-  if ( File::exists( storageInfoPath ) )
-    throw exWontOverwrite( storageInfoPath );
-
-  StorageInfoFile::save( storageInfoPath, storageInfo );
-}
-
-string ZBackupBase::deriveStorageDirFromBackupsFile( string const &
-                                                     backupsFile )
-{
-  // TODO: handle cases when there's a backup/ folder within the backup/ folder
-  // correctly
-  string realPath = Dir::getRealPath( Dir::getDirName( backupsFile ) );
-  size_t pos;
-  if ( realPath.size() >= 8 && strcmp( realPath.c_str() + realPath.size() - 8,
-                                       "/backups") == 0 )
-    pos = realPath.size() - 8;
-  else
-    pos = realPath.rfind( "/backups/" );
-  if ( pos == string::npos )
-    throw exCantDeriveStorageDir( backupsFile );
-  else
-    return realPath.substr( 0, pos );
-}
-
-ZBackup::ZBackup( string const & storageDir, string const & password,
-                  size_t threads ):
-  ZBackupBase( storageDir, password ),
-  chunkStorageWriter( storageInfo, encryptionkey, tmpMgr, chunkIndex,
-                      getBundlesPath(), getIndexPath(), threads )
-{
-}
-
-void ZBackup::backupFromStdin( string const & outputFileName )
-{
-  if ( isatty( fileno( stdin ) ) )
-    throw exWontReadFromTerminal();
-
-  if ( File::exists( outputFileName ) )
-    throw exWontOverwrite( outputFileName );
-
-  Sha256 sha256;
-  BackupCreator backupCreator( storageInfo, chunkIndex, chunkStorageWriter );
-
-  time_t startTime = time( 0 );
-  uint64_t totalDataSize = 0;
-
-  for ( ; ; )
-  {
-    size_t toRead = backupCreator.getInputBufferSize();
-//    dPrintf( "Reading up to %u bytes from stdin\n", toRead );
-
-    void * inputBuffer = backupCreator.getInputBuffer();
-    size_t rd = fread( inputBuffer, 1, toRead, stdin );
-
-    if ( !rd )
-    {
-      if ( feof( stdin ) )
-      {
-        dPrintf( "No more input on stdin\n" );
-        break;
-      }
-      else
-        throw exStdinError();
-    }
-
-    sha256.add( inputBuffer, rd );
-
-    backupCreator.handleMoreData( rd );
-
-    totalDataSize += rd;
-  }
-
-  // Finish up with the creator
-  backupCreator.finish();
-
-  string serialized;
-  backupCreator.getBackupData( serialized );
-
-  BackupInfo info;
-
-  info.set_sha256( sha256.finish() );
-  info.set_size( totalDataSize );
-
-  // Shrink the serialized data iteratively until it wouldn't shrink anymore
-  for ( ; ; )
-  {
-    BackupCreator backupCreator( storageInfo, chunkIndex, chunkStorageWriter );
-    char const * ptr = serialized.data();
-    size_t left = serialized.size();
-    while( left )
-    {
-      size_t bufferSize = backupCreator.getInputBufferSize();
-      size_t toCopy = bufferSize > left ? left : bufferSize;
-
-      memcpy( backupCreator.getInputBuffer(), ptr, toCopy );
-      backupCreator.handleMoreData( toCopy );
-      ptr += toCopy;
-      left -= toCopy;
-    }
-
-    backupCreator.finish();
-
-    string newGen;
-    backupCreator.getBackupData( newGen );
-
-    if ( newGen.size() < serialized.size() )
-    {
-      serialized.swap( newGen );
-      info.set_iterations( info.iterations() + 1 );
-    }
-    else
-      break;
-  }
-
-  dPrintf( "Iterations: %u\n", info.iterations() );
-
-  info.mutable_backup_data()->swap( serialized );
-
-  info.set_time( time( 0 ) - startTime );
-
-  // Commit the bundles to the disk before creating the final output file
-  chunkStorageWriter.commit();
-
-  // Now save the resulting BackupInfo
-
-  sptr< TemporaryFile > tmpFile = tmpMgr.makeTemporaryFile();
-  BackupFile::save( tmpFile->getFileName(), encryptionkey, info );
-  tmpFile->moveOverTo( outputFileName );
-}
-
-ZRestore::ZRestore( string const & storageDir, string const & password,
-                    size_t cacheSize ):
-  ZBackupBase( storageDir, password ),
-  chunkStorageReader( storageInfo, encryptionkey, chunkIndex, getBundlesPath(),
-                      cacheSize )
-{
-}
-
-void ZRestore::restoreToStdin( string const & inputFileName )
-{
-  if ( isatty( fileno( stdout ) ) )
-    throw exWontWriteToTerminal();
-
-  BackupInfo backupInfo;
-
-  BackupFile::load( inputFileName, encryptionkey, backupInfo );
-
-  string backupData;
-
-  // Perform the iterations needed to get to the actual user backup data
-  for ( ; ; )
-  {
-    backupData.swap( *backupInfo.mutable_backup_data() );
-
-    if ( backupInfo.iterations() )
-    {
-      struct StringWriter: public DataSink
-      {
-        string result;
-
-        virtual void saveData( void const * data, size_t size )
-        {
-          result.append( ( char const * ) data, size );
-        }
-      } stringWriter;
-
-      BackupRestorer::restore( chunkStorageReader, backupData, stringWriter );
-      backupInfo.mutable_backup_data()->swap( stringWriter.result );
-      backupInfo.set_iterations( backupInfo.iterations() - 1 );
-    }
-    else
-      break;
-  }
-
-  struct StdoutWriter: public DataSink
-  {
-    Sha256 sha256;
-
-    virtual void saveData( void const * data, size_t size )
-    {
-      sha256.add( data, size );
-      if ( fwrite( data, size, 1, stdout ) != 1 )
-        throw exStdoutError();
-    }
-  } stdoutWriter;
-
-  BackupRestorer::restore( chunkStorageReader, backupData, stdoutWriter );
-
-  if ( stdoutWriter.sha256.finish() != backupInfo.sha256() )
-    throw exChecksumError();
-}
+#include "version.hh"
+#include "utils.hh"
 
+DEF_EX( exSpecifyTwoKeys, "Specify password flag (--non-encrypted or --password-file)"
+  " for import/export/passwd operation twice (first for source and second for destination)", std::exception )
 DEF_EX( exNonEncryptedWithKey, "--non-encrypted and --password-file are incompatible", std::exception )
 DEF_EX( exSpecifyEncryptionOptions, "Specify either --password-file or --non-encrypted", std::exception )
-DEF_EX_STR( exInvalidThreadsValue, "Invalid threads value specified:", std::exception )
 
 int main( int argc, char *argv[] )
 {
   try
   {
-    char const * passwordFile = 0;
-    bool nonEncrypted = false;
-    size_t const defaultThreads = getNumberOfCpus();
-    size_t threads = defaultThreads;
-    size_t const defaultCacheSizeMb = 40;
-    size_t cacheSizeMb = defaultCacheSizeMb;
+    dPrintf( "ZBackup version %s\n", zbackup_version.c_str() );
+
+    bool printHelp = false;
     vector< char const * > args;
+    vector< string > passwords;
+    Config config;
 
     for( int x = 1; x < argc; ++x )
     {
+      string option;
+      Config::OptionType optionType = Config::Runtime;
+
       if ( strcmp( argv[ x ], "--password-file" ) == 0 && x + 1 < argc )
       {
-        passwordFile = argv[ x + 1 ];
+        // Read the password
+        char const * passwordFile = argv[ x + 1 ];
+        string passwordData;
+        if ( passwordFile )
+        {
+          File f( passwordFile, File::ReadOnly );
+          passwordData.resize( f.size() );
+          f.read( &passwordData[ 0 ], passwordData.size() );
+
+          // If the password ends with \n, remove that last \n. Many editors will
+          // add \n there even if a user doesn't want them to
+          if ( !passwordData.empty() &&
+               passwordData[ passwordData.size() - 1 ] == '\n' )
+            passwordData.resize( passwordData.size() - 1 );
+          passwords.push_back( passwordData );
+        }
         ++x;
       }
       else
       if ( strcmp( argv[ x ], "--non-encrypted" ) == 0 )
-        nonEncrypted = true;
+      {
+          passwords.push_back( "" );
+      }
       else
       if ( strcmp( argv[ x ], "--silent" ) == 0 )
         verboseMode = false;
       else
+      if ( strcmp( argv[ x ], "--exchange" ) == 0 && x + 1 < argc )
+      {
+        fprintf( stderr, "%s is deprecated, use -O exchange instead\n", argv[ x ] );
+        option = argv[ x ] + 2;//; + "=" + argv[ x + 1 ];
+        option += "=";
+        option += argv[ x + 1 ];
+        goto parse_option;
+      }
+      else
       if ( strcmp( argv[ x ], "--threads" ) == 0 && x + 1 < argc )
       {
-        int n;
-        if ( sscanf( argv[ x + 1 ], "%zu %n", &threads, &n ) != 1 ||
-             argv[ x + 1 ][ n ] || threads < 1 )
-          throw exInvalidThreadsValue( argv[ x + 1 ] );
-        ++x;
+        fprintf( stderr, "%s is deprecated, use -O threads instead\n", argv[ x ] );
+        option = argv[ x ] + 2;
+        option += "=";
+        option += argv[ x + 1 ];
+        goto parse_option;
       }
       else
       if ( strcmp( argv[ x ], "--cache-size" ) == 0 && x + 1 < argc )
       {
+        fprintf( stderr, "%s is deprecated, use -O cache-size instead\n", argv[ x ] );
+        size_t cacheSizeMb;
         char suffix[ 16 ];
         int n;
         if ( sscanf( argv[ x + 1 ], "%zu %15s %n",
-                     &cacheSizeMb, suffix, &n ) == 2 && !argv[ x + 1 ][ n ] )
+              &cacheSizeMb, suffix, &n ) == 2 && !argv[ x + 1][ n ] )
         {
-          // Check the suffix
-          for ( char * c = suffix; *c; ++c )
-            *c = tolower( *c );
+          option = argv[ x ] + 2;
+          option += "=" + Utils::numberToString( cacheSizeMb ) + "MiB";
+          goto parse_option;
+        }
+      }
+      else
+      if ( strcmp( argv[ x ], "--compression" ) == 0 && x + 1 < argc )
+      {
+        fprintf( stderr, "%s is deprecated, use -o bundle.compression_method instead\n", argv[ x ] );
+        option = argv[ x ] + 2;
+        option += "=";
+        option += argv[ x + 1 ];
+        optionType = Config::Storable;
+        goto parse_option;
+      }
+      else
+      if ( strcmp( argv[ x ], "--help" ) == 0 || strcmp( argv[ x ], "-h" ) == 0 )
+      {
+        printHelp = true;
+      }
+      else
+      if ( ( strcmp( argv[ x ], "-o" ) == 0 || strcmp( argv[ x ], "-O" ) == 0 )
+          && x + 1 < argc )
+      {
+        option = argv[ x + 1 ];
+        if ( !option.empty() )
+        {
+          if ( strcmp( argv[ x ], "-O" ) == 0 )
+            optionType = Config::Runtime;
+          else
+          if ( strcmp( argv[ x ], "-o" ) == 0 )
+            optionType = Config::Storable;
 
-          if ( strcmp( suffix, "mb" ) != 0 )
+          if ( strcmp( option.c_str(), "help" ) == 0 )
           {
-            fprintf( stderr, "Invalid suffix specified in cache size: %s. "
-                     "The only supported suffix is 'mb' for megabytes\n",
-                     argv[ x + 1 ] );
-            return EXIT_FAILURE;
+            config.showHelp( optionType );
+            return EXIT_SUCCESS;
+          }
+          else
+          {
+parse_option:
+            if ( !config.parseOrValidate( option, optionType ) )
+              goto invalid_option;
           }
-
-          ++x;
         }
         else
         {
-          fprintf( stderr, "Invalid cache size value specified: %s. "
-                   "Must be a number with the 'mb' suffix, e.g. '100mb'\n",
-                   argv[ x + 1 ] );
+invalid_option:
+          fprintf( stderr, "Invalid option specified: %s\n",
+                   option.c_str() );
           return EXIT_FAILURE;
         }
+        ++x;
       }
       else
         args.push_back( argv[ x ] );
     }
 
-    if ( nonEncrypted && passwordFile )
-      throw exNonEncryptedWithKey();
-
-    if ( args.size() < 1 )
+    if ( args.size() < 1 || printHelp )
     {
       fprintf( stderr,
-"ZBackup, a versatile deduplicating backup tool, version 1.0\n"
-"Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>\n"
-"Comes with no warranty. Licensed under GNU GPLv2 or later.\n"
+"ZBackup, a versatile deduplicating backup tool, version %s\n"
+"Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and\n"
+"ZBackup contributors\n"
+"Comes with no warranty. Licensed under GNU GPLv2 or later + OpenSSL.\n"
 "Visit the project's home page at http://zbackup.org/\n\n"
 
-"Usage: %s [flags] <command> [command args]\n"
+"Usage: %s [flags] <command [action]> [command args]\n"
 "  Flags: --non-encrypted|--password-file <file>\n"
+"          password flag should be specified twice if\n"
+"          import/export/passwd command specified\n"
 "         --silent (default is verbose)\n"
-"         --threads <number> (default is %zu on your system)\n"
-"         --cache-size <number> MB (default is %zu)\n"
+"         --help|-h show this message\n"
+"         -O <Option[=Value]> (overrides runtime configuration,\n"
+"          can be specified multiple times,\n"
+"          for detailed runtime options overview run with -O help)\n"
+"         -o <Option[=Value]> (overrides storable repository\n"
+"          configuration, can be specified multiple times,\n"
+"          for detailed storable options overview run with -o help)\n"
 "  Commands:\n"
-"    init <storage path> - initializes new storage;\n"
-"    backup <backup file name> - performs a backup from stdin;\n"
-"    restore <backup file name> - restores a backup to stdout.\n", *argv,
-               defaultThreads, defaultCacheSizeMb );
+"    init <storage path> - initializes new storage\n"
+"    backup <backup file name> - performs a backup from stdin\n"
+"    restore <backup file name> - restores a backup to stdout\n"
+"    restore <backup file name> <output file name> -\n"
+"            restores a backup to file using two-pass \"cacheless\" process\n"
+"    export <source storage path> <destination storage path> -\n"
+"            performs export from source to destination storage\n"
+"    import <source storage path> <destination storage path> -\n"
+"            performs import from source to destination storage,\n"
+"            for export/import storage path must be\n"
+"            a valid (initialized) storage\n"
+"    gc [fast|deep] <storage path> - performs garbage\n"
+"            collection (default is fast)\n"
+"    passwd <storage path> - changes repo info file passphrase\n"
+//"    info <storage path> - shows repo information\n"
+"    config [show|edit|set|reset] <storage path> - performs\n"
+"            configuration manipulations (default is show)\n"
+"", zbackup_version.c_str(), *argv );
       return EXIT_FAILURE;
     }
 
-    // Read the password
-    string passwordData;
-    if ( passwordFile )
-    {
-      File f( passwordFile, File::ReadOnly );
-      passwordData.resize( f.size() );
-      f.read( &passwordData[ 0 ], passwordData.size() );
-
-      // If the password ends with \n, remove that last \n. Many editors will
-      // add \n there even if a user doesn't want them to
-      if ( !passwordData.empty() &&
-           passwordData[ passwordData.size() - 1 ] == '\n' )
-        passwordData.resize( passwordData.size() - 1 );
-    }
+    if ( passwords.size() > 1 &&
+        ( ( passwords[ 0 ].empty() && !passwords[ 1 ].empty() ) ||
+          ( !passwords[ 0 ].empty() && passwords[ 1 ].empty() ) ) &&
+        ( strcmp( args[ 0 ], "export" ) != 0 &&
+	  strcmp( args[ 0 ], "import" ) != 0 &&
+	  strcmp( args[ 0 ], "passwd" )	) )
+      throw exNonEncryptedWithKey();
+    else
+      if ( passwords.size() != 2 &&
+          ( strcmp( args[ 0 ], "export" ) == 0 ||
+	    strcmp( args[ 0 ], "import" ) == 0 ||
+	    strcmp( args[ 0 ], "passwd" ) == 0 ) )
+        throw exSpecifyTwoKeys();
+    else
+      if ( passwords.size() < 1 )
+        throw exSpecifyEncryptionOptions();
 
     if ( strcmp( args[ 0 ], "init" ) == 0 )
     {
       // Perform the init
       if ( args.size() != 2 )
       {
-        fprintf( stderr, "Usage: %s init <storage path>\n", *argv );
+        fprintf( stderr, "Usage: %s %s <storage path>\n", *argv, args[ 0 ] );
         return EXIT_FAILURE;
       }
-      if ( !nonEncrypted && !passwordFile )
-          throw exSpecifyEncryptionOptions();
 
-      ZBackup::initStorage( args[ 1 ], passwordData, !nonEncrypted );
+      ZBackup::initStorage( args[ 1 ],
+          passwords[ 0 ], !passwords[ 0 ].empty(), config );
     }
     else
     if ( strcmp( args[ 0 ], "backup" ) == 0 )
@@ -424,27 +219,198 @@ int main( int argc, char *argv[] )
       // Perform the backup
       if ( args.size() != 2 )
       {
-        fprintf( stderr, "Usage: %s backup <backup file name>\n",
-                 *argv );
+        fprintf( stderr, "Usage: %s %s <backup file name>\n",
+                 *argv, args[ 0 ] );
         return EXIT_FAILURE;
       }
       ZBackup zb( ZBackup::deriveStorageDirFromBackupsFile( args[ 1 ] ),
-                  passwordData, threads );
+                  passwords[ 0 ], config );
       zb.backupFromStdin( args[ 1 ] );
     }
     else
     if ( strcmp( args[ 0 ], "restore" ) == 0 )
     {
       // Perform the restore
-      if ( args.size() != 2 )
+      if ( args.size() != 2 && args.size() != 3 )
       {
-        fprintf( stderr, "Usage: %s restore <backup file name>\n",
-                 *argv );
+        fprintf( stderr, "Usage: %s %s <backup file name> [output file name]\n",
+                 *argv , args[ 0 ] );
         return EXIT_FAILURE;
       }
       ZRestore zr( ZRestore::deriveStorageDirFromBackupsFile( args[ 1 ] ),
-                   passwordData, cacheSizeMb * 1048576 );
-      zr.restoreToStdin( args[ 1 ] );
+                   passwords[ 0 ], config );
+      if ( args.size() == 3 )
+        zr.restoreToFile( args[ 1 ], args[ 2 ] );
+      else
+        zr.restoreToStdin( args[ 1 ] );
+    }
+    else
+    if ( strcmp( args[ 0 ], "export" ) == 0 || strcmp( args[ 0 ], "import" ) == 0 )
+    {
+      if ( args.size() != 3 )
+      {
+        fprintf( stderr, "Usage: %s %s <source storage path> <destination storage path>\n",
+                 *argv, args[ 0 ] );
+        return EXIT_FAILURE;
+      }
+      if ( config.runtime.exchange.none() )
+      {
+        fprintf( stderr, "Specify any --exchange flag\n" );
+        return EXIT_FAILURE;
+      }
+
+      int src, dst;
+      if ( strcmp( args[ 0 ], "export" ) == 0 )
+      {
+        src = 1;
+        dst = 2;
+      }
+      else
+      if ( strcmp( args[ 0 ], "import" ) == 0 )
+      {
+        src = 2;
+        dst = 1;
+      }
+      dPrintf( "%s src: %s\n", args[ 0 ], args[ src ] );
+      dPrintf( "%s dst: %s\n", args[ 0 ], args[ dst ] );
+
+      ZExchange ze( ZBackupBase::deriveStorageDirFromBackupsFile( args[ src ], true ),
+                    passwords[ src - 1 ],
+                    ZBackupBase::deriveStorageDirFromBackupsFile( args[ dst ], true ),
+                    passwords[ dst - 1 ],
+                    config );
+      ze.exchange();
+    }
+    else
+    if ( strcmp( args[ 0 ], "gc" ) == 0 )
+    {
+      // Perform the garbage collection
+      if ( args.size() < 2 || args.size() > 3 )
+      {
+        fprintf( stderr, "Usage: %s %s [fast|deep] <storage path>\n",
+                 *argv, args[ 0 ] );
+        return EXIT_FAILURE;
+      }
+
+      int fieldStorage = 1;
+      int fieldAction = 2;
+
+      if ( args.size() == 3 )
+      {
+        fieldStorage = 2;
+        fieldAction = 1;
+      }
+
+      if ( args.size() > 2 && strcmp( args[ fieldAction ], "fast" ) == 0 )
+      {
+        ZCollector zc( ZBackupBase::deriveStorageDirFromBackupsFile( args[ fieldStorage ], true ),
+            passwords[ 0 ], config );
+        zc.gc( false );
+      }
+      else
+      if ( args.size() > 2 && strcmp( args[ fieldAction ], "deep" ) == 0 )
+      {
+        ZCollector zc( ZBackupBase::deriveStorageDirFromBackupsFile( args[ fieldStorage ], true ),
+            passwords[ 0 ], config );
+        zc.gc( true );
+      }
+      else
+      {
+        ZCollector zc( ZBackupBase::deriveStorageDirFromBackupsFile( args[ fieldStorage ], true ),
+            passwords[ 0 ], config );
+        zc.gc( false );
+      }
+    }
+    else
+    if ( strcmp( args[ 0 ], "passwd" ) == 0 )
+    {
+      // Perform the password change
+      if ( args.size() != 2 )
+      {
+        fprintf( stderr, "Usage: %s %s <storage path>\n",
+                 *argv, args[ 0 ] );
+        return EXIT_FAILURE;
+      }
+
+      ZBackupBase zbb( ZBackupBase::deriveStorageDirFromBackupsFile( args[ 1 ], true ),
+          passwords[ 0 ], true );
+
+      if ( passwords[ 0 ].empty() != passwords[ 1 ].empty() )
+      {
+        fprintf( stderr,
+"Changing repo encryption type (non-encrypted to encrypted and vice versa) is possible "
+"only via import/export operations.\n"
+"Current repo type: %s.\n", zbb.encryptionkey.hasKey() ? "encrypted" : "non-encrypted" );
+        return EXIT_FAILURE;
+      }
+      zbb.setPassword( passwords[ 1 ] );
+    }
+    else
+    if ( strcmp( args[ 0 ], "info" ) == 0 )
+    {
+      // Show repo info
+      if ( args.size() != 2 )
+      {
+        fprintf( stderr, "Usage: %s %s <storage path>\n",
+                 *argv, args[ 0 ] );
+        return EXIT_FAILURE;
+      }
+
+      // TODO: implementation in ZBackupBase
+      ZBackupBase zbb( ZBackupBase::deriveStorageDirFromBackupsFile( args[ 1 ], true ),
+          passwords[ 0 ], true );
+      fprintf( stderr, "NOT IMPLEMENTED YET!\n" );
+      return EXIT_FAILURE;
+    }
+    else
+    if ( strcmp( args[ 0 ], "config" ) == 0 )
+    {
+      if ( args.size() < 2 || args.size() > 3 )
+      {
+        fprintf( stderr, "Usage: %s %s [show|edit|set|reset] <storage path>\n",
+                 *argv, args[ 0 ] );
+        return EXIT_FAILURE;
+      }
+
+      int fieldStorage = 1;
+      int fieldAction = 2;
+
+      if ( args.size() == 3 )
+      {
+        fieldStorage = 2;
+        fieldAction = 1;
+      }
+
+      if ( args.size() > 2 && strcmp( args[ fieldAction ], "edit" ) == 0 )
+      {
+        ZBackupBase zbb( ZBackupBase::deriveStorageDirFromBackupsFile( args[ fieldStorage ], true ),
+            passwords[ 0 ], true );
+        if ( zbb.editConfigInteractively() )
+          zbb.saveExtendedStorageInfo();
+      }
+      else
+      if ( args.size() > 2 && strcmp( args[ fieldAction ], "set" ) == 0 )
+      {
+        ZBackupBase zbb( ZBackupBase::deriveStorageDirFromBackupsFile( args[ fieldStorage ], true ),
+            passwords[ 0 ], config, true );
+        zbb.config.show();
+        zbb.saveExtendedStorageInfo();
+      }
+      else
+      if ( args.size() > 2 && strcmp( args[ fieldAction ], "reset" ) == 0 )
+      {
+        ZBackupBase zbb( ZBackupBase::deriveStorageDirFromBackupsFile( args[ fieldStorage ], true ),
+            passwords[ 0 ], true );
+        zbb.config.reset_storable();
+        zbb.config.show();
+        zbb.saveExtendedStorageInfo();
+      }
+      else
+      {
+        ZBackupBase zbb( ZBackupBase::deriveStorageDirFromBackupsFile( args[ fieldStorage ], true ),
+            passwords[ 0 ], true );
+        zbb.config.show();
+      }
     }
     else
     {
@@ -457,5 +423,10 @@ int main( int argc, char *argv[] )
     fprintf( stderr, "%s\n", e.what() );
     return EXIT_FAILURE;
   }
+  catch( ... )
+  {
+    fprintf( stderr, "Unknown exception!\n" );
+    return EXIT_FAILURE;
+  }
   return EXIT_SUCCESS;
 }

zbackup.hh

@@ -1,97 +0,0 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
-
-#ifndef ZBACKUP_HH_INCLUDED__
-#define ZBACKUP_HH_INCLUDED__
-
-#include <stddef.h>
-#include <exception>
-#include <string>
-#include <vector>
-
-#include "chunk_id.hh"
-#include "chunk_index.hh"
-#include "chunk_storage.hh"
-#include "encryption_key.hh"
-#include "ex.hh"
-#include "tmp_mgr.hh"
-#include "zbackup.pb.h"
-
-using std::string;
-using std::vector;
-
-struct Paths
-{
-  string storageDir;
-
-  Paths( string const & storageDir );
-
-  string getTmpPath();
-  string getRestorePath();
-  string getCreatePath();
-  string getBundlesPath();
-  string getStorageInfoPath();
-  string getIndexPath();
-  string getBackupsPath();
-};
-
-class ZBackupBase: protected Paths
-{
-public:
-  DEF_EX( Ex, "ZBackup exception", std::exception )
-  DEF_EX_STR( exWontOverwrite, "Won't overwrite existing file", Ex )
-  DEF_EX( exStdinError, "Error reading from standard input", Ex )
-  DEF_EX( exWontReadFromTerminal, "Won't read data from a terminal", exStdinError )
-  DEF_EX( exStdoutError, "Error writing to standard output", Ex )
-  DEF_EX( exWontWriteToTerminal, "Won't write data to a terminal", exStdoutError )
-  DEF_EX( exSerializeError, "Failed to serialize data", Ex )
-  DEF_EX( exParseError, "Failed to parse data", Ex )
-  DEF_EX( exChecksumError, "Checksum error", Ex )
-  DEF_EX_STR( exCantDeriveStorageDir, "The path must be within the backups/ dir:", Ex )
-
-  /// Opens the storage
-  ZBackupBase( string const & storageDir, string const & password );
-
-  /// Creates new storage
-  static void initStorage( string const & storageDir, string const & password,
-                           bool isEncrypted );
-
-  /// For a given file within the backups/ dir in the storage, returns its
-  /// storage dir or throws an exception
-  static string deriveStorageDirFromBackupsFile( string const & backupsFile );
-
-protected:
-  StorageInfo storageInfo;
-  EncryptionKey encryptionkey;
-  TmpMgr tmpMgr;
-  ChunkIndex chunkIndex;
-
-private:
-  StorageInfo loadStorageInfo();
-};
-
-class ZBackup: public ZBackupBase
-{
-  ChunkStorage::Writer chunkStorageWriter;
-
-public:
-  ZBackup( string const & storageDir, string const & password,
-           size_t threads );
-
-  /// Backs up the data from stdin
-  void backupFromStdin( string const & outputFileName );
-};
-
-class ZRestore: public ZBackupBase
-{
-  ChunkStorage::Reader chunkStorageReader;
-
-public:
-  ZRestore( string const & storageDir, string const & password,
-            size_t cacheSize );
-
-  /// Restores the data to stdin
-  void restoreToStdin( string const & inputFileName );
-};
-
-#endif

zbackup.proto

@@ -1,5 +1,5 @@
-// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
-// Part of ZBackup. Licensed under GNU GPLv2 or later
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
 
 // Protobuffers used in zbackup
 
@@ -30,14 +30,53 @@ message EncryptionKeyInfo
 message StorageInfo
 {
   // Maximum chunk size used when storing chunks
-  required uint32 chunk_max_size = 1;
+  optional uint32 chunk_max_size = 1 [deprecated = true];
   // Maximum number of bytes a bundle can hold. Only real chunk bytes are
   // counted, not metadata. Any bundle should be able to contain at least
   // one arbitrary single chunk, so this should not be smaller than
   // chunk_max_size
-  required uint32 bundle_max_payload_size = 2;
+  optional uint32 bundle_max_payload_size = 2 [deprecated = true];
   // If present, used for encryption/decryption of all data
   optional EncryptionKeyInfo encryption_key = 3;
+  // Default compression for new bundles
+  optional string default_compression_method = 4 [default = "lzma", deprecated = true];
+}
+
+message LZMAConfigInfo
+{
+  // Compression level for new LZMA-compressed files
+  optional uint32 compression_level = 1 [default = 6];
+}
+
+message ChunkConfigInfo
+{
+  // Maximum chunk size used when storing chunks
+  required uint32 max_size = 1 [default = 65536];
+}
+
+message BundleConfigInfo
+{
+  // Maximum number of bytes a bundle can hold. Only real chunk bytes are
+  // counted, not metadata. Any bundle should be able to contain at least
+  // one arbitrary single chunk, so this should not be smaller than
+  // chunk_max_size
+  required uint32 max_payload_size = 2 [default = 0x200000];
+  // Compression method for new bundles
+  optional string compression_method = 3 [default = "lzma"];
+}
+
+// Storable config values should always have default values
+message ConfigInfo
+{
+  required ChunkConfigInfo chunk = 1;
+  required BundleConfigInfo bundle = 2;
+  required LZMAConfigInfo lzma = 3;
+}
+
+message ExtendedStorageInfo
+{
+  // Config data storage
+  optional ConfigInfo config = 1;
 }
 
 message BundleInfo
@@ -61,6 +100,18 @@ message FileHeader
   required uint32 version = 1;
 }
 
+message BundleFileHeader
+{
+  // File format version
+  required uint32 version = 1;
+
+  // Compression method that is used for this file
+  // If the program doesn't support that field, it will try LZMA. If it is
+  // LZMA, that will work. If it isn't, it will have aborted before because
+  // the version in FileHeader is higher than it can support.
+  optional string compression_method = 2 [default = "lzma"];
+}
+
 message IndexBundleHeader
 {
   // Id of the bundle following in the stream. If not present, indicates the

zbackup_base.cc

@@ -0,0 +1,462 @@
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
+
+#include <sys/wait.h>
+#include <cerrno>
+#include <unistd.h>
+#include <signal.h>
+
+#include "zbackup_base.hh"
+
+#include "storage_info_file.hh"
+#include "compression.hh"
+#include "debug.hh"
+
+// TODO: make configurable by cmake
+#if defined(PATH_VI)
+# define EDITOR PATH_VI
+#else
+# define EDITOR "/bin/vi"
+#endif
+
+#ifndef PATH_BSHELL
+# define PATH_BSHELL "/bin/sh"
+#endif
+
+using std::string;
+
+Paths::Paths( string const & storageDir ): storageDir( storageDir )
+{
+}
+
+Paths::Paths( string const & storageDir, Config const & config ):
+  storageDir( storageDir ), config( config )
+{
+}
+
+string Paths::getTmpPath()
+{
+  if ( config.runtime.pathsRespectTmp )
+  {
+    char * tmpdir;
+    if ( ( ( tmpdir = getenv( "TMPDIR" ) ) != NULL && *tmpdir != '\0' ) )
+      return string( tmpdir );
+  }
+  return string( Dir::addPath( storageDir, "tmp" ) );
+}
+
+string Paths::getBundlesPath()
+{
+  return string( Dir::addPath( storageDir, "bundles" ) );
+}
+
+string Paths::getStorageInfoPath()
+{
+  return string( Dir::addPath( storageDir, "info" ) );
+}
+
+string Paths::getExtendedStorageInfoPath()
+{
+  return string( Dir::addPath( storageDir, "info_extended" ) );
+}
+
+string Paths::getIndexPath()
+{
+  return string( Dir::addPath( storageDir, "index" ) );
+}
+
+string Paths::getBackupsPath()
+{
+  return string( Dir::addPath( storageDir, "backups" ) );
+}
+
+ZBackupBase::ZBackupBase( string const & storageDir, string const & password ):
+  Paths( storageDir ), storageInfo( loadStorageInfo() ),
+  encryptionkey( password, storageInfo.has_encryption_key() ?
+                   &storageInfo.encryption_key() : 0 ),
+  extendedStorageInfo( loadExtendedStorageInfo( encryptionkey ) ),
+  tmpMgr( getTmpPath() ),
+  chunkIndex( encryptionkey, tmpMgr, getIndexPath(), false ),
+  config( extendedStorageInfo.mutable_config() )
+{
+  propagateUpdate();
+  dPrintf("%s for %s is instantiated and initialized\n", __CLASS,
+      storageDir.c_str() );
+}
+
+ZBackupBase::ZBackupBase( string const & storageDir, string const & password,
+                          Config & configIn ):
+  Paths( storageDir, configIn ), storageInfo( loadStorageInfo() ),
+  encryptionkey( password, storageInfo.has_encryption_key() ?
+                   &storageInfo.encryption_key() : 0 ),
+  extendedStorageInfo( loadExtendedStorageInfo( encryptionkey ) ),
+  tmpMgr( getTmpPath() ),
+  chunkIndex( encryptionkey, tmpMgr, getIndexPath(), false ),
+  config( configIn, extendedStorageInfo.mutable_config() )
+{
+  propagateUpdate();
+  dPrintf("%s for %s is instantiated and initialized\n", __CLASS,
+      storageDir.c_str() );
+}
+
+ZBackupBase::ZBackupBase( string const & storageDir, string const & password,
+                          bool prohibitChunkIndexLoading ):
+  Paths( storageDir ), storageInfo( loadStorageInfo() ),
+  encryptionkey( password, storageInfo.has_encryption_key() ?
+                   &storageInfo.encryption_key() : 0 ),
+  extendedStorageInfo( loadExtendedStorageInfo( encryptionkey ) ),
+  tmpMgr( getTmpPath() ),
+  chunkIndex( encryptionkey, tmpMgr, getIndexPath(), prohibitChunkIndexLoading ),
+  config( extendedStorageInfo.mutable_config() )
+{
+  propagateUpdate();
+  dPrintf("%s for %s is instantiated and initialized\n", __CLASS,
+      storageDir.c_str() );
+}
+
+ZBackupBase::ZBackupBase( string const & storageDir, string const & password,
+                          Config & configIn, bool prohibitChunkIndexLoading ):
+  Paths( storageDir, configIn ), storageInfo( loadStorageInfo() ),
+  encryptionkey( password, storageInfo.has_encryption_key() ?
+                   &storageInfo.encryption_key() : 0 ),
+  extendedStorageInfo( loadExtendedStorageInfo( encryptionkey ) ),
+  tmpMgr( getTmpPath() ),
+  chunkIndex( encryptionkey, tmpMgr, getIndexPath(), prohibitChunkIndexLoading ),
+  config( configIn, extendedStorageInfo.mutable_config() )
+{
+  propagateUpdate();
+  dPrintf("%s for %s is instantiated and initialized\n", __CLASS,
+      storageDir.c_str() );
+}
+
+// Update all internal variables according to real configuration
+// Dunno why someone need to store duplicate information
+// in deduplication utility
+void ZBackupBase::propagateUpdate()
+{
+  const_sptr< Compression::CompressionMethod > compression =
+    Compression::CompressionMethod::findCompression(
+      config.GET_STORABLE( bundle, compression_method ) );
+  Compression::CompressionMethod::selectedCompression = compression;
+}
+
+StorageInfo ZBackupBase::loadStorageInfo()
+{
+  StorageInfo storageInfo;
+
+  StorageInfoFile::load( getStorageInfoPath(), storageInfo );
+
+  return storageInfo;
+}
+
+ExtendedStorageInfo ZBackupBase::loadExtendedStorageInfo(
+    EncryptionKey const & encryptionkey )
+{
+  try
+  {
+    ExtendedStorageInfo extendedStorageInfo;
+
+    ExtendedStorageInfoFile::load( getExtendedStorageInfoPath(), encryptionkey,
+        extendedStorageInfo );
+
+    return extendedStorageInfo;
+  }
+  catch ( UnbufferedFile::exCantOpen & ex )
+  {
+    verbosePrintf( "Can't open extended storage info (info_extended)!\n"
+                   "Attempting to start repo migration.\n" );
+
+    if ( !File::exists( getExtendedStorageInfoPath() ) )
+    {
+      ExtendedStorageInfo extendedStorageInfo;
+      Config config( extendedStorageInfo.mutable_config() );
+      config.reset_storable();
+      config.SET_STORABLE( chunk, max_size, storageInfo.chunk_max_size() );
+      config.SET_STORABLE( bundle, max_payload_size,
+          storageInfo.bundle_max_payload_size() );
+      config.SET_STORABLE( bundle, compression_method,
+          storageInfo.default_compression_method() );
+
+      ExtendedStorageInfoFile::save( getExtendedStorageInfoPath(), encryptionkey,
+          extendedStorageInfo );
+
+      verbosePrintf( "Done.\n" );
+
+      return loadExtendedStorageInfo( encryptionkey );
+    }
+    else
+    {
+      fprintf( stderr, "info_extended exists but can't be opened!\n"
+                       "Please check file permissions.\n" );
+    }
+  }
+}
+
+void ZBackupBase::initStorage( string const & storageDir,
+                               string const & password,
+                               bool isEncrypted,
+                               Config const & configIn )
+{
+  StorageInfo storageInfo;
+  ExtendedStorageInfo extendedStorageInfo;
+  Config config( extendedStorageInfo.mutable_config() );
+  config.reset_storable();
+  config.storable->MergeFrom( *configIn.storable );
+
+  EncryptionKey encryptionkey = EncryptionKey::noKey();
+
+  if ( isEncrypted )
+    EncryptionKey::generate( password,
+                             *storageInfo.mutable_encryption_key(),
+                             encryptionkey );
+
+  Paths paths( storageDir );
+
+  if ( !Dir::exists( storageDir ) )
+    Dir::create( storageDir );
+
+  if ( !Dir::exists( paths.getBundlesPath() ) )
+    Dir::create( paths.getBundlesPath() );
+
+  if ( !Dir::exists( paths.getBackupsPath() ) )
+    Dir::create( paths.getBackupsPath() );
+
+  if ( !Dir::exists( paths.getIndexPath() ) )
+    Dir::create( paths.getIndexPath() );
+
+  string storageInfoPath( paths.getStorageInfoPath() );
+  string extendedStorageInfoPath( paths.getExtendedStorageInfoPath() );
+
+  if ( File::exists( storageInfoPath ) )
+    throw exWontOverwrite( storageInfoPath );
+
+  encryptionkey = EncryptionKey( password, storageInfo.has_encryption_key() ?
+      &storageInfo.encryption_key() : 0 );
+
+  StorageInfoFile::save( storageInfoPath, storageInfo );
+  ExtendedStorageInfoFile::save( extendedStorageInfoPath, encryptionkey, extendedStorageInfo );
+}
+
+string ZBackupBase::deriveStorageDirFromBackupsFile( string const &
+                                                     backupsFile, bool allowOutside )
+{
+  // TODO: handle cases when there's a backup/ folder within the backup/ folder
+  // correctly
+  if ( allowOutside )
+    return Dir::getRealPath( backupsFile );
+
+  string realPath = Dir::getRealPath( Dir::getDirName( backupsFile ) );
+  size_t pos;
+  if ( realPath.size() >= 8 && strcmp( realPath.c_str() + realPath.size() - 8,
+                                       "/backups") == 0 )
+    pos = realPath.size() - 8;
+  else
+    pos = realPath.rfind( "/backups/" );
+  if ( pos == string::npos )
+    throw exCantDeriveStorageDir( backupsFile );
+  else
+    return realPath.substr( 0, pos );
+}
+
+void ZBackupBase::setPassword( string const & password )
+{
+  EncryptionKey::generate( password,
+                         *storageInfo.mutable_encryption_key(), encryptionkey );
+
+  StorageInfoFile::save( getStorageInfoPath(), storageInfo );
+
+  EncryptionKey encryptionkey( password, storageInfo.has_encryption_key() ?
+                   &storageInfo.encryption_key() : 0 );
+}
+
+void ZBackupBase::saveExtendedStorageInfo()
+{
+  ExtendedStorageInfoFile::save( getExtendedStorageInfoPath(), encryptionkey,
+      extendedStorageInfo );
+}
+
+bool ZBackupBase::spawnEditor( string & data, bool( * validator )
+    ( string const &, string const & ) )
+{
+  // Based on ideas found in cronie-1.4.4-12.el6
+  // Initially it was just a copy-paste from edit_cmd (crontab.c)
+
+  /* Turn off signals. */
+  (void) signal( SIGHUP, SIG_IGN );
+  (void) signal( SIGINT, SIG_IGN );
+  (void) signal( SIGQUIT, SIG_IGN );
+
+  sptr< TemporaryFile > tmpFile = tmpMgr.makeTemporaryFile();
+  const char * tmpFileName = tmpFile->getFileName().c_str();
+
+  sptr< File> tmpDataFile = new File( tmpFileName, File::WriteOnly );
+  tmpDataFile->writeRecords( data.c_str(), data.size(), 1 );
+
+again:
+  tmpDataFile->rewind();
+  if ( tmpDataFile->error() )
+  {
+    verbosePrintf( "Error while writing data to %s\n", tmpFileName );
+fatal:
+    tmpFile.reset();
+    exit( EXIT_FAILURE );
+  }
+
+  char * editorEnv;
+  string editor;
+  if ( ( ( editorEnv = getenv( "VISUAL" ) ) == NULL || *editorEnv == '\0' ) &&
+      ( ( editorEnv = getenv( "EDITOR" ) ) == NULL || *editorEnv == '\0' ) )
+    editor.assign( EDITOR );
+  else
+    editor.assign( editorEnv );
+
+  /* we still have the file open.  editors will generally rewrite the
+   * original file rather than renaming/unlinking it and starting a
+   * new one; even backup files are supposed to be made by copying
+   * rather than by renaming.  if some editor does not support this,
+   * then don't use it.  the security problems are more severe if we
+   * close and reopen the file around the edit.
+   */
+
+  string shellArgs;
+  shellArgs += editor;
+  shellArgs += " ";
+  shellArgs += tmpFileName;
+
+  pid_t pid, xpid;
+
+  switch ( pid = fork() )
+  {
+    case -1:
+      perror( "fork" );
+      goto fatal;
+    case 0:
+      /* child */
+      dPrintf( "Spawning editor: %s %s %s %s\n", PATH_BSHELL, PATH_BSHELL,
+          "-c", shellArgs.c_str() );
+      execlp( PATH_BSHELL, PATH_BSHELL, "-c", shellArgs.c_str(), (char *) 0 );
+      perror( editor.c_str() );
+      exit( EXIT_FAILURE );
+    /*NOTREACHED*/
+    default:
+      /* parent */
+      break;
+  }
+
+  /* parent */
+  int waiter;
+  for ( ; ; )
+  {
+    xpid = waitpid( pid, &waiter, 0 );
+    if ( xpid == -1 )
+    {
+      if ( errno != EINTR )
+        verbosePrintf( "waitpid() failed waiting for PID %ld from \"%s\": %s\n",
+            (long) pid, editor.c_str(), strerror( errno ) );
+    }
+    else
+    if (xpid != pid)
+    {
+      verbosePrintf( "wrong PID (%ld != %ld) from \"%s\"\n",
+          (long) xpid, (long) pid, editor.c_str() );
+      goto fatal;
+    }
+    else
+    if ( WIFEXITED( waiter ) && WEXITSTATUS( waiter ) )
+    {
+      verbosePrintf( "\"%s\" exited with status %d\n",
+          editor.c_str(), WEXITSTATUS( waiter ) );
+      goto fatal;
+    }
+    else
+    if ( WIFSIGNALED( waiter ) )
+    {
+      verbosePrintf( "\"%s\" killed; signal %d (%score dumped)\n",
+          editor.c_str(), WTERMSIG( waiter ),
+          WCOREDUMP( waiter ) ? "" : "no ");
+      goto fatal;
+    }
+    else
+      break;
+  }
+  (void) signal( SIGHUP, SIG_DFL );
+  (void) signal( SIGINT, SIG_DFL );
+  (void) signal( SIGQUIT, SIG_DFL );
+
+  tmpDataFile->close();
+  tmpDataFile = new File( tmpFileName, File::ReadOnly );
+
+  string newData;
+  newData.resize( tmpDataFile->size() );
+  tmpDataFile->read( &newData[ 0 ], newData.size() );
+  bool isChanged = false;
+
+  bool valid = validator( data, newData );
+
+  switch ( valid )
+  {
+    case true:
+      goto success;
+    case false:
+      for ( ; ; )
+      {
+        fprintf( stderr, "Supplied data is not valid\n" );
+        fflush( stderr );
+        printf( "Do you want to retry the same edit? " );
+        fflush( stdout );
+
+        string input;
+        input.resize( 131072 ); // Should I choose another magic value?
+        if ( fgets( &input[ 0 ], input.size(), stdin ) == 0L )
+          continue;
+
+        switch ( input[ 0 ] )
+        {
+          case 'y':
+          case 'Y':
+            goto again;
+          case 'n':
+          case 'N':
+            verbosePrintf( "Data is kept intact\n" );
+            goto end;
+          default:
+            fprintf( stderr, "Enter Y or N\n" );
+        }
+      }
+  }
+
+success:
+  isChanged = true;
+  data.assign( newData );
+
+end:
+  tmpDataFile.reset();
+  tmpFile.reset();
+
+  return isChanged;
+}
+
+bool ZBackupBase::editConfigInteractively()
+{
+  string configData( Config::toString( *config.storable ) );
+
+  if ( !spawnEditor( configData, &Config::validateProto ) )
+    return false;
+
+  ConfigInfo newConfig;
+  Config::parseProto( configData, &newConfig );
+  if ( Config::toString( *config.storable ) ==
+      Config::toString( newConfig ) )
+  {
+    verbosePrintf( "No changes made to config\n" );
+    return false;
+  }
+
+  verbosePrintf( "Updating configuration...\n" );
+  config.storable->MergeFrom( newConfig );
+  verbosePrintf(
+"Configuration successfully updated!\n"
+"Updated configuration:\n%s", Config::toString( *config.storable ).c_str() );
+
+  return true;
+}

zbackup_base.hh

@@ -0,0 +1,88 @@
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
+
+#ifndef ZBACKUP_BASE_HH_INCLUDED
+#define ZBACKUP_BASE_HH_INCLUDED
+
+#include <exception>
+#include <string>
+
+#include "ex.hh"
+#include "chunk_index.hh"
+#include "config.hh"
+
+struct Paths
+{
+  Config config;
+  std::string storageDir;
+
+  Paths( std::string const & storageDir );
+  Paths( std::string const & storageDir, Config const & );
+
+  std::string getTmpPath();
+  std::string getRestorePath();
+  std::string getCreatePath();
+  std::string getBundlesPath();
+  std::string getStorageInfoPath();
+  std::string getExtendedStorageInfoPath();
+  std::string getIndexPath();
+  std::string getBackupsPath();
+};
+
+class ZBackupBase: public Paths
+{
+public:
+  DEF_EX( Ex, "ZBackup exception", std::exception )
+  DEF_EX_STR( exWontOverwrite, "Won't overwrite existing file", Ex )
+  DEF_EX( exStdinError, "Error reading from standard input", Ex )
+  DEF_EX( exWontReadFromTerminal, "Won't read data from a terminal", exStdinError )
+  DEF_EX( exStdoutError, "Error writing to standard output", Ex )
+  DEF_EX( exWontWriteToTerminal, "Won't write data to a terminal", exStdoutError )
+  DEF_EX( exSerializeError, "Failed to serialize data", Ex )
+  DEF_EX( exParseError, "Failed to parse data", Ex )
+  DEF_EX( exChecksumError, "Checksum error", Ex )
+  DEF_EX_STR( exCantDeriveStorageDir, "The path must be within the backups/ dir:", Ex )
+
+  /// Opens the storage
+  ZBackupBase( std::string const & storageDir, std::string const & password );
+  ZBackupBase( std::string const & storageDir, std::string const & password, Config & configIn );
+  ZBackupBase( std::string const & storageDir, std::string const & password,
+      bool prohibitChunkIndexLoading );
+  ZBackupBase( std::string const & storageDir, std::string const & password, Config & configIn,
+      bool prohibitChunkIndexLoading );
+
+  /// Creates new storage
+  static void initStorage( std::string const & storageDir, std::string const & password,
+                           bool isEncrypted, Config const & );
+
+  /// For a given file within the backups/ dir in the storage, returns its
+  /// storage dir or throws an exception
+  static std::string deriveStorageDirFromBackupsFile( std::string const & backupsFile, bool allowOutside = false );
+
+  void propagateUpdate();
+
+  void saveExtendedStorageInfo();
+
+  void setPassword( std::string const & password );
+
+  // returns true if data is changed
+  bool spawnEditor( std::string & data, bool( * validator )
+      ( string const &, string const & ) );
+
+  // Edit current configuration
+  // returns true if configuration is changed
+  bool editConfigInteractively();
+
+  StorageInfo storageInfo;
+  EncryptionKey encryptionkey;
+  ExtendedStorageInfo extendedStorageInfo;
+  TmpMgr tmpMgr;
+  ChunkIndex chunkIndex;
+  Config config;
+
+private:
+  StorageInfo loadStorageInfo();
+  ExtendedStorageInfo loadExtendedStorageInfo( EncryptionKey const & );
+};
+
+#endif

zutils.cc

@@ -0,0 +1,413 @@
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
+
+#include "zutils.hh"
+#include "backup_creator.hh"
+#include "sha256.hh"
+#include "backup_collector.hh"
+#include <unistd.h>
+
+using std::vector;
+using std::bitset;
+using std::iterator;
+
+ZBackup::ZBackup( string const & storageDir, string const & password,
+                  Config & configIn ):
+  ZBackupBase( storageDir, password, configIn ),
+  chunkStorageWriter( config, encryptionkey, tmpMgr, chunkIndex,
+                      getBundlesPath(), getIndexPath(), config.runtime.threads )
+{
+}
+
+void ZBackup::backupFromStdin( string const & outputFileName )
+{
+  if ( isatty( fileno( stdin ) ) )
+    throw exWontReadFromTerminal();
+
+  if ( File::exists( outputFileName ) )
+    throw exWontOverwrite( outputFileName );
+
+  Sha256 sha256;
+  BackupCreator backupCreator( config, chunkIndex, chunkStorageWriter );
+
+  time_t startTime = time( 0 );
+  uint64_t totalDataSize = 0;
+
+  for ( ; ; )
+  {
+    size_t toRead = backupCreator.getInputBufferSize();
+//    dPrintf( "Reading up to %u bytes from stdin\n", toRead );
+
+    void * inputBuffer = backupCreator.getInputBuffer();
+    size_t rd = fread( inputBuffer, 1, toRead, stdin );
+
+    if ( !rd )
+    {
+      if ( feof( stdin ) )
+      {
+        dPrintf( "No more input on stdin\n" );
+        break;
+      }
+      else
+        throw exStdinError();
+    }
+
+    sha256.add( inputBuffer, rd );
+
+    backupCreator.handleMoreData( rd );
+
+    totalDataSize += rd;
+  }
+
+  // Finish up with the creator
+  backupCreator.finish();
+
+  string serialized;
+  backupCreator.getBackupData( serialized );
+
+  BackupInfo info;
+
+  info.set_sha256( sha256.finish() );
+  info.set_size( totalDataSize );
+
+  // Shrink the serialized data iteratively until it wouldn't shrink anymore
+  for ( ; ; )
+  {
+    BackupCreator backupCreator( config, chunkIndex, chunkStorageWriter );
+    char const * ptr = serialized.data();
+    size_t left = serialized.size();
+    while( left )
+    {
+      size_t bufferSize = backupCreator.getInputBufferSize();
+      size_t toCopy = bufferSize > left ? left : bufferSize;
+
+      memcpy( backupCreator.getInputBuffer(), ptr, toCopy );
+      backupCreator.handleMoreData( toCopy );
+      ptr += toCopy;
+      left -= toCopy;
+    }
+
+    backupCreator.finish();
+
+    string newGen;
+    backupCreator.getBackupData( newGen );
+
+    if ( newGen.size() < serialized.size() )
+    {
+      serialized.swap( newGen );
+      info.set_iterations( info.iterations() + 1 );
+    }
+    else
+      break;
+  }
+
+  dPrintf( "Iterations: %u\n", info.iterations() );
+
+  info.mutable_backup_data()->swap( serialized );
+
+  info.set_time( time( 0 ) - startTime );
+
+  // Commit the bundles to the disk before creating the final output file
+  chunkStorageWriter.commit();
+
+  // Now save the resulting BackupInfo
+
+  sptr< TemporaryFile > tmpFile = tmpMgr.makeTemporaryFile();
+  BackupFile::save( tmpFile->getFileName(), encryptionkey, info );
+  tmpFile->moveOverTo( outputFileName );
+}
+
+ZRestore::ZRestore( string const & storageDir, string const & password,
+                    Config & configIn ):
+  ZBackupBase( storageDir, password, configIn ),
+  chunkStorageReader( config, encryptionkey, chunkIndex, getBundlesPath(),
+                      config.runtime.cacheSize )
+{
+}
+
+void ZRestore::restoreToFile( string const & inputFileName, string const & outputFileName )
+{
+  BackupInfo backupInfo;
+
+  BackupFile::load( inputFileName, encryptionkey, backupInfo );
+
+  string backupData;
+
+  // Perform the iterations needed to get to the actual user backup data
+  BackupRestorer::restoreIterations( chunkStorageReader, backupInfo, backupData, NULL );
+
+  UnbufferedFile f( outputFileName.data(), UnbufferedFile::ReadWrite );
+
+  struct FileWriter: public SeekableSink
+  {
+    UnbufferedFile *f;
+
+    FileWriter( UnbufferedFile *f ):
+      f( f )
+    {
+    }
+
+    virtual void saveData( int64_t position, void const * data, size_t size )
+    {
+      f->seek( position );
+      f->write( data, size );
+    }
+  } seekWriter( &f );
+
+  BackupRestorer::ChunkMap map;
+  BackupRestorer::restore( chunkStorageReader, backupData, NULL, NULL, &map, &seekWriter );
+  BackupRestorer::restoreMap( chunkStorageReader, &map, &seekWriter );
+
+  Sha256 sha256;
+  string buf;
+  buf.resize( 0x100000 );
+  size_t r;
+  f.seek( 0 );
+  while ( ( r = f.read( (void*)buf.data(), buf.size() ) ) > 0 )
+    sha256.add( buf.data(), r );
+  if ( sha256.finish() != backupInfo.sha256() )
+    throw exChecksumError();
+}
+
+void ZRestore::restoreToStdin( string const & inputFileName )
+{
+  if ( isatty( fileno( stdout ) ) )
+    throw exWontWriteToTerminal();
+
+  BackupInfo backupInfo;
+
+  BackupFile::load( inputFileName, encryptionkey, backupInfo );
+
+  string backupData;
+
+  // Perform the iterations needed to get to the actual user backup data
+  BackupRestorer::restoreIterations( chunkStorageReader, backupInfo, backupData, NULL );
+
+  struct StdoutWriter: public DataSink
+  {
+    Sha256 sha256;
+
+    virtual void saveData( void const * data, size_t size )
+    {
+      sha256.add( data, size );
+      if ( fwrite( data, size, 1, stdout ) != 1 )
+        throw exStdoutError();
+    }
+  } stdoutWriter;
+
+  BackupRestorer::restore( chunkStorageReader, backupData, &stdoutWriter, NULL, NULL, NULL );
+
+  if ( stdoutWriter.sha256.finish() != backupInfo.sha256() )
+    throw exChecksumError();
+}
+
+ZExchange::ZExchange( string const & srcStorageDir, string const & srcPassword,
+                      string const & dstStorageDir, string const & dstPassword,
+                      Config & configIn ):
+  srcZBackupBase( srcStorageDir, srcPassword, configIn, true ),
+  dstZBackupBase( dstStorageDir, dstPassword, configIn, true ),
+  config( configIn )
+{
+}
+
+void ZExchange::exchange()
+{
+  vector< BackupExchanger::PendingExchangeRename > pendingExchangeRenames;
+
+  if ( config.runtime.exchange.test( BackupExchanger::bundles ) )
+  {
+    verbosePrintf( "Searching for bundles...\n" );
+
+    vector< string > bundles = BackupExchanger::findOrRebuild(
+        srcZBackupBase.getBundlesPath(), dstZBackupBase.getBundlesPath() );
+
+    for ( std::vector< string >::iterator it = bundles.begin(); it != bundles.end(); ++it )
+    {
+      verbosePrintf( "Processing bundle file %s... ", it->c_str() );
+      string outputFileName ( Dir::addPath( dstZBackupBase.getBundlesPath(), *it ) );
+      if ( !File::exists( outputFileName ) )
+      {
+        sptr< Bundle::Reader > reader = new Bundle::Reader( Dir::addPath (
+              srcZBackupBase.getBundlesPath(), *it ), srcZBackupBase.encryptionkey, true );
+        sptr< Bundle::Creator > creator = new Bundle::Creator;
+        sptr< TemporaryFile > bundleTempFile = dstZBackupBase.tmpMgr.makeTemporaryFile();
+        creator->write( bundleTempFile->getFileName(), dstZBackupBase.encryptionkey, *reader );
+
+        if ( creator.get() && reader.get() )
+        {
+          creator.reset();
+          reader.reset();
+          pendingExchangeRenames.push_back( BackupExchanger::PendingExchangeRename(
+                bundleTempFile, outputFileName ) );
+          verbosePrintf( "done.\n" );
+        }
+      }
+      else
+      {
+        verbosePrintf( "file exists - skipped.\n" );
+      }
+    }
+
+    verbosePrintf( "Bundle exchange completed.\n" );
+  }
+
+  if ( config.runtime.exchange.test( BackupExchanger::indexes ) )
+  {
+    verbosePrintf( "Searching for indexes...\n" );
+    vector< string > indexes = BackupExchanger::findOrRebuild(
+        srcZBackupBase.getIndexPath(), dstZBackupBase.getIndexPath() );
+
+    for ( std::vector< string >::iterator it = indexes.begin(); it != indexes.end(); ++it )
+    {
+      verbosePrintf( "Processing index file %s... ", it->c_str() );
+      string outputFileName ( Dir::addPath( dstZBackupBase.getIndexPath(), *it ) );
+      if ( !File::exists( outputFileName ) )
+      {
+        sptr< IndexFile::Reader > reader = new IndexFile::Reader( srcZBackupBase.encryptionkey,
+                                 Dir::addPath( srcZBackupBase.getIndexPath(), *it ) );
+        sptr< TemporaryFile > indexTempFile = dstZBackupBase.tmpMgr.makeTemporaryFile();
+        sptr< IndexFile::Writer > writer = new IndexFile::Writer( dstZBackupBase.encryptionkey,
+            indexTempFile->getFileName() );
+
+        BundleInfo bundleInfo;
+        Bundle::Id bundleId;
+        while( reader->readNextRecord( bundleInfo, bundleId ) )
+        {
+          writer->add( bundleInfo, bundleId );
+        }
+
+        if ( writer.get() && reader.get() )
+        {
+          writer.reset();
+          reader.reset();
+          pendingExchangeRenames.push_back( BackupExchanger::PendingExchangeRename(
+                indexTempFile, outputFileName ) );
+          verbosePrintf( "done.\n" );
+        }
+      }
+      else
+      {
+        verbosePrintf( "file exists - skipped.\n" );
+      }
+    }
+
+    verbosePrintf( "Index exchange completed.\n" );
+  }
+
+  if ( config.runtime.exchange.test( BackupExchanger::backups ) )
+  {
+    BackupInfo backupInfo;
+
+    verbosePrintf( "Searching for backups...\n" );
+    vector< string > backups = BackupExchanger::findOrRebuild(
+        srcZBackupBase.getBackupsPath(), dstZBackupBase.getBackupsPath() );
+
+    for ( std::vector< string >::iterator it = backups.begin(); it != backups.end(); ++it )
+    {
+      verbosePrintf( "Processing backup file %s... ", it->c_str() );
+      string outputFileName ( Dir::addPath( dstZBackupBase.getBackupsPath(), *it ) );
+      if ( !File::exists( outputFileName ) )
+      {
+        BackupFile::load( Dir::addPath( srcZBackupBase.getBackupsPath(), *it ),
+            srcZBackupBase.encryptionkey, backupInfo );
+        sptr< TemporaryFile > tmpFile = dstZBackupBase.tmpMgr.makeTemporaryFile();
+        BackupFile::save( tmpFile->getFileName(), dstZBackupBase.encryptionkey,
+            backupInfo );
+        pendingExchangeRenames.push_back( BackupExchanger::PendingExchangeRename(
+                tmpFile, outputFileName ) );
+        verbosePrintf( "done.\n" );
+      }
+      else
+      {
+        verbosePrintf( "file exists - skipped.\n" );
+      }
+    }
+
+    verbosePrintf( "Backup exchange completed.\n" );
+  }
+
+  if ( pendingExchangeRenames.size() > 0 )
+  {
+    verbosePrintf( "Moving files from temp directory to appropriate places... " );
+    for ( size_t x = pendingExchangeRenames.size(); x--; )
+    {
+      BackupExchanger::PendingExchangeRename & r = pendingExchangeRenames[ x ];
+      r.first->moveOverTo( r.second );
+      if ( r.first.get() )
+      {
+        r.first.reset();
+      }
+    }
+    pendingExchangeRenames.clear();
+    verbosePrintf( "done.\n" );
+  }
+}
+
+ZCollector::ZCollector( string const & storageDir, string const & password,
+                    Config & configIn ):
+  ZBackupBase( storageDir, password, configIn ),
+  chunkStorageReader( config, encryptionkey, chunkIndex, getBundlesPath(),
+                      config.runtime.cacheSize )
+{
+}
+
+void ZCollector::gc( bool gcDeep )
+{
+  ChunkIndex chunkReindex( encryptionkey, tmpMgr, getIndexPath(), true );
+
+  ChunkStorage::Writer chunkStorageWriter( config, encryptionkey, tmpMgr,
+      chunkReindex, getBundlesPath(), getIndexPath(), config.runtime.threads );
+
+  string fileName;
+
+  BundleCollector collector;
+  collector.bundlesPath = getBundlesPath();
+  collector.chunkStorageReader = &this->chunkStorageReader;
+  collector.chunkStorageWriter = &chunkStorageWriter;
+  collector.gcRepack = config.runtime.gcRepack;
+  collector.gcDeep = gcDeep;
+
+  verbosePrintf( "Performing garbage collection...\n" );
+
+  verbosePrintf( "Searching for backups...\n" );
+  vector< string > backups = BackupExchanger::findOrRebuild( getBackupsPath() );
+
+  for ( std::vector< string >::iterator it = backups.begin(); it != backups.end(); ++it )
+  {
+    string backup( Dir::addPath( getBackupsPath(), *it ) );
+
+    verbosePrintf( "Checking backup %s...\n", backup.c_str() );
+
+    BackupInfo backupInfo;
+
+    BackupFile::load( backup, encryptionkey, backupInfo );
+
+    string backupData;
+
+    BackupRestorer::restoreIterations( chunkStorageReader, backupInfo, backupData, &collector.usedChunkSet );
+
+    BackupRestorer::restore( chunkStorageReader, backupData, NULL, &collector.usedChunkSet, NULL, NULL );
+  }
+
+  verbosePrintf( "Checking bundles...\n" );
+
+  chunkIndex.loadIndex( collector );
+
+  collector.commit();
+
+  verbosePrintf( "Cleaning up...\n" );
+
+  string bundlesPath = getBundlesPath();
+  Dir::Listing bundleLst( bundlesPath );
+  Dir::Entry entry;
+  while( bundleLst.getNext( entry ) )
+  {
+    const string dirPath = Dir::addPath( bundlesPath, entry.getFileName());
+    if ( entry.isDir() && Dir::isDirEmpty( dirPath ) )
+    {
+      Dir::remove( dirPath );
+    }
+  }
+
+  verbosePrintf( "Garbage collection complete\n" );
+}

zutils.hh

@@ -0,0 +1,64 @@
+// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
+// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
+
+#ifndef ZUTILS_HH_INCLUDED
+#define ZUTILS_HH_INCLUDED
+
+#include "chunk_storage.hh"
+#include "zbackup_base.hh"
+
+class ZBackup: public ZBackupBase
+{
+  ChunkStorage::Writer chunkStorageWriter;
+
+public:
+  ZBackup( string const & storageDir, string const & password,
+           Config & configIn );
+
+  /// Backs up the data from stdin
+  void backupFromStdin( string const & outputFileName );
+};
+
+class ZRestore: public ZBackupBase
+{
+  ChunkStorage::Reader chunkStorageReader;
+
+public:
+  ZRestore( string const & storageDir, string const & password,
+            Config & configIn );
+
+  /// Restores the data to file
+  void restoreToFile( string const & inputFileName, string const & outputFileName );
+
+  /// Restores the data to stdout
+  void restoreToStdin( string const & inputFileName );
+};
+
+class ZExchange
+{
+  ZBackupBase srcZBackupBase;
+  ZBackupBase dstZBackupBase;
+
+public:
+  ZExchange( string const & srcStorageDir, string const & srcPassword,
+             string const & dstStorageDir, string const & dstPassword,
+             Config & configIn );
+
+  Config config;
+
+  /// Exchanges the data between storages
+  void exchange();
+};
+
+class ZCollector : public ZBackupBase
+{
+  ChunkStorage::Reader chunkStorageReader;
+
+public:
+  ZCollector( std::string const & storageDir, std::string const & password,
+              Config & configIn );
+
+  void gc( bool );
+};
+
+#endif