Create a --cache option, for using a binary cache

9 files changed, 483 insertions, 47 deletions

diff --git a/Makefile.am b/Makefile.am
index d4494fa7..13ea4e7f 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -59,6 +59,7 @@ libledger_data_la_SOURCES  =			\
 	src/timelog.cc				\
 	src/textual.cc				\
 	src/journal.cc				\
+	src/archive.cc				\
 	src/account.cc				\
 	src/xact.cc				\
 	src/post.cc				\
@@ -118,6 +119,7 @@ pkginclude_HEADERS =				\
 	src/xact.h				\
 	src/account.h				\
 	src/journal.h				\
+	src/archive.h				\
 	src/timelog.h				\
 	src/iterators.h				\
 	src/compare.h				\
diff --git a/src/archive.cc b/src/archive.cc
new file mode 100644
index 00000000..d631651f
--- /dev/null
+++ b/src/archive.cc
@@ -0,0 +1,247 @@
+/*
+ * Copyright (c) 2003-2009, John Wiegley.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above copyright
+ *   notice, this list of conditions and the following disclaimer in the
+ *   documentation and/or other materials provided with the distribution.
+ *
+ * - Neither the name of New Artisans LLC nor the names of its
+ *   contributors may be used to endorse or promote products derived from
+ *   this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <system.hh>
+
+#if defined(HAVE_BOOST_SERIALIZATION)
+
+#include "archive.h"
+#include "amount.h"
+#include "commodity.h"
+#include "pool.h"
+#include "scope.h"
+#include "account.h"
+#include "post.h"
+#include "xact.h"
+
+#define ARCHIVE_VERSION 0x03000001
+
+//BOOST_IS_ABSTRACT(ledger::scope_t)
+BOOST_CLASS_EXPORT(ledger::scope_t)
+BOOST_CLASS_EXPORT(ledger::child_scope_t)
+BOOST_CLASS_EXPORT(ledger::symbol_scope_t)
+BOOST_CLASS_EXPORT(ledger::call_scope_t)
+BOOST_CLASS_EXPORT(ledger::account_t)
+BOOST_CLASS_EXPORT(ledger::item_t)
+BOOST_CLASS_EXPORT(ledger::post_t)
+BOOST_CLASS_EXPORT(ledger::xact_base_t)
+BOOST_CLASS_EXPORT(ledger::xact_t)
+BOOST_CLASS_EXPORT(ledger::auto_xact_t)
+BOOST_CLASS_EXPORT(ledger::period_xact_t)
+
+template void ledger::journal_t::serialize(boost::archive::binary_oarchive&,
+					   const unsigned int);
+template void ledger::journal_t::serialize(boost::archive::binary_iarchive&,
+					   const unsigned int);
+namespace ledger {
+
+void archive_t::read_header()
+{
+  if (exists(file)) {
+    // Open the stream, read the version number and the list of sources
+    ifstream stream(file, std::ios::binary);
+    boost::archive::binary_iarchive iarchive(stream);
+
+    DEBUG("archive.journal", "Reading header from archive");
+    iarchive >> *this;
+
+    DEBUG("archive.journal",
+	  "Version number:    " << std::hex << version << std::dec);
+    DEBUG("archive.journal", "Number of sources: " << sources.size());
+
+    foreach (const journal_t::fileinfo_t& i, sources)
+      DEBUG("archive.journal", "Loaded source: " << *i.filename);
+  }
+}
+
+bool archive_t::should_load(const std::list<path>& data_files)
+{
+  std::size_t found = 0;
+
+  DEBUG("archive.journal", "Should the archive be loaded?");
+
+  if (! exists(file)) {
+    DEBUG("archive.journal", "No, it does not exist");
+    return false;
+  }
+
+  if (version != ARCHIVE_VERSION) {
+    DEBUG("archive.journal", "No, it fails the version check");
+    return false;
+  }
+
+  if (data_files.empty()) {
+    DEBUG("archive.journal", "No, there were no data files!");
+    return false;
+  }
+
+  if (sources.empty()) {
+    DEBUG("archive.journal", "No, there were no sources!");
+    return false;
+  }
+
+  if (data_files.size() != sources.size()) {
+    DEBUG("archive.journal", "No, number of sources doesn't match: "
+	  << data_files.size() << " != " << sources.size());
+    return false;
+  }
+
+  foreach (const path& p, data_files) {
+    DEBUG("archive.journal", "Scanning for data file: " << p);
+
+    if (! exists(p)) {
+      DEBUG("archive.journal", "No, an input source no longer exists: " << p);
+      return false;
+    }
+
+    foreach (const journal_t::fileinfo_t& i, sources) {
+      assert(! i.from_stream);
+      assert(i.filename);
+
+      DEBUG("archive.journal", "Comparing against source file: " << *i.filename);
+
+      if (*i.filename == p) {
+	if (! exists(*i.filename)) {
+	  DEBUG("archive.journal",
+		"No, a referent source no longer exists: " << *i.filename);
+	  return false;
+	}
+
+	if (i.modtime != posix_time::from_time_t(last_write_time(p))) {
+	  DEBUG("archive.journal", "No, a source's modtime has changed: " << p);
+	  return false;
+	}
+
+	if (i.size != file_size(p)) {
+	  DEBUG("archive.journal", "No, a source's size has changed: " << p);
+	  return false;
+	}
+
+	found++;
+      }
+    }
+  }
+
+  if (found != data_files.size()) {
+    DEBUG("archive.journal", "No, not every source's name matched");
+    return false;
+  }
+
+  DEBUG("archive.journal", "Yes, it should be loaded!");
+  return true;
+}
+
+bool archive_t::should_save(shared_ptr<journal_t> journal)
+{
+  std::list<path> data_files;
+
+  DEBUG("archive.journal", "Should the archive be saved?");
+
+  if (journal->was_loaded) {
+    DEBUG("archive.journal", "No, it's one we loaded before");
+    return false;
+  }
+
+  if (journal->sources.empty()) {
+    DEBUG("archive.journal", "No, there were no sources!");
+    return false;
+  }
+
+  foreach (const journal_t::fileinfo_t& i, journal->sources) {
+    if (i.from_stream) {
+      DEBUG("archive.journal", "No, one source was from a stream");
+      return false;
+    }
+
+    if (! exists(*i.filename)) {
+      DEBUG("archive.journal",
+	    "No, a source no longer exists: " << *i.filename);
+      return false;
+    }
+
+    data_files.push_back(*i.filename);
+  }
+
+  if (should_load(data_files)) {
+    DEBUG("archive.journal", "No, because it's still loadable");
+    return false;
+  }
+
+  DEBUG("archive.journal", "Yes, it should be saved!");
+  return true;
+}
+
+void archive_t::save(shared_ptr<journal_t> journal)
+{
+  INFO_START(archive, "Saved journal file cache");
+
+  ofstream archive(file, std::ios::binary);
+  boost::archive::binary_oarchive oa(archive);
+
+  version = ARCHIVE_VERSION;
+  sources = journal->sources;
+
+  foreach (const journal_t::fileinfo_t& i, sources)
+    DEBUG("archive.journal", "Saving source: " << *i.filename);
+
+  DEBUG("archive.journal",
+	"Creating archive with version " << std::hex << version << std::dec);
+  oa << *this;
+
+  DEBUG("archive.journal",
+	"Archiving journal with " << sources.size() << " sources");
+  oa << *journal;
+
+  INFO_FINISH(archive);
+}
+
+bool archive_t::load(shared_ptr<journal_t> journal)
+{
+  INFO_START(archive, "Read cached journal file");
+
+  ifstream stream(file, std::ios::binary);
+  boost::archive::binary_iarchive iarchive(stream);
+
+  // Skip past the archive header, it was already read in before
+  archive_t temp;
+  iarchive >> temp;
+
+  iarchive >> *journal.get();
+  journal->was_loaded = true;
+
+  INFO_FINISH(archive);
+
+  return true;
+}
+
+} // namespace ledger
+
+#endif // HAVE_BOOST_SERIALIZATION
diff --git a/src/archive.h b/src/archive.h
new file mode 100644
index 00000000..77272dbe
--- /dev/null
+++ b/src/archive.h
@@ -0,0 +1,105 @@
+/*
+ * Copyright (c) 2003-2009, John Wiegley.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above copyright
+ *   notice, this list of conditions and the following disclaimer in the
+ *   documentation and/or other materials provided with the distribution.
+ *
+ * - Neither the name of New Artisans LLC nor the names of its
+ *   contributors may be used to endorse or promote products derived from
+ *   this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * @defgroup report Reporting
+ */
+
+/**
+ * @file   archive.h
+ * @author John Wiegley
+ *
+ * @ingroup report
+ *
+ * @brief Brief
+ *
+ * Long.
+ */
+#ifndef _ARCHIVE_H
+#define _ARCHIVE_H
+
+#include "journal.h"
+
+namespace ledger {
+
+/**
+ * @brief Brief
+ *
+ * Long.
+ */
+class archive_t
+{
+  path	   file;
+  uint32_t version;
+
+  std::list<journal_t::fileinfo_t> sources;
+
+public:
+  archive_t() {
+    TRACE_CTOR(archive_t, "");
+  }
+  archive_t(const path& _file)
+    : file(_file), version(0) {
+    TRACE_CTOR(archive_t, "const path&");
+  }
+  archive_t(const archive_t& ar)
+    : file(ar.file), version(0) {
+    TRACE_CTOR(archive_t, "copy");
+  }
+  ~archive_t() {
+    TRACE_DTOR(archive_t);
+  }
+
+  void read_header();
+
+  bool should_load(const std::list<path>& data_files);
+  bool should_save(shared_ptr<journal_t> journal);
+
+  void save(shared_ptr<journal_t> journal);
+  bool load(shared_ptr<journal_t> journal);
+
+#if defined(HAVE_BOOST_SERIALIZATION)
+private:
+  /** Serialization. */
+
+  friend class boost::serialization::access;
+
+  template<class Archive>
+  void serialize(Archive & ar, const unsigned int /* version */) {
+    ar & version;
+    ar & sources;
+  }
+#endif // HAVE_BOOST_SERIALIZATION
+};
+
+} // namespace ledger
+
+#endif // _ARCHIVE_H
diff --git a/src/journal.cc b/src/journal.cc
index d73d0fb8..7dbc2907 100644
--- a/src/journal.cc
+++ b/src/journal.cc
@@ -41,7 +41,7 @@
 namespace ledger {
 
 journal_t::journal_t()
-  : master(new account_t),
+  : master(new account_t), was_loaded(false),
     commodity_pool(new commodity_pool_t)
 {
   TRACE_CTOR(journal_t, "");
diff --git a/src/journal.h b/src/journal.h
index 4ad10fd3..03a465f8 100644
--- a/src/journal.h
+++ b/src/journal.h
@@ -48,11 +48,10 @@
 
 #include "utils.h"
 #include "hooks.h"
+#include "times.h"
 
 namespace ledger {
 
-typedef std::list<path>	paths_list;
-
 class commodity_pool_t;
 class xact_t;
 class auto_xact_t;
@@ -73,11 +72,55 @@ typedef std::list<period_xact_t *> period_xacts_list;
 class journal_t : public noncopyable
 {
 public:
-  account_t *	     master;
-  account_t *	     basket;
-  xacts_list	     xacts;
-  auto_xacts_list    auto_xacts;
-  period_xacts_list  period_xacts;
+  struct fileinfo_t
+  {
+    optional<path> filename;
+    std::size_t	   size;
+    datetime_t	   modtime;
+    bool	   from_stream;
+
+    fileinfo_t() : size(0), from_stream(true) {
+      TRACE_CTOR(journal_t::fileinfo_t, "");
+    }
+    fileinfo_t(const path& _filename)
+      : filename(_filename), from_stream(false) {
+      TRACE_CTOR(journal_t::fileinfo_t, "const path&");
+      size    = file_size(*filename);
+      modtime = posix_time::from_time_t(last_write_time(*filename));
+    }
+    fileinfo_t(const fileinfo_t& info)
+      : filename(info.filename), size(info.size),
+	modtime(info.modtime), from_stream(info.from_stream)
+    {
+      TRACE_CTOR(journal_t::fileinfo_t, "copy");
+    }
+    ~fileinfo_t() throw() {
+      TRACE_DTOR(journal_t::fileinfo_t);
+    }
+
+#if defined(HAVE_BOOST_SERIALIZATION)
+  private:
+    /** Serialization. */
+
+    friend class boost::serialization::access;
+
+    template<class Archive>
+    void serialize(Archive & ar, const unsigned int /* version */) {
+      ar & filename;
+      ar & size;
+      ar & modtime;
+      ar & from_stream;
+    }
+#endif // HAVE_BOOST_SERIALIZATION
+  };
+
+  account_t *		master;
+  account_t *		basket;
+  xacts_list		xacts;
+  auto_xacts_list	auto_xacts;
+  period_xacts_list	period_xacts;
+  std::list<fileinfo_t> sources;
+  bool                  was_loaded;
 
   shared_ptr<commodity_pool_t> commodity_pool;
   hooks_t<xact_finalizer_t, xact_t> xact_finalize_hooks;
@@ -123,6 +166,7 @@ private:
     ar & xacts;
     ar & auto_xacts;
     ar & period_xacts;
+    ar & sources;
   }
 #endif // HAVE_BOOST_SERIALIZATION
 };
diff --git a/src/session.cc b/src/session.cc
index ad735976..b7fdf275 100644
--- a/src/session.cc
+++ b/src/session.cc
@@ -37,7 +37,7 @@
 #include "journal.h"
 #include "iterators.h"
 #include "filters.h"
-#include "pstream.h"
+#include "archive.h"
 
 namespace ledger {
 
@@ -105,6 +105,9 @@ std::size_t session_t::read_journal(const path& pathname,
 
 std::size_t session_t::read_data(const string& master_account)
 {
+  bool populated_data_files = false;
+  bool populated_price_db   = false;
+
   if (HANDLER(file_).data_files.empty()) {
     path file;
     if (const char * home_var = std::getenv("HOME"))
@@ -114,6 +117,8 @@ std::size_t session_t::read_data(const string& master_account)
       HANDLER(file_).data_files.push_back(file);
     else
       throw_(parse_error, "No journal file was specified (please use -f)");
+
+    populated_data_files = true;
   }
 
   std::size_t xact_count = 0;
@@ -122,43 +127,75 @@ std::size_t session_t::read_data(const string& master_account)
   if (! master_account.empty())
     acct = journal->find_account(master_account);
 
-  if (HANDLED(price_db_)) {
-    path price_db_path = resolve_path(HANDLER(price_db_).str());
-    if (exists(price_db_path) && read_journal(price_db_path) > 0)
-	throw_(parse_error, _("Transactions not allowed in price history file"));
-  }
+  optional<path> price_db_path;
+  if (HANDLED(price_db_))
+    price_db_path = resolve_path(HANDLER(price_db_).str());
 
-  foreach (const path& pathname, HANDLER(file_).data_files) {
-    path filename = resolve_path(pathname);
-    if (filename == "-") {
-      // To avoid problems with stdin and pipes, etc., we read the entire
-      // file in beforehand into a memory buffer, and then parcel it out
-      // from there.
-      std::ostringstream buffer;
-
-      while (std::cin.good() && ! std::cin.eof()) {
-	char line[8192];
-	std::cin.read(line, 8192);
-	std::streamsize count = std::cin.gcount();
-	buffer.write(line, count);
-      }
-      buffer.flush();
+  optional<archive_t> cache;
+  if (HANDLED(cache_) && master_account.empty()) {
+    cache = archive_t(HANDLED(cache_).str());
+    cache->read_header();
 
-      std::istringstream buf_in(buffer.str());
-
-      xact_count += read_journal(buf_in, "/dev/stdin", acct);
+    if (price_db_path) {
+      HANDLER(file_).data_files.push_back(*price_db_path);
+      populated_price_db = true;
     }
-    else if (exists(filename)) {
-      xact_count += read_journal(filename, acct);
+  }
+
+  if (! (cache &&
+	 cache->should_load(HANDLER(file_).data_files) &&
+	 cache->load(journal))) {
+    if (price_db_path) {
+      if (exists(*price_db_path) && read_journal(*price_db_path) > 0)
+	throw_(parse_error, _("Transactions not allowed in price history file"));
+      journal->sources.push_back(journal_t::fileinfo_t(*price_db_path));
+      HANDLER(file_).data_files.remove(*price_db_path);
     }
-    else {
-      throw_(parse_error, _("Could not read journal file '%1'") << filename);
+
+    foreach (const path& pathname, HANDLER(file_).data_files) {
+      path filename = resolve_path(pathname);
+      if (filename == "-") {
+	// To avoid problems with stdin and pipes, etc., we read the entire
+	// file in beforehand into a memory buffer, and then parcel it out
+	// from there.
+	std::ostringstream buffer;
+
+	while (std::cin.good() && ! std::cin.eof()) {
+	  char line[8192];
+	  std::cin.read(line, 8192);
+	  std::streamsize count = std::cin.gcount();
+	  buffer.write(line, count);
+	}
+	buffer.flush();
+
+	std::istringstream buf_in(buffer.str());
+
+	xact_count += read_journal(buf_in, "/dev/stdin", acct);
+	journal->sources.push_back(journal_t::fileinfo_t());
+      }
+      else if (exists(filename)) {
+	xact_count += read_journal(filename, acct);
+	journal->sources.push_back(journal_t::fileinfo_t(filename));
+      }
+      else {
+	throw_(parse_error, _("Could not read journal file '%1'") << filename);
+      }
     }
+
+    assert(xact_count == journal->xacts.size());
+
+    if (cache && cache->should_save(journal))
+      cache->save(journal);
   }
 
+  if (populated_data_files)
+    HANDLER(file_).data_files.clear();
+  else if (populated_price_db)
+    HANDLER(file_).data_files.remove(*price_db_path);
+
   VERIFY(journal->valid());
 
-  return xact_count;
+  return journal->xacts.size();
 }
 
 void session_t::read_journal_files()
@@ -219,6 +256,9 @@ option_t<session_t> * session_t::lookup_option(const char * p)
   case 'a':
     OPT_(account_); // -a
     break;
+  case 'c':
+    OPT(cache_);
+    break;
   case 'd':
     OPT(download); // -Q
     break;
diff --git a/src/session.h b/src/session.h
index 48960745..c1e0243b 100644
--- a/src/session.h
+++ b/src/session.h
@@ -103,6 +103,7 @@ public:
   void report_options(std::ostream& out)
   {
     HANDLER(account_).report(out);
+    HANDLER(cache_).report(out);
     HANDLER(download).report(out);
     HANDLER(file_).report(out);
     HANDLER(input_date_format_).report(out);
@@ -120,6 +121,7 @@ public:
    */
 
   OPTION(session_t, account_); // -a
+  OPTION(session_t, cache_);
   OPTION(session_t, download); // -Q
 
   OPTION__
diff --git a/src/system.hh.in b/src/system.hh.in
index 8495600e..4a7dc55f 100644
--- a/src/system.hh.in
+++ b/src/system.hh.in
@@ -189,20 +189,14 @@ typedef std::ostream::pos_type ostream_pos_type;
 #include <boost/date_time/posix_time/time_serialize.hpp>
 #include <boost/date_time/gregorian/greg_serialize.hpp>
 
-BOOST_CLASS_IMPLEMENTATION(boost::filesystem::path, boost::serialization::primitive_type)
-#ifndef BOOST_NO_STD_WSTRING
-BOOST_CLASS_IMPLEMENTATION(boost::filesystem::wpath, boost::serialization::primitive_type)
-#endif
-
 namespace boost {
 namespace serialization {
 
-template <class Archive, class String, class Traits>
-void serialize(Archive& ar, boost::filesystem::basic_path<String, Traits>& p,
-	       const unsigned int)
+template <class Archive>
+void serialize(Archive& ar, boost::filesystem::path& p, const unsigned int)
 {
-  String s; 
- if (Archive::is_saving::value)
+  std::string s; 
+  if (Archive::is_saving::value)
     s = p.string();
 
   ar & s;
diff --git a/tools/Makefile.am b/tools/Makefile.am
index d4494fa7..13ea4e7f 100644
--- a/tools/Makefile.am
+++ b/tools/Makefile.am
@@ -59,6 +59,7 @@ libledger_data_la_SOURCES  =			\
 	src/timelog.cc				\
 	src/textual.cc				\
 	src/journal.cc				\
+	src/archive.cc				\
 	src/account.cc				\
 	src/xact.cc				\
 	src/post.cc				\
@@ -118,6 +119,7 @@ pkginclude_HEADERS =				\
 	src/xact.h				\
 	src/account.h				\
 	src/journal.h				\
+	src/archive.h				\
 	src/timelog.h				\
 	src/iterators.h				\
 	src/compare.h				\