Added some preliminary code for convert CSV to Ledger

7 files changed, 449 insertions, 2 deletions

diff --git a/src/convert.cc b/src/convert.cc
new file mode 100644
index 00000000..e47fb6c0
--- /dev/null
+++ b/src/convert.cc
@@ -0,0 +1,136 @@
+/*
+ * Copyright (c) 2003-2010, John Wiegley.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above copyright
+ *   notice, this list of conditions and the following disclaimer in the
+ *   documentation and/or other materials provided with the distribution.
+ *
+ * - Neither the name of New Artisans LLC nor the names of its
+ *   contributors may be used to endorse or promote products derived from
+ *   this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <system.hh>
+
+#include "convert.h"
+#include "csv.h"
+#include "scope.h"
+#include "interactive.h"
+#include "iterators.h"
+#include "report.h"
+#include "xact.h"
+#include "output.h"
+
+namespace ledger {
+
+value_t convert_command(call_scope_t& scope)
+{
+  interactive_t args(scope, "s");
+  report_t&	report(find_scope<report_t>(scope));
+  journal_t&	journal(*report.session.journal.get());
+
+  string bucket_name;
+  if (report.HANDLED(account_))
+    bucket_name = report.HANDLER(account_).str();
+  else
+    bucket_name = "Equity:Unknown";
+
+  account_t * bucket = journal.master->find_account(bucket_name);
+
+  // Make an amounts mapping for the account under consideration
+
+  typedef std::map<value_t, std::list<post_t *> > post_map_t;
+  post_map_t post_map;
+
+  xacts_iterator journal_iter(journal);
+  while (xact_t * xact = journal_iter()) {
+    post_t * post = NULL;
+    xact_posts_iterator xact_iter(*xact);
+    while ((post = xact_iter()) != NULL) {
+      if (post->account == bucket)
+	break;
+    }
+    if (post) {
+      post_map_t::iterator i = post_map.find(post->amount);
+      if (i == post_map.end()) {
+	std::list<post_t *> post_list;
+	post_list.push_back(post);
+	post_map.insert(post_map_t::value_type(post->amount, post_list));
+      } else {
+	(*i).second.push_back(post);
+      }
+    }
+  }
+
+  // Read in the series of transactions from the CSV file
+
+  format_posts formatter(report,
+			 report.report_format(report.HANDLER(print_format_)),
+			 false);
+
+  ifstream   data(path(args.get<string>(0)));
+  csv_reader reader(data);
+  while (xact_t * xact = reader.read_xact(journal, bucket)) {
+    bool matched = false;
+    post_map_t::iterator i = post_map.find(- xact->posts.front()->amount);
+    if (i != post_map.end()) {
+      std::list<post_t *>& post_list((*i).second);
+      foreach (post_t * post, post_list) {
+	if (xact->code && post->xact->code &&
+	    *xact->code == *post->xact->code) {
+	  matched = true;
+	  break;
+	}
+	else if (xact->actual_date() == post->actual_date()) {
+	  matched = true;
+	  break;
+	}
+      }
+    }
+
+    if (matched) {
+      DEBUG("convert.csv", "Ignored xact with code: " << *xact->code);
+      delete xact;		// ignore it
+    }
+    else if (! journal.add_xact(xact)) {
+      delete xact;
+      throw_(std::runtime_error,
+	     _("Failed to finalize derived transaction (check commodities)"));
+    }
+    else {
+      xact_posts_iterator xact_iter(*xact);
+      while (post_t * post = xact_iter())
+	formatter(*post);
+    }
+  }
+
+  // If not, transform the payee according to regexps
+
+  // Set the account to a default vaule, then transform the account according
+  // to the payee
+
+  // Print out the final form of the transaction
+
+  return true;
+}
+
+} // namespace ledger
diff --git a/src/convert.h b/src/convert.h
new file mode 100644
index 00000000..6d02f24a
--- /dev/null
+++ b/src/convert.h
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2003-2010, John Wiegley.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above copyright
+ *   notice, this list of conditions and the following disclaimer in the
+ *   documentation and/or other materials provided with the distribution.
+ *
+ * - Neither the name of New Artisans LLC nor the names of its
+ *   contributors may be used to endorse or promote products derived from
+ *   this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * @addtogroup data
+ */
+
+/**
+ * @file   convert.h
+ * @author John Wiegley
+ *
+ * @ingroup data
+ */
+#ifndef _CONVERT_H
+#define _CONVERT_H
+
+#include "value.h"
+
+namespace ledger {
+
+class call_scope_t;
+
+value_t convert_command(call_scope_t& scope);
+
+} // namespace ledger
+
+#endif // _CONVERT_H
diff --git a/src/csv.cc b/src/csv.cc
new file mode 100644
index 00000000..c0f8cd0e
--- /dev/null
+++ b/src/csv.cc
@@ -0,0 +1,181 @@
+/*
+ * Copyright (c) 2003-2010, John Wiegley.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above copyright
+ *   notice, this list of conditions and the following disclaimer in the
+ *   documentation and/or other materials provided with the distribution.
+ *
+ * - Neither the name of New Artisans LLC nor the names of its
+ *   contributors may be used to endorse or promote products derived from
+ *   this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <system.hh>
+
+#include "csv.h"
+#include "xact.h"
+#include "post.h"
+#include "account.h"
+#include "journal.h"
+#include "pool.h"
+
+namespace ledger {
+
+string csv_reader::read_field()
+{
+  string field;
+
+  char c;
+  if (in.peek() == '"' || in.peek() == '|') {
+    in.get(c);
+    char x;
+    while (in.good() && ! in.eof()) {
+      in.get(x);
+      if (x == '\\') {
+	in.get(x);
+      }
+      else if (x == c) {
+	if (x == '|')
+	  in.unget();
+	else if (in.peek() == ',')
+	  in.get(c);
+	break;
+      }
+      field += x;
+    }
+  }
+  else {
+    
+  }
+  return field;
+}
+
+xact_t * csv_reader::read_xact(journal_t& journal, account_t * bucket)
+{
+  static char linebuf[MAX_LINE + 1];
+
+  if (! in.good() || in.eof())
+    return NULL;
+
+  std::auto_ptr<xact_t> xact;
+
+  while (in.good() && ! in.eof() && in.peek() == '#')
+    in.getline(linebuf, MAX_LINE);
+
+  xact.reset(new xact_t);
+
+  xact->pos	      = position_t();
+  xact->pos->pathname = "jww (2010-03-05): unknown";
+  xact->pos->beg_pos  = in.tellg();
+  xact->pos->beg_line = 0;
+  xact->pos->sequence = 0;
+
+  string date	= read_field(); trim(date);
+  string code	= read_field(); trim(code);
+  string payee	= read_field(); trim(payee);
+
+  if (date.empty())
+    return NULL;
+
+  xact->set_state(item_t::CLEARED);
+  xact->_date = parse_date(date);
+  if (! code.empty())
+    xact->code  = code;
+
+  bool found = false;
+  foreach (payee_mapping_t& value, journal.payee_mappings) {
+    DEBUG("csv.mappings", "Looking for payee mapping: " << value.first);
+    if (value.first.match(payee)) {
+      xact->payee = value.second;
+      found = true;
+      break;
+    }
+  }
+  if (! found)
+    xact->payee = payee;
+
+  string amount = read_field(); trim(amount);
+  string total	= read_field(); trim(total);
+  in.getline(linebuf, MAX_LINE); // skip to the next line
+
+  std::auto_ptr<post_t> post(new post_t);
+
+  post->xact = xact.get();
+
+#if 0
+  post->pos	      = position_t();
+  post->pos->pathname = pathname;
+  post->pos->beg_pos  = line_beg_pos;
+  post->pos->beg_line = linenum;
+  post->pos->sequence = context.sequence++;
+#endif
+
+  post->set_state(item_t::CLEARED);
+  post->account = journal.master->find_account(_("Expenses:Unknown"));
+
+  foreach (account_mapping_t& value, journal.account_mappings) {
+    if (value.first.match(xact->payee)) {
+      post->account = value.second;
+      break;
+    }
+  }
+
+  std::istringstream amount_str(amount);
+  amount_t amt;
+  amt.parse(amount_str, PARSE_NO_REDUCE);
+  if (! amt.has_commodity() &&
+      commodity_pool_t::current_pool->default_commodity)
+    amt.set_commodity
+      (*commodity_pool_t::current_pool->default_commodity);
+  post->amount = amt;
+
+  xact->add_post(post.release());
+
+  post.reset(new post_t);
+
+  post->xact = xact.get();
+
+#if 0
+  post->pos	      = position_t();
+  post->pos->pathname = pathname;
+  post->pos->beg_pos  = line_beg_pos;
+  post->pos->beg_line = linenum;
+  post->pos->sequence = context.sequence++;
+#endif
+
+  post->set_state(item_t::CLEARED);
+  post->account = bucket;
+  post->amount	= - amt;
+
+  if (! total.empty()) {
+    std::istringstream assigned_amount_str(total);
+    amount_t assigned_amount;
+    assigned_amount.parse(assigned_amount_str, PARSE_NO_REDUCE);
+    post->assigned_amount = assigned_amount;
+  }
+  
+  xact->add_post(post.release());
+
+  return xact.release();
+}
+
+} // namespace ledger
diff --git a/src/csv.h b/src/csv.h
new file mode 100644
index 00000000..7029d482
--- /dev/null
+++ b/src/csv.h
@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2003-2010, John Wiegley.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above copyright
+ *   notice, this list of conditions and the following disclaimer in the
+ *   documentation and/or other materials provided with the distribution.
+ *
+ * - Neither the name of New Artisans LLC nor the names of its
+ *   contributors may be used to endorse or promote products derived from
+ *   this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * @addtogroup data
+ */
+
+/**
+ * @file   csv.h
+ * @author John Wiegley
+ *
+ * @ingroup data
+ */
+#ifndef _CSV_H
+#define _CSV_H
+
+#include "value.h"
+
+namespace ledger {
+
+class xact_t;
+class journal_t;
+class account_t;
+
+class csv_reader
+{
+  static const std::size_t MAX_LINE = 1024;
+
+  std::istream& in;
+
+public:
+  csv_reader(std::istream& _in) : in(_in) {}
+
+  string read_field();
+
+  xact_t * read_xact(journal_t& journal, account_t * bucket);
+};
+
+} // namespace ledger
+
+#endif // _CSV_H
diff --git a/src/report.cc b/src/report.cc
index f4ebfb72..d3e24379 100644
--- a/src/report.cc
+++ b/src/report.cc
@@ -43,6 +43,7 @@
 #include "stats.h"
 #include "generate.h"
 #include "draft.h"
+#include "convert.h"
 #include "xml.h"
 #include "emacs.h"
 
@@ -1206,6 +1207,8 @@ expr_t::ptr_op_t report_t::lookup(const symbol_t::kind_t kind,
 				maybe_format(HANDLER(prepend_format_))),
 	    *this, "#cleared"));
       }
+      else if (is_eq(p, "convert"))
+	return WRAP_FUNCTOR(convert_command);
       break;
 
     case 'e':
diff --git a/src/system.hh.in b/src/system.hh.in
index d68ac1a5..5de7150e 100644
--- a/src/system.hh.in
+++ b/src/system.hh.in
@@ -136,8 +136,7 @@ typedef std::ostream::pos_type ostream_pos_type;
 #include <editline/readline.h>
 #endif
 
-#include <boost/algorithm/string/classification.hpp>
-#include <boost/algorithm/string/predicate.hpp>
+#include <boost/algorithm/string.hpp>
 #include <boost/bind.hpp>
 #include <boost/cast.hpp>
 #include <boost/current_function.hpp>
diff --git a/tools/Makefile.am b/tools/Makefile.am
index 53caf565..88d84231 100644
--- a/tools/Makefile.am
+++ b/tools/Makefile.am
@@ -72,6 +72,8 @@ libledger_data_la_LDFLAGS    = -release $(VERSION)
 libledger_report_la_SOURCES  =			\
 	src/stats.cc				\
 	src/generate.cc				\
+	src/csv.cc				\
+	src/convert.cc				\
 	src/draft.cc				\
 	src/emacs.cc                            \
 	src/xml.cc                              \
@@ -133,6 +135,8 @@ pkginclude_HEADERS =				\
 	src/temps.h				\
 	src/chain.h				\
 	src/precmd.h				\
+	src/csv.h				\
+	src/convert.h				\
 	src/draft.h				\
 	src/generate.h				\
 	src/stats.h				\